From 84125af4cd99d960f274522a04faeffe9b9be3a6 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Fri, 23 Oct 2026 22:04:20 -0400 Subject: [PATCH 001/103] Share pure directory records and canonical coordination schemas. Keep legacy lease decisions and JSON behavior intact while making the shared records usable without HTTP or SQLite. Define a strict typed v2 boundary with exact uint64 and sealed-byte round trips across Go and JavaScript before consensus code depends on it. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterstate/README.md | 55 +++ internal/clusterstate/codec.go | 343 ++++++++++++++++++ internal/clusterstate/codec_test.go | 238 ++++++++++++ internal/clusterstate/imports_test.go | 39 ++ internal/clusterstate/records.go | 127 +++++++ internal/clusterstate/rules.go | 143 ++++++++ internal/clusterstate/rules_test.go | 108 ++++++ internal/clusterstate/testdata/canonical.json | 27 ++ internal/clusterstate/types.go | 167 +++++++++ internal/directory/client.go | 25 +- internal/directory/clusterstate_test.go | 31 ++ internal/directory/feed.go | 38 +- internal/directory/handler.go | 17 +- internal/directory/records.go | 128 +------ internal/directory/requests.go | 15 +- internal/directory/rotation.go | 10 +- internal/directory/rules.go | 126 +------ relay/hosted/src/raft/codec.js | 57 +++ relay/hosted/test/raft-codec.test.js | 46 +++ 19 files changed, 1456 insertions(+), 284 deletions(-) create mode 100644 internal/clusterstate/README.md create mode 100644 internal/clusterstate/codec.go create mode 100644 internal/clusterstate/codec_test.go create mode 100644 internal/clusterstate/imports_test.go create mode 100644 internal/clusterstate/records.go create mode 100644 internal/clusterstate/rules.go create mode 100644 internal/clusterstate/rules_test.go create mode 100644 internal/clusterstate/testdata/canonical.json create mode 100644 internal/clusterstate/types.go create mode 100644 internal/directory/clusterstate_test.go create mode 100644 relay/hosted/src/raft/codec.js create mode 100644 relay/hosted/test/raft-codec.test.js diff --git a/internal/clusterstate/README.md b/internal/clusterstate/README.md new file mode 100644 index 0000000000..1950d9660c --- /dev/null +++ b/internal/clusterstate/README.md @@ -0,0 +1,55 @@ +# Pure coordination boundary + +`clusterstate` is shared by native members and the future WASI reactor. It has +no directory, HTTP, database, identity-key, wall-clock or random-source imports. +`directory` keeps aliases to its records/errors and forwards its existing rules. +The old v1 JSON representation and pure rule behavior remain unchanged. + +P01 defines schema 1 for commands, results and individual application rows. This +is internal development scaffolding, not a production routing change. Later +packets add their typed operation/row variants before the single feature ships. +An older member must reject unknown versions and variants and stop applying; +it cannot turn an unknown committed entry into a successful empty operation. + +`EncodeCommand`/`DecodeCommand`, `EncodeResult`/`DecodeResult`, and +`EncodeRow`/`DecodeRow` validate their schema and closed unions. The underlying +`Encode`/`Decode` handle canonical typed values; they do not authorize an actor, +verify a signature, validate upload receipts, implement deduplication, or apply a +mutation. The host still authenticates the original signed request, and apply +must check committed enrollment, revocation, rotation, dependencies and scope. +`RequestDigest` and `Signature` are original authentication/session inputs; P07 +and P09 supply their cryptographic and deduplication validation. + +The v2 canonical encoding has these rules: + +- Every Go `uint64`, including zero, becomes an unsigned decimal string without + leading zeros. Values through `18446744073709551615` round trip exactly. +- Other integers remain JSON integers within ±(2^53−1). Millisecond times are + integers, never floats. V1 fence admission must retain its 2^53−1 ceiling; + the directory adapter must reject overflow before calling the legacy rule. +- Object keys sort by UTF-8 bytes, with no insignificant whitespace. Strings + use Go JSON escaping, including HTML and U+2028/U+2029. Invalid UTF-8 is refused. + Ciphertext strings and wrapped-key map values remain opaque strings even if + a map key happens to say `fence` or `size`. +- Byte slices use padded standard base64. Nil and empty values retain the + record's JSON semantics; fields marked `omitempty` are omitted when empty. +- Decode accepts exactly canonical bytes. Unknown, duplicate, missing required, + noncanonical and trailing fields/data are refused without changing the target. + API ingress must normalize external requests before signing/submitting them. + +The hosted codec is a serialization boundary only. It preserves decimal strings +and checks canonical syntax; typed schema validation remains in Go. The shared +`testdata/canonical.json` goldens are independently specified wire values, also +read by the targeted hosted codec test. The existing relay rule vectors are +consumed unchanged by the pure Go test and the hosted vector test. + +`Row` holds one record, so a host can supply bounded read-sets and stream sorted +rows at a fixed applied index. It is not a whole-identity snapshot format. P04/P05 +must atomically persist applied index, visible version, logical time, dedup and +membership metadata alongside rows. They must not use `Listing` JSON as a state +export: its visible version deliberately lives outside the v1 body. + +`raftsim/model` is an independent sequential ownership/vault skeleton. It imports +no production types or apply functions. P06 adds the Porcupine adapter, unknown +timeout outcomes and replay/history machinery; later assignment packets extend +the independent domain. P01 does not claim simulator or consensus proof. diff --git a/internal/clusterstate/codec.go b/internal/clusterstate/codec.go new file mode 100644 index 0000000000..2a2f42e502 --- /dev/null +++ b/internal/clusterstate/codec.go @@ -0,0 +1,343 @@ +package clusterstate + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "unicode/utf8" +) + +var ( + ErrSchema = errors.New("clusterstate: unsupported schema") + ErrCodec = errors.New("clusterstate: invalid canonical encoding") +) + +// Encode is the v2 canonical JSON boundary, separate from legacy json.Marshal. +// All uint64 fields are plain decimal strings, even below 2^53; other integers +// must fit JavaScript exactly. Maps sort by UTF-8 key bytes, sealed strings stay +// opaque, and byte slices use Go's padded standard base64. HTML and U+2028/2029 +// escaping follows encoding/json. Hosts must pass these bytes without rewriting. +func Encode(v any) ([]byte, error) { + b, err := json.Marshal(v) + if err != nil { + return nil, ErrCodec + } + if !validStrings(reflect.ValueOf(v)) { + return nil, ErrCodec + } + var tree any + d := json.NewDecoder(bytes.NewReader(b)) + d.UseNumber() + if err = d.Decode(&tree); err != nil { + return nil, ErrCodec + } + tree, err = integers(tree, reflect.TypeOf(v), true) + if err != nil { + return nil, err + } + return json.Marshal(tree) +} + +// JSON's replacement of invalid UTF-8 would alter an opaque field. Reject it +// before it can become a different canonical request or persisted record. +func validStrings(v reflect.Value) bool { + switch v.Kind() { + case reflect.Pointer, reflect.Interface: + return v.IsNil() || validStrings(v.Elem()) + case reflect.String: + return utf8.ValidString(v.String()) + case reflect.Struct: + for i := 0; i < v.NumField(); i++ { + if v.Type().Field(i).IsExported() && !validStrings(v.Field(i)) { + return false + } + } + case reflect.Map: + iter := v.MapRange() + for iter.Next() { + if !validStrings(iter.Key()) || !validStrings(iter.Value()) { + return false + } + } + case reflect.Slice, reflect.Array: + if v.Type().Elem().Kind() != reflect.Uint8 { + for i := 0; i < v.Len(); i++ { + if !validStrings(v.Index(i)) { + return false + } + } + } + } + return true +} + +// Decode accepts only canonical bytes into a typed destination. Re-encoding +// rejects duplicate/unknown/missing fields, alternate integer spellings, lossy +// Unicode, and trailing data. It updates the destination only after validation. +func Decode(b []byte, into any) error { + v := reflect.ValueOf(into) + if v.Kind() != reflect.Pointer || v.IsNil() { + return ErrCodec + } + d := json.NewDecoder(bytes.NewReader(b)) + d.UseNumber() + var tree any + if err := d.Decode(&tree); err != nil { + return ErrCodec + } + tree, err := integers(tree, v.Type().Elem(), false) + if err != nil { + return err + } + plain, err := json.Marshal(tree) + if err != nil { + return ErrCodec + } + candidate := reflect.New(v.Type().Elem()) + d = json.NewDecoder(bytes.NewReader(plain)) + d.DisallowUnknownFields() + if err = d.Decode(candidate.Interface()); err != nil { + return ErrCodec + } + canonical, err := Encode(candidate.Elem().Interface()) + if err != nil || !bytes.Equal(b, canonical) { + return ErrCodec + } + v.Elem().Set(candidate.Elem()) + return nil +} + +// integers uses the declared Go shape rather than field names: a sealed string +// named "fence" in a wrapped-key map must never be parsed as a counter. Floats, +// interfaces, custom marshalers and non-string map keys are outside the schema. +func integers(tree any, typ reflect.Type, encode bool) (any, error) { + if typ == nil { + return nil, ErrCodec + } + for typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + if tree == nil { + return nil, nil + } + switch typ.Kind() { + case reflect.Uint64: + if encode { + n, ok := tree.(json.Number) + if !ok { + return nil, ErrCodec + } + return string(n), nil + } + s, ok := tree.(string) + if !ok { + return nil, ErrCodec + } + n, err := strconv.ParseUint(s, 10, 64) + if err != nil || strconv.FormatUint(n, 10) != s { + return nil, ErrCodec + } + return json.Number(s), nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32: + n, ok := tree.(json.Number) + if !ok { + return nil, ErrCodec + } + i, err := strconv.ParseInt(string(n), 10, 64) + if err != nil || i < -int64(MaxSafeInteger) || i > int64(MaxSafeInteger) { + return nil, ErrCodec + } + case reflect.Struct: + m, ok := tree.(map[string]any) + if !ok { + return nil, ErrCodec + } + for i := 0; i < typ.NumField(); i++ { + f := typ.Field(i) + name := strings.Split(f.Tag.Get("json"), ",")[0] + if name == "-" || !f.IsExported() { + continue + } + if name == "" { + name = f.Name + } + if value, exists := m[name]; exists { + next, err := integers(value, f.Type, encode) + if err != nil { + return nil, err + } + m[name] = next + } + } + case reflect.Map: + m, ok := tree.(map[string]any) + if !ok || typ.Key().Kind() != reflect.String { + return nil, ErrCodec + } + for k, value := range m { + next, err := integers(value, typ.Elem(), encode) + if err != nil { + return nil, err + } + m[k] = next + } + case reflect.Slice, reflect.Array: + if typ.Kind() == reflect.Slice && typ.Elem().Kind() == reflect.Uint8 { + if _, ok := tree.(string); !ok { + return nil, ErrCodec + } + return tree, nil + } + list, ok := tree.([]any) + if !ok { + return nil, ErrCodec + } + for i, value := range list { + next, err := integers(value, typ.Elem(), encode) + if err != nil { + return nil, err + } + list[i] = next + } + case reflect.String: + if _, ok := tree.(string); !ok { + return nil, ErrCodec + } + case reflect.Bool: + if _, ok := tree.(bool); !ok { + return nil, ErrCodec + } + default: + return nil, ErrCodec + } + return tree, nil +} + +// EncodeCommand and DecodeCommand additionally check the versioned union. No +// unsupported operation can decode as an empty successful command. +func EncodeCommand(c Command) ([]byte, error) { + if err := c.Validate(); err != nil { + return nil, err + } + return Encode(c) +} + +func DecodeCommand(b []byte) (Command, error) { + var c Command + if err := Decode(b, &c); err != nil { + return Command{}, err + } + if err := c.Validate(); err != nil { + return Command{}, err + } + return c, nil +} + +func (c Command) Validate() error { + if c.Schema != SchemaVersion { + return ErrSchema + } + if c.Cluster == "" || c.Actor == "" || c.Session == "" || c.Sequence == 0 || c.RequestDigest == "" { + return ErrCodec + } + return oneOf(c.Change) +} + +// oneOf checks closed unions without a second hand-maintained operation table. +func oneOf(v any) error { + value := reflect.ValueOf(v) + n := 0 + for i := 0; i < value.NumField(); i++ { + if !value.Field(i).IsNil() { + n++ + } + } + if n != 1 { + return fmt.Errorf("%w: expected one record or operation", ErrCodec) + } + return nil +} + +func EncodeRow(r Row) ([]byte, error) { + if err := r.Validate(); err != nil { + return nil, err + } + return Encode(r) +} + +func DecodeRow(b []byte) (Row, error) { + var r Row + if err := Decode(b, &r); err != nil { + return Row{}, err + } + if err := r.Validate(); err != nil { + return Row{}, err + } + return r, nil +} + +func (r Row) Validate() error { + if r.Schema != SchemaVersion { + return ErrSchema + } + if r.Key == "" { + return ErrCodec + } + n := 0 + for _, present := range []bool{r.Identity != nil, r.Device != nil, r.Cell != nil} { + if present { + n++ + } + } + if n != 1 { + return ErrCodec + } + return nil +} + +func EncodeResult(r Result) ([]byte, error) { + if err := r.Validate(); err != nil { + return nil, err + } + return Encode(r) +} + +func DecodeResult(b []byte) (Result, error) { + var r Result + if err := Decode(b, &r); err != nil { + return Result{}, err + } + if err := r.Validate(); err != nil { + return Result{}, err + } + return r, nil +} + +func (r Result) Validate() error { + if r.Schema != SchemaVersion { + return ErrSchema + } + if !r.Code.valid() { + return ErrCodec + } + for _, item := range r.Items { + if !item.Code.valid() { + return ErrCodec + } + } + return nil +} + +func (c Code) valid() bool { + switch c { + case OK, NotFound, Exists, LeaseHeld, FenceStale, HeadMoved, CAS, Revoked, + SelfRevoke, Unauthorized, Rotated, Conflict, Capacity: + return true + } + return false +} diff --git a/internal/clusterstate/codec_test.go b/internal/clusterstate/codec_test.go new file mode 100644 index 0000000000..f9dd0230ee --- /dev/null +++ b/internal/clusterstate/codec_test.go @@ -0,0 +1,238 @@ +package clusterstate + +import ( + "bytes" + "encoding/json" + "errors" + "math" + "os" + "reflect" + "strings" + "testing" +) + +type golden struct { + Name string `json:"name"` + Kind string `json:"kind"` + Canonical string `json:"canonical"` +} + +func goldens(t *testing.T) []golden { + t.Helper() + b, err := os.ReadFile("testdata/canonical.json") + if err != nil { + t.Fatal(err) + } + var cases []golden + if err := json.Unmarshal(b, &cases); err != nil { + t.Fatal(err) + } + return cases +} + +func TestR14CanonicalGoldenVectors(t *testing.T) { + for _, tc := range goldens(t) { + t.Run(tc.Name, func(t *testing.T) { + var b []byte + var err error + switch tc.Kind { + case "command": + var c Command + c, err = DecodeCommand([]byte(tc.Canonical)) + if err == nil { + b, err = EncodeCommand(c) + } + case "row": + var r Row + r, err = DecodeRow([]byte(tc.Canonical)) + if err == nil { + b, err = EncodeRow(r) + } + case "result": + var r Result + r, err = DecodeResult([]byte(tc.Canonical)) + if err == nil { + b, err = EncodeResult(r) + } + default: + t.Fatalf("unknown fixture kind %s", tc.Kind) + } + if err != nil || string(b) != tc.Canonical { + t.Fatalf("round trip: %v\ngot %s\nwant %s", err, b, tc.Canonical) + } + }) + } +} + +func TestR14Uint64BoundariesAndLegacyJSONStayDistinct(t *testing.T) { + for _, n := range []uint64{0, 1, MaxSafeInteger, MaxSafeInteger + 1, math.MaxUint64} { + in := Lease{Fence: n} + b, err := Encode(in) + if err != nil { + t.Fatal(err) + } + var out Lease + if err := Decode(b, &out); err != nil || in != out { + t.Fatalf("%d: %s, %+v, %v", n, b, out, err) + } + legacy, _ := json.Marshal(in) + if bytes.Equal(legacy, b) || bytes.Contains(legacy, []byte(`"fence":"`)) { + t.Fatalf("v1 numeric JSON changed: %s", legacy) + } + } + for _, n := range []int64{-int64(MaxSafeInteger), int64(MaxSafeInteger)} { + in := CellView{Now: n} + b, err := Encode(in) + var out CellView + if err != nil || Decode(b, &out) != nil || out.Now != n { + t.Fatalf("safe timestamp %d: %s, %v", n, b, err) + } + } + if _, err := Encode(CellView{Now: int64(MaxSafeInteger) + 1}); !errors.Is(err, ErrCodec) { + t.Fatalf("rounded timestamp accepted: %v", err) + } +} + +func TestR14RejectAmbiguousCountersAndJSON(t *testing.T) { + base := goldens(t)[0].Canonical + for name, b := range map[string]string{ + "numeric uint64": strings.Replace(base, `"9007199254740992"`, `9007199254740992`, 1), + "overflow": strings.Replace(base, `"18446744073709551615"`, `"18446744073709551616"`, 1), + "leading zero": strings.Replace(base, `"9007199254740992"`, `"09007199254740992"`, 1), + "negative": strings.Replace(base, `"9007199254740992"`, `"-1"`, 1), + "plus": strings.Replace(base, `"9007199254740992"`, `"+1"`, 1), + "exponent": strings.Replace(base, `"9007199254740992"`, `"1e3"`, 1), + "fraction": strings.Replace(base, `1700000000000`, `1700000000000.0`, 1), + "unsafe time": strings.Replace(base, `1700000000000`, `9007199254740992`, 1), + "unknown": strings.Replace(base, `"actor":`, `"unknown":0,"actor":`, 1), + "duplicate": strings.Replace(base, `"actor":`, `"actor":"other","actor":`, 1), + "trailing": base + `{}`, + "whitespace": base + "\n", + "null scalar": strings.Replace(base, `"actor":"dev_a"`, `"actor":null`, 1), + "unpaired surrogate": strings.Replace(base, `"dev_a"`, `"\ud800"`, 1), + } { + t.Run(name, func(t *testing.T) { + if _, err := DecodeCommand([]byte(b)); err == nil { + t.Fatalf("accepted %s", b) + } + }) + } +} + +func TestR14UnknownSchemaAndOperationsStopDecoding(t *testing.T) { + base := goldens(t)[0].Canonical + if _, err := DecodeCommand([]byte(strings.Replace(base, `"schema":1`, `"schema":2`, 1))); !errors.Is(err, ErrSchema) { + t.Fatalf("unknown version: %v", err) + } + c, err := DecodeCommand([]byte(base)) + if err != nil { + t.Fatal(err) + } + c.Change.AdvanceTime = &AdvanceTime{} + if _, err := EncodeCommand(c); err == nil { + t.Fatal("two operations accepted") + } + c.Change = Mutation{} + if _, err := EncodeCommand(c); err == nil { + t.Fatal("no operation accepted") + } + unknown := strings.Replace(base, `"acquire":`, `"unknown_operation":`, 1) + if _, err := DecodeCommand([]byte(unknown)); err == nil { + t.Fatal("unknown operation accepted") + } +} + +func TestR14EveryInitialCommandVariantRoundTrips(t *testing.T) { + for _, change := range []Mutation{ + {Create: &CreateCell{Cell: "c", Init: CellInit{Head: "h", Frames: []string{"f"}}, Receipt: "r"}}, + {Acquire: &AcquireCell{Cell: "c", Expect: LeaseExpectation{Device: "a", Fence: 1}}}, + {Beat: &BeatCell{Cell: "c", Beat: Beat{Fence: 1, Pending: 2}}}, + {BeatBatch: &BeatBatch{Items: []BeatCell{{Cell: "c", Beat: Beat{Fence: 1}}, {Cell: "d", Beat: Beat{Fence: 2}}}}}, + {BeatBatch: &BeatBatch{}}, + {BeatBatch: &BeatBatch{Items: []BeatCell{}}}, + {Publish: &PublishCell{Cell: "c", Publish: Publish{Fence: 1, OldHead: "h", Head: "h2"}, Receipt: "r"}}, + {Release: &ReleaseCell{Cell: "c", Fence: 1, Quiet: true}}, + {Archive: &CellID{Cell: "c"}}, + {PutDevice: &PutDevice{ID: "d", Device: Device{Name: "sealed"}}}, + {Revoke: &DeviceID{Device: "d"}}, + {SetVault: &VaultCAS{Old: "v", New: "v2"}}, + {AdvanceTime: &AdvanceTime{}}, + {DeviceSeen: &DeviceSeen{Device: "d"}}, + } { + c := Command{Schema: SchemaVersion, Cluster: "cluster", Actor: "a", Session: "s", Sequence: 1, RequestDigest: "digest", Change: change} + b, err := EncodeCommand(c) + if err != nil { + t.Fatal(err) + } + out, err := DecodeCommand(b) + if err != nil || !reflect.DeepEqual(c, out) { + t.Fatalf("command round trip: %s, %v", b, err) + } + } +} + +func TestR15SealedBytesAndOpaqueMapKeysRemainExact(t *testing.T) { + if _, err := Encode(Cell{Title: string([]byte{0xff})}); !errors.Is(err, ErrCodec) { + t.Fatalf("lossy opaque string accepted: %v", err) + } + c, err := DecodeCommand([]byte(goldens(t)[0].Canonical)) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 256; i++ { + if len(c.Signature) != 256 || c.Signature[i] != byte(i) { + t.Fatal("binary payload changed") + } + } + r, err := DecodeRow([]byte(goldens(t)[1].Canonical)) + if err != nil { + t.Fatal(err) + } + if r.Cell.Size != math.MaxUint64 || r.Cell.Lease.Fence != MaxSafeInteger || r.Cell.Keys["key"]["fence"] != "9007199254740993" { + t.Fatalf("typed or opaque counter changed: %+v", r.Cell) + } + if r.Cell.Title != jsonBase64(t, c.Signature) || r.Cell.Keys["key"]["id"] != r.Cell.Title { + t.Fatal("sealed fields changed") + } +} + +func jsonBase64(t *testing.T, b []byte) string { + t.Helper() + encoded, err := json.Marshal(b) + if err != nil { + t.Fatal(err) + } + var s string + if err := json.Unmarshal(encoded, &s); err != nil { + t.Fatal(err) + } + return s +} + +func TestR13CanonicalRowsRetainAllApplicationFields(t *testing.T) { + for _, tc := range goldens(t) { + if tc.Kind != "row" { + continue + } + r, err := DecodeRow([]byte(tc.Canonical)) + if err != nil { + t.Fatal(err) + } + legacy, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + var restored Row + if err := json.Unmarshal(legacy, &restored); err != nil || !reflect.DeepEqual(r, restored) { + t.Fatalf("legacy record fields changed: %v", err) + } + } +} + +func TestR14DecodeFailureDoesNotChangeDestination(t *testing.T) { + in := Lease{Device: "keep", Fence: 5} + out := in + if err := Decode([]byte(`{"fence":"6","unknown":0}`), &out); err == nil || out != in { + t.Fatalf("invalid decode changed destination: %+v, %v", out, err) + } +} diff --git a/internal/clusterstate/imports_test.go b/internal/clusterstate/imports_test.go new file mode 100644 index 0000000000..baeb95a97d --- /dev/null +++ b/internal/clusterstate/imports_test.go @@ -0,0 +1,39 @@ +package clusterstate + +import ( + "go/parser" + "go/token" + "os" + "strconv" + "strings" + "testing" +) + +// R14 needs the same application package in the native and wasm builds. Keep +// its import boundary explicit so a storage/auth/HTTP helper cannot silently +// enter the pure graph when another packet adds a record or rule. +func TestR14PureImportBoundary(t *testing.T) { + allowed := map[string]bool{ + "bytes": true, "encoding/json": true, "errors": true, "fmt": true, + "reflect": true, "strconv": true, "strings": true, "unicode/utf8": true, + } + files, err := os.ReadDir(".") + if err != nil { + t.Fatal(err) + } + for _, file := range files { + if !strings.HasSuffix(file.Name(), ".go") || strings.HasSuffix(file.Name(), "_test.go") { + continue + } + parsed, err := parser.ParseFile(token.NewFileSet(), file.Name(), nil, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, imp := range parsed.Imports { + path, err := strconv.Unquote(imp.Path.Value) + if err != nil || !allowed[path] { + t.Errorf("%s imports %q outside the pure dependency boundary", file.Name(), path) + } + } + } +} diff --git a/internal/clusterstate/records.go b/internal/clusterstate/records.go new file mode 100644 index 0000000000..e079b1392f --- /dev/null +++ b/internal/clusterstate/records.go @@ -0,0 +1,127 @@ +// Package clusterstate holds the pure directory records, rules and versioned +// coordination encoding shared by native members and the wasm reactor. It has +// no storage, transport, clock or identity-key dependencies. +package clusterstate + +// IdentityRec is the per-identity root record. +type IdentityRec struct { + V uint16 `json:"V"` + Identity string `json:"identity"` // id_…, derived from the signing key + Vault string `json:"vault,omitempty"` // rid of the current vault object + // Rotation is set once the identity has been replaced by a rotation: it + // takes no writes from then on. Nil means live. + Rotation *Rotation `json:"rotation,omitempty"` +} + +// Caps says what a device can do. +type Caps struct { + OS string `json:"os"` // runtime.GOOS + Arch string `json:"arch"` // runtime.GOARCH + Sandbox *string `json:"sandbox"` // "seatbelt" | "landlock" | null + Container *string `json:"container"` // null in Stage 1 + GPU *string `json:"gpu"` // null in Stage 1 + Cow string `json:"cow"` // "apfs" | "btrfs" | "xfs-reflink" | "overlayfs" | "none" +} + +// Device is keyed by device id (dev_…). +type Device struct { + V uint16 `json:"V"` + Name string `json:"name"` // b64, sealed under the metadata key + AddedBy string `json:"added_by"` // identity id that signed the device cert + Revoked bool `json:"revoked"` // set only by Revoke; a fresh pairing makes a new device id + Caps Caps `json:"caps"` + + // Added by link pairing. Old records decode with zero values, and a client + // value for Created or LastSeen is ignored: the directory sets both. + Platform string `json:"platform,omitempty"` // darwin|linux|windows|ios|android|other + Created int64 `json:"created,omitempty"` // directory ms when the device joined + LastSeen int64 `json:"last_seen,omitempty"` // directory ms of the last watch socket close or hello +} + +// Lease says who may write a cell. Only the directory clock sets Expires. +type Lease struct { + Device string `json:"device"` // device id of the last holder, kept after release + Fence uint64 `json:"fence"` // +1 on every acquire, never decreases + Expires int64 `json:"expires"` // directory ms; held iff Expires > now; release sets 0 + Pending uint32 `json:"pending"` // turns sealed on the holder and not yet durable + // ReleasedAt is directory time of an explicit release, separate from the last turn. + ReleasedAt int64 `json:"released_at,omitempty"` +} + +// Cell is keyed by cell ULID. It exists only once it has a durable head. +type Cell struct { + V uint16 `json:"V"` + Head string `json:"head"` // last durable turn id (hex64) + DurableAt int64 `json:"durable_at"` // directory ms + Class string `json:"class"` + Size uint64 `json:"size"` + ParentCell string `json:"parent_cell,omitempty"` + Title string `json:"title,omitempty"` // b64, sealed + Keys map[string]map[string]string `json:"keys"` // cell_key_id -> identity -> wrapped + Lease Lease `json:"lease"` + OrphanTurns uint32 `json:"orphan_turns,omitempty"` // turns in a death branch + Archived bool `json:"archived,omitempty"` // a merged or discarded branch + Frames []string `json:"frames,omitempty"` // hex64 ids of the frames holding the head's closure; a hint, never a condition +} + +// Listing is everything a home list needs, in one read. +type Listing struct { + Now int64 `json:"now"` // directory ms when read + Identity IdentityRec `json:"identity"` + Devices map[string]Device `json:"devices"` + Cells map[string]Cell `json:"cells"` + // Version is the directory version the records were read at, which is + // sent as the VersionHeader and not in the body. + Version uint64 `json:"-"` +} + +// CellView is one cell with the directory time it was read at. +type CellView struct { + Now int64 `json:"now"` + Cell Cell `json:"cell"` +} + +// CellInit creates a cell. The calling device holds the new lease at fence 1. +type CellInit struct { + Head string `json:"head"` + Class string `json:"class"` + Title string `json:"title,omitempty"` + ParentCell string `json:"parent_cell,omitempty"` + Size uint64 `json:"size"` + Keys map[string]map[string]string `json:"keys"` + OrphanTurns uint32 `json:"orphan_turns,omitempty"` + Frames []string `json:"frames,omitempty"` +} + +// Beat renews a lease the caller holds. +type Beat struct { + Fence uint64 `json:"fence"` + Pending uint32 `json:"pending"` +} + +// Publish moves a cell's durable head under a lease the caller holds. +type Publish struct { + Fence uint64 `json:"fence"` + OldHead string `json:"old_head"` + Head string `json:"head"` + Size uint64 `json:"size"` + Class string `json:"class"` + Title string `json:"title,omitempty"` // empty keeps the current title + Pending uint32 `json:"pending"` // turns still not durable after this one + Frames []string `json:"frames,omitempty"` // empty or absent replaces the record's plan with none +} + +// Rotation is what an identity's record says about its replacement. +type Rotation struct { + State string `json:"state"` + By string `json:"by"` // the device that froze it; only it may retire + At int64 `json:"at"` // directory ms of the freeze + RetireAt int64 `json:"retire_at,omitempty"` +} + +// LeaseExpectation binds a continuation to the lease the taking device read, +// so another continuation during its fetch wins rather than being displaced. +type LeaseExpectation struct { + Device string `json:"device"` + Fence uint64 `json:"fence"` +} diff --git a/internal/clusterstate/rules.go b/internal/clusterstate/rules.go new file mode 100644 index 0000000000..d3e3163a22 --- /dev/null +++ b/internal/clusterstate/rules.go @@ -0,0 +1,143 @@ +package clusterstate + +// This file is the only place lease logic lives. Each rule takes a cell and +// returns the changed cell or an error, and never touches a clock or a store: +// the caller passes the directory time in and keeps the result. + +const ttlMs = LeaseTTLMillis + +// Lifted is c with its lease kept live until vouchedUntil, the time a watch +// socket's sign of life lasts (0 for none). A released lease (expires 0) stays +// released, and a stored expiry already later is left alone. It is the one +// place evidence enters a rule: every reader of a lease asks Lifted first and +// then applies the pure rules to the answer. +func Lifted(c Cell, vouchedUntil int64) Cell { + if c.Lease.Expires != 0 && vouchedUntil > c.Lease.Expires { + c.Lease.Expires = vouchedUntil + } + return c +} + +// Acquire takes the lease for device unless another device still holds it. The +// same device may re-take its own live lease; the fence still goes up. A forced +// acquire is a person's choice to displace a live holder: it skips the refusal, +// and the fence still goes up, so the old holder's next write fails as stale. +func Acquire(c Cell, device string, now int64, force bool, expected ...*LeaseExpectation) (Cell, error) { + if len(expected) > 0 && expected[0] != nil { + seen := expected[0] + if c.Lease.Device != seen.Device || c.Lease.Fence != seen.Fence { + return c, ErrLeaseHeld + } + } + if !force && c.Lease.Expires > now && c.Lease.Device != device { + return c, ErrLeaseHeld + } + c.Lease = Lease{Device: device, Fence: c.Lease.Fence + 1, Expires: now + ttlMs} + return c, nil +} + +// Heartbeat renews the lease. An expired lease nobody took may be renewed: the +// unchanged fence proves nobody did. +func Heartbeat(c Cell, device string, b Beat, now int64) (Cell, error) { + if err := holder(c, device, b.Fence); err != nil { + return c, err + } + c.Lease.Expires = now + ttlMs + c.Lease.Pending = b.Pending + return c, nil +} + +// PublishTo moves the durable head and renews the lease, because a publish is +// proof that the holder is alive. The fence is checked before the head, so a +// superseded holder always learns it lost the lease first. +func PublishTo(c Cell, device string, p Publish, now int64) (Cell, error) { + if err := holder(c, device, p.Fence); err != nil { + return c, err + } + if p.OldHead != c.Head { + return c, ErrHeadMoved + } + c.Head, c.Size, c.Class = p.Head, p.Size, p.Class + // The frames describe the head they were set with, so every publish replaces + // the list wholesale: an empty or absent list is a true statement ("the + // publisher did not say"), a stale one is not kept. + c.Frames = p.Frames + c.Lease.Pending, c.DurableAt = p.Pending, now + c.Lease.Expires = now + ttlMs + c.Title = keepIfEmpty(p.Title, c.Title) + return c, nil +} + +// ReleaseOf gives the lease up. The fence stays, so the next acquire still +// goes up by one. +func ReleaseOf(c Cell, device string, fence uint64, at ...int64) (Cell, error) { + if err := holder(c, device, fence); err != nil { + return c, err + } + c.Lease.Expires, c.Lease.Pending = 0, 0 + if len(at) > 0 { + c.Lease.ReleasedAt = at[0] + } + return c, nil +} + +// Created is the first record of a cell: the creating device holds the lease +// at fence 1. The caller has already checked that the id is free. +func Created(in CellInit, device string, now int64) Cell { + return Cell{ + V: 1, Head: in.Head, DurableAt: now, Class: in.Class, Size: in.Size, + ParentCell: in.ParentCell, Title: in.Title, Keys: in.Keys, OrphanTurns: in.OrphanTurns, + Frames: in.Frames, + Lease: Lease{Device: device, Fence: 1, Expires: now + ttlMs}, + } +} + +// holder says whether device still holds the lease at fence. +func holder(c Cell, device string, fence uint64) error { + if c.Lease.Fence != fence || c.Lease.Device != device { + return ErrFenceStale + } + return nil +} + +func keepIfEmpty(next, current string) string { + if next == "" { + return current + } + return next +} + +// RevokeOf marks target revoked on behalf of caller. A device cannot revoke +// itself: that is nearly always a slip, and a revoked device can no longer say +// so, so the mistake would be permanent. Revoking twice changes nothing. +func RevokeOf(target Device, caller, targetID string) (Device, error) { + if caller == targetID { + return target, ErrSelfRevoke + } + target.Revoked = true + return target, nil +} + +// Stamped is d as the directory stores it, given the record already stored +// under its id (had says whether there was one) and the directory time. The +// relay-owned fields are never taken from the client: Revoked and LastSeen stay +// as stored, and Created is the time of the first write. +func Stamped(old Device, had bool, d Device, now int64) Device { + d.Revoked, d.LastSeen, d.Created = old.Revoked, old.LastSeen, old.Created + if !had { + d.Created = now + } + d.Platform = PlatformOf(d.Platform) + return d +} + +// PlatformOf bounds advisory platform metadata without inventing a value for +// an old record that did not supply it. +func PlatformOf(p string) string { + switch p { + case "", "darwin", "linux", "windows", "ios", "android", "other": + return p + default: + return "other" + } +} diff --git a/internal/clusterstate/rules_test.go b/internal/clusterstate/rules_test.go new file mode 100644 index 0000000000..3608c58772 --- /dev/null +++ b/internal/clusterstate/rules_test.go @@ -0,0 +1,108 @@ +package clusterstate + +import ( + "encoding/json" + "os" + "reflect" + "testing" +) + +// These are the existing cross-relay contract vectors, consumed unchanged. +// R1/R2/R22/R25 require extraction to preserve their observable decisions. +func TestR25ExistingPureRuleVectorsRemainIdentical(t *testing.T) { + b, err := os.ReadFile("../../relay/hosted/vectors.json") + if err != nil { + t.Fatal(err) + } + var vectors struct { + Rules []struct { + Name string `json:"name"` + Op string `json:"op"` + Device string `json:"device"` + Now int64 `json:"now"` + Cell Cell `json:"cell"` + Arg json.RawMessage `json:"arg"` + Want Cell `json:"want"` + Err string `json:"err"` + } `json:"rules"` + } + if err := json.Unmarshal(b, &vectors); err != nil { + t.Fatal(err) + } + if len(vectors.Rules) == 0 { + t.Fatal("no contract vectors found") + } + for _, tc := range vectors.Rules { + t.Run(tc.Name, func(t *testing.T) { + arg := func(v any) { + if len(tc.Arg) > 0 { + if err := json.Unmarshal(tc.Arg, v); err != nil { + t.Fatal(err) + } + } + } + var got Cell + var err error + switch tc.Op { + case "acquire": + var in struct{ Force bool } + arg(&in) + got, err = Acquire(tc.Cell, tc.Device, tc.Now, in.Force) + case "heartbeat": + var in Beat + arg(&in) + got, err = Heartbeat(tc.Cell, tc.Device, in, tc.Now) + case "publish": + var in Publish + arg(&in) + got, err = PublishTo(tc.Cell, tc.Device, in, tc.Now) + case "release": + var in Beat + arg(&in) + got, err = ReleaseOf(tc.Cell, tc.Device, in.Fence) + case "create": + var in CellInit + arg(&in) + got = Created(in, tc.Device, tc.Now) + default: + t.Fatalf("unknown operation %s", tc.Op) + } + wantErr, known := map[string]error{"": nil, "lease_held": ErrLeaseHeld, "fence_stale": ErrFenceStale, "head_moved": ErrHeadMoved}[tc.Err] + if !known || err != wantErr { + t.Fatalf("error = %v, want %q", err, tc.Err) + } + if err == nil && !reflect.DeepEqual(got, tc.Want) { + t.Fatalf("got %+v, want %+v", got, tc.Want) + } + if err != nil && !reflect.DeepEqual(got, tc.Cell) { + t.Fatal("a refusal changed the cell") + } + }) + } +} + +func TestR1ConditionalAcquireComparesBeforeLivenessAndForce(t *testing.T) { + c := Cell{Lease: Lease{Device: "a", Fence: 7, Expires: 100}} + expect := &LeaseExpectation{Device: "a", Fence: 7} + for _, force := range []bool{false, true} { + taken, err := Acquire(c, "b", 101, force, expect) + if err != nil || taken.Lease.Fence != 8 { + t.Fatalf("first take force=%v: %+v %v", force, taken, err) + } + refused, err := Acquire(taken, "c", 1_000_000, force, expect) + if err != ErrLeaseHeld || !reflect.DeepEqual(taken, refused) { + t.Fatalf("changed observation force=%v: %+v %v", force, refused, err) + } + } +} + +func TestR16DeviceStampCannotUndoRevocationOrTrustClientTimes(t *testing.T) { + old := Device{Revoked: true, Created: 10, LastSeen: 20} + got := Stamped(old, true, Device{Created: 100, LastSeen: 200, Platform: "arbitrary"}, 300) + if !got.Revoked || got.Created != 10 || got.LastSeen != 20 || got.Platform != "other" { + t.Fatalf("stamp changed authority: %+v", got) + } + if _, err := RevokeOf(old, "a", "a"); err != ErrSelfRevoke { + t.Fatalf("self revoke: %v", err) + } +} diff --git a/internal/clusterstate/testdata/canonical.json b/internal/clusterstate/testdata/canonical.json new file mode 100644 index 0000000000..cf1e8b8b07 --- /dev/null +++ b/internal/clusterstate/testdata/canonical.json @@ -0,0 +1,27 @@ +[ + { + "name": "R14 full-width command and bytes", + "kind": "command", + "canonical": "{\"actor\":\"dev_a\",\"change\":{\"acquire\":{\"cell\":\"chat\",\"expect\":{\"device\":\"dev_b\",\"fence\":\"9007199254740992\"},\"force\":true}},\"cluster\":\"cluster-1\",\"now\":1700000000000,\"request_digest\":\"opaque-digest\",\"schema\":1,\"sequence\":\"18446744073709551615\",\"session\":\"s1\",\"signature\":\"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w==\"}" + }, + { + "name": "R13 R15 every cell field and sealed bytes", + "kind": "row", + "canonical": "{\"cell\":{\"V\":1,\"archived\":true,\"class\":\"host-bound\",\"durable_at\":1700000000000,\"frames\":[\"f1\",\"f2\"],\"head\":\"head\",\"keys\":{\"key\":{\"10\":\"ten\",\"2\":\"two\",\"fence\":\"9007199254740993\",\"id\":\"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w==\",\"x\\u003c\\u003e\\u0026\\u2028\\u2029\":\"escaped\",\"\":\"bmp\",\"𐀀\":\"astral\"}},\"lease\":{\"device\":\"dev_a\",\"expires\":1700000090000,\"fence\":\"9007199254740991\",\"pending\":4294967295,\"released_at\":1699999999000},\"orphan_turns\":3,\"parent_cell\":\"parent\",\"size\":\"18446744073709551615\",\"title\":\"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w==\"},\"key\":\"chat\",\"schema\":1}" + }, + { + "name": "R14 result and per-item refusal", + "kind": "result", + "canonical": "{\"code\":\"ok\",\"items\":[{\"cell\":\"other\",\"code\":\"fence_stale\"}],\"now\":1700000000000,\"schema\":1,\"version\":\"9007199254740993\",\"view\":{\"cell\":{\"V\":1,\"archived\":true,\"class\":\"host-bound\",\"durable_at\":1700000000000,\"frames\":[\"f1\",\"f2\"],\"head\":\"head\",\"keys\":{\"key\":{\"10\":\"ten\",\"2\":\"two\",\"fence\":\"9007199254740993\",\"id\":\"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w==\",\"x\\u003c\\u003e\\u0026\\u2028\\u2029\":\"escaped\",\"\":\"bmp\",\"𐀀\":\"astral\"}},\"lease\":{\"device\":\"dev_a\",\"expires\":1700000090000,\"fence\":\"9007199254740991\",\"pending\":4294967295,\"released_at\":1699999999000},\"orphan_turns\":3,\"parent_cell\":\"parent\",\"size\":\"18446744073709551615\",\"title\":\"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w==\"},\"now\":1700000000000}}" + }, + { + "name": "R16 revoked device", + "kind": "row", + "canonical": "{\"device\":{\"V\":1,\"added_by\":\"id\",\"caps\":{\"arch\":\"arm64\",\"container\":null,\"cow\":\"none\",\"gpu\":null,\"os\":\"linux\",\"sandbox\":null},\"created\":1700000000000,\"last_seen\":1700000000001,\"name\":\"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w==\",\"platform\":\"linux\",\"revoked\":true},\"key\":\"dev_a\",\"schema\":1}" + }, + { + "name": "R13 rotated identity", + "kind": "row", + "canonical": "{\"identity\":{\"V\":1,\"identity\":\"id\",\"rotation\":{\"at\":1699999999000,\"by\":\"dev_a\",\"retire_at\":1700000090000,\"state\":\"retired\"},\"vault\":\"vault\"},\"key\":\"id\",\"schema\":1}" + } +] diff --git a/internal/clusterstate/types.go b/internal/clusterstate/types.go new file mode 100644 index 0000000000..e3836cb3c6 --- /dev/null +++ b/internal/clusterstate/types.go @@ -0,0 +1,167 @@ +package clusterstate + +import "errors" + +// SchemaVersion versions commands, results and application rows together. A +// member must stop application on an unknown schema, rather than skip an entry. +const SchemaVersion uint16 = 1 + +// LeaseTTLMillis is shared with the legacy directory without importing clocks. +const LeaseTTLMillis int64 = 90_000 + +// MaxSafeInteger is the v1 fence ceiling and the limit for numeric JSON values +// that cross JavaScript. V2 uint64 fields instead use decimal strings. +const MaxSafeInteger uint64 = 1<<53 - 1 + +// These sentinels retain the existing directory error identity through aliases. +var ( + ErrNotFound = errors.New("directory: not found") + ErrExists = errors.New("directory: exists") + ErrLeaseHeld = errors.New("directory: lease held by another device") + ErrFenceStale = errors.New("directory: fence superseded") + ErrHeadMoved = errors.New("directory: head moved") + ErrCAS = errors.New("directory: compare-and-swap failed") + ErrSelfRevoke = errors.New("directory: a device cannot revoke itself") +) + +// Command binds an original signed request to one identity cluster and durable +// client sequence. Signature authentication and committed authorization belong +// to ingress and apply respectively; decoding alone grants no authority. +type Command struct { + Schema uint16 `json:"schema"` + Cluster string `json:"cluster"` + Actor string `json:"actor"` + Session string `json:"session"` + Sequence uint64 `json:"sequence"` + RequestDigest string `json:"request_digest"` + Now int64 `json:"now"` + Change Mutation `json:"change"` + Signature []byte `json:"signature"` +} + +// Mutation is a closed union: exactly one field is present. Future packets add +// their typed operations here; unknown fields fail decoding on older members. +// There is no arbitrary JSON payload in which task text or secrets can hide. +type Mutation struct { + Create *CreateCell `json:"create,omitempty"` + Acquire *AcquireCell `json:"acquire,omitempty"` + Beat *BeatCell `json:"beat,omitempty"` + BeatBatch *BeatBatch `json:"beat_batch,omitempty"` + Publish *PublishCell `json:"publish,omitempty"` + Release *ReleaseCell `json:"release,omitempty"` + Archive *CellID `json:"archive,omitempty"` + PutDevice *PutDevice `json:"put_device,omitempty"` + Revoke *DeviceID `json:"revoke,omitempty"` + SetVault *VaultCAS `json:"set_vault,omitempty"` + AdvanceTime *AdvanceTime `json:"advance_time,omitempty"` + DeviceSeen *DeviceSeen `json:"device_seen,omitempty"` +} + +type CellID struct { + Cell string `json:"cell"` +} + +type DeviceID struct { + Device string `json:"device"` +} + +type CreateCell struct { + Cell string `json:"cell"` + Init CellInit `json:"init"` + Receipt string `json:"receipt"` +} + +// AcquireCell always names the observed tuple on v2. The legacy adapter will +// normalize older requests before they become commands in the same log. +type AcquireCell struct { + Cell string `json:"cell"` + Force bool `json:"force"` + Expect LeaseExpectation `json:"expect"` +} + +type BeatCell struct { + Cell string `json:"cell"` + Beat Beat `json:"beat"` +} + +// BeatBatch has a record envelope so an empty item set cannot collapse the +// operation's presence into a null union member during JSON decoding. +type BeatBatch struct { + Items []BeatCell `json:"items"` +} + +type PublishCell struct { + Cell string `json:"cell"` + Publish Publish `json:"publish"` + Receipt string `json:"receipt"` +} + +type ReleaseCell struct { + Cell string `json:"cell"` + Fence uint64 `json:"fence"` + Quiet bool `json:"quiet"` +} + +type PutDevice struct { + ID string `json:"id"` + Device Device `json:"device"` +} + +type VaultCAS struct { + Old string `json:"old"` + New string `json:"new"` +} + +// AdvanceTime has no host clock dependency: Command.Now supplies its sample. +type AdvanceTime struct{} + +type DeviceSeen struct { + Device string `json:"device"` +} + +// Result stores the exact application response for deduplication. Codes carry +// no free-form diagnostic text, so logging a refusal cannot disclose payloads. +// Now and Version are application time/version, never a local clock/Raft index. +type Result struct { + Schema uint16 `json:"schema"` + Code Code `json:"code"` + Now int64 `json:"now"` + Version uint64 `json:"version"` + View *CellView `json:"view,omitempty"` + Items []ItemResult `json:"items,omitempty"` +} + +type ItemResult struct { + Cell string `json:"cell"` + Code Code `json:"code"` + View *CellView `json:"view,omitempty"` +} + +type Code string + +const ( + OK Code = "ok" + NotFound Code = "not_found" + Exists Code = "exists" + LeaseHeld Code = "lease_held" + FenceStale Code = "fence_stale" + HeadMoved Code = "head_moved" + CAS Code = "cas" + Revoked Code = "revoked" + SelfRevoke Code = "self_revoke" + Unauthorized Code = "unauthorized" + Rotated Code = "rotated" + Conflict Code = "conflict" + Capacity Code = "capacity" +) + +// Row encodes one application record for paged reads and streamed snapshots. +// Logical time, visible version and applied index belong to the host's atomic +// application transaction; they must not be hidden in Listing's json:"-" field. +type Row struct { + Schema uint16 `json:"schema"` + Key string `json:"key"` + Identity *IdentityRec `json:"identity,omitempty"` + Device *Device `json:"device,omitempty"` + Cell *Cell `json:"cell,omitempty"` +} diff --git a/internal/directory/client.go b/internal/directory/client.go index 90f4d7f23e..387c4f90c5 100644 --- a/internal/directory/client.go +++ b/internal/directory/client.go @@ -5,6 +5,7 @@ import ( "errors" "time" + "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/wireauth" ) @@ -61,12 +62,8 @@ type AcquireOpts struct { Expect *LeaseExpectation `json:"expect,omitempty"` } -// LeaseExpectation binds a continuation to the lease the taking device read, -// so another continuation during its fetch wins rather than being displaced. -type LeaseExpectation struct { - Device string `json:"device"` - Fence uint64 `json:"fence"` -} +// LeaseExpectation retains the conditional-take tuple shared by every runtime. +type LeaseExpectation = clusterstate.LeaseExpectation // Directory is one identity's shared directory; For gives each device its own // Client. The relay handler calls For(deviceFromSignature) once per request. @@ -79,20 +76,20 @@ type Directory interface { // relay sees one request per HeartbeatEvery at most and the TTL still covers // two missed beats. const ( - LeaseTTL = 90 * time.Second + LeaseTTL = time.Duration(clusterstate.LeaseTTLMillis) * time.Millisecond HeartbeatEvery = 30 * time.Second ) // The errors a Client returns. Callers compare with errors.Is. var ( - ErrNotFound = errors.New("directory: not found") - ErrExists = errors.New("directory: exists") - ErrLeaseHeld = errors.New("directory: lease held by another device") - ErrFenceStale = errors.New("directory: fence superseded") // L7 - ErrHeadMoved = errors.New("directory: head moved") - ErrCAS = errors.New("directory: compare-and-swap failed") + ErrNotFound = clusterstate.ErrNotFound + ErrExists = clusterstate.ErrExists + ErrLeaseHeld = clusterstate.ErrLeaseHeld + ErrFenceStale = clusterstate.ErrFenceStale // L7 + ErrHeadMoved = clusterstate.ErrHeadMoved + ErrCAS = clusterstate.ErrCAS ErrRevoked = wireauth.ErrRevoked // every verb of a revoked device; the wires share this one sentinel - ErrSelfRevoke = errors.New("directory: a device cannot revoke itself") + ErrSelfRevoke = clusterstate.ErrSelfRevoke ErrUnauthorized = errors.New("directory: unauthorized") ErrUnreachable = errors.New("directory: unreachable") // transport; callers degrade ErrTooOld = errors.New("directory: this relay is too old for that") diff --git a/internal/directory/clusterstate_test.go b/internal/directory/clusterstate_test.go new file mode 100644 index 0000000000..66a1ce1da2 --- /dev/null +++ b/internal/directory/clusterstate_test.go @@ -0,0 +1,31 @@ +package directory + +import ( + "bytes" + "net/http/httptest" + "testing" +) + +// Aliased records must retain the legacy presentation projections for both +// values and pointers without putting HTTP/presentation code into the core. +func TestR25SharedRecordProjectionPreservesLegacyVersions(t *testing.T) { + d := Device{LastSeen: 10} + seen := d + seen.LastSeen = 20 + if !bytes.Equal(fingerprint(d, 100), fingerprint(&seen, 100)) { + t.Fatal("device presence changed the visible record") + } + c := Cell{Lease: Lease{Expires: 200}} + later := c + later.Lease.Expires = 300 + if !bytes.Equal(fingerprint(c, 100), fingerprint(&later, 100)) { + t.Fatal("expiry-only heartbeat changed the visible record") + } + for _, value := range []any{Listing{Version: 42}, &Listing{Version: 42}, PresenceView{Version: 42}} { + w := httptest.NewRecorder() + reply(w, value) + if w.Header().Get(VersionHeader) != "42" { + t.Fatalf("%T lost the directory version header", value) + } + } +} diff --git a/internal/directory/feed.go b/internal/directory/feed.go index 24ae0955d7..e5f1f40afb 100644 --- a/internal/directory/feed.go +++ b/internal/directory/feed.go @@ -316,25 +316,29 @@ func (s *Sub) leave() { } } -// visible is a record that can say which of its fields a person sees. -type visible interface{ visibleAt(now int64) any } - -// visibleAt is the Cell as the home list shows it: whether the lease is held -// counts, and when it runs out does not, so a heartbeat that only pushes the -// expiry later changes nothing a person could see. -func (c Cell) visibleAt(now int64) any { - if c.Lease.Expires > now { - c.Lease.Expires = 1 - } else { - c.Lease.Expires = 0 - } - return c -} - // fingerprint is the bytes two records are compared by. func fingerprint(v any, now int64) []byte { - if p, ok := v.(visible); ok { - v = p.visibleAt(now) + // Aliased shared records cannot carry private directory methods. Keep the + // presentation projection here so it never becomes replicated authority. + switch r := v.(type) { + case Device: + r.LastSeen = 0 + v = r + case *Device: + if r != nil { + return fingerprint(*r, now) + } + case Cell: + if r.Lease.Expires > now { + r.Lease.Expires = 1 + } else { + r.Lease.Expires = 0 + } + v = r + case *Cell: + if r != nil { + return fingerprint(*r, now) + } } b, _ := json.Marshal(v) return b diff --git a/internal/directory/handler.go b/internal/directory/handler.go index 4bef18af18..171f362fd8 100644 --- a/internal/directory/handler.go +++ b/internal/directory/handler.go @@ -106,13 +106,18 @@ func refuse(w http.ResponseWriter, err error) { // headed is an answer that also sets headers, as a list sets its version. type headed interface{ header(http.Header) } -func (l Listing) header(h http.Header) { - h.Set(VersionHeader, strconv.FormatUint(l.Version, 10)) -} - func reply(w http.ResponseWriter, out any) { - if h, ok := out.(headed); ok { - h.header(w.Header()) + // Listing is a shared record alias; keep its HTTP projection here rather + // than importing HTTP into the pure record package. + switch l := out.(type) { + case Listing: + w.Header().Set(VersionHeader, strconv.FormatUint(l.Version, 10)) + case *Listing: + if l != nil { + w.Header().Set(VersionHeader, strconv.FormatUint(l.Version, 10)) + } + case headed: + l.header(w.Header()) } if out == nil { w.WriteHeader(http.StatusNoContent) diff --git a/internal/directory/records.go b/internal/directory/records.go index af331f9ac2..7727c328e6 100644 --- a/internal/directory/records.go +++ b/internal/directory/records.go @@ -33,84 +33,20 @@ // retry); the numbers are LinkLimits, read with LinkHTTP.Limits. package directory -// IdentityRec is the per-identity root record. -type IdentityRec struct { - V uint16 `json:"V"` - Identity string `json:"identity"` // id_…, derived from the signing key - Vault string `json:"vault,omitempty"` // rid of the current vault object - // Rotation is set once the identity has been replaced by a rotation: it - // takes no writes from then on. Nil means live. - Rotation *Rotation `json:"rotation,omitempty"` -} - -// Caps says what a device can do. -type Caps struct { - OS string `json:"os"` // runtime.GOOS - Arch string `json:"arch"` // runtime.GOARCH - Sandbox *string `json:"sandbox"` // "seatbelt" | "landlock" | null - Container *string `json:"container"` // null in Stage 1 - GPU *string `json:"gpu"` // null in Stage 1 - Cow string `json:"cow"` // "apfs" | "btrfs" | "xfs-reflink" | "overlayfs" | "none" -} - -// Device is keyed by device id (dev_…). -type Device struct { - V uint16 `json:"V"` - Name string `json:"name"` // b64, sealed under the metadata key - AddedBy string `json:"added_by"` // identity id that signed the device cert - Revoked bool `json:"revoked"` // set only by Revoke; a fresh pairing makes a new device id - Caps Caps `json:"caps"` - - // Added by link pairing. Old records decode with zero values, and a client - // value for Created or LastSeen is ignored: the directory sets both. - Platform string `json:"platform,omitempty"` // darwin|linux|windows|ios|android|other - Created int64 `json:"created,omitempty"` // directory ms when the device joined - LastSeen int64 `json:"last_seen,omitempty"` // directory ms of the last watch socket close or hello -} - -// visibleAt hides LastSeen: a device coming and going is not a change a person -// sees in the list, so it moves no directory version. -func (d Device) visibleAt(int64) any { - d.LastSeen = 0 - return d -} - -// Lease says who may write a cell. Only the directory clock sets Expires. -type Lease struct { - Device string `json:"device"` // device id of the last holder, kept after release - Fence uint64 `json:"fence"` // +1 on every acquire, never decreases - Expires int64 `json:"expires"` // directory ms; held iff Expires > now; release sets 0 - Pending uint32 `json:"pending"` // turns sealed on the holder and not yet durable - // ReleasedAt is directory time of an explicit release, separate from the last turn. - ReleasedAt int64 `json:"released_at,omitempty"` -} - -// Cell is keyed by cell ULID. It exists only once it has a durable head. -type Cell struct { - V uint16 `json:"V"` - Head string `json:"head"` // last durable turn id (hex64) - DurableAt int64 `json:"durable_at"` // directory ms - Class string `json:"class"` - Size uint64 `json:"size"` - ParentCell string `json:"parent_cell,omitempty"` - Title string `json:"title,omitempty"` // b64, sealed - Keys map[string]map[string]string `json:"keys"` // cell_key_id -> identity -> wrapped - Lease Lease `json:"lease"` - OrphanTurns uint32 `json:"orphan_turns,omitempty"` // turns in a death branch - Archived bool `json:"archived,omitempty"` // a merged or discarded branch - Frames []string `json:"frames,omitempty"` // hex64 ids of the frames holding the head's closure; a hint, never a condition -} - -// Listing is everything a home list needs, in one read. -type Listing struct { - Now int64 `json:"now"` // directory ms when read - Identity IdentityRec `json:"identity"` - Devices map[string]Device `json:"devices"` - Cells map[string]Cell `json:"cells"` - // Version is the directory version the records were read at, which is - // sent as the VersionHeader and not in the body. - Version uint64 `json:"-"` -} +import "github.com/Agent-Field/codeaf/internal/clusterstate" + +// These aliases preserve the public API and legacy JSON while the pure records +// remain usable without linking the directory transports and stores. +type IdentityRec = clusterstate.IdentityRec +type Caps = clusterstate.Caps +type Device = clusterstate.Device +type Lease = clusterstate.Lease +type Cell = clusterstate.Cell +type Listing = clusterstate.Listing +type CellView = clusterstate.CellView +type CellInit = clusterstate.CellInit +type Beat = clusterstate.Beat +type Publish = clusterstate.Publish // withoutFrames drops every cell's frames from a list answer: the list is the // home screen's hot read, and frame plans are for takers, who read the cell. @@ -121,39 +57,3 @@ func withoutFrames(l Listing) Listing { } return l } - -// CellView is one cell with the directory time it was read at. -type CellView struct { - Now int64 `json:"now"` - Cell Cell `json:"cell"` -} - -// CellInit creates a cell. The calling device holds the new lease at fence 1. -type CellInit struct { - Head string `json:"head"` - Class string `json:"class"` - Title string `json:"title,omitempty"` - ParentCell string `json:"parent_cell,omitempty"` - Size uint64 `json:"size"` - Keys map[string]map[string]string `json:"keys"` - OrphanTurns uint32 `json:"orphan_turns,omitempty"` - Frames []string `json:"frames,omitempty"` -} - -// Beat renews a lease the caller holds. -type Beat struct { - Fence uint64 `json:"fence"` - Pending uint32 `json:"pending"` -} - -// Publish moves a cell's durable head under a lease the caller holds. -type Publish struct { - Fence uint64 `json:"fence"` - OldHead string `json:"old_head"` - Head string `json:"head"` - Size uint64 `json:"size"` - Class string `json:"class"` - Title string `json:"title,omitempty"` // empty keeps the current title - Pending uint32 `json:"pending"` // turns still not durable after this one - Frames []string `json:"frames,omitempty"` // empty or absent replaces the record's plan with none -} diff --git a/internal/directory/requests.go b/internal/directory/requests.go index ba3ec4319f..a4408aadc2 100644 --- a/internal/directory/requests.go +++ b/internal/directory/requests.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/identity" "github.com/Agent-Field/codeaf/internal/wireauth" ) @@ -111,18 +112,8 @@ var b64u = base64.RawURLEncoding // NormalizeCode is the one spelling a code is looked up by. func NormalizeCode(code string) string { return strings.ToLower(strings.TrimSpace(code)) } -// platforms are the values a platform may take; anything else is "other". -var platforms = map[string]bool{"darwin": true, "linux": true, "windows": true, "ios": true, "android": true, "other": true} - -// platformOf keeps a known platform, maps an unknown one to "other" and leaves -// an unset one unset. -func platformOf(p string) string { - switch { - case p == "" || platforms[p]: - return p - } - return "other" -} +// platformOf shares normalization with the replicated device rule. +func platformOf(p string) string { return clusterstate.PlatformOf(p) } // CheckOf is the four digits both ends show so a substituted key is visible. func CheckOf(pubkey []byte) string { diff --git a/internal/directory/rotation.go b/internal/directory/rotation.go index 2c39e9c679..3e331ae532 100644 --- a/internal/directory/rotation.go +++ b/internal/directory/rotation.go @@ -5,6 +5,7 @@ import ( "errors" "time" + "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/wireauth" ) @@ -14,13 +15,8 @@ const ( StateRetired = "retired" // read-only until RetireAt, then deleted by the relay ) -// Rotation is what an identity's record says about its replacement. -type Rotation struct { - State string `json:"state"` - By string `json:"by"` // the device that froze it; only it may retire - At int64 `json:"at"` // directory ms of the freeze - RetireAt int64 `json:"retire_at,omitempty"` -} +// Rotation retains the shared record without importing the directory into wasm. +type Rotation = clusterstate.Rotation // RotationOp is one move of the rotation state machine. type RotationOp string diff --git a/internal/directory/rules.go b/internal/directory/rules.go index cee53a2a35..3d58930bd0 100644 --- a/internal/directory/rules.go +++ b/internal/directory/rules.go @@ -1,132 +1,30 @@ package directory -// This file is the only place lease logic lives. Each rule takes a cell and -// returns the changed cell or an error, and never touches a clock or a store: -// the caller passes the directory time in and keeps the result. +import "github.com/Agent-Field/codeaf/internal/clusterstate" -const ttlMs = int64(LeaseTTL / 1e6) +// Lease rules live in clusterstate so every runtime applies the same decisions. +// These forwarding functions retain the directory API used by existing clients. +const ttlMs = clusterstate.LeaseTTLMillis -// Lifted is c with its lease kept live until vouchedUntil, the time a watch -// socket's sign of life lasts (0 for none). A released lease (expires 0) stays -// released, and a stored expiry already later is left alone. It is the one -// place evidence enters a rule: every reader of a lease asks Lifted first and -// then applies the pure rules to the answer. -func Lifted(c Cell, vouchedUntil int64) Cell { - if c.Lease.Expires != 0 && vouchedUntil > c.Lease.Expires { - c.Lease.Expires = vouchedUntil - } - return c -} - -// Acquire takes the lease for device unless another device still holds it. The -// same device may re-take its own live lease; the fence still goes up. A forced -// acquire is a person's choice to displace a live holder: it skips the refusal, -// and the fence still goes up, so the old holder's next write fails as stale. +func Lifted(c Cell, until int64) Cell { return clusterstate.Lifted(c, until) } func Acquire(c Cell, device string, now int64, force bool, expected ...*LeaseExpectation) (Cell, error) { - if len(expected) > 0 && expected[0] != nil { - seen := expected[0] - if c.Lease.Device != seen.Device || c.Lease.Fence != seen.Fence { - return c, ErrLeaseHeld - } - } - if !force && c.Lease.Expires > now && c.Lease.Device != device { - return c, ErrLeaseHeld - } - c.Lease = Lease{Device: device, Fence: c.Lease.Fence + 1, Expires: now + ttlMs} - return c, nil + return clusterstate.Acquire(c, device, now, force, expected...) } - -// Heartbeat renews the lease. An expired lease nobody took may be renewed: the -// unchanged fence proves nobody did. func Heartbeat(c Cell, device string, b Beat, now int64) (Cell, error) { - if err := holder(c, device, b.Fence); err != nil { - return c, err - } - c.Lease.Expires = now + ttlMs - c.Lease.Pending = b.Pending - return c, nil + return clusterstate.Heartbeat(c, device, b, now) } - -// PublishTo moves the durable head and renews the lease, because a publish is -// proof that the holder is alive. The fence is checked before the head, so a -// superseded holder always learns it lost the lease first. func PublishTo(c Cell, device string, p Publish, now int64) (Cell, error) { - if err := holder(c, device, p.Fence); err != nil { - return c, err - } - if p.OldHead != c.Head { - return c, ErrHeadMoved - } - c.Head, c.Size, c.Class = p.Head, p.Size, p.Class - // The frames describe the head they were set with, so every publish replaces - // the list wholesale: an empty or absent list is a true statement ("the - // publisher did not say"), a stale one is not kept. - c.Frames = p.Frames - c.Lease.Pending, c.DurableAt = p.Pending, now - c.Lease.Expires = now + ttlMs - c.Title = keepIfEmpty(p.Title, c.Title) - return c, nil + return clusterstate.PublishTo(c, device, p, now) } - -// ReleaseOf gives the lease up. The fence stays, so the next acquire still -// goes up by one. func ReleaseOf(c Cell, device string, fence uint64, at ...int64) (Cell, error) { - if err := holder(c, device, fence); err != nil { - return c, err - } - c.Lease.Expires, c.Lease.Pending = 0, 0 - if len(at) > 0 { - c.Lease.ReleasedAt = at[0] - } - return c, nil + return clusterstate.ReleaseOf(c, device, fence, at...) } - -// Created is the first record of a cell: the creating device holds the lease -// at fence 1. The caller has already checked that the id is free. func Created(in CellInit, device string, now int64) Cell { - return Cell{ - V: 1, Head: in.Head, DurableAt: now, Class: in.Class, Size: in.Size, - ParentCell: in.ParentCell, Title: in.Title, Keys: in.Keys, OrphanTurns: in.OrphanTurns, - Frames: in.Frames, - Lease: Lease{Device: device, Fence: 1, Expires: now + ttlMs}, - } + return clusterstate.Created(in, device, now) } - -// holder says whether device still holds the lease at fence. -func holder(c Cell, device string, fence uint64) error { - if c.Lease.Fence != fence || c.Lease.Device != device { - return ErrFenceStale - } - return nil -} - -func keepIfEmpty(next, current string) string { - if next == "" { - return current - } - return next -} - -// RevokeOf marks target revoked on behalf of caller. A device cannot revoke -// itself: that is nearly always a slip, and a revoked device can no longer say -// so, so the mistake would be permanent. Revoking twice changes nothing. func RevokeOf(target Device, caller, targetID string) (Device, error) { - if caller == targetID { - return target, ErrSelfRevoke - } - target.Revoked = true - return target, nil + return clusterstate.RevokeOf(target, caller, targetID) } - -// Stamped is d as the directory stores it, given the record already stored -// under its id (had says whether there was one) and the directory time. The -// relay-owned fields are never taken from the client: Revoked and LastSeen stay -// as stored, and Created is the time of the first write. func Stamped(old Device, had bool, d Device, now int64) Device { - d.Revoked, d.LastSeen, d.Created = old.Revoked, old.LastSeen, old.Created - if !had { - d.Created = now - } - d.Platform = platformOf(d.Platform) - return d + return clusterstate.Stamped(old, had, d, now) } diff --git a/relay/hosted/src/raft/codec.js b/relay/hosted/src/raft/codec.js new file mode 100644 index 0000000000..ee931fbee9 --- /dev/null +++ b/relay/hosted/src/raft/codec.js @@ -0,0 +1,57 @@ +// This module only preserves the Go core's canonical boundary. It grants no +// authority and implements no directory or consensus decisions. +const maxUint64 = (1n << 64n) - 1n; +const encoder = new TextEncoder(); + +// uint64 keeps counters out of JavaScript's rounded number domain. Callers +// retain the original decimal string when handing a command to the core. +export function uint64(value) { + if (typeof value !== 'string' || value.length > 20 || !/^(0|[1-9][0-9]*)$/.test(value)) throw new TypeError('invalid uint64'); + const n = BigInt(value); + if (n > maxUint64) throw new TypeError('invalid uint64'); + return n; +} + +function quoted(s) { + for (const c of s) { + const cp = c.codePointAt(0); + if (cp >= 0xd800 && cp <= 0xdfff) throw new TypeError('invalid Unicode'); + } + return JSON.stringify(s).replace(/[<>&\u2028\u2029]/g, + (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, '0')}`); +} + +// Go sorts map keys by UTF-8 bytes. JavaScript's default UTF-16 sort differs +// for supplementary characters, and object enumeration reorders integer keys. +function compareKeys(a, b) { + const x = encoder.encode(a), y = encoder.encode(b); + for (let i = 0; i < Math.min(x.length, y.length); i++) if (x[i] !== y[i]) return x[i] - y[i]; + return x.length - y.length; +} + +export function encodeCanonical(value) { + if (value === null) return 'null'; + switch (typeof value) { + case 'string': return quoted(value); + case 'boolean': return String(value); + case 'bigint': + if (value < 0n || value > maxUint64) throw new TypeError('invalid uint64'); + return quoted(String(value)); + case 'number': + if (!Number.isSafeInteger(value) || Object.is(value, -0)) throw new TypeError('invalid integer'); + return String(value); + case 'object': + if (Array.isArray(value)) return `[${Array.from(value, encodeCanonical).join(',')}]`; + if (Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) { + throw new TypeError('invalid object'); + } + return `{${Object.keys(value).sort(compareKeys).map((k) => `${quoted(k)}:${encodeCanonical(value[k])}`).join(',')}}`; + default: throw new TypeError('invalid canonical value'); + } +} + +export function decodeCanonical(text) { + const value = JSON.parse(text); + if (encodeCanonical(value) !== text) throw new TypeError('invalid canonical encoding'); + return value; +} diff --git a/relay/hosted/test/raft-codec.test.js b/relay/hosted/test/raft-codec.test.js new file mode 100644 index 0000000000..c14148836c --- /dev/null +++ b/relay/hosted/test/raft-codec.test.js @@ -0,0 +1,46 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { encodeCanonical, decodeCanonical, uint64 } from '../src/raft/codec.js'; + +const fixtures = JSON.parse(readFileSync(new URL('../../../internal/clusterstate/testdata/canonical.json', import.meta.url))); + +for (const c of fixtures) { + test(`R14 canonical Go/JS round trip: ${c.name}`, () => { + const value = decodeCanonical(c.canonical); + assert.equal(encodeCanonical(value), c.canonical); + assert.equal(value.schema, 1); + if (c.kind === 'command') { + assert.equal(uint64(value.sequence), (1n << 64n) - 1n); + assert.equal(uint64(value.change.acquire.expect.fence), 1n << 53n); + assert.deepEqual(Buffer.from(value.signature, 'base64'), Buffer.from(Array.from({ length: 256 }, (_, i) => i))); + } + if (value.cell) { + assert.equal(uint64(value.cell.size), (1n << 64n) - 1n); + assert.equal(value.cell.keys.key.fence, '9007199254740993'); + } + }); +} + +test('R14 full-width counters never pass through Number', () => { + for (const n of [0n, 1n, (1n << 53n) - 1n, 1n << 53n, (1n << 64n) - 1n]) { + assert.equal(uint64(decodeCanonical(encodeCanonical(n))), n); + } + for (const value of [1, -1n, 1n << 64n, '', '-1', '+1', '01', '1e3', '1.0', '18446744073709551616']) { + assert.throws(() => uint64(value)); + } +}); + +test('R14 ambiguous JSON and unsafe integers are refused', () => { + for (const value of [1.5, Number.MAX_SAFE_INTEGER + 1, NaN, Infinity, -0, undefined, '\ud800']) { + assert.throws(() => encodeCanonical(value)); + } + for (const text of ['{"x":1,"x":2}', '{"x":1} ', '{"x":1e3}', '{"x":-0}', '{"x":9007199254740992}', '"\\ud800"']) { + assert.throws(() => decodeCanonical(text)); + } +}); + +test('R15 opaque strings are preserved even when named like counters', () => { + const value = { fence: 'not a counter: <>&\u2028\u2029', title: 'AAEC/w==', name: 'sealed' }; + assert.deepEqual(decodeCanonical(encodeCanonical(value)), value); +}); From eec4dc62001e56ab8b0fdb315539c460ae260372 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Fri, 23 Oct 2026 22:04:20 -0400 Subject: [PATCH 002/103] Start an independent sequential coordination oracle. Give the fault simulator an ownership, vault and revocation model that cannot inherit defects from production apply functions. Keep candidate history states independent and pin observed-take and displaced-write behavior before the history checker is added. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/raftsim/model/model.go | 134 +++++++++++++++++++++++++++ internal/raftsim/model/model_test.go | 45 +++++++++ 2 files changed, 179 insertions(+) create mode 100644 internal/raftsim/model/model.go create mode 100644 internal/raftsim/model/model_test.go diff --git a/internal/raftsim/model/model.go b/internal/raftsim/model/model.go new file mode 100644 index 0000000000..743989798e --- /dev/null +++ b/internal/raftsim/model/model.go @@ -0,0 +1,134 @@ +// Package model is the independent sequential ownership/vault oracle for the +// fault simulator. It deliberately imports no production state types or rules. +// P06 supplies history checking, unknown timeout outcomes and assignment cases. +package model + +// State describes observable coordination, independent of Raft internals. +// Calls advance committed time explicitly; the model never reads a host clock. +type State struct { + Now int64 + Vault string + Cells map[string]Cell + Revoked map[string]bool +} + +type Cell struct { + Holder string + Fence uint64 + Until int64 + Head string + Pending uint32 + ReleasedAt int64 +} + +// Call contains only the small ownership/vault domain. Expected is an observed +// holder/fence, not a fresh read substituted by the checker behind the caller. +type Call struct { + Kind string + Actor string + Cell string + At int64 + Expected Cell + Force bool + Fence uint64 + Head string + OldHead string + Pending uint32 + Quiet bool + OldVault string + NewVault string + Target string +} + +type Outcome struct { + Code string + Cell Cell + Vault string +} + +// Step returns a new state so alternative history orders cannot mutate one +// another. The independent 90-second lease specification is intentional: using +// the production constant or apply function would hide a shared defect. +func Step(s State, op Call) (State, Outcome) { + next := State{Now: s.Now, Vault: s.Vault, Cells: map[string]Cell{}, Revoked: map[string]bool{}} + for k, v := range s.Cells { + next.Cells[k] = v + } + for k, v := range s.Revoked { + next.Revoked[k] = v + } + if op.At > next.Now { + next.Now = op.At + } + if next.Revoked[op.Actor] { + return next, Outcome{Code: "revoked"} + } + c, exists := next.Cells[op.Cell] + out := Outcome{Code: "ok"} + switch op.Kind { + case "advance_time": + return next, out + case "set_vault": + if op.OldVault != next.Vault { + return next, Outcome{Code: "cas"} + } + next.Vault = op.NewVault + return next, out + case "read_vault": + return next, Outcome{Code: "ok", Vault: next.Vault} + case "revoke": + if op.Target == op.Actor { + return next, Outcome{Code: "self_revoke"} + } + next.Revoked[op.Target] = true + return next, out + case "create": + if exists { + return next, Outcome{Code: "exists"} + } + c = Cell{Holder: op.Actor, Fence: 1, Head: op.Head, Until: next.Now + 90_000} + default: + if !exists { + return next, Outcome{Code: "not_found"} + } + switch op.Kind { + case "read": + return next, Outcome{Code: "ok", Cell: c} + case "acquire": + if op.Expected.Holder != c.Holder || op.Expected.Fence != c.Fence || + (!op.Force && c.Until > next.Now && c.Holder != op.Actor) { + return next, Outcome{Code: "lease_held"} + } + if c.Fence == ^uint64(0) { + return next, Outcome{Code: "capacity"} + } + c.Holder, c.Fence, c.Until = op.Actor, c.Fence+1, next.Now+90_000 + c.Pending, c.ReleasedAt = 0, 0 + case "beat", "publish", "release": + if c.Holder != op.Actor || c.Fence != op.Fence { + return next, Outcome{Code: "fence_stale"} + } + switch op.Kind { + case "publish": + if c.Head != op.OldHead { + return next, Outcome{Code: "head_moved"} + } + c.Head = op.Head + fallthrough + case "beat": + c.Until, c.Pending = next.Now+90_000, op.Pending + case "release": + c.Until, c.Pending = 0, 0 + c.ReleasedAt = 0 + if !op.Quiet { + c.ReleasedAt = next.Now + } + } + default: + return next, Outcome{Code: "unsupported"} + } + } + next.Cells[op.Cell] = c + out.Cell = c + return next, out +} diff --git a/internal/raftsim/model/model_test.go b/internal/raftsim/model/model_test.go new file mode 100644 index 0000000000..1dd0795cc3 --- /dev/null +++ b/internal/raftsim/model/model_test.go @@ -0,0 +1,45 @@ +package model + +import "testing" + +func TestR1ObservedTakeHasOneWinnerInEitherOrder(t *testing.T) { + s, created := Step(State{}, Call{Kind: "create", Actor: "a", Cell: "c", Head: "h", At: 100}) + for _, order := range [][2]string{{"b", "c"}, {"c", "b"}} { + first, won := Step(s, Call{Kind: "acquire", Actor: order[0], Cell: "c", Expected: created.Cell, Force: true}) + last, lost := Step(first, Call{Kind: "acquire", Actor: order[1], Cell: "c", Expected: created.Cell, Force: true}) + if won.Code != "ok" || lost.Code != "lease_held" || last.Cells["c"].Holder != order[0] || last.Cells["c"].Fence != 2 { + t.Fatalf("order %v: won=%+v lost=%+v state=%+v", order, won, lost, last) + } + if s.Cells["c"].Holder != "a" { + t.Fatal("a candidate history mutated the input state") + } + } +} + +func TestR2FencePrecedesHeadAndDisplacedWritesChangeNothing(t *testing.T) { + s, created := Step(State{}, Call{Kind: "create", Actor: "a", Cell: "c", Head: "h"}) + s, _ = Step(s, Call{Kind: "acquire", Actor: "b", Cell: "c", Force: true, Expected: created.Cell}) + for _, kind := range []string{"publish", "beat", "release"} { + next, out := Step(s, Call{Kind: kind, Actor: "a", Cell: "c", Fence: 1, OldHead: "wrong", Head: "wrong"}) + if out.Code != "fence_stale" || next.Cells["c"] != s.Cells["c"] { + t.Fatalf("%s: %+v, %+v", kind, out, next) + } + } +} + +func TestR16RevokeAppliesBeforeAlreadySubmittedMutation(t *testing.T) { + s, _ := Step(State{}, Call{Kind: "revoke", Actor: "a", Target: "b", At: 100}) + next, out := Step(s, Call{Kind: "create", Actor: "b", Cell: "c", At: 50}) + if out.Code != "revoked" || len(next.Cells) != 0 || next.Now != 100 { + t.Fatalf("revoked mutation: %+v, %+v", next, out) + } +} + +func TestR4VaultCASAndReadAreOneSequentialHistory(t *testing.T) { + s, ok := Step(State{}, Call{Kind: "set_vault", NewVault: "v1"}) + s, lost := Step(s, Call{Kind: "set_vault", OldVault: "", NewVault: "v2"}) + _, read := Step(s, Call{Kind: "read_vault"}) + if ok.Code != "ok" || lost.Code != "cas" || read.Vault != "v1" { + t.Fatalf("CAS history: %+v %+v %+v", ok, lost, read) + } +} From 46ab8640dd5d3cb4d62b5b9e1d0f78ce1ac2de0c Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Fri, 23 Oct 2026 22:13:10 -0400 Subject: [PATCH 003/103] Pin the shared Raft source with per-node election randomness. Native and WASI members must use the same reviewed election seam and protobuf wire implementation. Retain upstream source and notices, record the narrow patch and source hashes, and expose only these dependency directories through the existing ignore rule. Co-Authored-By: Claude Opus 5.5 (1M context) --- .gitignore | 4 +- go.mod | 5 + go.sum | 2 + third_party/raft-patch/PROTOBUF-LICENSE | 27 + third_party/raft-patch/PROTOBUF-PATENTS | 22 + third_party/raft-patch/README.md | 34 + third_party/raft-patch/SOURCES.json | 180 + third_party/raft-patch/election-random.patch | 47 + third_party/raft-patch/verify.py | 36 + third_party/raft/.github/dependabot.yml | 11 + .../.github/workflows/codeql-analysis.yml | 71 + .../workflows/gh-workflow-approve.yaml | 42 + .../raft/.github/workflows/govuln.yaml | 14 + .../.github/workflows/static-analysis.yaml | 21 + .../raft/.github/workflows/test_amd64.yaml | 26 + .../raft/.github/workflows/test_template.yaml | 49 + third_party/raft/.gitignore | 1 + third_party/raft/.go-version | 1 + third_party/raft/.golangci.yaml | 60 + third_party/raft/CHANGELOG/CHANGELOG-3.6.md | 41 + third_party/raft/CHANGELOG/CHANGELOG-3.7.md | 11 + third_party/raft/LICENSE | 202 + third_party/raft/Makefile | 28 + third_party/raft/OWNERS | 11 + third_party/raft/README.md | 201 + third_party/raft/bootstrap.go | 82 + third_party/raft/code-of-conduct.md | 3 + third_party/raft/confchange/confchange.go | 419 ++ .../raft/confchange/datadriven_test.go | 110 + third_party/raft/confchange/quick_test.go | 177 + third_party/raft/confchange/restore.go | 155 + third_party/raft/confchange/restore_test.go | 139 + .../confchange/testdata/joint_autoleave.txt | 29 + .../confchange/testdata/joint_idempotency.txt | 23 + .../testdata/joint_learners_next.txt | 24 + .../raft/confchange/testdata/joint_safety.txt | 81 + .../testdata/simple_idempotency.txt | 69 + .../testdata/simple_promote_demote.txt | 60 + .../confchange/testdata/simple_safety.txt | 64 + .../raft/confchange/testdata/update.txt | 23 + third_party/raft/confchange/testdata/zero.txt | 6 + third_party/raft/design.md | 57 + third_party/raft/diff_test.go | 67 + third_party/raft/doc.go | 399 ++ third_party/raft/example_test.go | 47 + third_party/raft/go.mod | 17 + third_party/raft/go.sum | 16 + third_party/raft/interaction_test.go | 38 + third_party/raft/log.go | 576 ++ third_party/raft/log_test.go | 1049 ++++ third_party/raft/log_unstable.go | 249 + third_party/raft/log_unstable_test.go | 581 ++ third_party/raft/logger.go | 142 + third_party/raft/node.go | 616 +++ third_party/raft/node_bench_test.go | 51 + third_party/raft/node_test.go | 1061 ++++ third_party/raft/node_util_test.go | 110 + third_party/raft/quorum/bench_test.go | 40 + third_party/raft/quorum/datadriven_test.go | 247 + third_party/raft/quorum/joint.go | 75 + third_party/raft/quorum/majority.go | 198 + third_party/raft/quorum/majority_test.go | 110 + third_party/raft/quorum/quick_test.go | 122 + third_party/raft/quorum/quorum.go | 58 + .../raft/quorum/testdata/joint_commit.txt | 481 ++ .../raft/quorum/testdata/joint_vote.txt | 165 + .../raft/quorum/testdata/majority_commit.txt | 153 + .../raft/quorum/testdata/majority_vote.txt | 97 + third_party/raft/quorum/voteresult_string.go | 26 + third_party/raft/raft.go | 2177 ++++++++ third_party/raft/raft_flow_control_test.go | 138 + third_party/raft/raft_paper_test.go | 829 +++ third_party/raft/raft_snap_test.go | 122 + third_party/raft/raft_test.go | 4261 ++++++++++++++ third_party/raft/raftpb/alias.go | 55 + third_party/raft/raftpb/confchange.go | 177 + third_party/raft/raftpb/confchange_test.go | 130 + third_party/raft/raftpb/confstate.go | 52 + third_party/raft/raftpb/confstate_test.go | 59 + third_party/raft/raftpb/raft.pb.go | 1233 +++++ third_party/raft/raftpb/raft.proto | 205 + third_party/raft/raftpb/raft_test.go | 60 + third_party/raft/raftpb/util.go | 67 + third_party/raft/rafttest/doc.go | 16 + third_party/raft/rafttest/interaction_env.go | 107 + .../raft/rafttest/interaction_env_handler.go | 241 + .../interaction_env_handler_add_nodes.go | 167 + .../interaction_env_handler_campaign.go | 31 + .../interaction_env_handler_compact.go | 40 + .../interaction_env_handler_deliver_msgs.go | 104 + .../interaction_env_handler_forget_leader.go | 32 + .../interaction_env_handler_log_level.go | 36 + ...ction_env_handler_process_append_thread.go | 100 + ...action_env_handler_process_apply_thread.go | 112 + .../interaction_env_handler_process_ready.go | 82 + .../interaction_env_handler_propose.go | 34 + ...raction_env_handler_propose_conf_change.go | 83 + .../interaction_env_handler_raft_log.go | 55 + .../interaction_env_handler_raftstate.go | 49 + ...eraction_env_handler_report_unreachable.go | 30 + .../interaction_env_handler_send_snapshot.go | 51 + ...handler_set_randomized_election_timeout.go | 34 + .../interaction_env_handler_stabilize.go | 132 + .../interaction_env_handler_status.go | 43 + .../rafttest/interaction_env_handler_tick.go | 39 + ...raction_env_handler_transfer_leadership.go | 41 + .../raft/rafttest/interaction_env_logger.go | 138 + third_party/raft/rafttest/network.go | 167 + third_party/raft/rafttest/network_test.go | 71 + third_party/raft/rafttest/node.go | 158 + third_party/raft/rafttest/node_bench_test.go | 52 + third_party/raft/rafttest/node_test.go | 171 + third_party/raft/rawnode.go | 557 ++ third_party/raft/rawnode_test.go | 1068 ++++ third_party/raft/read_only.go | 101 + third_party/raft/scripts/fix.sh | 22 + third_party/raft/scripts/genproto.sh | 44 + third_party/raft/scripts/test.sh | 155 + third_party/raft/scripts/test_lib.sh | 109 + third_party/raft/scripts/verify_genproto.sh | 27 + third_party/raft/state_trace.go | 339 ++ third_party/raft/state_trace_nop.go | 50 + third_party/raft/status.go | 105 + third_party/raft/storage.go | 326 ++ third_party/raft/storage_test.go | 277 + .../raft/testdata/async_storage_writes.txt | 869 +++ .../async_storage_writes_append_aba_race.txt | 517 ++ third_party/raft/testdata/campaign.txt | 118 + .../testdata/campaign_learner_must_vote.txt | 159 + third_party/raft/testdata/checkquorum.txt | 236 + .../confchange_disable_validation.txt | 76 + .../testdata/confchange_v1_add_single.txt | 96 + .../testdata/confchange_v1_remove_leader.txt | 250 + .../confchange_v1_remove_leader_stepdown.txt | 190 + .../confchange_v2_add_double_auto.txt | 404 ++ .../confchange_v2_add_double_implicit.txt | 136 + .../confchange_v2_add_single_auto.txt | 97 + .../confchange_v2_add_single_explicit.txt | 205 + .../testdata/confchange_v2_replace_leader.txt | 419 ++ .../confchange_v2_replace_leader_stepdown.txt | 192 + third_party/raft/testdata/forget_leader.txt | 192 + .../forget_leader_prevote_checkquorum.txt | 235 + .../forget_leader_read_only_lease_based.txt | 30 + .../heartbeat_resp_recovers_from_probing.txt | 90 + third_party/raft/testdata/lagging_commit.txt | 174 + third_party/raft/testdata/prevote.txt | 265 + .../raft/testdata/prevote_checkquorum.txt | 345 ++ .../raft/testdata/probe_and_replicate.txt | 770 +++ third_party/raft/testdata/replicate_pause.txt | 190 + third_party/raft/testdata/single_node.txt | 34 + .../slow_follower_after_compaction.txt | 121 + .../snapshot_succeed_via_app_resp.txt | 147 + .../snapshot_succeed_via_app_resp_behind.txt | 167 + third_party/raft/tla/MCetcdraft.cfg | 69 + third_party/raft/tla/MCetcdraft.tla | 120 + third_party/raft/tla/README.md | 94 + third_party/raft/tla/Traceetcdraft.cfg | 75 + third_party/raft/tla/Traceetcdraft.tla | 478 ++ third_party/raft/tla/etcdraft.cfg | 53 + third_party/raft/tla/etcdraft.tla | 926 ++++ third_party/raft/tla/example.ndjson | 4888 +++++++++++++++++ third_party/raft/tla/validate-model.sh | 51 + third_party/raft/tla/validate.sh | 191 + third_party/raft/tracker/inflights.go | 143 + third_party/raft/tracker/inflights_test.go | 256 + third_party/raft/tracker/progress.go | 314 ++ third_party/raft/tracker/progress_test.go | 211 + third_party/raft/tracker/state.go | 42 + third_party/raft/tracker/tracker.go | 281 + third_party/raft/types.go | 106 + third_party/raft/types_test.go | 94 + third_party/raft/util.go | 343 ++ third_party/raft/util_test.go | 152 + 173 files changed, 40230 insertions(+), 1 deletion(-) create mode 100644 third_party/raft-patch/PROTOBUF-LICENSE create mode 100644 third_party/raft-patch/PROTOBUF-PATENTS create mode 100644 third_party/raft-patch/README.md create mode 100644 third_party/raft-patch/SOURCES.json create mode 100644 third_party/raft-patch/election-random.patch create mode 100644 third_party/raft-patch/verify.py create mode 100644 third_party/raft/.github/dependabot.yml create mode 100644 third_party/raft/.github/workflows/codeql-analysis.yml create mode 100644 third_party/raft/.github/workflows/gh-workflow-approve.yaml create mode 100644 third_party/raft/.github/workflows/govuln.yaml create mode 100644 third_party/raft/.github/workflows/static-analysis.yaml create mode 100644 third_party/raft/.github/workflows/test_amd64.yaml create mode 100644 third_party/raft/.github/workflows/test_template.yaml create mode 100644 third_party/raft/.gitignore create mode 100644 third_party/raft/.go-version create mode 100644 third_party/raft/.golangci.yaml create mode 100644 third_party/raft/CHANGELOG/CHANGELOG-3.6.md create mode 100644 third_party/raft/CHANGELOG/CHANGELOG-3.7.md create mode 100644 third_party/raft/LICENSE create mode 100644 third_party/raft/Makefile create mode 100644 third_party/raft/OWNERS create mode 100644 third_party/raft/README.md create mode 100644 third_party/raft/bootstrap.go create mode 100644 third_party/raft/code-of-conduct.md create mode 100644 third_party/raft/confchange/confchange.go create mode 100644 third_party/raft/confchange/datadriven_test.go create mode 100644 third_party/raft/confchange/quick_test.go create mode 100644 third_party/raft/confchange/restore.go create mode 100644 third_party/raft/confchange/restore_test.go create mode 100644 third_party/raft/confchange/testdata/joint_autoleave.txt create mode 100644 third_party/raft/confchange/testdata/joint_idempotency.txt create mode 100644 third_party/raft/confchange/testdata/joint_learners_next.txt create mode 100644 third_party/raft/confchange/testdata/joint_safety.txt create mode 100644 third_party/raft/confchange/testdata/simple_idempotency.txt create mode 100644 third_party/raft/confchange/testdata/simple_promote_demote.txt create mode 100644 third_party/raft/confchange/testdata/simple_safety.txt create mode 100644 third_party/raft/confchange/testdata/update.txt create mode 100644 third_party/raft/confchange/testdata/zero.txt create mode 100644 third_party/raft/design.md create mode 100644 third_party/raft/diff_test.go create mode 100644 third_party/raft/doc.go create mode 100644 third_party/raft/example_test.go create mode 100644 third_party/raft/go.mod create mode 100644 third_party/raft/go.sum create mode 100644 third_party/raft/interaction_test.go create mode 100644 third_party/raft/log.go create mode 100644 third_party/raft/log_test.go create mode 100644 third_party/raft/log_unstable.go create mode 100644 third_party/raft/log_unstable_test.go create mode 100644 third_party/raft/logger.go create mode 100644 third_party/raft/node.go create mode 100644 third_party/raft/node_bench_test.go create mode 100644 third_party/raft/node_test.go create mode 100644 third_party/raft/node_util_test.go create mode 100644 third_party/raft/quorum/bench_test.go create mode 100644 third_party/raft/quorum/datadriven_test.go create mode 100644 third_party/raft/quorum/joint.go create mode 100644 third_party/raft/quorum/majority.go create mode 100644 third_party/raft/quorum/majority_test.go create mode 100644 third_party/raft/quorum/quick_test.go create mode 100644 third_party/raft/quorum/quorum.go create mode 100644 third_party/raft/quorum/testdata/joint_commit.txt create mode 100644 third_party/raft/quorum/testdata/joint_vote.txt create mode 100644 third_party/raft/quorum/testdata/majority_commit.txt create mode 100644 third_party/raft/quorum/testdata/majority_vote.txt create mode 100644 third_party/raft/quorum/voteresult_string.go create mode 100644 third_party/raft/raft.go create mode 100644 third_party/raft/raft_flow_control_test.go create mode 100644 third_party/raft/raft_paper_test.go create mode 100644 third_party/raft/raft_snap_test.go create mode 100644 third_party/raft/raft_test.go create mode 100644 third_party/raft/raftpb/alias.go create mode 100644 third_party/raft/raftpb/confchange.go create mode 100644 third_party/raft/raftpb/confchange_test.go create mode 100644 third_party/raft/raftpb/confstate.go create mode 100644 third_party/raft/raftpb/confstate_test.go create mode 100644 third_party/raft/raftpb/raft.pb.go create mode 100644 third_party/raft/raftpb/raft.proto create mode 100644 third_party/raft/raftpb/raft_test.go create mode 100644 third_party/raft/raftpb/util.go create mode 100644 third_party/raft/rafttest/doc.go create mode 100644 third_party/raft/rafttest/interaction_env.go create mode 100644 third_party/raft/rafttest/interaction_env_handler.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_add_nodes.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_campaign.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_compact.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_deliver_msgs.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_forget_leader.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_log_level.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_process_append_thread.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_process_apply_thread.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_process_ready.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_propose.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_propose_conf_change.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_raft_log.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_raftstate.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_report_unreachable.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_send_snapshot.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_set_randomized_election_timeout.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_stabilize.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_status.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_tick.go create mode 100644 third_party/raft/rafttest/interaction_env_handler_transfer_leadership.go create mode 100644 third_party/raft/rafttest/interaction_env_logger.go create mode 100644 third_party/raft/rafttest/network.go create mode 100644 third_party/raft/rafttest/network_test.go create mode 100644 third_party/raft/rafttest/node.go create mode 100644 third_party/raft/rafttest/node_bench_test.go create mode 100644 third_party/raft/rafttest/node_test.go create mode 100644 third_party/raft/rawnode.go create mode 100644 third_party/raft/rawnode_test.go create mode 100644 third_party/raft/read_only.go create mode 100644 third_party/raft/scripts/fix.sh create mode 100644 third_party/raft/scripts/genproto.sh create mode 100644 third_party/raft/scripts/test.sh create mode 100644 third_party/raft/scripts/test_lib.sh create mode 100644 third_party/raft/scripts/verify_genproto.sh create mode 100644 third_party/raft/state_trace.go create mode 100644 third_party/raft/state_trace_nop.go create mode 100644 third_party/raft/status.go create mode 100644 third_party/raft/storage.go create mode 100644 third_party/raft/storage_test.go create mode 100644 third_party/raft/testdata/async_storage_writes.txt create mode 100644 third_party/raft/testdata/async_storage_writes_append_aba_race.txt create mode 100644 third_party/raft/testdata/campaign.txt create mode 100644 third_party/raft/testdata/campaign_learner_must_vote.txt create mode 100644 third_party/raft/testdata/checkquorum.txt create mode 100644 third_party/raft/testdata/confchange_disable_validation.txt create mode 100644 third_party/raft/testdata/confchange_v1_add_single.txt create mode 100644 third_party/raft/testdata/confchange_v1_remove_leader.txt create mode 100644 third_party/raft/testdata/confchange_v1_remove_leader_stepdown.txt create mode 100644 third_party/raft/testdata/confchange_v2_add_double_auto.txt create mode 100644 third_party/raft/testdata/confchange_v2_add_double_implicit.txt create mode 100644 third_party/raft/testdata/confchange_v2_add_single_auto.txt create mode 100644 third_party/raft/testdata/confchange_v2_add_single_explicit.txt create mode 100644 third_party/raft/testdata/confchange_v2_replace_leader.txt create mode 100644 third_party/raft/testdata/confchange_v2_replace_leader_stepdown.txt create mode 100644 third_party/raft/testdata/forget_leader.txt create mode 100644 third_party/raft/testdata/forget_leader_prevote_checkquorum.txt create mode 100644 third_party/raft/testdata/forget_leader_read_only_lease_based.txt create mode 100644 third_party/raft/testdata/heartbeat_resp_recovers_from_probing.txt create mode 100644 third_party/raft/testdata/lagging_commit.txt create mode 100644 third_party/raft/testdata/prevote.txt create mode 100644 third_party/raft/testdata/prevote_checkquorum.txt create mode 100644 third_party/raft/testdata/probe_and_replicate.txt create mode 100644 third_party/raft/testdata/replicate_pause.txt create mode 100644 third_party/raft/testdata/single_node.txt create mode 100644 third_party/raft/testdata/slow_follower_after_compaction.txt create mode 100644 third_party/raft/testdata/snapshot_succeed_via_app_resp.txt create mode 100644 third_party/raft/testdata/snapshot_succeed_via_app_resp_behind.txt create mode 100644 third_party/raft/tla/MCetcdraft.cfg create mode 100644 third_party/raft/tla/MCetcdraft.tla create mode 100644 third_party/raft/tla/README.md create mode 100644 third_party/raft/tla/Traceetcdraft.cfg create mode 100644 third_party/raft/tla/Traceetcdraft.tla create mode 100644 third_party/raft/tla/etcdraft.cfg create mode 100644 third_party/raft/tla/etcdraft.tla create mode 100644 third_party/raft/tla/example.ndjson create mode 100644 third_party/raft/tla/validate-model.sh create mode 100644 third_party/raft/tla/validate.sh create mode 100644 third_party/raft/tracker/inflights.go create mode 100644 third_party/raft/tracker/inflights_test.go create mode 100644 third_party/raft/tracker/progress.go create mode 100644 third_party/raft/tracker/progress_test.go create mode 100644 third_party/raft/tracker/state.go create mode 100644 third_party/raft/tracker/tracker.go create mode 100644 third_party/raft/types.go create mode 100644 third_party/raft/types_test.go create mode 100644 third_party/raft/util.go create mode 100644 third_party/raft/util_test.go diff --git a/.gitignore b/.gitignore index 761c61d0eb..ab5b3da550 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,9 @@ bin/ # The furrow the build fetches, and the copy it stages for go:embed. Both are # six megabytes of somebody else's release; internal/furrowbin/pin.json is the # committed, auditable half and these are rebuilt from it by `make furrow`. -third_party/ +third_party/* +!third_party/raft/ +!third_party/raft-patch/ internal/furrowbin/cache/furrow-*.gz .obs/ *.log diff --git a/go.mod b/go.mod index a89d0d78ed..64a13f7fc1 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/rivo/uniseg v0.4.7 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/yuin/goldmark v1.8.5 + go.etcd.io/raft/v3 v3.7.0 golang.org/x/crypto v0.54.0 golang.org/x/image v0.32.0 golang.org/x/net v0.57.0 @@ -31,6 +32,7 @@ require ( golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 golang.org/x/text v0.40.0 + google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.29.1 ) @@ -113,3 +115,6 @@ require ( modernc.org/strutil v1.2.0 // indirect modernc.org/token v1.1.0 // indirect ) + +// Both native and WASI members use this reviewed per-node election patch. +replace go.etcd.io/raft/v3 => ./third_party/raft diff --git a/go.sum b/go.sum index 6e7a9939f4..a7ea105399 100644 --- a/go.sum +++ b/go.sum @@ -303,6 +303,8 @@ golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxb golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/third_party/raft-patch/PROTOBUF-LICENSE b/third_party/raft-patch/PROTOBUF-LICENSE new file mode 100644 index 0000000000..49ea0f9288 --- /dev/null +++ b/third_party/raft-patch/PROTOBUF-LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2018 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/third_party/raft-patch/PROTOBUF-PATENTS b/third_party/raft-patch/PROTOBUF-PATENTS new file mode 100644 index 0000000000..733099041f --- /dev/null +++ b/third_party/raft-patch/PROTOBUF-PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/third_party/raft-patch/README.md b/third_party/raft-patch/README.md new file mode 100644 index 0000000000..eadd0d1d8a --- /dev/null +++ b/third_party/raft-patch/README.md @@ -0,0 +1,34 @@ +# Shared Raft dependency + +Both native and WASI targets import the root module's local replacement +`third_party/raft`, an isolated `go.etcd.io/raft/v3` module at **v3.7.0**. Its +complete upstream source, tests, fixtures and Apache-2.0 license are retained. +Only `raft.go` is modified. `election-random.patch` is the complete upstream +diff, including the local-change comments; no election algorithm is rewritten. + +The added optional `Config.ElectionRandom func(int) int` belongs to one node. +It is consulted only by `resetRandomizedElectionTimeout`. Nil preserves the +upstream behavior for upstream tests/users; invalid bounds panic. codeaf always +sets it to the core-owned SplitMix64 provider, so the upstream global RNG is +never reached by that path. No global RNG replacement or linker trick is used. +This patch is local and has not been submitted or accepted upstream. + +Protobuf source is pinned to **google.golang.org/protobuf v1.36.11** through the +root and isolated module requirements and sums. It is the unmodified Go module, +not a second copy/fork; Go downloads exactly that checksummed source for either +target. `PROTOBUF-LICENSE` and `PROTOBUF-PATENTS` retain its notices. Do not +silently upgrade either source or apply a runtime-only replacement for wasm. +A future separately built ABI module must use these same requirements and local +Raft replacement; a replacement in a dependency's go.mod is not inherited. + +`SOURCES.json` records upstream module sums, archive SHA-256 values, and hashes +of every upstream Raft file. `verify.py` checks that the checked-in tree is +exactly that source plus the reviewed patch, and checks the root pins/notices. +The source archive can be obtained by `go mod download -json + go.etcd.io/raft/v3@v3.7.0` without using git or modifying the fork. + +Replay compatibility comprises this patch, the raft/protobuf source versions, +the deterministic protobuf marshal mode, and the SplitMix64 algorithm and +consumption order in `internal/raftcore`. The fixed core trace and published +PRNG vectors intentionally fail if that behavior changes. P03 owns the actual +wasm runtime comparison; native replay alone does not prove it. diff --git a/third_party/raft-patch/SOURCES.json b/third_party/raft-patch/SOURCES.json new file mode 100644 index 0000000000..ab5d5c3a31 --- /dev/null +++ b/third_party/raft-patch/SOURCES.json @@ -0,0 +1,180 @@ +{ + "raft": { + "module": "go.etcd.io/raft/v3", + "version": "v3.7.0", + "sum": "h1:BGzlwx07bLv8PW6OU5HObuz1y4hlPZUXA07pM1mPUh4=", + "zip_sha256": "1dd9bfb2dc99832d7a236d2a4a39f4a8cdb3fbbbe28187ec8c0a478f7a96cec7" + }, + "protobuf": { + "module": "google.golang.org/protobuf", + "version": "v1.36.11", + "sum": "h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=", + "zip_sha256": "14983d36c56a814ed91b6d652f2b8f895baba1b84eb43b28a0b132c8637cd274" + }, + "upstream_files": { + ".github/dependabot.yml": "474ea34f20a1555d46866c9989dec5e82b21605d74cb277860b469537d8f22fa", + ".github/workflows/codeql-analysis.yml": "c9d9d3627afe32974b7336df11a9c7cdaa85ad7f3a0f9622cd63bb8822a12e18", + ".github/workflows/gh-workflow-approve.yaml": "91cb3fcbbf0df8b1989e082d6a1b4f899174d40b23df2fc3f2353864db6b29b2", + ".github/workflows/govuln.yaml": "f7db43c29b23e829f5a492f9214fd459378da16acf2be372cc6d461cb8328828", + ".github/workflows/static-analysis.yaml": "e6f87d3bfa165dd5c02db3a02639d98ca804290e1078d7aaef1c670086550abc", + ".github/workflows/test_amd64.yaml": "f231e51cad3baefde9440704e3e0f9c60a997f46afca89b7447f7570022577b2", + ".github/workflows/test_template.yaml": "f3ea66b65913afa3b7ba1f88b606fc275b3155b4405a2071c9e52887705cc34d", + ".gitignore": "1d15bdb66e2d42cd0ecddeb6926d7a97dbf7a82a055067b7b9afc05112ee0437", + ".go-version": "aeb5ed3472a8898812d4cdca053cb6ef6e0132cc2db76e6f84d1e5e7cf0ebb72", + ".golangci.yaml": "231fec473a9e7da3abb31e2f47b846df3384ccc0ed5133494e5333189e006430", + "CHANGELOG/CHANGELOG-3.6.md": "ba67bc061293af7bad97c003169fa76178ca0e310b2f7a89af1991143b9fda2b", + "CHANGELOG/CHANGELOG-3.7.md": "a803f20f79242424b5c6cc2606eb495e94199a888efa6abab6564c9dde1dc648", + "LICENSE": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30", + "Makefile": "f1e4e044302f5f76de7df3c2f387ee3b71a371783b129f279b73b3d15ba11224", + "OWNERS": "b9a04fd770bc858d17c0768a584eada282525ecd83bad94457c2a220d66855fe", + "README.md": "208cb573a1a2d5155d14d8f6ba02bf7c1517f7e04f22f25d49d6d851717f5d5d", + "bootstrap.go": "92dee0b87b8a9ab0239514b3bb2e6c562b5be39645a5d4a7e575584179d40998", + "code-of-conduct.md": "ad7aa6f04cf0b889b543f472ec12ea0b1b23492bd6ced75e3588a896e281a8e0", + "confchange/confchange.go": "b4e082d86bb67bff630a5f1465d5fedf794a6806d47bd420c705b9de8291d6b3", + "confchange/datadriven_test.go": "061a509ba2fbcc1006bff49889102457c6b2e2aa9e1295bb0361ce0628b73628", + "confchange/quick_test.go": "46c5ed18396f19141c38e011a87ac11d392712a29e79a20f5c309fdc9ac543fe", + "confchange/restore.go": "744a5f97b4a9cc7d561ef96a155f8759102fa6553b651cff4f6901dada2007d1", + "confchange/restore_test.go": "e0e23551cce8cce90c9ac41992b074731561a77b88b101bc6624cebb11e7d147", + "confchange/testdata/joint_autoleave.txt": "bafae85d800985c4330d493ae88c821a27da8c0aafbeb68bd8ede42ebe399a27", + "confchange/testdata/joint_idempotency.txt": "73fb044a6626a3eb99974de77a62919f4b37a2a6bbe84e331ff6f90d13b22784", + "confchange/testdata/joint_learners_next.txt": "e768cbe86409273d311b1ac339d5e5d6dd316b6a7293e4dcea32f0f43833ccf0", + "confchange/testdata/joint_safety.txt": "dfabd98d6cd98913454e34063a27beddceb3ebbe2135d5dd9203e73df51c472a", + "confchange/testdata/simple_idempotency.txt": "72f4f5549e63bdd2d286735461686d2617dcc056275899944c64464a7da4136b", + "confchange/testdata/simple_promote_demote.txt": "1a37b5b72a2294bb1a1dc5f48cb5c397c3e8d7b8091dd07569a8589a5a9ceecf", + "confchange/testdata/simple_safety.txt": "4b5934ad4fea1c5e3d5746413ded662a7ded9d45c316baf3a69b48e43c6428c1", + "confchange/testdata/update.txt": "366dab3d4643b7adb554996fc0795fd37c4eefedb2534e929c09728c64260527", + "confchange/testdata/zero.txt": "90250d76bd923c41778f070e3a3bf81d4fefb4933b90c20e58b709df602fa8d6", + "design.md": "476fec21c454dc1c1c3727788063836d291d66f3fa82b5c69d0ecc73f53a6ead", + "diff_test.go": "b300fcd2daed178c81760b0cdf759f9fc68bacfaa7ec86e7df43880e6c87798d", + "doc.go": "5f9d35582ee1a4fe88facc4428dd833430278b5bc1a9f86f4d8d12aceb531490", + "example_test.go": "0e6b1894f87212d61d2b1e816784174c7d13043d9edd56a35d02eeaee5470fc7", + "go.mod": "1775c8fda51ca3bd28a5fdba571d841a2193a302a9bc809516fa1fdf9ff2c48d", + "go.sum": "ca86c1b1f31ef921e42e5651f9cc2cd661cc6e8ad5acf98bac052dfd950c0de3", + "interaction_test.go": "ad56e56e718de2f3197586c5ea27abd907eefc293790f7ce16198460d0acfb4f", + "log.go": "58316f5ae5a7e02067f83b9715af04ad375e1ba3e96f89ccd166e5cf947544d2", + "log_test.go": "74f77013a19dc08e589b98c3afdb7b715732e13b49dadc1ea4a3a09e7aa0e648", + "log_unstable.go": "9cdbd352eefb755342819adb7503f99f41a82259181e0ab929c801947179f3d0", + "log_unstable_test.go": "2d3d329e19465814d3c98cf7ef6e5aea0a47d970237fe2fff24bcfd1799c900f", + "logger.go": "6a342fe46a0de800f361bb517c2e0e823b3b2408f3f0be9874217e20c661e22a", + "node.go": "311ece789019299836c97ca24908a52ca0e96ea11d3c81109444b41f625607a9", + "node_bench_test.go": "7eb23ad07fb26e918bcd4a09577b273c7d2116d8326f25a0f6ecaf1cfcf75096", + "node_test.go": "bd69d9a2d030768927da28a3eac17454471a67721e72b56e7f21bd82ed58fd07", + "node_util_test.go": "9e3a244868507a925ea7c8b2298bd4dd864a6a6ea24c35afaac146995cb9fe0c", + "quorum/bench_test.go": "82e4d65761b96d4210ed74582a06caccbf2de3014b1b137a4833030e986b62be", + "quorum/datadriven_test.go": "65c2c51b8decc82b112ad6c5fb36c9bfeceb3359de233d5e912c3dba83676a4f", + "quorum/joint.go": "974987a423a79d06014a562543507375377a25a24abe39265df9ef0819b68d49", + "quorum/majority.go": "dbe87c11688eddab3a44c047356c3696fc5e835be19bcb7771e4cbe631e16ea0", + "quorum/majority_test.go": "9c0e7a2635760ae46b59c9774f83f4c27af555bf5fb68d3a04221b5cd22b9194", + "quorum/quick_test.go": "1baf4340e22b7a534cc762bcefe641180f26f7736a53db05d94389da04bd31f7", + "quorum/quorum.go": "63d6aa6b319b49b16afc3d741cf57dc41a040637d66001f8477f4e90f6a61331", + "quorum/testdata/joint_commit.txt": "3b70b68b2da1853ac9b58bfe07f6f623fc7b20a6570640c11d51c6ec69137c89", + "quorum/testdata/joint_vote.txt": "70d50c324fa0fc1ab5dc544b29d774e51269daf115c06af9165f784fbd55acf1", + "quorum/testdata/majority_commit.txt": "ed758fe8d1c97a27239d0496abe8cd723fd8009d36e6c47ba1e4cb65079a9434", + "quorum/testdata/majority_vote.txt": "f6c81808892de2ce21f39cafce10bc60b9df799fbaaf465c34453668a8d0543b", + "quorum/voteresult_string.go": "10c458e638369f50c5e7da0e05bc5468c1976c73bba022a0c6cd6703570e2f6b", + "raft.go": "d3d8fa573e1488e3aa35b9b997ba943454f3f357740a550d4bcc44d81975f07f", + "raft_flow_control_test.go": "60ad53d3dfbdc543934eee9ec2ec8c1ba741d9980fac53d49d96dd2e8726247a", + "raft_paper_test.go": "2bfc528490e59ec53ec0780a2972d5ae336b15eefc460ccdd331f4868f759047", + "raft_snap_test.go": "e39e33e51c92e137ec8d2ad166c4bbe00726da75abe49d08937e470da24f78e3", + "raft_test.go": "5f48681ef9c4fe99995662ce5e997bee7f52176515024b8fcbd1acd730e26311", + "raftpb/alias.go": "cf30bf89bdc663381fe1cedb7600ee8a8e6eeb2eac66fbaa39b612adcb19721a", + "raftpb/confchange.go": "0d40cf8a7f45e0791bd1c287e7b7de99a3a814418d40ecee1be4cbee7a02876b", + "raftpb/confchange_test.go": "06a35840bb1a4ad6a28504c90b569fb043e76264805147d0121a96df99a2c558", + "raftpb/confstate.go": "834b784909b8789e26f163b5879e9c282858e0392a68c9853998635fedf56240", + "raftpb/confstate_test.go": "0c3d28bbf2a75d80102d60c4f7293af02c7e2102a2e693472466a518f154adf6", + "raftpb/raft.pb.go": "d94c220250d54147f849f7c416787be4971903b4c95d93570e7906041513ff99", + "raftpb/raft.proto": "bd0fcbc7b76b85843c038af64bc6ea6ef0863d06335ecab79cb66144ad1aebd1", + "raftpb/raft_test.go": "d3685820829bb6216359f315b411b8b750e75f8e3bd38001169758be7f52fe82", + "raftpb/util.go": "acdf57a935b4a011c32519c6efe236a352f906423651deaea422d269a9a2ac1f", + "rafttest/doc.go": "27489bdf0cba11a615e91055701bfaae7931b211f08aec2b70c17d2828e8c7d6", + "rafttest/interaction_env.go": "7e0fcd59b146f1b8c1c0ec1ca8f7ee01264f5c19b48e55b3829d852aba9f2439", + "rafttest/interaction_env_handler.go": "0ffc48d2b64f56de68821038b8ac492021168cbcbb4e0bb37b436451ece55c89", + "rafttest/interaction_env_handler_add_nodes.go": "42a7d1f27c565669e4b154d9e393a4d58a360a90174ae9e120eaca66e5f54546", + "rafttest/interaction_env_handler_campaign.go": "c929bc07506698007cee42ef49046087721a34dfaa2a288227ddbcd2f1aa2c3e", + "rafttest/interaction_env_handler_compact.go": "14b0cc7c74cf161e487f6e48a53ff96701929bc4902385d85bd8872203cd0bf7", + "rafttest/interaction_env_handler_deliver_msgs.go": "8f28004334b86846ebc4d68ba5edf4848bf9c04c684a1a609079a628bcb44ea9", + "rafttest/interaction_env_handler_forget_leader.go": "7e29d53ba45ce1c1ffe42583c354501c63a67d998a024ce1d4239fd76433f25b", + "rafttest/interaction_env_handler_log_level.go": "c74f3a15b2ef2450a0d34d2933fd2a2731fa3815ed123f46f62964d8201e3392", + "rafttest/interaction_env_handler_process_append_thread.go": "d17517fbd97cff2a00c254675fddf1d2eb477ea15ffcb593648206a37df78517", + "rafttest/interaction_env_handler_process_apply_thread.go": "ceb5b3de1dadeeddee636ab4cf2530d1949621c4f4a58f1fd38dbe06d34127c9", + "rafttest/interaction_env_handler_process_ready.go": "52d534536138b6ea0664c59bdf9c69b4fe50a2b0f4337a61c56c15cf0461ff83", + "rafttest/interaction_env_handler_propose.go": "35e233fb125e452c0e937d950f69f62f79dae1219392d079d61b2500f2419d59", + "rafttest/interaction_env_handler_propose_conf_change.go": "1948993f7654412d9fce7eaf3de545a346f3a93ef1e0a397707fbcc86e7a96e7", + "rafttest/interaction_env_handler_raft_log.go": "6583b4067795788db42454e5ec265f784bc10059e6d69a273608d2a336aab7de", + "rafttest/interaction_env_handler_raftstate.go": "d89bc10ef58639813021d5cc46d8492afddadea0b647509cfd58ecbad04e2c02", + "rafttest/interaction_env_handler_report_unreachable.go": "1063ce218c4456eeeb291fb251996833415f3b39f3e26d5a1973b5d5188b50b7", + "rafttest/interaction_env_handler_send_snapshot.go": "d8551ae50f5750e8d283596b88f266c3fbaf77184e01050fee4ac02f9ef55b19", + "rafttest/interaction_env_handler_set_randomized_election_timeout.go": "37936648ae1f6890d8420c46a9413efdb814ee1d5c6892541f37b766574ff0a0", + "rafttest/interaction_env_handler_stabilize.go": "814e46a48897fd6590acd712d6b1edff01a19478cd45c7f0b80f3ec369291485", + "rafttest/interaction_env_handler_status.go": "37526fc346a193aeabc5a5fea37a04c0148a044aea0217aab361fc421484a95b", + "rafttest/interaction_env_handler_tick.go": "a8290ef8d59b984b145b2b8fec82b51f8e2b92798f62b855213d9b07390c89c3", + "rafttest/interaction_env_handler_transfer_leadership.go": "425f999bb45896b9e2d25d1e0f9126a4c6ace16e30344781b81d03e87220c876", + "rafttest/interaction_env_logger.go": "31ce0917a48d01841c0ef72907ddaec701b64a7406a8aad71a4f2bd0c9e336ff", + "rafttest/network.go": "06140961d78b5711c4057083d9c91949ba8c03969958959574ee9fdc8d1118ed", + "rafttest/network_test.go": "e9851033f886ceac85fb08d84c5c4dc6b685bd290c18920998acb222a6a6f436", + "rafttest/node.go": "220e756cfa50d8ea9be0fc206e600747e7575c0e57ef077dc43fe90d3a3f03d5", + "rafttest/node_bench_test.go": "240b4132100d27d4661afc0b23e6d39006f9f52c424bba47ef6fac1da24a1752", + "rafttest/node_test.go": "b96763f277e452dc40d373c8ef762b8f51213a06748b3b1abc57f6e4a78771a2", + "rawnode.go": "531cac8b286fd6e0bcd430c12053590325b9b00d113a584b8d3e6d1314c50f05", + "rawnode_test.go": "4f55f146abfb2dd39c3ea5b0ec659196f13420d808aac23e549f9ddd8a1ca18f", + "read_only.go": "7277fd552348bee2dcc7f9afe8ce24860f35bb9693b64d293b3d343ca38d669a", + "scripts/fix.sh": "ab50829e792500c4b4a02f5b9476aa8208e7049583546bc43679d149dfc5ea44", + "scripts/genproto.sh": "cad694f4bfa591f6bee56e0ed9ad7ad1075eace18eb4cb35459e278f58780c9f", + "scripts/test.sh": "05b4375acabecb35caebd6c88eca49592aca6a35f1074b78be2ef7d16dd3aac0", + "scripts/test_lib.sh": "05f1f3afcf003ba064263e1e9bca1c239a94c43ced23531dd9a834749f6acdfd", + "scripts/verify_genproto.sh": "d76a46aae3d32211447493b44d5f7748f14fac739fce1dfabd5bff44c46eae21", + "state_trace.go": "21b27746f2ca67584fc8543b5768a9523b3477249df4c39e4b5d3dee13a0c78a", + "state_trace_nop.go": "59df0abfddc9674b75edb2f700bd49ddf15768a9a95bbab764b28287f2ae8323", + "status.go": "67cea0e18c32d29f7b6f65e3d3450d4e92ee6e6cb3961122a7a2863eccd24c59", + "storage.go": "22020183114fe7555bc44ebda1b5b0d67de6534b0c8735c08c8667a0bd245403", + "storage_test.go": "aaa615729514d7341c3624bfe9ec5e15b7f67965dd47fe8be30dde2662871101", + "testdata/async_storage_writes.txt": "1e7f15961900e18e6c4d5e35f05ae6fb1d6f408e04b34f3f742805124ad874dc", + "testdata/async_storage_writes_append_aba_race.txt": "5fd1ed5d6d31d9c67e29489cbeaf77dab0c25381859075c544ecdb8b53bdac43", + "testdata/campaign.txt": "6d028c67780bccf0b5556badc2b575390aff7ba1058572b3f62ce3faed4b322f", + "testdata/campaign_learner_must_vote.txt": "82e18a052b18f7e9e82291500da55ae9c5ff121d9f580a9fd75a87484616e566", + "testdata/checkquorum.txt": "ae0440d545351cfdec8b969eae7d36fe761a394deacb103c0219b6acf02bf003", + "testdata/confchange_disable_validation.txt": "f1a511a622f76372e2a67a6c0da463802c5eb2732e6a5b0936455c9bb9749973", + "testdata/confchange_v1_add_single.txt": "f81f2b995da0bcffc485205d606526077c44e9c542f78ac08711fa87403bf995", + "testdata/confchange_v1_remove_leader.txt": "6b32021b069170a39073119ff2ac1e9f9adedde7c5d454802ec6135fc1a126fe", + "testdata/confchange_v1_remove_leader_stepdown.txt": "a142bf8605f15fb5a789cbd63a63ccc8d4c61529d940c7952ed8e2b7702be0ec", + "testdata/confchange_v2_add_double_auto.txt": "740e333b7af9fd942d365165ccaec7a315ef64c03ce23d59d0b2251d44b6099d", + "testdata/confchange_v2_add_double_implicit.txt": "e3c638ddb1c79ccd96d80e1dab1a8f1bd0f7486941b77f9628d19ee071b60ec1", + "testdata/confchange_v2_add_single_auto.txt": "ae896114e3bfe98c3c2e3a7665c85e490257a689bb2d55e117a83a0f521027f7", + "testdata/confchange_v2_add_single_explicit.txt": "6b304b15a4ee3fd2e06f3dbf2b74230db3a0b3097aecc0590eac69715bbae030", + "testdata/confchange_v2_replace_leader.txt": "32bb4fb9bc4d87c081ac6b156e3d08e7456991765f97306dfa7a07401ae39371", + "testdata/confchange_v2_replace_leader_stepdown.txt": "862afc142336ca141d2881406e3cedcd6a7b0fd3a36f40371a823c95e05c79ba", + "testdata/forget_leader.txt": "eb59f9b25f77397419bd5ec70f0c73e540100b4b68515b05cce60eadf48f7ed2", + "testdata/forget_leader_prevote_checkquorum.txt": "d542257736c1158a740cf76125675180ceb78e7ce9addc6d9bff6211ada28978", + "testdata/forget_leader_read_only_lease_based.txt": "ac344b338b9aa460b31da861a0255c7cea9bccd368135b88df5c460beee3df63", + "testdata/heartbeat_resp_recovers_from_probing.txt": "5a455ecf4d8a45b2ab048f43c7ff004777513dbc8d655fd9db49092d624f8a2c", + "testdata/lagging_commit.txt": "d624e20daa2413d41fdff2e2e6efb952770c4d07f1b06f26c190fd7c01ab7f3d", + "testdata/prevote.txt": "b27019f37948c31f1d58b5c2c0e97383c763d2974cbee82b783c7ee921361fcc", + "testdata/prevote_checkquorum.txt": "36d77800bf76df254c9255f8a97999c12c753850aa1901a2540bd52e962ad3dc", + "testdata/probe_and_replicate.txt": "92bb36b65c99b4b3055c3e22f6f3cc6a6cda4c379412e209815ae3e609bcd7be", + "testdata/replicate_pause.txt": "ac42fa5a1fefbd0a29df9d0fdb83598cf03937ea92cbb9866c8775ced512e6c2", + "testdata/single_node.txt": "fab19716b19e8361a0726df94897e896d87b6a3637712a4f86cf58ccf752b9d2", + "testdata/slow_follower_after_compaction.txt": "766d6322464f8655391e1d692d6453232ffb21ab891012debf08f70de9477c66", + "testdata/snapshot_succeed_via_app_resp.txt": "2f40418238cd1f290dd65bcb4c770394f660eed19b4c6850ef92d083f5ae675f", + "testdata/snapshot_succeed_via_app_resp_behind.txt": "c438b8565cedff6bdf5982b9e8460215c6709790da23c0a0b7839b0f49b831f3", + "tla/MCetcdraft.cfg": "5cc769549deb7acdf280a8b2600d87c88810c299f5102b15f5176f64a9ae04a9", + "tla/MCetcdraft.tla": "3b206581c4d1e8795b543f6bec1c293ea69241183acbdec6537c0265234a72b6", + "tla/README.md": "97a96cd4fcc5429da35a7d36e2c8fbca01cebbac844c0c88188a75a1d9fd1e78", + "tla/Traceetcdraft.cfg": "f146d25a0311f6944f5f3b3a89dcd66409d7fc3aac9326373b7cebf70a0b1795", + "tla/Traceetcdraft.tla": "93e46f774c6602674d6f6db90c75ddc4144cfadc101c2f1cb31fd1ce2e90b0bc", + "tla/etcdraft.cfg": "0279cc6c7c100f622e30230ad56d383eb9d9bd8ac99c996fe707710106a9960e", + "tla/etcdraft.tla": "15c445684529c59a59877951c7fb7df9a40da250aaa445f056ef48f56bb27044", + "tla/example.ndjson": "cd02768bd1849a717d9abaf29612d7901efe8048d2b3fa25d9a66ba3df7f7db7", + "tla/validate-model.sh": "212f61a008bc1f1bc808bdc280b5d5f08d9fd156b4f0bf9d4ecc812379ebe1ee", + "tla/validate.sh": "f3e9cd5de811a76072a3973850778c9172fcd91723f6105f0df19c4a8f8447b5", + "tracker/inflights.go": "4c1302b2c1937baf93c49e2265528a7b61184af8f3b3bbd1596b80c346e887a8", + "tracker/inflights_test.go": "2b1904bfc6a4d5292a61e53ea2a0697606e2510604dd5a336fe855965574c234", + "tracker/progress.go": "1062711d8f2693bc56d14bdb73cac71f9a4cb041f00b09253948e422f4027eed", + "tracker/progress_test.go": "0d4aa9a43224ec9d9b7af77b7a162617a4eb4e0f156d6f6f4d1e8c8a4a3b8ad7", + "tracker/state.go": "b4a19e82ddc422d15fb6cfa2738c16abe42cdf14166bc788d8fd536d79652fe8", + "tracker/tracker.go": "1ffda5213765af23030c9a7b640478ff69c070231307adcd2bc9ecac6675d452", + "types.go": "60068200885b00bbaff3daa09d468db89473c61c2150867f33d90ccfe16b438f", + "types_test.go": "352ba52936ab230bcb6034a91dc4768e2bb8fc48367fed101f545e765c8cf605", + "util.go": "b85b6fd2915e7d09eb534df528c4ede99890ec3770b60b2ff75a69f2ad17b33e", + "util_test.go": "0b6a9792c4129982a55bdba9dac2bcfa063a56e83179fe2bb2f35ca22adef3d4" + } +} diff --git a/third_party/raft-patch/election-random.patch b/third_party/raft-patch/election-random.patch new file mode 100644 index 0000000000..c0b6c642b4 --- /dev/null +++ b/third_party/raft-patch/election-random.patch @@ -0,0 +1,47 @@ +--- a/raft.go ++++ b/raft.go +@@ -126,6 +126,11 @@ + type Config struct { + // ID is the identity of the local raft. ID cannot be 0. + ID uint64 ++ ++ // ElectionRandom optionally supplies per-node bounded election randomness. ++ // It must return a value in [0, n) and must not be shared between nodes. ++ // A nil provider preserves the upstream default for existing callers. ++ ElectionRandom func(n int) int + + // ElectionTick is the number of Node.Tick invocations that must pass between + // elections. That is, if a follower does not receive any message from the +@@ -419,6 +424,7 @@ + // [electiontimeout, 2 * electiontimeout - 1]. It gets reset + // when raft changes its state to follower or candidate. + randomizedElectionTimeout int ++ electionRandom func(int) int + disableProposalForwarding bool + stepDownOnRemoval bool + +@@ -448,6 +454,7 @@ + + r := &raft{ + id: c.ID, ++ electionRandom: c.ElectionRandom, + lead: None, + isLearner: false, + raftLog: raftlog, +@@ -2051,7 +2058,15 @@ + } + + func (r *raft) resetRandomizedElectionTimeout() { +- r.randomizedElectionTimeout = r.electionTimeout + globalRand.Intn(r.electionTimeout) ++ random := r.electionRandom ++ if random == nil { ++ random = globalRand.Intn ++ } ++ v := random(r.electionTimeout) ++ if v < 0 || v >= r.electionTimeout { ++ panic("raft: election random value outside bound") ++ } ++ r.randomizedElectionTimeout = r.electionTimeout + v + } + + func (r *raft) sendTimeoutNow(to uint64) { diff --git a/third_party/raft-patch/verify.py b/third_party/raft-patch/verify.py new file mode 100644 index 0000000000..293ab45165 --- /dev/null +++ b/third_party/raft-patch/verify.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Check the R14 source boundary without git, downloads or the caller's home.""" +import hashlib +import json +from pathlib import Path +import subprocess +import tempfile + +meta = Path(__file__).resolve().parent +repo = meta.parent.parent +pin = json.loads((meta / "SOURCES.json").read_text()) +source = repo / "third_party/raft" +expected = pin["upstream_files"] +actual = {str(p.relative_to(source)) for p in source.rglob("*") if p.is_file()} +assert actual == set(expected), "Raft source inventory changed" +for name, digest in expected.items(): + if name == "raft.go": + continue + assert hashlib.sha256((source / name).read_bytes()).hexdigest() == digest, name +# Reverse the exact recorded patch into a throwaway file and compare with the +# independently pinned upstream bytes. The working tree is never modified. +with tempfile.TemporaryDirectory(prefix="p02-pin-", dir="/tmp") as temp: + path = Path(temp) / "raft.go" + path.write_bytes((source / "raft.go").read_bytes()) + subprocess.run(["patch", "--silent", "--batch", "--reverse", str(path), + str(meta / "election-random.patch")], check=True) + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected["raft.go"] +rootmod = (repo / "go.mod").read_text() +rootsum = (repo / "go.sum").read_text() +assert "replace go.etcd.io/raft/v3 => ./third_party/raft" in rootmod +for key in ["raft", "protobuf"]: + item = pin[key] + assert f'{item["module"]} {item["version"]}' in rootmod +assert f'google.golang.org/protobuf v1.36.11 {pin["protobuf"]["sum"]}' in rootsum +assert (meta / "PROTOBUF-LICENSE").read_text().startswith("Copyright") +print("R14 source, patch, protobuf pins and notices: PASS") diff --git a/third_party/raft/.github/dependabot.yml b/third_party/raft/.github/dependabot.yml new file mode 100644 index 0000000000..aafb8a27bc --- /dev/null +++ b/third_party/raft/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + + - package-ecosystem: gomod + directory: / + schedule: + interval: weekly diff --git a/third_party/raft/.github/workflows/codeql-analysis.yml b/third_party/raft/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000000..f1019c02b8 --- /dev/null +++ b/third_party/raft/.github/workflows/codeql-analysis.yml @@ -0,0 +1,71 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ main ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ main ] + schedule: + - cron: '20 14 * * 5' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v3.29.5 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v3.29.5 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v3.29.5 diff --git a/third_party/raft/.github/workflows/gh-workflow-approve.yaml b/third_party/raft/.github/workflows/gh-workflow-approve.yaml new file mode 100644 index 0000000000..d0be9220ec --- /dev/null +++ b/third_party/raft/.github/workflows/gh-workflow-approve.yaml @@ -0,0 +1,42 @@ +--- +name: Approve GitHub Workflows + +on: + pull_request_target: + types: + - labeled + - synchronize + branches: + - main + +jobs: + approve: + name: Approve ok-to-test + if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Update PR + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + continue-on-error: true + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + debug: ${{ secrets.ACTIONS_RUNNER_DEBUG == 'true' }} + script: | + const result = await github.rest.actions.listWorkflowRunsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + event: "pull_request", + status: "action_required", + head_sha: context.payload.pull_request.head.sha, + per_page: 100 + }); + + for (var run of result.data.workflow_runs) { + await github.rest.actions.approveWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: run.id + }); + } diff --git a/third_party/raft/.github/workflows/govuln.yaml b/third_party/raft/.github/workflows/govuln.yaml new file mode 100644 index 0000000000..f77af2f157 --- /dev/null +++ b/third_party/raft/.github/workflows/govuln.yaml @@ -0,0 +1,14 @@ +name: Go Vulnerability Checker +on: [push, pull_request] +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - run: date + - run: go install golang.org/x/vuln/cmd/govulncheck@latest && govulncheck ./... diff --git a/third_party/raft/.github/workflows/static-analysis.yaml b/third_party/raft/.github/workflows/static-analysis.yaml new file mode 100644 index 0000000000..db0cb8e98e --- /dev/null +++ b/third_party/raft/.github/workflows/static-analysis.yaml @@ -0,0 +1,21 @@ +name: Static Analysis +on: [push, pull_request] +jobs: + run: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - name: golangci-lint + uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee # v9.2.1 + with: + version: v2.10.1 + - name: protoc + uses: arduino/setup-protoc@149f6c87b92550901b26acd1632e11c3662e381f # v1.3.0 + with: + version: '3.20.3' + - run: make verify diff --git a/third_party/raft/.github/workflows/test_amd64.yaml b/third_party/raft/.github/workflows/test_amd64.yaml new file mode 100644 index 0000000000..179bd294aa --- /dev/null +++ b/third_party/raft/.github/workflows/test_amd64.yaml @@ -0,0 +1,26 @@ +name: Test AMD64 +permissions: read-all +on: [push, pull_request] +jobs: + test-linux-amd64: + uses: ./.github/workflows/test_template.yaml + with: + targets: "['linux-amd64-unit-4-cpu-race']" + test-linux-386: + uses: ./.github/workflows/test_template.yaml + with: + targets: "['linux-386-unit-1-cpu']" + + coverage: + needs: + - test-linux-amd64 + - test-linux-386 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - run: make test diff --git a/third_party/raft/.github/workflows/test_template.yaml b/third_party/raft/.github/workflows/test_template.yaml new file mode 100644 index 0000000000..587e5d8f00 --- /dev/null +++ b/third_party/raft/.github/workflows/test_template.yaml @@ -0,0 +1,49 @@ +name: Workflow Test Template +on: + workflow_call: + inputs: + runs-on: + required: false + type: string + default: ubuntu-latest + targets: + required: false + type: string + default: "[]" +permissions: read-all + +jobs: + run: + runs-on: ${{ inputs.runs-on }} + # this is to prevent arm64 jobs from running at forked projects + if: inputs.arch != 'arm64' || github.repository == 'etcd-io/raft' + strategy: + fail-fast: false + matrix: + target: ${{ fromJSON(inputs.targets) }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - env: + TARGET: ${{ matrix.target }} + run: | + go clean -testcache + case "${TARGET}" in + linux-amd64-unit-4-cpu-race) + GOARCH=amd64 PASSES='unit' RACE='true' CPU='4' ./scripts/test.sh -p=2 + ;; + linux-386-unit-1-cpu) + GOARCH=386 PASSES='unit' RACE='false' CPU='1' ./scripts/test.sh -p=4 + ;; + linux-arm64-unit-4-cpu-race) + GOARCH=arm64 PASSES='unit' RACE='true' CPU='4' ./scripts/test.sh -p=2 + ;; + *) + echo "Failed to find target" + exit 1 + ;; + esac diff --git a/third_party/raft/.gitignore b/third_party/raft/.gitignore new file mode 100644 index 0000000000..485dee64bc --- /dev/null +++ b/third_party/raft/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/third_party/raft/.go-version b/third_party/raft/.go-version new file mode 100644 index 0000000000..ea0928cedf --- /dev/null +++ b/third_party/raft/.go-version @@ -0,0 +1 @@ +1.26.4 diff --git a/third_party/raft/.golangci.yaml b/third_party/raft/.golangci.yaml new file mode 100644 index 0000000000..7eacd7f881 --- /dev/null +++ b/third_party/raft/.golangci.yaml @@ -0,0 +1,60 @@ +version: "2" +run: + timeout: 30m +linters: + default: none + # please keep this alphabetized + enable: + # Don't use soon to deprecated[1] linters that lead to false + # https://github.com/golangci/golangci-lint/issues/1841 + # - deadcode + # - structcheck + # - varcheck + - govet + - ineffassign + - revive + - staticcheck + - unused + - unconvert # Remove unnecessary type conversions + - usetesting + # please keep this alphabetized + settings: + staticcheck: + checks: + - all + - -SA1019 # TODO(fix) Using a deprecated function, variable, constant or field + - -SA2002 # TODO(fix) Called testing.T.FailNow or SkipNow in a goroutine, which isn’t allowed + - -QF1008 + - -QF1003 + - -QF1001 + - -S1019 + # Excluding configuration per-path, per-linter, per-text and per-source + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + rules: + # exclude ineffassing linter for generated files for conversion + - linters: + - ineffassign + path: conversion\.go +issues: + max-same-issues: 0 +formatters: + enable: + - gci + - goimports + settings: + gci: + sections: + - standard + - default + - prefix(go.etcd.io) + goimports: + local-prefixes: + - go.etcd.io + exclusions: + generated: lax diff --git a/third_party/raft/CHANGELOG/CHANGELOG-3.6.md b/third_party/raft/CHANGELOG/CHANGELOG-3.6.md new file mode 100644 index 0000000000..4e867f6741 --- /dev/null +++ b/third_party/raft/CHANGELOG/CHANGELOG-3.6.md @@ -0,0 +1,41 @@ +Note that we start to track changes starting from v3.6. + +
+ +## v3.6.0(2025-02-05) + +### Changelog since v3.6.0-beta.0 +There isn't any production code change since v3.6.0-beta.0. Only some dependencies +are bumped, and also a minor update on readme in https://github.com/etcd-io/raft/pull/227. + +
+ +## v3.6.0-beta.0(2024-11-20) + +### Changelog since v3.6.0-alpha.0 +- [Minor refactoring `raft.maybeSendAppend`](https://github.com/etcd-io/raft/pull/136) +- [Add entryID and logSlice types](https://github.com/etcd-io/raft/pull/145) +- [Fix next index might be smaller than match index](https://github.com/etcd-io/raft/pull/149) +- [Minor refactoring raftLog initialization](https://github.com/etcd-io/raft/pull/151) +- [cleanup Match, Next and MaybeUpdate](https://github.com/etcd-io/raft/pull/165) +- [tracker: track in-flight commit index](https://github.com/etcd-io/raft/pull/171) +- [Replace sort.Slice with slices.Sort, slices.SortFunc](https://github.com/etcd-io/raft/pull/221) + +### Others +- [Introduce TLA+ trace validation](https://github.com/etcd-io/raft/pull/113) + +
+ +## v3.6.0-alpha.0(2024-01-12) + +### Features +- [Add MaxInflightBytes setting in `raft.Config` for better flow control of entries](https://github.com/etcd-io/etcd/pull/14624) +- [Send empty `MsgApp` when entry in-flight limits are exceeded](https://github.com/etcd-io/etcd/pull/14633) +- [Support asynchronous storage writes](https://github.com/etcd-io/raft/pull/8) +- [Paginate the unapplied config changes scan](https://github.com/etcd-io/raft/pull/32) +- [Add ForgetLeader](https://github.com/etcd-io/raft/pull/78) +- [Add StepDownOnRemoval](https://github.com/etcd-io/raft/pull/79) +- [Accept any snapshot that allows replication](https://github.com/etcd-io/raft/pull/110) + +### Others +- [Deprecate RawNode.TickQuiesced()](https://github.com/etcd-io/raft/pull/62) diff --git a/third_party/raft/CHANGELOG/CHANGELOG-3.7.md b/third_party/raft/CHANGELOG/CHANGELOG-3.7.md new file mode 100644 index 0000000000..64f70abad7 --- /dev/null +++ b/third_party/raft/CHANGELOG/CHANGELOG-3.7.md @@ -0,0 +1,11 @@ + +
+ +## v3.7.0-alpha.0(TBD) + +### Features +- [Allow users to pass in a snapshot with only the ConfState initialized during bootstrap](https://github.com/etcd-io/raft/pull/370) +- [Improve the ReadIndex flow to prevent stale read index caused by RequestIndex retries](https://github.com/etcd-io/raft/pull/397) + +### Minor changes +- [Align formatting of node ID in MajorityConfig](https://github.com/etcd-io/raft/pull/414) \ No newline at end of file diff --git a/third_party/raft/LICENSE b/third_party/raft/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/third_party/raft/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/raft/Makefile b/third_party/raft/Makefile new file mode 100644 index 0000000000..3ae9d6a3c2 --- /dev/null +++ b/third_party/raft/Makefile @@ -0,0 +1,28 @@ +GO_TEST_FLAGS?= + +.PHONY: verify +verify: verify-gofmt verify-dep verify-lint verify-mod-tidy verify-genproto + +.PHONY: verify-gofmt +verify-gofmt: + PASSES="gofmt" ./scripts/test.sh + +.PHONY: verify-dep +verify-dep: + PASSES="dep" ./scripts/test.sh + +.PHONY: verify-lint +verify-lint: + golangci-lint run + +.PHONY: verify-mod-tidy +verify-mod-tidy: + PASSES="mod_tidy" ./scripts/test.sh + +.PHONY: verify-genproto +verify-genproto: + PASSES="genproto" ./scripts/test.sh + +.PHONY: test +test: + PASSES="unit" ./scripts/test.sh $(GO_TEST_FLAGS) diff --git a/third_party/raft/OWNERS b/third_party/raft/OWNERS new file mode 100644 index 0000000000..288f19949d --- /dev/null +++ b/third_party/raft/OWNERS @@ -0,0 +1,11 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - ahrtr # Benjamin Wang + - serathius # Marek Siarkowicz + - ptabor # Piotr Tabor + - spzala # Sahdev Zala +reviewers: + - pav-kv # Pavel Kalinnikov +emeritus_approvers: + - tbg # Tobias Grieger diff --git a/third_party/raft/README.md b/third_party/raft/README.md new file mode 100644 index 0000000000..9bddeb1b9b --- /dev/null +++ b/third_party/raft/README.md @@ -0,0 +1,201 @@ +# Raft library + +Raft is a protocol with which a cluster of nodes can maintain a replicated state machine. +The state machine is kept in sync through the use of a replicated log. +For more details on Raft, see "In Search of an Understandable Consensus Algorithm" +(https://raft.github.io/raft.pdf) by Diego Ongaro and John Ousterhout. + +This Raft library is stable and feature complete. As of 2016, it is **the most widely used** Raft library in production, serving tens of thousands clusters each day. It powers distributed systems such as etcd, Kubernetes, Docker Swarm, Cloud Foundry Diego, CockroachDB, TiDB, Project Calico, Flannel, Hyperledger and more. + +Most Raft implementations have a monolithic design, including storage handling, messaging serialization, and network transport. This library instead follows a minimalistic design philosophy by only implementing the core raft algorithm. This minimalism buys flexibility, determinism, and performance. + +To keep the codebase small as well as provide flexibility, the library only implements the Raft algorithm; both network and disk IO are left to the user. Library users must implement their own transportation layer for message passing between Raft peers over the wire. Similarly, users must implement their own storage layer to persist the Raft log and state. + +In order to easily test the Raft library, its behavior should be deterministic. To achieve this determinism, the library models Raft as a state machine. The state machine takes a `Message` as input. A message can either be a local timer update or a network message sent from a remote peer. The state machine's output is a 3-tuple `{[]Messages, []LogEntries, NextState}` consisting of an array of `Messages`, `log entries`, and `Raft state changes`. For state machines with the same state, the same state machine input should always generate the same state machine output. + +A simple example application, _raftexample_, is also available to help illustrate how to use this package in practice: https://github.com/etcd-io/etcd/tree/main/contrib/raftexample + +# Features + +This raft implementation is a full feature implementation of Raft protocol. Features includes: + +- Leader election +- Log replication +- Log compaction +- Membership changes +- Leadership transfer extension +- Efficient linearizable read-only queries served by both the leader and followers + - leader checks with quorum and bypasses Raft log before processing read-only queries + - followers asks leader to get a safe read index before processing read-only queries +- More efficient lease-based linearizable read-only queries served by both the leader and followers + - leader bypasses Raft log and processing read-only queries locally + - followers asks leader to get a safe read index before processing read-only queries + - this approach relies on the clock of the all the machines in raft group + +This raft implementation also includes a few optional enhancements: + +- Optimistic pipelining to reduce log replication latency +- Flow control for log replication +- Batching Raft messages to reduce synchronized network I/O calls +- Batching log entries to reduce disk synchronized I/O +- Writing to leader's disk in parallel +- Internal proposal redirection from followers to leader +- Automatic stepping down when the leader loses quorum +- Protection against unbounded log growth when quorum is lost + +## Notable Users + +- [cockroachdb](https://github.com/cockroachdb/cockroach) A Scalable, Survivable, Strongly-Consistent SQL Database +- [dgraph](https://github.com/dgraph-io/dgraph) A Scalable, Distributed, Low Latency, High Throughput Graph Database +- [etcd](https://github.com/etcd-io/etcd) A distributed reliable key-value store +- [tikv](https://github.com/pingcap/tikv) A Distributed transactional key value database powered by Rust and Raft +- [swarmkit](https://github.com/docker/swarmkit) A toolkit for orchestrating distributed systems at any scale. +- [chain core](https://github.com/chain/chain) Software for operating permissioned, multi-asset blockchain networks + +## Usage + +The primary object in raft is a Node. Either start a Node from scratch using raft.StartNode or start a Node from some initial state using raft.RestartNode. + +To start a three-node cluster +```go + storage := raft.NewMemoryStorage() + c := &raft.Config{ + ID: 0x01, + ElectionTick: 10, + HeartbeatTick: 1, + Storage: storage, + MaxSizePerMsg: 4096, + MaxInflightMsgs: 256, + } + // Set peer list to all nodes in the cluster. + // Note that other nodes in the cluster, apart from this node, need to be started separately. + n := raft.StartNode(c, []raft.Peer{{ID: 0x01}, {ID: 0x02}, {ID: 0x03}}) +``` + +Start a single node cluster, like so: +```go + // Create storage and config as shown above. + // Set peer list to itself, so this node can become the leader of this single-node cluster. + peers := []raft.Peer{{ID: 0x01}} + n := raft.StartNode(c, peers) +``` + +To allow a new node to join this cluster, do not pass in any peers. First, add the node to the existing cluster by calling `ProposeConfChange` on any existing node inside the cluster. Then, start the node with an empty peer list, like so: +```go + // Create storage and config as shown above. + n := raft.StartNode(c, nil) +``` + +To restart a node from previous state: +```go + storage := raft.NewMemoryStorage() + + // Recover the in-memory storage from persistent snapshot, state and entries. + storage.ApplySnapshot(snapshot) + storage.SetHardState(state) + storage.Append(entries) + + c := &raft.Config{ + ID: 0x01, + ElectionTick: 10, + HeartbeatTick: 1, + Storage: storage, + MaxSizePerMsg: 4096, + MaxInflightMsgs: 256, + } + + // Restart raft without peer information. + // Peer information is already included in the storage. + n := raft.RestartNode(c) +``` + +After creating a Node, the user has a few responsibilities: + +First, read from the Node.Ready() channel and process the updates it contains. These steps may be performed in parallel, except as noted in step 2. + +1. Write Entries, HardState and Snapshot to persistent storage in order, i.e. Entries first, then HardState and Snapshot if they are not empty. If persistent storage supports atomic writes then all of them can be written together. Note that when writing an Entry with Index i, any previously-persisted entries with Index >= i must be discarded. + +2. Send all Messages to the nodes named in the To field. It is important that no messages be sent until the latest HardState has been persisted to disk, and all Entries written by any previous Ready batch (Messages may be sent while entries from the same batch are being persisted). To reduce the I/O latency, an optimization can be applied to make leader write to disk in parallel with its followers (as explained at section 10.2.1 in Raft thesis). If any Message has type MsgSnap, call Node.ReportSnapshot() after it has been sent (these messages may be large). Note: Marshalling messages is not thread-safe; it is important to make sure that no new entries are persisted while marshalling. The easiest way to achieve this is to serialise the messages directly inside the main raft loop. + +3. Apply Snapshot (if any) and CommittedEntries to the state machine. If any committed Entry has Type EntryConfChange, call Node.ApplyConfChange() to apply it to the node. The configuration change may be cancelled at this point by setting the NodeId field to zero before calling ApplyConfChange (but ApplyConfChange must be called one way or the other, and the decision to cancel must be based solely on the state machine and not external information such as the observed health of the node). + +4. Call Node.Advance() to signal readiness for the next batch of updates. This may be done at any time after step 1, although all updates must be processed in the order they were returned by Ready. + +Second, all persisted log entries must be made available via an implementation of the Storage interface. The provided MemoryStorage type can be used for this (if repopulating its state upon a restart), or a custom disk-backed implementation can be supplied. + +Third, after receiving a message from another node, pass it to Node.Step: + +```go + func recvRaftRPC(ctx context.Context, m *raftpb.Message) { + n.Step(ctx, m) + } +``` + +Finally, call `Node.Tick()` at regular intervals (probably via a `time.Ticker`). Raft has two important timeouts: heartbeat and the election timeout. However, internally to the raft package time is represented by an abstract "tick". + +The total state machine handling loop will look something like this: + +```go + for { + select { + case <-s.Ticker: + n.Tick() + case rd := <-s.Node.Ready(): + saveToStorage(rd.HardState, rd.Entries, rd.Snapshot) + send(rd.Messages) + if !raft.IsEmptySnap(rd.Snapshot) { + processSnapshot(rd.Snapshot) + } + for _, entry := range rd.CommittedEntries { + process(entry) + if entry.GetType() == raftpb.EntryConfChange { + var cc raftpb.ConfChange + proto.Unmarshal(entry.GetData(), &cc) + s.Node.ApplyConfChange(cc) + } + } + s.Node.Advance() + case <-s.done: + return + } + } +``` + +To propose changes to the state machine from the node to take application data, serialize it into a byte slice and call: + +```go + n.Propose(ctx, data) +``` + +If the proposal is committed, data will appear in committed entries with type raftpb.EntryNormal. There is no guarantee that a proposed command will be committed; the command may have to be reproposed after a timeout. + +To add or remove node in a cluster, build ConfChange struct 'cc' and call: + +```go + n.ProposeConfChange(ctx, cc) +``` + +After config change is committed, some committed entry with type raftpb.EntryConfChange will be returned. This must be applied to node through: + +```go + var cc raftpb.ConfChange + proto.Unmarshal(data, &cc) + n.ApplyConfChange(cc) +``` + +Note: An ID represents a unique node in a cluster for all time. A +given ID MUST be used only once even if the old node has been removed. +This means that for example IP addresses make poor node IDs since they +may be reused. Node IDs must be non-zero. + +## Implementation notes + +This implementation is up to date with the final Raft thesis (https://github.com/ongardie/dissertation/blob/master/stanford.pdf), although this implementation of the membership change protocol differs somewhat from that described in chapter 4. The key invariant that membership changes happen one node at a time is preserved, but in our implementation the membership change takes effect when its entry is applied, not when it is added to the log (so the entry is committed under the old membership instead of the new). This is equivalent in terms of safety, since the old and new configurations are guaranteed to overlap. + +To ensure there is no attempt to commit two membership changes at once by matching log positions (which would be unsafe since they should have different quorum requirements), any proposed membership change is simply disallowed while any uncommitted change appears in the leader's log. + +This approach introduces a problem when removing a member from a two-member cluster: If one of the members dies before the other one receives the commit of the confchange entry, then the member cannot be removed any more since the cluster cannot make progress. For this reason it is highly recommended to use three or more nodes in every cluster. + +## Go docs + +More detailed development documentation can be found in go docs: https://pkg.go.dev/go.etcd.io/raft/v3. diff --git a/third_party/raft/bootstrap.go b/third_party/raft/bootstrap.go new file mode 100644 index 0000000000..ba50b14892 --- /dev/null +++ b/third_party/raft/bootstrap.go @@ -0,0 +1,82 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "errors" + + "google.golang.org/protobuf/proto" + + pb "go.etcd.io/raft/v3/raftpb" +) + +// Bootstrap initializes the RawNode for first use by appending configuration +// changes for the supplied peers. This method returns an error if the Storage +// is nonempty. +// +// It is recommended that instead of calling this method, applications bootstrap +// their state manually by setting up a Storage that has a first index > 1 and +// which stores the desired ConfState as its InitialState. +func (rn *RawNode) Bootstrap(peers []Peer) error { + if len(peers) == 0 { + return errors.New("must provide at least one peer to Bootstrap") + } + lastIndex, err := rn.raft.raftLog.storage.LastIndex() + if err != nil { + return err + } + + if lastIndex != 0 { + return errors.New("can't bootstrap a nonempty Storage") + } + + // We've faked out initial entries above, but nothing has been + // persisted. Start with an empty HardState (thus the first Ready will + // emit a HardState update for the app to persist). + rn.prevHardSt = emptyState + + // TODO(tbg): remove StartNode and give the application the right tools to + // bootstrap the initial membership in a cleaner way. + rn.raft.becomeFollower(1, None) + ents := make([]*pb.Entry, len(peers)) + for i, peer := range peers { + cc := &pb.ConfChange{Type: pb.ConfChangeAddNode.Enum(), NodeId: new(peer.ID), Context: peer.Context} + data, err := proto.Marshal(cc) + if err != nil { + return err + } + + ents[i] = &pb.Entry{Type: pb.EntryConfChange.Enum(), Term: new(uint64(1)), Index: new(uint64(i + 1)), Data: data} + } + rn.raft.raftLog.append(ents...) + + // Now apply them, mainly so that the application can call Campaign + // immediately after StartNode in tests. Note that these nodes will + // be added to raft twice: here and when the application's Ready + // loop calls ApplyConfChange. The calls to addNode must come after + // all calls to raftLog.append so progress.next is set after these + // bootstrapping entries (it is an error if we try to append these + // entries since they have already been committed). + // We do not set raftLog.applied so the application will be able + // to observe all conf changes via Ready.CommittedEntries. + // + // TODO(bdarnell): These entries are still unstable; do we need to preserve + // the invariant that committed < unstable? + rn.raft.raftLog.committed = uint64(len(ents)) + for _, peer := range peers { + rn.raft.applyConfChange((&pb.ConfChange{NodeId: new(peer.ID), Type: pb.ConfChangeAddNode.Enum()}).AsV2()) + } + return nil +} diff --git a/third_party/raft/code-of-conduct.md b/third_party/raft/code-of-conduct.md new file mode 100644 index 0000000000..f78dd84bc2 --- /dev/null +++ b/third_party/raft/code-of-conduct.md @@ -0,0 +1,3 @@ +# etcd Community Code of Conduct + +Please refer to [etcd Community Code of Conduct](https://github.com/etcd-io/etcd/blob/main/code-of-conduct.md). diff --git a/third_party/raft/confchange/confchange.go b/third_party/raft/confchange/confchange.go new file mode 100644 index 0000000000..7694bd1bff --- /dev/null +++ b/third_party/raft/confchange/confchange.go @@ -0,0 +1,419 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package confchange + +import ( + "errors" + "fmt" + "strings" + + "go.etcd.io/raft/v3/quorum" + pb "go.etcd.io/raft/v3/raftpb" + "go.etcd.io/raft/v3/tracker" +) + +// Changer facilitates configuration changes. It exposes methods to handle +// simple and joint consensus while performing the proper validation that allows +// refusing invalid configuration changes before they affect the active +// configuration. +type Changer struct { + Tracker tracker.ProgressTracker + LastIndex uint64 +} + +// EnterJoint verifies that the outgoing (=right) majority config of the joint +// config is empty and initializes it with a copy of the incoming (=left) +// majority config. That is, it transitions from +// +// (1 2 3)&&() +// +// to +// +// (1 2 3)&&(1 2 3). +// +// The supplied changes are then applied to the incoming majority config, +// resulting in a joint configuration that in terms of the Raft thesis[1] +// (Section 4.3) corresponds to `C_{new,old}`. +// +// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf +func (c Changer) EnterJoint(autoLeave bool, ccs ...*pb.ConfChangeSingle) (tracker.Config, tracker.ProgressMap, error) { + cfg, trk, err := c.checkAndCopy() + if err != nil { + return c.err(err) + } + if joint(cfg) { + err := errors.New("config is already joint") + return c.err(err) + } + if len(incoming(cfg.Voters)) == 0 { + // We allow adding nodes to an empty config for convenience (testing and + // bootstrap), but you can't enter a joint state. + err := errors.New("can't make a zero-voter config joint") + return c.err(err) + } + // Clear the outgoing config. + *outgoingPtr(&cfg.Voters) = quorum.MajorityConfig{} + // Copy incoming to outgoing. + for id := range incoming(cfg.Voters) { + outgoing(cfg.Voters)[id] = struct{}{} + } + + if err := c.apply(&cfg, trk, ccs...); err != nil { + return c.err(err) + } + cfg.AutoLeave = autoLeave + return checkAndReturn(cfg, trk) +} + +// LeaveJoint transitions out of a joint configuration. It is an error to call +// this method if the configuration is not joint, i.e. if the outgoing majority +// config Voters[1] is empty. +// +// The outgoing majority config of the joint configuration will be removed, +// that is, the incoming config is promoted as the sole decision maker. In the +// notation of the Raft thesis[1] (Section 4.3), this method transitions from +// `C_{new,old}` into `C_new`. +// +// At the same time, any staged learners (LearnersNext) the addition of which +// was held back by an overlapping voter in the former outgoing config will be +// inserted into Learners. +// +// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf +func (c Changer) LeaveJoint() (tracker.Config, tracker.ProgressMap, error) { + cfg, trk, err := c.checkAndCopy() + if err != nil { + return c.err(err) + } + if !joint(cfg) { + err := errors.New("can't leave a non-joint config") + return c.err(err) + } + for id := range cfg.LearnersNext { + nilAwareAdd(&cfg.Learners, id) + trk[id].IsLearner = true + } + cfg.LearnersNext = nil + + for id := range outgoing(cfg.Voters) { + _, isVoter := incoming(cfg.Voters)[id] + _, isLearner := cfg.Learners[id] + + if !isVoter && !isLearner { + delete(trk, id) + } + } + *outgoingPtr(&cfg.Voters) = nil + cfg.AutoLeave = false + + return checkAndReturn(cfg, trk) +} + +// Simple carries out a series of configuration changes that (in aggregate) +// mutates the incoming majority config Voters[0] by at most one. This method +// will return an error if that is not the case, if the resulting quorum is +// zero, or if the configuration is in a joint state (i.e. if there is an +// outgoing configuration). +func (c Changer) Simple(ccs ...*pb.ConfChangeSingle) (tracker.Config, tracker.ProgressMap, error) { + cfg, trk, err := c.checkAndCopy() + if err != nil { + return c.err(err) + } + if joint(cfg) { + err := errors.New("can't apply simple config change in joint config") + return c.err(err) + } + if err := c.apply(&cfg, trk, ccs...); err != nil { + return c.err(err) + } + if n := symdiff(incoming(c.Tracker.Voters), incoming(cfg.Voters)); n > 1 { + return tracker.Config{}, nil, errors.New("more than one voter changed without entering joint config") + } + + return checkAndReturn(cfg, trk) +} + +// apply a change to the configuration. By convention, changes to voters are +// always made to the incoming majority config Voters[0]. Voters[1] is either +// empty or preserves the outgoing majority configuration while in a joint state. +func (c Changer) apply(cfg *tracker.Config, trk tracker.ProgressMap, ccs ...*pb.ConfChangeSingle) error { + for _, cc := range ccs { + if cc.GetNodeId() == 0 { + // etcd replaces the NodeId with zero if it decides (downstream of + // raft) to not apply a change, so we have to have explicit code + // here to ignore these. + continue + } + switch cc.GetType() { + case pb.ConfChangeAddNode: + c.makeVoter(cfg, trk, cc.GetNodeId()) + case pb.ConfChangeAddLearnerNode: + c.makeLearner(cfg, trk, cc.GetNodeId()) + case pb.ConfChangeRemoveNode: + c.remove(cfg, trk, cc.GetNodeId()) + case pb.ConfChangeUpdateNode: + default: + return fmt.Errorf("unexpected conf type %d", cc.GetType()) + } + } + if len(incoming(cfg.Voters)) == 0 { + return errors.New("removed all voters") + } + return nil +} + +// makeVoter adds or promotes the given ID to be a voter in the incoming +// majority config. +func (c Changer) makeVoter(cfg *tracker.Config, trk tracker.ProgressMap, id uint64) { + pr := trk[id] + if pr == nil { + c.initProgress(cfg, trk, id, false /* isLearner */) + return + } + + pr.IsLearner = false + nilAwareDelete(&cfg.Learners, id) + nilAwareDelete(&cfg.LearnersNext, id) + incoming(cfg.Voters)[id] = struct{}{} +} + +// makeLearner makes the given ID a learner or stages it to be a learner once +// an active joint configuration is exited. +// +// The former happens when the peer is not a part of the outgoing config, in +// which case we either add a new learner or demote a voter in the incoming +// config. +// +// The latter case occurs when the configuration is joint and the peer is a +// voter in the outgoing config. In that case, we do not want to add the peer +// as a learner because then we'd have to track a peer as a voter and learner +// simultaneously. Instead, we add the learner to LearnersNext, so that it will +// be added to Learners the moment the outgoing config is removed by +// LeaveJoint(). +func (c Changer) makeLearner(cfg *tracker.Config, trk tracker.ProgressMap, id uint64) { + pr := trk[id] + if pr == nil { + c.initProgress(cfg, trk, id, true /* isLearner */) + return + } + if pr.IsLearner { + return + } + // Remove any existing voter in the incoming config... + c.remove(cfg, trk, id) + // ... but save the Progress. + trk[id] = pr + // Use LearnersNext if we can't add the learner to Learners directly, i.e. + // if the peer is still tracked as a voter in the outgoing config. It will + // be turned into a learner in LeaveJoint(). + // + // Otherwise, add a regular learner right away. + if _, onRight := outgoing(cfg.Voters)[id]; onRight { + nilAwareAdd(&cfg.LearnersNext, id) + } else { + pr.IsLearner = true + nilAwareAdd(&cfg.Learners, id) + } +} + +// remove this peer as a voter or learner from the incoming config. +func (c Changer) remove(cfg *tracker.Config, trk tracker.ProgressMap, id uint64) { + if _, ok := trk[id]; !ok { + return + } + + delete(incoming(cfg.Voters), id) + nilAwareDelete(&cfg.Learners, id) + nilAwareDelete(&cfg.LearnersNext, id) + + // If the peer is still a voter in the outgoing config, keep the Progress. + if _, onRight := outgoing(cfg.Voters)[id]; !onRight { + delete(trk, id) + } +} + +// initProgress initializes a new progress for the given node or learner. +func (c Changer) initProgress(cfg *tracker.Config, trk tracker.ProgressMap, id uint64, isLearner bool) { + if !isLearner { + incoming(cfg.Voters)[id] = struct{}{} + } else { + nilAwareAdd(&cfg.Learners, id) + } + trk[id] = &tracker.Progress{ + // Initializing the Progress with the last index means that the follower + // can be probed (with the last index). + // + // TODO(tbg): seems awfully optimistic. Using the first index would be + // better. The general expectation here is that the follower has no log + // at all (and will thus likely need a snapshot), though the app may + // have applied a snapshot out of band before adding the replica (thus + // making the first index the better choice). + Match: 0, + Next: max(c.LastIndex, 1), // invariant: Match < Next + Inflights: tracker.NewInflights(c.Tracker.MaxInflight, c.Tracker.MaxInflightBytes), + IsLearner: isLearner, + // When a node is first added, we should mark it as recently active. + // Otherwise, CheckQuorum may cause us to step down if it is invoked + // before the added node has had a chance to communicate with us. + RecentActive: true, + } +} + +// checkInvariants makes sure that the config and progress are compatible with +// each other. This is used to check both what the Changer is initialized with, +// as well as what it returns. +func checkInvariants(cfg tracker.Config, trk tracker.ProgressMap) error { + // NB: intentionally allow the empty config. In production we'll never see a + // non-empty config (we prevent it from being created) but we will need to + // be able to *create* an initial config, for example during bootstrap (or + // during tests). Instead of having to hand-code this, we allow + // transitioning from an empty config into any other legal and non-empty + // config. + for _, ids := range []map[uint64]struct{}{ + cfg.Voters.IDs(), + cfg.Learners, + cfg.LearnersNext, + } { + for id := range ids { + if _, ok := trk[id]; !ok { + return fmt.Errorf("no progress for %d", id) + } + } + } + + // Any staged learner was staged because it could not be directly added due + // to a conflicting voter in the outgoing config. + for id := range cfg.LearnersNext { + if _, ok := outgoing(cfg.Voters)[id]; !ok { + return fmt.Errorf("%d is in LearnersNext, but not Voters[1]", id) + } + if trk[id].IsLearner { + return fmt.Errorf("%d is in LearnersNext, but is already marked as learner", id) + } + } + // Conversely Learners and Voters doesn't intersect at all. + for id := range cfg.Learners { + if _, ok := outgoing(cfg.Voters)[id]; ok { + return fmt.Errorf("%d is in Learners and Voters[1]", id) + } + if _, ok := incoming(cfg.Voters)[id]; ok { + return fmt.Errorf("%d is in Learners and Voters[0]", id) + } + if !trk[id].IsLearner { + return fmt.Errorf("%d is in Learners, but is not marked as learner", id) + } + } + + if !joint(cfg) { + // We enforce that empty maps are nil instead of zero. + if outgoing(cfg.Voters) != nil { + return fmt.Errorf("cfg.Voters[1] must be nil when not joint") + } + if cfg.LearnersNext != nil { + return fmt.Errorf("cfg.LearnersNext must be nil when not joint") + } + if cfg.AutoLeave { + return fmt.Errorf("AutoLeave must be false when not joint") + } + } + + return nil +} + +// checkAndCopy copies the tracker's config and progress map (deeply enough for +// the purposes of the Changer) and returns those copies. It returns an error +// if checkInvariants does. +func (c Changer) checkAndCopy() (tracker.Config, tracker.ProgressMap, error) { + cfg := c.Tracker.Config.Clone() + trk := tracker.ProgressMap{} + + for id, pr := range c.Tracker.Progress { + // A shallow copy is enough because we only mutate the Learner field. + ppr := *pr + trk[id] = &ppr + } + return checkAndReturn(cfg, trk) +} + +// checkAndReturn calls checkInvariants on the input and returns either the +// resulting error or the input. +func checkAndReturn(cfg tracker.Config, trk tracker.ProgressMap) (tracker.Config, tracker.ProgressMap, error) { + if err := checkInvariants(cfg, trk); err != nil { + return tracker.Config{}, tracker.ProgressMap{}, err + } + return cfg, trk, nil +} + +// err returns zero values and an error. +func (c Changer) err(err error) (tracker.Config, tracker.ProgressMap, error) { + return tracker.Config{}, nil, err +} + +// nilAwareAdd populates a map entry, creating the map if necessary. +func nilAwareAdd(m *map[uint64]struct{}, id uint64) { + if *m == nil { + *m = map[uint64]struct{}{} + } + (*m)[id] = struct{}{} +} + +// nilAwareDelete deletes from a map, nil'ing the map itself if it is empty after. +func nilAwareDelete(m *map[uint64]struct{}, id uint64) { + if *m == nil { + return + } + delete(*m, id) + if len(*m) == 0 { + *m = nil + } +} + +// symdiff returns the count of the symmetric difference between the sets of +// uint64s, i.e. len( (l - r) \union (r - l)). +func symdiff(l, r map[uint64]struct{}) int { + var n int + pairs := [][2]quorum.MajorityConfig{ + {l, r}, // count elems in l but not in r + {r, l}, // count elems in r but not in l + } + for _, p := range pairs { + for id := range p[0] { + if _, ok := p[1][id]; !ok { + n++ + } + } + } + return n +} + +func joint(cfg tracker.Config) bool { + return len(outgoing(cfg.Voters)) > 0 +} + +func incoming(voters quorum.JointConfig) quorum.MajorityConfig { return voters[0] } +func outgoing(voters quorum.JointConfig) quorum.MajorityConfig { return voters[1] } +func outgoingPtr(voters *quorum.JointConfig) *quorum.MajorityConfig { return &voters[1] } + +// Describe prints the type and NodeId of the configuration changes as a +// space-delimited string. +func Describe(ccs ...*pb.ConfChangeSingle) string { + var buf strings.Builder + for _, cc := range ccs { + if buf.Len() > 0 { + buf.WriteByte(' ') + } + fmt.Fprintf(&buf, "%s(%d)", cc.GetType(), cc.GetNodeId()) + } + return buf.String() +} diff --git a/third_party/raft/confchange/datadriven_test.go b/third_party/raft/confchange/datadriven_test.go new file mode 100644 index 0000000000..f814272278 --- /dev/null +++ b/third_party/raft/confchange/datadriven_test.go @@ -0,0 +1,110 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package confchange + +import ( + "errors" + "fmt" + "strconv" + "strings" + "testing" + + "github.com/cockroachdb/datadriven" + + pb "go.etcd.io/raft/v3/raftpb" + "go.etcd.io/raft/v3/tracker" +) + +func TestConfChangeDataDriven(t *testing.T) { + datadriven.Walk(t, "testdata", func(t *testing.T, path string) { + tr := tracker.MakeProgressTracker(10, 0) + c := Changer{ + Tracker: tr, + LastIndex: 0, // incremented in this test with each cmd + } + + // The test files use the commands + // - simple: run a simple conf change (i.e. no joint consensus), + // - enter-joint: enter a joint config, and + // - leave-joint: leave a joint config. + // The first two take a list of config changes, which have the following + // syntax: + // - vn: make n a voter, + // - ln: make n a learner, + // - rn: remove n, and + // - un: update n. + datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string { + defer func() { + c.LastIndex++ + }() + var ccs []*pb.ConfChangeSingle + toks := strings.Split(strings.TrimSpace(d.Input), " ") + if toks[0] == "" { + toks = nil + } + for _, tok := range toks { + if len(tok) < 2 { + return fmt.Sprintf("unknown token %s", tok) + } + cc := &pb.ConfChangeSingle{} + switch tok[0] { + case 'v': + cc.Type = pb.ConfChangeAddNode.Enum() + case 'l': + cc.Type = pb.ConfChangeAddLearnerNode.Enum() + case 'r': + cc.Type = pb.ConfChangeRemoveNode.Enum() + case 'u': + cc.Type = pb.ConfChangeUpdateNode.Enum() + default: + return fmt.Sprintf("unknown input: %s", tok) + } + id, err := strconv.ParseUint(tok[1:], 10, 64) + if err != nil { + return err.Error() + } + cc.NodeId = new(id) + ccs = append(ccs, cc) + } + + var cfg tracker.Config + var trk tracker.ProgressMap + var err error + switch d.Cmd { + case "simple": + cfg, trk, err = c.Simple(ccs...) + case "enter-joint": + var autoLeave bool + if len(d.CmdArgs) > 0 { + d.ScanArgs(t, "autoleave", &autoLeave) + } + cfg, trk, err = c.EnterJoint(autoLeave, ccs...) + case "leave-joint": + if len(ccs) > 0 { + err = errors.New("this command takes no input") + } else { + cfg, trk, err = c.LeaveJoint() + } + default: + return "unknown command" + } + if err != nil { + return err.Error() + "\n" + } + c.Tracker.Config, c.Tracker.Progress = cfg, trk + return fmt.Sprintf("%s\n%s", c.Tracker.Config, c.Tracker.Progress) + }) + }) +} diff --git a/third_party/raft/confchange/quick_test.go b/third_party/raft/confchange/quick_test.go new file mode 100644 index 0000000000..31e8f7b352 --- /dev/null +++ b/third_party/raft/confchange/quick_test.go @@ -0,0 +1,177 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package confchange + +import ( + "math/rand" + "reflect" + "testing" + "testing/quick" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + pb "go.etcd.io/raft/v3/raftpb" + "go.etcd.io/raft/v3/tracker" +) + +// TestConfChangeQuick uses quickcheck to verify that simple and joint config +// changes arrive at the same result. +func TestConfChangeQuick(t *testing.T) { + cfg := &quick.Config{ + MaxCount: 1000, + } + + // Log the first couple of runs to give some indication of things working + // as intended. + const infoCount = 5 + + runWithJoint := func(c *Changer, ccs []*pb.ConfChangeSingle) error { + cfg, trk, err := c.EnterJoint(false /* autoLeave */, ccs...) + if err != nil { + return err + } + // Also do this with autoLeave on, just to check that we'd get the same + // result. + cfg2a, trk2a, err := c.EnterJoint(true /* autoLeave */, ccs...) + if err != nil { + return err + } + cfg2a.AutoLeave = false + assert.Equal(t, cfg, cfg2a) + assert.Equal(t, trk, trk2a) + c.Tracker.Config = cfg + c.Tracker.Progress = trk + cfg2b, trk2b, err := c.LeaveJoint() + if err != nil { + return err + } + // Reset back to the main branch with autoLeave=false. + c.Tracker.Config = cfg + c.Tracker.Progress = trk + cfg, trk, err = c.LeaveJoint() + if err != nil { + return err + } + assert.Equal(t, cfg, cfg2b) + assert.Equal(t, trk, trk2b) + c.Tracker.Config = cfg + c.Tracker.Progress = trk + return nil + } + + runWithSimple := func(c *Changer, ccs []*pb.ConfChangeSingle) error { + for _, cc := range ccs { + cfg, trk, err := c.Simple(cc) + if err != nil { + return err + } + c.Tracker.Config, c.Tracker.Progress = cfg, trk + } + return nil + } + + type testFunc func(*Changer, []*pb.ConfChangeSingle) error + + wrapper := func(invoke testFunc) func(setup initialChanges, ccs confChanges) (*Changer, error) { + return func(setup initialChanges, ccs confChanges) (*Changer, error) { + tr := tracker.MakeProgressTracker(10, 0) + c := &Changer{ + Tracker: tr, + LastIndex: 10, + } + + if err := runWithSimple(c, setup); err != nil { + return nil, err + } + + err := invoke(c, ccs) + return c, err + } + } + + var n int + f1 := func(setup initialChanges, ccs confChanges) *Changer { + c, err := wrapper(runWithSimple)(setup, ccs) + require.NoError(t, err) + if n < infoCount { + t.Log("initial setup:", Describe(setup...)) + t.Log("changes:", Describe(ccs...)) + t.Log(c.Tracker.Config) + t.Log(c.Tracker.Progress) + } + n++ + return c + } + f2 := func(setup initialChanges, ccs confChanges) *Changer { + c, err := wrapper(runWithJoint)(setup, ccs) + require.NoError(t, err) + return c + } + err := quick.CheckEqual(f1, f2, cfg) + if err == nil { + return + } + cErr, ok := err.(*quick.CheckEqualError) + require.True(t, ok, err) + + t.Error("setup:", Describe(cErr.In[0].([]*pb.ConfChangeSingle)...)) + t.Error("ccs:", Describe(cErr.In[1].([]*pb.ConfChangeSingle)...)) + t.Errorf("out1: %+v\nout2: %+v", cErr.Out1, cErr.Out2) +} + +type confChanges []*pb.ConfChangeSingle + +func genCC(num func() int, id func() uint64, typ func() pb.ConfChangeType) []*pb.ConfChangeSingle { + var ccs []*pb.ConfChangeSingle + n := num() + for i := 0; i < n; i++ { + ccs = append(ccs, &pb.ConfChangeSingle{Type: typ().Enum(), NodeId: new(id())}) + } + return ccs +} + +func (confChanges) Generate(rand *rand.Rand, _ int) reflect.Value { + num := func() int { + return 1 + rand.Intn(9) + } + id := func() uint64 { + // Note that num() >= 1, so we're never returning 1 from this method, + // meaning that we'll never touch NodeId one, which is special to avoid + // voterless configs altogether in this test. + return 1 + uint64(num()) + } + typ := func() pb.ConfChangeType { + return pb.ConfChangeType(rand.Intn(len(pb.ConfChangeType_name))) + } + return reflect.ValueOf(genCC(num, id, typ)) +} + +type initialChanges []*pb.ConfChangeSingle + +func (initialChanges) Generate(rand *rand.Rand, _ int) reflect.Value { + num := func() int { + return 1 + rand.Intn(5) + } + id := func() uint64 { return uint64(num()) } + typ := func() pb.ConfChangeType { + return pb.ConfChangeAddNode + } + // NodeId one is special - it's in the initial config and will be a voter + // always (this is to avoid uninteresting edge cases where the simple conf + // changes can't easily make progress). + ccs := append([]*pb.ConfChangeSingle{{Type: pb.ConfChangeAddNode.Enum(), NodeId: new(uint64(1))}}, genCC(num, id, typ)...) + return reflect.ValueOf(ccs) +} diff --git a/third_party/raft/confchange/restore.go b/third_party/raft/confchange/restore.go new file mode 100644 index 0000000000..1cc72e5016 --- /dev/null +++ b/third_party/raft/confchange/restore.go @@ -0,0 +1,155 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package confchange + +import ( + pb "go.etcd.io/raft/v3/raftpb" + "go.etcd.io/raft/v3/tracker" +) + +// toConfChangeSingle translates a conf state into 1) a slice of operations creating +// first the config that will become the outgoing one, and then the incoming one, and +// b) another slice that, when applied to the config resulted from 1), represents the +// ConfState. +func toConfChangeSingle(cs *pb.ConfState) (out []*pb.ConfChangeSingle, in []*pb.ConfChangeSingle) { + // Example to follow along this code: + // voters=(1 2 3) learners=(5) outgoing=(1 2 4 6) learners_next=(4) + // + // This means that before entering the joint config, the configuration + // had voters (1 2 4 6) and perhaps some learners that are already gone. + // The new set of voters is (1 2 3), i.e. (1 2) were kept around, and (4 6) + // are no longer voters; however 4 is poised to become a learner upon leaving + // the joint state. + // We can't tell whether 5 was a learner before entering the joint config, + // but it doesn't matter (we'll pretend that it wasn't). + // + // The code below will construct + // outgoing = add 1; add 2; add 4; add 6 + // incoming = remove 1; remove 2; remove 4; remove 6 + // add 1; add 2; add 3; + // add-learner 5; + // add-learner 4; + // + // So, when starting with an empty config, after applying 'outgoing' we have + // + // quorum=(1 2 4 6) + // + // From which we enter a joint state via 'incoming' + // + // quorum=(1 2 3)&&(1 2 4 6) learners=(5) learners_next=(4) + // + // as desired. + + for _, id := range cs.VotersOutgoing { + // If there are outgoing voters, first add them one by one so that the + // (non-joint) config has them all. + out = append(out, &pb.ConfChangeSingle{ + Type: pb.ConfChangeAddNode.Enum(), + NodeId: new(id), + }) + + } + + // We're done constructing the outgoing slice, now on to the incoming one + // (which will apply on top of the config created by the outgoing slice). + + // First, we'll remove all of the outgoing voters. + for _, id := range cs.VotersOutgoing { + in = append(in, &pb.ConfChangeSingle{ + Type: pb.ConfChangeRemoveNode.Enum(), + NodeId: new(id), + }) + } + // Then we'll add the incoming voters and learners. + for _, id := range cs.Voters { + in = append(in, &pb.ConfChangeSingle{ + Type: pb.ConfChangeAddNode.Enum(), + NodeId: new(id), + }) + } + for _, id := range cs.Learners { + in = append(in, &pb.ConfChangeSingle{ + Type: pb.ConfChangeAddLearnerNode.Enum(), + NodeId: new(id), + }) + } + // Same for LearnersNext; these are nodes we want to be learners but which + // are currently voters in the outgoing config. + for _, id := range cs.LearnersNext { + in = append(in, &pb.ConfChangeSingle{ + Type: pb.ConfChangeAddLearnerNode.Enum(), + NodeId: new(id), + }) + } + return out, in +} + +func chain(chg Changer, ops ...func(Changer) (tracker.Config, tracker.ProgressMap, error)) (tracker.Config, tracker.ProgressMap, error) { + for _, op := range ops { + cfg, trk, err := op(chg) + if err != nil { + return tracker.Config{}, nil, err + } + chg.Tracker.Config = cfg + chg.Tracker.Progress = trk + } + return chg.Tracker.Config, chg.Tracker.Progress, nil +} + +// Restore takes a Changer (which must represent an empty configuration), and +// runs a sequence of changes enacting the configuration described in the +// ConfState. +// +// TODO(tbg) it's silly that this takes a Changer. Unravel this by making sure +// the Changer only needs a ProgressMap (not a whole Tracker) at which point +// this can just take LastIndex and MaxInflight directly instead and cook up +// the results from that alone. +func Restore(chg Changer, cs *pb.ConfState) (tracker.Config, tracker.ProgressMap, error) { + outgoing, incoming := toConfChangeSingle(cs) + + var ops []func(Changer) (tracker.Config, tracker.ProgressMap, error) + + if len(outgoing) == 0 { + // No outgoing config, so just apply the incoming changes one by one. + for _, cc := range incoming { + cc := cc // loop-local copy + ops = append(ops, func(chg Changer) (tracker.Config, tracker.ProgressMap, error) { + return chg.Simple(cc) + }) + } + } else { + // The ConfState describes a joint configuration. + // + // First, apply all of the changes of the outgoing config one by one, so + // that it temporarily becomes the incoming active config. For example, + // if the config is (1 2 3)&(2 3 4), this will establish (2 3 4)&(). + for _, cc := range outgoing { + cc := cc // loop-local copy + ops = append(ops, func(chg Changer) (tracker.Config, tracker.ProgressMap, error) { + return chg.Simple(cc) + }) + } + // Now enter the joint state, which rotates the above additions into the + // outgoing config, and adds the incoming config in. Continuing the + // example above, we'd get (1 2 3)&(2 3 4), i.e. the incoming operations + // would be removing 2,3,4 and then adding in 1,2,3 while transitioning + // into a joint state. + ops = append(ops, func(chg Changer) (tracker.Config, tracker.ProgressMap, error) { + return chg.EnterJoint(cs.GetAutoLeave(), incoming...) + }) + } + + return chain(chg, ops...) +} diff --git a/third_party/raft/confchange/restore_test.go b/third_party/raft/confchange/restore_test.go new file mode 100644 index 0000000000..21423b0f43 --- /dev/null +++ b/third_party/raft/confchange/restore_test.go @@ -0,0 +1,139 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package confchange + +import ( + "math/rand" + "reflect" + "slices" + "testing" + "testing/quick" + + "github.com/stretchr/testify/assert" + + pb "go.etcd.io/raft/v3/raftpb" + "go.etcd.io/raft/v3/tracker" +) + +type rndConfChange pb.ConfState + +// Generate creates a random (valid) ConfState for use with quickcheck. +func (*rndConfChange) Generate(rand *rand.Rand, _ int) reflect.Value { + conv := func(sl []int) []uint64 { + // We want IDs but the incoming slice is zero-indexed, so add one to + // each. + out := make([]uint64, len(sl)) + for i := range sl { + out[i] = uint64(sl[i] + 1) + } + return out + } + cs := &pb.ConfState{} + // NB: never generate the empty ConfState, that one should be unit tested. + nVoters := 1 + rand.Intn(5) + + nLearners := rand.Intn(5) + // The number of voters that are in the outgoing config but not in the + // incoming one. (We'll additionally retain a random number of the + // incoming voters below). + nRemovedVoters := rand.Intn(3) + + // Voters, learners, and removed voters must not overlap. A "removed voter" + // is one that we have in the outgoing config but not the incoming one. + ids := conv(rand.Perm(2 * (nVoters + nLearners + nRemovedVoters))) + + cs.Voters = ids[:nVoters] + ids = ids[nVoters:] + + if nLearners > 0 { + cs.Learners = ids[:nLearners] + ids = ids[nLearners:] + } + + // Roll the dice on how many of the incoming voters we decide were also + // previously voters. + // + // NB: this code avoids creating non-nil empty slices (here and below). + nOutgoingRetainedVoters := rand.Intn(nVoters + 1) + if nOutgoingRetainedVoters > 0 || nRemovedVoters > 0 { + cs.VotersOutgoing = append([]uint64(nil), cs.Voters[:nOutgoingRetainedVoters]...) + cs.VotersOutgoing = append(cs.VotersOutgoing, ids[:nRemovedVoters]...) + } + // Only outgoing voters that are not also incoming voters can be in + // LearnersNext (they represent demotions). + if nRemovedVoters > 0 { + if nLearnersNext := rand.Intn(nRemovedVoters + 1); nLearnersNext > 0 { + cs.LearnersNext = ids[:nLearnersNext] + } + } + + cs.AutoLeave = new(len(cs.VotersOutgoing) > 0 && rand.Intn(2) == 1) + return reflect.ValueOf((*rndConfChange)(cs)) +} + +func TestRestore(t *testing.T) { + cfg := quick.Config{MaxCount: 1000} + + f := func(cs *pb.ConfState) bool { + chg := Changer{ + Tracker: tracker.MakeProgressTracker(20, 0), + LastIndex: 10, + } + cfg, trk, err := Restore(chg, cs) + if !assert.NoError(t, err) { + return false + } + chg.Tracker.Config = cfg + chg.Tracker.Progress = trk + + for _, sl := range [][]uint64{ + cs.Voters, + cs.Learners, + cs.VotersOutgoing, + cs.LearnersNext, + } { + slices.Sort(sl) + } + + cs2 := chg.Tracker.ConfState() + // NB: cs.Equivalent does the same "sorting" dance internally, but let's + // test it a bit here instead of relying on it. + if assert.Equal(t, cs, cs2) && assert.NoError(t, cs.Equivalent(cs2)) && assert.NoError(t, cs2.Equivalent(cs)) { + return true // success + } + return false + } + + ids := func(sl ...uint64) []uint64 { + return sl + } + + // Unit tests. + for _, cs := range []*pb.ConfState{ + {}, + {Voters: ids(1, 2, 3)}, + {Voters: ids(1, 2, 3), Learners: ids(4, 5, 6)}, + {Voters: ids(1, 2, 3), Learners: ids(5), VotersOutgoing: ids(1, 2, 4, 6), LearnersNext: ids(4)}, + } { + pb.EnsureConfState(cs) + if !f(cs) { + t.FailNow() // f() already logged a nice t.Error() + } + } + + assert.NoError(t, quick.Check(func(cs *rndConfChange) bool { + return f((*pb.ConfState)(cs)) + }, &cfg)) +} diff --git a/third_party/raft/confchange/testdata/joint_autoleave.txt b/third_party/raft/confchange/testdata/joint_autoleave.txt new file mode 100644 index 0000000000..be138df3cf --- /dev/null +++ b/third_party/raft/confchange/testdata/joint_autoleave.txt @@ -0,0 +1,29 @@ +# Test the autoleave argument to EnterJoint. It defaults to false in the +# datadriven tests. The flag has no associated semantics in this package, +# it is simply passed through. +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=1 + +# Autoleave is reflected in the config. +enter-joint autoleave=true +v2 v3 +---- +voters=(1 2 3)&&(1) autoleave +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=1 +3: StateProbe match=0 next=1 + +# Can't enter-joint twice, even if autoleave changes. +enter-joint autoleave=false +---- +config is already joint + +leave-joint +---- +voters=(1 2 3) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=1 +3: StateProbe match=0 next=1 diff --git a/third_party/raft/confchange/testdata/joint_idempotency.txt b/third_party/raft/confchange/testdata/joint_idempotency.txt new file mode 100644 index 0000000000..a47f3a662c --- /dev/null +++ b/third_party/raft/confchange/testdata/joint_idempotency.txt @@ -0,0 +1,23 @@ +# Verify that operations upon entering the joint state are idempotent, i.e. +# removing an absent node is fine, etc. + +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=1 + +enter-joint +r1 r2 r9 v2 v3 v4 v2 v3 v4 l2 l2 r4 r4 l1 l1 +---- +voters=(3)&&(1) learners=(2) learners_next=(1) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=1 learner +3: StateProbe match=0 next=1 + +leave-joint +---- +voters=(3) learners=(1 2) +1: StateProbe match=0 next=1 learner +2: StateProbe match=0 next=1 learner +3: StateProbe match=0 next=1 diff --git a/third_party/raft/confchange/testdata/joint_learners_next.txt b/third_party/raft/confchange/testdata/joint_learners_next.txt new file mode 100644 index 0000000000..6faddfe7e9 --- /dev/null +++ b/third_party/raft/confchange/testdata/joint_learners_next.txt @@ -0,0 +1,24 @@ +# Verify that when a voter is demoted in a joint config, it will show up in +# learners_next until the joint config is left, and only then will the progress +# turn into that of a learner, without resetting the progress. Note that this +# last fact is verified by `next`, which can tell us which "round" the progress +# was originally created in. + +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=1 + +enter-joint +v2 l1 +---- +voters=(2)&&(1) learners_next=(1) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=1 + +leave-joint +---- +voters=(2) learners=(1) +1: StateProbe match=0 next=1 learner +2: StateProbe match=0 next=1 diff --git a/third_party/raft/confchange/testdata/joint_safety.txt b/third_party/raft/confchange/testdata/joint_safety.txt new file mode 100644 index 0000000000..75d11b199e --- /dev/null +++ b/third_party/raft/confchange/testdata/joint_safety.txt @@ -0,0 +1,81 @@ +leave-joint +---- +can't leave a non-joint config + +enter-joint +---- +can't make a zero-voter config joint + +enter-joint +v1 +---- +can't make a zero-voter config joint + +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=3 + +leave-joint +---- +can't leave a non-joint config + +# Can enter into joint config. +enter-joint +---- +voters=(1)&&(1) +1: StateProbe match=0 next=3 + +enter-joint +---- +config is already joint + +leave-joint +---- +voters=(1) +1: StateProbe match=0 next=3 + +leave-joint +---- +can't leave a non-joint config + +# Can enter again, this time with some ops. +enter-joint +r1 v2 v3 l4 +---- +voters=(2 3)&&(1) learners=(4) +1: StateProbe match=0 next=3 +2: StateProbe match=0 next=9 +3: StateProbe match=0 next=9 +4: StateProbe match=0 next=9 learner + +enter-joint +---- +config is already joint + +enter-joint +v12 +---- +config is already joint + +simple +l15 +---- +can't apply simple config change in joint config + +leave-joint +---- +voters=(2 3) learners=(4) +2: StateProbe match=0 next=9 +3: StateProbe match=0 next=9 +4: StateProbe match=0 next=9 learner + +simple +l9 +---- +voters=(2 3) learners=(4 9) +2: StateProbe match=0 next=9 +3: StateProbe match=0 next=9 +4: StateProbe match=0 next=9 learner +9: StateProbe match=0 next=14 learner diff --git a/third_party/raft/confchange/testdata/simple_idempotency.txt b/third_party/raft/confchange/testdata/simple_idempotency.txt new file mode 100644 index 0000000000..e31c43b7ff --- /dev/null +++ b/third_party/raft/confchange/testdata/simple_idempotency.txt @@ -0,0 +1,69 @@ +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=1 + +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=1 + +simple +v2 +---- +voters=(1 2) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=2 + +simple +l1 +---- +voters=(2) learners=(1) +1: StateProbe match=0 next=1 learner +2: StateProbe match=0 next=2 + +simple +l1 +---- +voters=(2) learners=(1) +1: StateProbe match=0 next=1 learner +2: StateProbe match=0 next=2 + +simple +r1 +---- +voters=(2) +2: StateProbe match=0 next=2 + +simple +r1 +---- +voters=(2) +2: StateProbe match=0 next=2 + +simple +v3 +---- +voters=(2 3) +2: StateProbe match=0 next=2 +3: StateProbe match=0 next=7 + +simple +r3 +---- +voters=(2) +2: StateProbe match=0 next=2 + +simple +r3 +---- +voters=(2) +2: StateProbe match=0 next=2 + +simple +r4 +---- +voters=(2) +2: StateProbe match=0 next=2 diff --git a/third_party/raft/confchange/testdata/simple_promote_demote.txt b/third_party/raft/confchange/testdata/simple_promote_demote.txt new file mode 100644 index 0000000000..b4b770de7c --- /dev/null +++ b/third_party/raft/confchange/testdata/simple_promote_demote.txt @@ -0,0 +1,60 @@ +# Set up three voters for this test. + +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=1 + +simple +v2 +---- +voters=(1 2) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=1 + +simple +v3 +---- +voters=(1 2 3) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=1 +3: StateProbe match=0 next=2 + +# Can atomically demote and promote without a hitch. +# This is pointless, but possible. +simple +l1 v1 +---- +voters=(1 2 3) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=1 +3: StateProbe match=0 next=2 + +# Can demote a voter. +simple +l2 +---- +voters=(1 3) learners=(2) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=1 learner +3: StateProbe match=0 next=2 + +# Can atomically promote and demote the same voter. +# This is pointless, but possible. +simple +v2 l2 +---- +voters=(1 3) learners=(2) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=1 learner +3: StateProbe match=0 next=2 + +# Can promote a voter. +simple +v2 +---- +voters=(1 2 3) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=1 +3: StateProbe match=0 next=2 diff --git a/third_party/raft/confchange/testdata/simple_safety.txt b/third_party/raft/confchange/testdata/simple_safety.txt new file mode 100644 index 0000000000..6566c5fccf --- /dev/null +++ b/third_party/raft/confchange/testdata/simple_safety.txt @@ -0,0 +1,64 @@ +simple +l1 +---- +removed all voters + +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=1 + +simple +v2 l3 +---- +voters=(1 2) learners=(3) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=2 +3: StateProbe match=0 next=2 learner + +simple +r1 v5 +---- +more than one voter changed without entering joint config + +simple +r1 r2 +---- +removed all voters + +simple +v3 v4 +---- +more than one voter changed without entering joint config + +simple +l1 v5 +---- +more than one voter changed without entering joint config + +simple +l1 l2 +---- +removed all voters + +simple +l2 l3 l4 l5 +---- +voters=(1) learners=(2 3 4 5) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=2 learner +3: StateProbe match=0 next=2 learner +4: StateProbe match=0 next=8 learner +5: StateProbe match=0 next=8 learner + +simple +r1 +---- +removed all voters + +simple +r2 r3 r4 r5 +---- +voters=(1) +1: StateProbe match=0 next=1 diff --git a/third_party/raft/confchange/testdata/update.txt b/third_party/raft/confchange/testdata/update.txt new file mode 100644 index 0000000000..ac47bf3e8c --- /dev/null +++ b/third_party/raft/confchange/testdata/update.txt @@ -0,0 +1,23 @@ +# Nobody cares about ConfChangeUpdateNode, but at least use it once. It is used +# by etcd as a convenient way to pass a blob through their conf change machinery +# that updates information tracked outside of raft. + +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=1 + +simple +v2 u1 +---- +voters=(1 2) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=1 + +simple +u1 u2 u3 u1 u2 u3 +---- +voters=(1 2) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=1 diff --git a/third_party/raft/confchange/testdata/zero.txt b/third_party/raft/confchange/testdata/zero.txt new file mode 100644 index 0000000000..a0161dc14f --- /dev/null +++ b/third_party/raft/confchange/testdata/zero.txt @@ -0,0 +1,6 @@ +# NodeId zero is ignored. +simple +v1 r0 v0 l0 +---- +voters=(1) +1: StateProbe match=0 next=1 diff --git a/third_party/raft/design.md b/third_party/raft/design.md new file mode 100644 index 0000000000..7bc0531dce --- /dev/null +++ b/third_party/raft/design.md @@ -0,0 +1,57 @@ +## Progress + +Progress represents a follower’s progress in the view of the leader. Leader maintains progresses of all followers, and sends `replication message` to the follower based on its progress. + +`replication message` is a `msgApp` with log entries. + +A progress has two attribute: `match` and `next`. `match` is the index of the highest known matched entry. If leader knows nothing about follower’s replication status, `match` is set to zero. `next` is the index of the first entry that will be replicated to the follower. Leader puts entries from `next` to its latest one in next `replication message`. + +A progress is in one of the three state: `probe`, `replicate`, `snapshot`. + +``` + +--------------------------------------------------------+ + | send snapshot | + | | + +---------+----------+ +----------v---------+ + +---> probe | | snapshot | + | | max inflight = 1 <----------------------------------+ max inflight = 0 | + | +---------+----------+ +--------------------+ + | | 1. snapshot success + | | (next=snapshot.index + 1) + | | 2. snapshot failure + | | (no change) + | | 3. receives msgAppResp(rej=false&&index>lastsnap.index) + | | (match=m.index,next=match+1) +receives msgAppResp(rej=true) +(next=match+1)| | + | | + | | + | | receives msgAppResp(rej=false&&index>match) + | | (match=m.index,next=match+1) + | | + | | + | | + | +---------v----------+ + | | replicate | + +---+ max inflight = n | + +--------------------+ +``` + +When the progress of a follower is in `probe` state, leader sends at most one `replication message` per heartbeat interval. The leader sends `replication message` slowly and probing the actual progress of the follower. A `msgHeartbeatResp` or a `msgAppResp` with reject might trigger the sending of the next `replication message`. + +When the progress of a follower is in `replicate` state, leader sends `replication message`, then optimistically increases `next` to the latest entry sent. This is an optimized state for fast replicating log entries to the follower. + +When the progress of a follower is in `snapshot` state, leader stops sending any `replication message`. + +A newly elected leader sets the progresses of all the followers to `probe` state with `match` = 0 and `next` = last index. The leader slowly (at most once per heartbeat) sends `replication message` to the follower and probes its progress. + +A progress changes to `replicate` when the follower replies with a non-rejection `msgAppResp`, which implies that it has matched the index sent. At this point, leader starts to stream log entries to the follower fast. The progress will fall back to `probe` when the follower replies a rejection `msgAppResp` or the link layer reports the follower is unreachable. We aggressively reset `next` to `match`+1 since if we receive any `msgAppResp` soon, both `match` and `next` will increase directly to the `index` in `msgAppResp`. (We might end up with sending some duplicate entries when aggressively reset `next` too low. see open question) + +A progress changes from `probe` to `snapshot` when the follower falls very far behind and requires a snapshot. After sending `msgSnap`, the leader waits until the success, failure or abortion of the previous snapshot sent. The progress will go back to `probe` after the sending result is applied. + +### Flow Control + +1. limit the max size of message sent per message. Max should be configurable. +Lower the cost at probing state as we limit the size per message; lower the penalty when aggressively decreased to a too low `next` + +2. limit the # of in flight messages < N when in `replicate` state. N should be configurable. Most implementation will have a sending buffer on top of its actual network transport layer (not blocking raft node). We want to make sure raft does not overflow that buffer, which can cause message dropping and triggering a bunch of unnecessary resending repeatedly. diff --git a/third_party/raft/diff_test.go b/third_party/raft/diff_test.go new file mode 100644 index 0000000000..21ae6e9365 --- /dev/null +++ b/third_party/raft/diff_test.go @@ -0,0 +1,67 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "fmt" + "io" + "os" + "os/exec" + "strings" +) + +func diffu(a, b string) string { + if a == b { + return "" + } + aname, bname := mustTemp("base", a), mustTemp("other", b) + defer os.Remove(aname) + defer os.Remove(bname) + cmd := exec.Command("diff", "-u", aname, bname) + buf, err := cmd.CombinedOutput() + if err != nil { + if _, ok := err.(*exec.ExitError); ok { + // do nothing + return string(buf) + } + panic(err) + } + return string(buf) +} + +func mustTemp(pre, body string) string { + f, err := os.CreateTemp("", pre) + if err != nil { + panic(err) + } + _, err = io.Copy(f, strings.NewReader(body)) + if err != nil { + panic(err) + } + f.Close() + return f.Name() +} + +func ltoa(l *raftLog) string { + s := fmt.Sprintf("lastIndex: %d\n", l.lastIndex()) + s += fmt.Sprintf("applied: %d\n", l.applied) + s += fmt.Sprintf("applying: %d\n", l.applying) + for i, e := range l.allEntries() { + // TODO: after migrating to protoc-gen-go, marsh entry using code like below, + // es, _ := proto.MarshalOptions{Deterministic: true}.Marshal(e) + s += fmt.Sprintf("#%d: %+v\n", i, e.String()) + } + return s +} diff --git a/third_party/raft/doc.go b/third_party/raft/doc.go new file mode 100644 index 0000000000..b600ac6afa --- /dev/null +++ b/third_party/raft/doc.go @@ -0,0 +1,399 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package raft sends and receives messages in the Protocol Buffer format +defined in the raftpb package. + +Raft is a protocol with which a cluster of nodes can maintain a replicated state machine. +The state machine is kept in sync through the use of a replicated log. +For more details on Raft, see "In Search of an Understandable Consensus Algorithm" +(https://raft.github.io/raft.pdf) by Diego Ongaro and John Ousterhout. + +A simple example application, _raftexample_, is also available to help illustrate +how to use this package in practice: +https://github.com/etcd-io/etcd/tree/main/contrib/raftexample + +# Usage + +The primary object in raft is a Node. You either start a Node from scratch +using raft.StartNode or start a Node from some initial state using raft.RestartNode. + +To start a node from scratch: + + storage := raft.NewMemoryStorage() + c := &Config{ + ID: 0x01, + ElectionTick: 10, + HeartbeatTick: 1, + Storage: storage, + MaxSizePerMsg: 4096, + MaxInflightMsgs: 256, + } + n := raft.StartNode(c, []raft.Peer{{ID: 0x02}, {ID: 0x03}}) + +To restart a node from previous state: + + storage := raft.NewMemoryStorage() + + // recover the in-memory storage from persistent + // snapshot, state and entries. + storage.ApplySnapshot(snapshot) + storage.SetHardState(state) + storage.Append(entries) + + c := &Config{ + ID: 0x01, + ElectionTick: 10, + HeartbeatTick: 1, + Storage: storage, + MaxSizePerMsg: 4096, + MaxInflightMsgs: 256, + } + + // restart raft without peer information. + // peer information is already included in the storage. + n := raft.RestartNode(c) + +Now that you are holding onto a Node you have a few responsibilities: + +First, you must read from the Node.Ready() channel and process the updates +it contains. These steps may be performed in parallel, except as noted in step +2. + +1. Write HardState, Entries, and Snapshot to persistent storage if they are +not empty. Note that when writing an Entry with Index i, any +previously-persisted entries with Index >= i must be discarded. + +2. Send all Messages to the nodes named in the To field. It is important that +no messages be sent until the latest HardState has been persisted to disk, +and all Entries written by any previous Ready batch (Messages may be sent while +entries from the same batch are being persisted). To reduce the I/O latency, an +optimization can be applied to make leader write to disk in parallel with its +followers (as explained at section 10.2.1 in Raft thesis). If any Message has type +MsgSnap, call Node.ReportSnapshot() after it has been sent (these messages may be +large). + +Note: Marshalling messages is not thread-safe; it is important that you +make sure that no new entries are persisted while marshalling. +The easiest way to achieve this is to serialize the messages directly inside +your main raft loop. + +3. Apply Snapshot (if any) and CommittedEntries to the state machine. +If any committed Entry has Type EntryConfChange, call Node.ApplyConfChange() +to apply it to the node. The configuration change may be cancelled at this point +by setting the NodeId field to zero before calling ApplyConfChange +(but ApplyConfChange must be called one way or the other, and the decision to cancel +must be based solely on the state machine and not external information such as +the observed health of the node). + +4. Call Node.Advance() to signal readiness for the next batch of updates. +This may be done at any time after step 1, although all updates must be processed +in the order they were returned by Ready. + +Second, all persisted log entries must be made available via an +implementation of the Storage interface. The provided MemoryStorage +type can be used for this (if you repopulate its state upon a +restart), or you can supply your own disk-backed implementation. + +Third, when you receive a message from another node, pass it to Node.Step: + + func recvRaftRPC(ctx context.Context, m *raftpb.Message) { + n.Step(ctx, m) + } + +Finally, you need to call Node.Tick() at regular intervals (probably +via a time.Ticker). Raft has two important timeouts: heartbeat and the +election timeout. However, internally to the raft package time is +represented by an abstract "tick". + +The total state machine handling loop will look something like this: + + for { + select { + case <-s.Ticker: + n.Tick() + case rd := <-s.Node.Ready(): + saveToStorage(rd.State, rd.Entries, rd.Snapshot) + send(rd.Messages) + if !raft.IsEmptySnap(rd.Snapshot) { + processSnapshot(rd.Snapshot) + } + for _, entry := range rd.CommittedEntries { + process(entry) + if entry.GetType() == raftpb.EntryConfChange { + var cc raftpb.ConfChange + proto.Unmarshal(entry.GetData(), &cc) + s.Node.ApplyConfChange(cc) + } + } + s.Node.Advance() + case <-s.done: + return + } + } + +To propose changes to the state machine from your node take your application +data, serialize it into a byte slice and call: + + n.Propose(ctx, data) + +If the proposal is committed, data will appear in committed entries with type +raftpb.EntryNormal. There is no guarantee that a proposed command will be +committed; you may have to re-propose after a timeout. + +To add or remove a node in a cluster, build ConfChange struct 'cc' and call: + + n.ProposeConfChange(ctx, cc) + +After config change is committed, some committed entry with type +raftpb.EntryConfChange will be returned. You must apply it to node through: + + var cc raftpb.ConfChange + proto.Unmarshal(data, &cc) + n.ApplyConfChange(cc) + +Note: An ID represents a unique node in a cluster for all time. A +given ID MUST be used only once even if the old node has been removed. +This means that for example IP addresses make poor node IDs since they +may be reused. Node IDs must be non-zero. + +# Usage with Asynchronous Storage Writes + +The library can be configured with an alternate interface for local storage +writes that can provide better performance in the presence of high proposal +concurrency by minimizing interference between proposals. This feature is called +AsynchronousStorageWrites, and can be enabled using the flag on the Config +struct with the same name. + +When Asynchronous Storage Writes is enabled, the responsibility of code using +the library is different from what was presented above. Users still read from +the Node.Ready() channel. However, they process the updates it contains in a +different manner. Users no longer consult the HardState, Entries, and Snapshot +fields (steps 1 and 3 above). They also no longer call Node.Advance() to +indicate that they have processed all entries in the Ready (step 4 above). +Instead, all local storage operations are also communicated through messages +present in the Ready.Message slice. + +The local storage messages come in two flavors. The first flavor is log append +messages, which target a LocalAppendThread and carry Entries, HardState, and a +Snapshot. The second flavor is entry application messages, which target a +LocalApplyThread and carry CommittedEntries. Messages to the same target must be +reliably processed in order. Messages to different targets can be processed in +any order. + +Each local storage message carries a slice of response messages that must +delivered after the corresponding storage write has been completed. These +responses may target the same node or may target other nodes. + +With Asynchronous Storage Writes enabled, the total state machine handling loop +will look something like this: + + for { + select { + case <-s.Ticker: + n.Tick() + case rd := <-s.Node.Ready(): + for _, m := range rd.Messages { + switch m.GetTo() { + case raft.LocalAppendThread: + toAppend <- m + case raft.LocalApplyThread: + toApply <-m + default: + sendOverNetwork(m) + } + } + case <-s.done: + return + } + } + +Usage of Asynchronous Storage Writes will typically also contain a pair of +storage handler threads, one for log writes (append) and one for entry +application to the local state machine (apply). Those will look something like: + + // append thread + go func() { + for { + select { + case m := <-toAppend: + saveToStorage(m.State, m.GetEntries(), m.GetSnapshot()) + send(m.GetResponses()) + case <-s.done: + return + } + } + } + + // apply thread + go func() { + for { + select { + case m := <-toApply: + for _, entry := range m.CommittedEntries { + process(entry) + if entry.GetType() == raftpb.EntryConfChange { + var cc raftpb.ConfChange + proto.Unmarshal(entry.GetData(), &cc) + s.Node.ApplyConfChange(cc) + } + } + send(m.GetResponses()) + case <-s.done: + return + } + } + } + +# Implementation notes + +This implementation is up to date with the final Raft thesis +(https://github.com/ongardie/dissertation/blob/master/stanford.pdf), although our +implementation of the membership change protocol differs somewhat from +that described in chapter 4. The key invariant that membership changes +happen one node at a time is preserved, but in our implementation the +membership change takes effect when its entry is applied, not when it +is added to the log (so the entry is committed under the old +membership instead of the new). This is equivalent in terms of safety, +since the old and new configurations are guaranteed to overlap. + +To ensure that we do not attempt to commit two membership changes at +once by matching log positions (which would be unsafe since they +should have different quorum requirements), we simply disallow any +proposed membership change while any uncommitted change appears in +the leader's log. + +This approach introduces a problem when you try to remove a member +from a two-member cluster: If one of the members dies before the +other one receives the commit of the confchange entry, then the member +cannot be removed any more since the cluster cannot make progress. +For this reason it is highly recommended to use three or more nodes in +every cluster. + +# MessageType + +Package raft sends and receives message in Protocol Buffer format (defined +in raftpb package). Each state (follower, candidate, leader) implements its +own 'step' method ('stepFollower', 'stepCandidate', 'stepLeader') when +advancing with the given raftpb.Message. Each step is determined by its +raftpb.MessageType. Note that every step is checked by one common method +'Step' that safety-checks the terms of node and incoming message to prevent +stale log entries: + + 'MsgHup' is used for election. If a node is a follower or candidate, the + 'tick' function in 'raft' struct is set as 'tickElection'. If a follower or + candidate has not received any heartbeat before the election timeout, it + passes 'MsgHup' to its Step method and becomes (or remains) a candidate to + start a new election. + + 'MsgBeat' is an internal type that signals the leader to send a heartbeat of + the 'MsgHeartbeat' type. If a node is a leader, the 'tick' function in + the 'raft' struct is set as 'tickHeartbeat', and triggers the leader to + send periodic 'MsgHeartbeat' messages to its followers. + + 'MsgProp' proposes to append data to its log entries. This is a special + type to redirect proposals to leader. Therefore, send method overwrites + raftpb.Message's term with its HardState's term to avoid attaching its + local term to 'MsgProp'. When 'MsgProp' is passed to the leader's 'Step' + method, the leader first calls the 'appendEntry' method to append entries + to its log, and then calls 'bcastAppend' method to send those entries to + its peers. When passed to candidate, 'MsgProp' is dropped. When passed to + follower, 'MsgProp' is stored in follower's mailbox(msgs) by the send + method. It is stored with sender's ID and later forwarded to leader by + rafthttp package. + + 'MsgApp' contains log entries to replicate. A leader calls bcastAppend, + which calls sendAppend, which sends soon-to-be-replicated logs in 'MsgApp' + type. When 'MsgApp' is passed to candidate's Step method, candidate reverts + back to follower, because it indicates that there is a valid leader sending + 'MsgApp' messages. Candidate and follower respond to this message in + 'MsgAppResp' type. + + 'MsgAppResp' is response to log replication request('MsgApp'). When + 'MsgApp' is passed to candidate or follower's Step method, it responds by + calling 'handleAppendEntries' method, which sends 'MsgAppResp' to raft + mailbox. + + 'MsgVote' requests votes for election. When a node is a follower or + candidate and 'MsgHup' is passed to its Step method, then the node calls + 'campaign' method to campaign itself to become a leader. Once 'campaign' + method is called, the node becomes candidate and sends 'MsgVote' to peers + in cluster to request votes. When passed to leader or candidate's Step + method and the message's Term is lower than leader's or candidate's, + 'MsgVote' will be rejected ('MsgVoteResp' is returned with Reject true). + If leader or candidate receives 'MsgVote' with higher term, it will revert + back to follower. When 'MsgVote' is passed to follower, it votes for the + sender only when sender's last term is greater than MsgVote's term or + sender's last term is equal to MsgVote's term but sender's last committed + index is greater than or equal to follower's. + + 'MsgVoteResp' contains responses from voting request. When 'MsgVoteResp' is + passed to candidate, the candidate calculates how many votes it has won. If + it's more than majority (quorum), it becomes leader and calls 'bcastAppend'. + If candidate receives majority of votes of denials, it reverts back to + follower. + + 'MsgPreVote' and 'MsgPreVoteResp' are used in an optional two-phase election + protocol. When Config.PreVote is true, a pre-election is carried out first + (using the same rules as a regular election), and no node increases its term + number unless the pre-election indicates that the campaigning node would win. + This minimizes disruption when a partitioned node rejoins the cluster. + + 'MsgSnap' requests to install a snapshot message. When a node has just + become a leader or the leader receives 'MsgProp' message, it calls + 'bcastAppend' method, which then calls 'sendAppend' method to each + follower. In 'sendAppend', if a leader fails to get term or entries, + the leader requests snapshot by sending 'MsgSnap' type message. + + 'MsgSnapStatus' tells the result of snapshot install message. When a + follower rejected 'MsgSnap', it indicates the snapshot request with + 'MsgSnap' had failed from network issues which causes the network layer + to fail to send out snapshots to its followers. Then leader considers + follower's progress as probe. When 'MsgSnap' were not rejected, it + indicates that the snapshot succeeded and the leader sets follower's + progress to probe and resumes its log replication. + + 'MsgHeartbeat' sends heartbeat from leader. When 'MsgHeartbeat' is passed + to candidate and message's term is higher than candidate's, the candidate + reverts back to follower and updates its committed index from the one in + this heartbeat. And it sends the message to its mailbox. When + 'MsgHeartbeat' is passed to follower's Step method and message's term is + higher than follower's, the follower updates its leaderID with the ID + from the message. + + 'MsgHeartbeatResp' is a response to 'MsgHeartbeat'. When 'MsgHeartbeatResp' + is passed to leader's Step method, the leader knows which follower + responded. And only when the leader's last committed index is greater than + follower's Match index, the leader runs 'sendAppend` method. + + 'MsgUnreachable' tells that request(message) wasn't delivered. When + 'MsgUnreachable' is passed to leader's Step method, the leader discovers + that the follower that sent this 'MsgUnreachable' is not reachable, often + indicating 'MsgApp' is lost. When follower's progress state is replicate, + the leader sets it back to probe. + + 'MsgStorageAppend' is a message from a node to its local append storage + thread to write entries, hard state, and/or a snapshot to stable storage. + The message will carry one or more responses, one of which will be a + 'MsgStorageAppendResp' back to itself. The responses can also contain + 'MsgAppResp', 'MsgVoteResp', and 'MsgPreVoteResp' messages. Used with + AsynchronousStorageWrites. + + 'MsgStorageApply' is a message from a node to its local apply storage + thread to apply committed entries. The message will carry one response, + which will be a 'MsgStorageApplyResp' back to itself. Used with + AsynchronousStorageWrites. +*/ +package raft diff --git a/third_party/raft/example_test.go b/third_party/raft/example_test.go new file mode 100644 index 0000000000..91dfbb2d70 --- /dev/null +++ b/third_party/raft/example_test.go @@ -0,0 +1,47 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + pb "go.etcd.io/raft/v3/raftpb" +) + +func applyToStore(_ []*pb.Entry) {} +func sendMessages(_ []*pb.Message) {} +func saveStateToDisk(_ *pb.HardState) {} +func saveToDisk(_ []*pb.Entry) {} + +func ExampleNode() { + c := &Config{} + n := StartNode(c, nil) + defer n.Stop() + + // stuff to n happens in other goroutines + + // the last known state + var prev *pb.HardState + for { + // Ready blocks until there is new state ready. + rd := <-n.Ready() + if !isHardStateEqual(prev, rd.HardState) { + saveStateToDisk(rd.HardState) + prev = rd.HardState + } + + saveToDisk(rd.Entries) + go applyToStore(rd.CommittedEntries) + sendMessages(rd.Messages) + } +} diff --git a/third_party/raft/go.mod b/third_party/raft/go.mod new file mode 100644 index 0000000000..a3c11d07af --- /dev/null +++ b/third_party/raft/go.mod @@ -0,0 +1,17 @@ +module go.etcd.io/raft/v3 + +go 1.26 + +toolchain go1.26.4 + +require ( + github.com/cockroachdb/datadriven v1.0.2 + github.com/stretchr/testify v1.11.1 + google.golang.org/protobuf v1.36.11 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/third_party/raft/go.sum b/third_party/raft/go.sum new file mode 100644 index 0000000000..60403f8e98 --- /dev/null +++ b/third_party/raft/go.sum @@ -0,0 +1,16 @@ +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/third_party/raft/interaction_test.go b/third_party/raft/interaction_test.go new file mode 100644 index 0000000000..56d18200fc --- /dev/null +++ b/third_party/raft/interaction_test.go @@ -0,0 +1,38 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft_test + +import ( + "testing" + + "github.com/cockroachdb/datadriven" + + "go.etcd.io/raft/v3" + "go.etcd.io/raft/v3/rafttest" +) + +func TestInteraction(t *testing.T) { + // NB: if this test fails, run `go test ./raft -rewrite` and inspect the + // diff. Only commit the changes if you understand what caused them and if + // they are desired. + datadriven.Walk(t, "testdata", func(t *testing.T, path string) { + env := rafttest.NewInteractionEnv(&rafttest.InteractionOpts{ + SetRandomizedElectionTimeout: raft.SetRandomizedElectionTimeout, + }) + datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string { + return env.Handle(t, *d) + }) + }) +} diff --git a/third_party/raft/log.go b/third_party/raft/log.go new file mode 100644 index 0000000000..02e2bb752d --- /dev/null +++ b/third_party/raft/log.go @@ -0,0 +1,576 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "fmt" + + "google.golang.org/protobuf/proto" + + pb "go.etcd.io/raft/v3/raftpb" +) + +type raftLog struct { + // storage contains all stable entries since the last snapshot. + storage Storage + + // unstable contains all unstable entries and snapshot. + // they will be saved into storage. + unstable unstable + + // committed is the highest log position that is known to be in + // stable storage on a quorum of nodes. + committed uint64 + // applying is the highest log position that the application has + // been instructed to apply to its state machine. Some of these + // entries may be in the process of applying and have not yet + // reached applied. + // Use: The field is incremented when accepting a Ready struct. + // Invariant: applied <= applying && applying <= committed + applying uint64 + // applied is the highest log position that the application has + // successfully applied to its state machine. + // Use: The field is incremented when advancing after the committed + // entries in a Ready struct have been applied (either synchronously + // or asynchronously). + // Invariant: applied <= committed + applied uint64 + + logger Logger + + // maxApplyingEntsSize limits the outstanding byte size of the messages + // returned from calls to nextCommittedEnts that have not been acknowledged + // by a call to appliedTo. + maxApplyingEntsSize entryEncodingSize + // applyingEntsSize is the current outstanding byte size of the messages + // returned from calls to nextCommittedEnts that have not been acknowledged + // by a call to appliedTo. + applyingEntsSize entryEncodingSize + // applyingEntsPaused is true when entry application has been paused until + // enough progress is acknowledged. + applyingEntsPaused bool +} + +// newLog returns log using the given storage and default options. It +// recovers the log to the state that it just commits and applies the +// latest snapshot. +func newLog(storage Storage, logger Logger) *raftLog { + return newLogWithSize(storage, logger, noLimit) +} + +// newLogWithSize returns a log using the given storage and max +// message size. +func newLogWithSize(storage Storage, logger Logger, maxApplyingEntsSize entryEncodingSize) *raftLog { + firstIndex, err := storage.FirstIndex() + if err != nil { + panic(err) // TODO(bdarnell) + } + lastIndex, err := storage.LastIndex() + if err != nil { + panic(err) // TODO(bdarnell) + } + return &raftLog{ + storage: storage, + unstable: unstable{ + offset: lastIndex + 1, + offsetInProgress: lastIndex + 1, + logger: logger, + }, + maxApplyingEntsSize: maxApplyingEntsSize, + + // Initialize our committed and applied pointers to the time of the last compaction. + committed: firstIndex - 1, + applying: firstIndex - 1, + applied: firstIndex - 1, + + logger: logger, + } +} + +func (l *raftLog) String() string { + return fmt.Sprintf("committed=%d, applied=%d, applying=%d, unstable.offset=%d, unstable.offsetInProgress=%d, len(unstable.Entries)=%d", + l.committed, l.applied, l.applying, l.unstable.offset, l.unstable.offsetInProgress, len(l.unstable.entries)) +} + +// maybeAppend returns (0, false) if the entries cannot be appended. Otherwise, +// it returns (last index of new entries, true). +func (l *raftLog) maybeAppend(a logSlice, committed uint64) (lastnewi uint64, ok bool) { + if !l.matchTerm(a.prev) { + return 0, false + } + // TODO(pav-kv): propagate logSlice down the stack. It will be used all the + // way down in unstable, for safety checks, and for useful bookkeeping. + + lastnewi = a.prev.index + uint64(len(a.entries)) + ci := l.findConflict(a.entries) + switch { + case ci == 0: + case ci <= l.committed: + l.logger.Panicf("entry %d conflict with committed entry [committed(%d)]", ci, l.committed) + default: + offset := a.prev.index + 1 + if ci-offset > uint64(len(a.entries)) { + l.logger.Panicf("index, %d, is out of range [%d]", ci-offset, len(a.entries)) + } + l.append(a.entries[ci-offset:]...) + } + l.commitTo(min(committed, lastnewi)) + return lastnewi, true +} + +func (l *raftLog) append(ents ...*pb.Entry) uint64 { + if len(ents) == 0 { + return l.lastIndex() + } + if after := ents[0].GetIndex() - 1; after < l.committed { + l.logger.Panicf("after(%d) is out of range [committed(%d)]", after, l.committed) + } + l.unstable.truncateAndAppend(ents) + return l.lastIndex() +} + +// findConflict finds the index of the conflict. +// It returns the first pair of conflicting entries between the existing +// entries and the given entries, if there are any. +// If there is no conflicting entries, and the existing entries contains +// all the given entries, zero will be returned. +// If there is no conflicting entries, but the given entries contains new +// entries, the index of the first new entry will be returned. +// An entry is considered to be conflicting if it has the same index but +// a different term. +// The index of the given entries MUST be continuously increasing. +func (l *raftLog) findConflict(ents []*pb.Entry) uint64 { + for i := range ents { + if id := pbEntryID(ents[i]); !l.matchTerm(id) { + if id.index <= l.lastIndex() { + // TODO(pav-kv): can simply print %+v of the id. This will change the + // log format though. + l.logger.Infof("found conflict at index %d [existing term: %d, conflicting term: %d]", + id.index, l.zeroTermOnOutOfBounds(l.term(id.index)), id.term) + } + return id.index + } + } + return 0 +} + +// findConflictByTerm returns a best guess on where this log ends matching +// another log, given that the only information known about the other log is the +// (index, term) of its single entry. +// +// Specifically, the first returned value is the max guessIndex <= index, such +// that term(guessIndex) <= term or term(guessIndex) is not known (because this +// index is compacted or not yet stored). +// +// The second returned value is the term(guessIndex), or 0 if it is unknown. +// +// This function is used by a follower and leader to resolve log conflicts after +// an unsuccessful append to a follower, and ultimately restore the steady flow +// of appends. +func (l *raftLog) findConflictByTerm(index uint64, term uint64) (uint64, uint64) { + for ; index > 0; index-- { + // If there is an error (likely ErrCompacted or ErrUnavailable), we don't + // know whether it's a match or not, so assume a possible match and return + // the index, with 0 term indicating an unknown term. + if ourTerm, err := l.term(index); err != nil { + return index, 0 + } else if ourTerm <= term { + return index, ourTerm + } + } + return 0, 0 +} + +// nextUnstableEnts returns all entries that are available to be written to the +// local stable log and are not already in-progress. +func (l *raftLog) nextUnstableEnts() []*pb.Entry { + return l.unstable.nextEntries() +} + +// hasNextUnstableEnts returns if there are any entries that are available to be +// written to the local stable log and are not already in-progress. +func (l *raftLog) hasNextUnstableEnts() bool { + return len(l.nextUnstableEnts()) > 0 +} + +// hasNextOrInProgressUnstableEnts returns if there are any entries that are +// available to be written to the local stable log or in the process of being +// written to the local stable log. +func (l *raftLog) hasNextOrInProgressUnstableEnts() bool { + return len(l.unstable.entries) > 0 +} + +// nextCommittedEnts returns all the available entries for execution. +// Entries can be committed even when the local raft instance has not durably +// appended them to the local raft log yet. If allowUnstable is true, committed +// entries from the unstable log may be returned; otherwise, only entries known +// to reside locally on stable storage will be returned. +func (l *raftLog) nextCommittedEnts(allowUnstable bool) (ents []*pb.Entry) { + if l.applyingEntsPaused { + // Entry application outstanding size limit reached. + return nil + } + if l.hasNextOrInProgressSnapshot() { + // See comment in hasNextCommittedEnts. + return nil + } + lo, hi := l.applying+1, l.maxAppliableIndex(allowUnstable)+1 // [lo, hi) + if lo >= hi { + // Nothing to apply. + return nil + } + maxSize := l.maxApplyingEntsSize - l.applyingEntsSize + if maxSize <= 0 { + l.logger.Panicf("applying entry size (%d-%d)=%d not positive", + l.maxApplyingEntsSize, l.applyingEntsSize, maxSize) + } + ents, err := l.slice(lo, hi, maxSize) + if err != nil { + l.logger.Panicf("unexpected error when getting unapplied entries (%v)", err) + } + return ents +} + +// hasNextCommittedEnts returns if there is any available entries for execution. +// This is a fast check without heavy raftLog.slice() in nextCommittedEnts(). +func (l *raftLog) hasNextCommittedEnts(allowUnstable bool) bool { + if l.applyingEntsPaused { + // Entry application outstanding size limit reached. + return false + } + if l.hasNextOrInProgressSnapshot() { + // If we have a snapshot to apply, don't also return any committed + // entries. Doing so raises questions about what should be applied + // first. + return false + } + lo, hi := l.applying+1, l.maxAppliableIndex(allowUnstable)+1 // [lo, hi) + return lo < hi +} + +// maxAppliableIndex returns the maximum committed index that can be applied. +// If allowUnstable is true, committed entries from the unstable log can be +// applied; otherwise, only entries known to reside locally on stable storage +// can be applied. +func (l *raftLog) maxAppliableIndex(allowUnstable bool) uint64 { + hi := l.committed + if !allowUnstable { + hi = min(hi, l.unstable.offset-1) + } + return hi +} + +// nextUnstableSnapshot returns the snapshot, if present, that is available to +// be applied to the local storage and is not already in-progress. +func (l *raftLog) nextUnstableSnapshot() *pb.Snapshot { + return l.unstable.nextSnapshot() +} + +// hasNextUnstableSnapshot returns if there is a snapshot that is available to +// be applied to the local storage and is not already in-progress. +func (l *raftLog) hasNextUnstableSnapshot() bool { + return l.unstable.nextSnapshot() != nil +} + +// hasNextOrInProgressSnapshot returns if there is pending snapshot waiting for +// applying or in the process of being applied. +func (l *raftLog) hasNextOrInProgressSnapshot() bool { + return l.unstable.snapshot != nil +} + +func (l *raftLog) snapshot() (*pb.Snapshot, error) { + if l.unstable.snapshot != nil { + return proto.Clone(l.unstable.snapshot).(*pb.Snapshot), nil + } + return l.storage.Snapshot() +} + +func (l *raftLog) firstIndex() uint64 { + if i, ok := l.unstable.maybeFirstIndex(); ok { + return i + } + index, err := l.storage.FirstIndex() + if err != nil { + panic(err) // TODO(bdarnell) + } + return index +} + +func (l *raftLog) lastIndex() uint64 { + if i, ok := l.unstable.maybeLastIndex(); ok { + return i + } + i, err := l.storage.LastIndex() + if err != nil { + panic(err) // TODO(bdarnell) + } + return i +} + +func (l *raftLog) commitTo(tocommit uint64) { + // never decrease commit + if l.committed < tocommit { + if l.lastIndex() < tocommit { + l.logger.Panicf("tocommit(%d) is out of range [lastIndex(%d)]. Was the raft log corrupted, truncated, or lost?", tocommit, l.lastIndex()) + } + l.committed = tocommit + } +} + +func (l *raftLog) appliedTo(i uint64, size entryEncodingSize) { + if l.committed < i || i < l.applied { + l.logger.Panicf("applied(%d) is out of range [prevApplied(%d), committed(%d)]", i, l.applied, l.committed) + } + l.applied = i + l.applying = max(l.applying, i) + if l.applyingEntsSize > size { + l.applyingEntsSize -= size + } else { + // Defense against underflow. + l.applyingEntsSize = 0 + } + l.applyingEntsPaused = l.applyingEntsSize >= l.maxApplyingEntsSize +} + +func (l *raftLog) acceptApplying(i uint64, size entryEncodingSize, allowUnstable bool) { + if l.committed < i { + l.logger.Panicf("applying(%d) is out of range [prevApplying(%d), committed(%d)]", i, l.applying, l.committed) + } + l.applying = i + l.applyingEntsSize += size + // Determine whether to pause entry application until some progress is + // acknowledged. We pause in two cases: + // 1. the outstanding entry size equals or exceeds the maximum size. + // 2. the outstanding entry size does not equal or exceed the maximum size, + // but we determine that the next entry in the log will push us over the + // limit. We determine this by comparing the last entry returned from + // raftLog.nextCommittedEnts to the maximum entry that the method was + // allowed to return had there been no size limit. If these indexes are + // not equal, then the returned entries slice must have been truncated to + // adhere to the memory limit. + l.applyingEntsPaused = l.applyingEntsSize >= l.maxApplyingEntsSize || + i < l.maxAppliableIndex(allowUnstable) +} + +func (l *raftLog) stableTo(id entryID) { l.unstable.stableTo(id) } + +func (l *raftLog) stableSnapTo(i uint64) { l.unstable.stableSnapTo(i) } + +// acceptUnstable indicates that the application has started persisting the +// unstable entries in storage, and that the current unstable entries are thus +// to be marked as being in-progress, to avoid returning them with future calls +// to Ready(). +func (l *raftLog) acceptUnstable() { l.unstable.acceptInProgress() } + +// lastEntryID returns the ID of the last entry in the log. +func (l *raftLog) lastEntryID() entryID { + index := l.lastIndex() + t, err := l.term(index) + if err != nil { + l.logger.Panicf("unexpected error when getting the last term at %d: %v", index, err) + } + return entryID{term: t, index: index} +} + +func (l *raftLog) term(i uint64) (uint64, error) { + // Check the unstable log first, even before computing the valid term range, + // which may need to access stable Storage. If we find the entry's term in + // the unstable log, we know it was in the valid range. + if t, ok := l.unstable.maybeTerm(i); ok { + return t, nil + } + + // The valid term range is [firstIndex-1, lastIndex]. Even though the entry at + // firstIndex-1 is compacted away, its term is available for matching purposes + // when doing log appends. + if i+1 < l.firstIndex() { + return 0, ErrCompacted + } + if i > l.lastIndex() { + return 0, ErrUnavailable + } + + t, err := l.storage.Term(i) + if err == nil { + return t, nil + } + if err == ErrCompacted || err == ErrUnavailable { + return 0, err + } + panic(err) // TODO(bdarnell) +} + +func (l *raftLog) entries(i uint64, maxSize entryEncodingSize) ([]*pb.Entry, error) { + if i > l.lastIndex() { + return nil, nil + } + return l.slice(i, l.lastIndex()+1, maxSize) +} + +// allEntries returns all entries in the log. +func (l *raftLog) allEntries() []*pb.Entry { + ents, err := l.entries(l.firstIndex(), noLimit) + if err == nil { + return ents + } + if err == ErrCompacted { // try again if there was a racing compaction + return l.allEntries() + } + // TODO (xiangli): handle error? + panic(err) +} + +// isUpToDate determines if a log with the given last entry is more up-to-date +// by comparing the index and term of the last entries in the existing logs. +// +// If the logs have last entries with different terms, then the log with the +// later term is more up-to-date. If the logs end with the same term, then +// whichever log has the larger lastIndex is more up-to-date. If the logs are +// the same, the given log is up-to-date. +func (l *raftLog) isUpToDate(their entryID) bool { + our := l.lastEntryID() + return their.term > our.term || their.term == our.term && their.index >= our.index +} + +func (l *raftLog) matchTerm(id entryID) bool { + t, err := l.term(id.index) + if err != nil { + return false + } + return t == id.term +} + +func (l *raftLog) maybeCommit(at entryID) bool { + // NB: term should never be 0 on a commit because the leader campaigned at + // least at term 1. But if it is 0 for some reason, we don't consider this a + // term match. + if at.term != 0 && at.index > l.committed && l.matchTerm(at) { + l.commitTo(at.index) + return true + } + return false +} + +func (l *raftLog) restore(s *pb.Snapshot) { + l.logger.Infof("log [%s] starts to restore snapshot [index: %d, term: %d]", l, s.GetMetadata().GetIndex(), s.GetMetadata().GetTerm()) + l.committed = s.GetMetadata().GetIndex() + l.unstable.restore(s) +} + +// scan visits all log entries in the [lo, hi) range, returning them via the +// given callback. The callback can be invoked multiple times, with consecutive +// sub-ranges of the requested range. Returns up to pageSize bytes worth of +// entries at a time. May return more if a single entry size exceeds the limit. +// +// The entries in [lo, hi) must exist, otherwise scan() eventually returns an +// error (possibly after passing some entries through the callback). +// +// If the callback returns an error, scan terminates and returns this error +// immediately. This can be used to stop the scan early ("break" the loop). +func (l *raftLog) scan(lo, hi uint64, pageSize entryEncodingSize, v func([]*pb.Entry) error) error { + for lo < hi { + ents, err := l.slice(lo, hi, pageSize) + if err != nil { + return err + } else if len(ents) == 0 { + return fmt.Errorf("got 0 entries in [%d, %d)", lo, hi) + } + if err := v(ents); err != nil { + return err + } + lo += uint64(len(ents)) + } + return nil +} + +// slice returns a slice of log entries from lo through hi-1, inclusive. +func (l *raftLog) slice(lo, hi uint64, maxSize entryEncodingSize) ([]*pb.Entry, error) { + if err := l.mustCheckOutOfBounds(lo, hi); err != nil { + return nil, err + } + if lo == hi { + return nil, nil + } + if lo >= l.unstable.offset { + ents := limitSize(l.unstable.slice(lo, hi), maxSize) + // NB: use the full slice expression to protect the unstable slice from + // appends to the returned ents slice. + return ents[:len(ents):len(ents)], nil + } + + cut := min(hi, l.unstable.offset) + ents, err := l.storage.Entries(lo, cut, uint64(maxSize)) + if err == ErrCompacted { + return nil, err + } else if err == ErrUnavailable { + l.logger.Panicf("entries[%d:%d) is unavailable from storage", lo, cut) + } else if err != nil { + panic(err) // TODO(pavelkalinnikov): handle errors uniformly + } + if hi <= l.unstable.offset { + return ents, nil + } + + // Fast path to check if ents has reached the size limitation. Either the + // returned slice is shorter than requested (which means the next entry would + // bring it over the limit), or a single entry reaches the limit. + if uint64(len(ents)) < cut-lo { + return ents, nil + } + // Slow path computes the actual total size, so that unstable entries are cut + // optimally before being copied to ents slice. + size := entsSize(ents) + if size >= maxSize { + return ents, nil + } + + unstable := limitSize(l.unstable.slice(l.unstable.offset, hi), maxSize-size) + // Total size of unstable may exceed maxSize-size only if len(unstable) == 1. + // If this happens, ignore this extra entry. + if len(unstable) == 1 && size+entsSize(unstable) > maxSize { + return ents, nil + } + // Otherwise, total size of unstable does not exceed maxSize-size, so total + // size of ents+unstable does not exceed maxSize. Simply concatenate them. + return extend(ents, unstable), nil +} + +// l.firstIndex <= lo <= hi <= l.firstIndex + len(l.entries) +func (l *raftLog) mustCheckOutOfBounds(lo, hi uint64) error { + if lo > hi { + l.logger.Panicf("invalid slice %d > %d", lo, hi) + } + fi := l.firstIndex() + if lo < fi { + return ErrCompacted + } + + length := l.lastIndex() + 1 - fi + if hi > fi+length { + l.logger.Panicf("slice[%d,%d) out of bound [%d,%d]", lo, hi, fi, l.lastIndex()) + } + return nil +} + +func (l *raftLog) zeroTermOnOutOfBounds(t uint64, err error) uint64 { + if err == nil { + return t + } + if err == ErrCompacted || err == ErrUnavailable { + return 0 + } + l.logger.Panicf("unexpected error (%v)", err) + return 0 +} diff --git a/third_party/raft/log_test.go b/third_party/raft/log_test.go new file mode 100644 index 0000000000..353def5cb2 --- /dev/null +++ b/third_party/raft/log_test.go @@ -0,0 +1,1049 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + pb "go.etcd.io/raft/v3/raftpb" +) + +func TestFindConflict(t *testing.T) { + previousEnts := index(1).terms(1, 2, 3) + tests := []struct { + ents []*pb.Entry + wconflict uint64 + }{ + // no conflict, empty ent + {nil, 0}, + // no conflict + {index(1).terms(1, 2, 3), 0}, + {index(2).terms(2, 3), 0}, + {index(3).terms(3), 0}, + // no conflict, but has new entries + {index(1).terms(1, 2, 3, 4, 4), 4}, + {index(2).terms(2, 3, 4, 5), 4}, + {index(3).terms(3, 4, 4), 4}, + {index(4).terms(4, 4), 4}, + // conflicts with existing entries + {index(1).terms(4, 4), 1}, + {index(2).terms(1, 4, 4), 2}, + {index(3).terms(1, 2, 4, 4), 3}, + } + + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + raftLog := newLog(NewMemoryStorage(), raftLogger) + raftLog.append(previousEnts...) + require.Equal(t, tt.wconflict, raftLog.findConflict(tt.ents)) + }) + } +} + +func TestFindConflictByTerm(t *testing.T) { + for _, tt := range []struct { + ents []*pb.Entry // ents[0] contains the (index, term) of the snapshot + index uint64 + term uint64 + want uint64 + }{ + // Log starts from index 1. + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 100, term: 2, want: 100}, // ErrUnavailable + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 5, term: 6, want: 5}, + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 5, term: 5, want: 5}, + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 5, term: 4, want: 2}, + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 5, term: 2, want: 2}, + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 5, term: 1, want: 0}, + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 1, term: 2, want: 1}, + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 1, term: 1, want: 0}, + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 0, term: 0, want: 0}, + // Log with compacted entries. + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 30, term: 3, want: 30}, // ErrUnavailable + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 14, term: 9, want: 14}, + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 14, term: 4, want: 14}, + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 14, term: 3, want: 12}, + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 14, term: 2, want: 9}, + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 11, term: 5, want: 11}, + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 10, term: 5, want: 10}, + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 10, term: 3, want: 10}, + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 10, term: 2, want: 9}, + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 9, term: 2, want: 9}, // ErrCompacted + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 4, term: 2, want: 4}, // ErrCompacted + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 0, term: 0, want: 0}, // ErrCompacted + } { + t.Run("", func(t *testing.T) { + st := NewMemoryStorage() + require.NotEmpty(t, tt.ents) + st.ApplySnapshot(&pb.Snapshot{Metadata: &pb.SnapshotMetadata{ + Index: new(tt.ents[0].GetIndex()), + Term: new(tt.ents[0].GetTerm()), + }}) + l := newLog(st, raftLogger) + l.append(tt.ents[1:]...) + + index, term := l.findConflictByTerm(tt.index, tt.term) + require.Equal(t, tt.want, index) + wantTerm, err := l.term(index) + wantTerm = l.zeroTermOnOutOfBounds(wantTerm, err) + require.Equal(t, wantTerm, term) + }) + } +} + +func TestIsUpToDate(t *testing.T) { + previousEnts := index(1).terms(1, 2, 3) + raftLog := newLog(NewMemoryStorage(), raftLogger) + raftLog.append(previousEnts...) + tests := []struct { + lastIndex uint64 + term uint64 + wUpToDate bool + }{ + // greater term, ignore lastIndex + {raftLog.lastIndex() - 1, 4, true}, + {raftLog.lastIndex(), 4, true}, + {raftLog.lastIndex() + 1, 4, true}, + // smaller term, ignore lastIndex + {raftLog.lastIndex() - 1, 2, false}, + {raftLog.lastIndex(), 2, false}, + {raftLog.lastIndex() + 1, 2, false}, + // equal term, equal or lager lastIndex wins + {raftLog.lastIndex() - 1, 3, false}, + {raftLog.lastIndex(), 3, true}, + {raftLog.lastIndex() + 1, 3, true}, + } + + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + require.Equal(t, tt.wUpToDate, raftLog.isUpToDate(entryID{term: tt.term, index: tt.lastIndex})) + }) + } +} + +func TestAppend(t *testing.T) { + previousEnts := index(1).terms(1, 2) + tests := []struct { + ents []*pb.Entry + windex uint64 + wents []*pb.Entry + wunstable uint64 + }{ + { + nil, + 2, + index(1).terms(1, 2), + 3, + }, + { + index(3).terms(2), + 3, + index(1).terms(1, 2, 2), + 3, + }, + // conflicts with index 1 + { + index(1).terms(2), + 1, + index(1).terms(2), + 1, + }, + // conflicts with index 2 + { + index(2).terms(3, 3), + 3, + index(1).terms(1, 3, 3), + 2, + }, + } + + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + storage := NewMemoryStorage() + storage.Append(previousEnts) + raftLog := newLog(storage, raftLogger) + require.Equal(t, tt.windex, raftLog.append(tt.ents...)) + g, err := raftLog.entries(1, noLimit) + require.NoError(t, err) + requireEqualEntries(t, tt.wents, g) + require.Equal(t, tt.wunstable, raftLog.unstable.offset) + }) + } +} + +// TestLogMaybeAppend ensures: +// If the given (index, term) matches with the existing log: +// 1. If an existing entry conflicts with a new one (same index +// but different terms), delete the existing entry and all that +// follow it +// 2.Append any new entries not already in the log +// +// If the given (index, term) does not match with the existing log: +// +// return false +func TestLogMaybeAppend(t *testing.T) { + previousEnts := index(1).terms(1, 2, 3) + lastindex := uint64(3) + lastterm := uint64(3) + commit := uint64(1) + + // TODO(pav-kv): clean-up this test. + tests := []struct { + prev entryID + committed uint64 + ents []*pb.Entry + + wlasti uint64 + wappend bool + wcommit uint64 + wpanic bool + }{ + // not match: term is different + { + entryID{term: lastterm - 1, index: lastindex}, lastindex, + index(lastindex + 1).terms(4), + 0, false, commit, false, + }, + // not match: index out of bound + { + entryID{term: lastterm, index: lastindex + 1}, lastindex, + index(lastindex + 2).terms(4), + 0, false, commit, false, + }, + // match with the last existing entry + { + entryID{term: lastterm, index: lastindex}, lastindex, nil, + lastindex, true, lastindex, false, + }, + { + entryID{term: lastterm, index: lastindex}, lastindex + 1, nil, + lastindex, true, lastindex, false, // do not increase commit higher than lastnewi + }, + { + entryID{term: lastterm, index: lastindex}, lastindex - 1, nil, + lastindex, true, lastindex - 1, false, // commit up to the commit in the message + }, + { + entryID{term: lastterm, index: lastindex}, 0, nil, + lastindex, true, commit, false, // commit do not decrease + }, + { + entryID{}, lastindex, nil, + 0, true, commit, false, // commit do not decrease + }, + { + entryID{term: lastterm, index: lastindex}, lastindex, + index(lastindex + 1).terms(4), + lastindex + 1, true, lastindex, false, + }, + { + entryID{term: lastterm, index: lastindex}, lastindex + 1, + index(lastindex + 1).terms(4), + lastindex + 1, true, lastindex + 1, false, + }, + { + entryID{term: lastterm, index: lastindex}, lastindex + 2, + index(lastindex + 1).terms(4), + lastindex + 1, true, lastindex + 1, false, // do not increase commit higher than lastnewi + }, + { + entryID{term: lastterm, index: lastindex}, lastindex + 2, + index(lastindex+1).terms(4, 4), + lastindex + 2, true, lastindex + 2, false, + }, + // match with the entry in the middle + { + entryID{term: lastterm - 1, index: lastindex - 1}, lastindex, + index(lastindex).terms(4), + lastindex, true, lastindex, false, + }, + { + entryID{term: lastterm - 2, index: lastindex - 2}, lastindex, + index(lastindex - 1).terms(4), + lastindex - 1, true, lastindex - 1, false, + }, + { + entryID{term: lastterm - 3, index: lastindex - 3}, lastindex, + index(lastindex - 2).terms(4), + lastindex - 2, true, lastindex - 2, true, // conflict with existing committed entry + }, + { + entryID{term: lastterm - 2, index: lastindex - 2}, lastindex, + index(lastindex-1).terms(4, 4), + lastindex, true, lastindex, false, + }, + } + + for i, tt := range tests { + // TODO(pav-kv): for now, we pick a high enough app.term so that it + // represents a valid append message. The maybeAppend currently ignores it, + // but it must check that the append does not regress the term. + app := logSlice{ + term: 100, + prev: tt.prev, + entries: tt.ents, + } + require.NoError(t, app.valid()) + + raftLog := newLog(NewMemoryStorage(), raftLogger) + raftLog.append(previousEnts...) + raftLog.committed = commit + + t.Run(fmt.Sprint(i), func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + require.True(t, tt.wpanic) + } + }() + glasti, gappend := raftLog.maybeAppend(app, tt.committed) + require.Equal(t, tt.wlasti, glasti) + require.Equal(t, tt.wappend, gappend) + require.Equal(t, tt.wcommit, raftLog.committed) + if gappend && len(tt.ents) != 0 { + gents, err := raftLog.slice(raftLog.lastIndex()-uint64(len(tt.ents))+1, raftLog.lastIndex()+1, noLimit) + require.NoError(t, err) + require.Equal(t, tt.ents, gents) + } + }) + } +} + +// TestCompactionSideEffects ensures that all the log related functionality works correctly after +// a compaction. +func TestCompactionSideEffects(t *testing.T) { + // Populate the log with 1000 entries; 750 in stable storage and 250 in unstable. + lastIndex := uint64(1000) + unstableIndex := uint64(750) + storage := NewMemoryStorage() + require.NoError(t, storage.Append(index(1).termRange(1, unstableIndex+1))) + raftLog := newLog(storage, raftLogger) + raftLog.append(index(unstableIndex+1).termRange(unstableIndex+1, lastIndex+1)...) + + require.True(t, raftLog.maybeCommit(raftLog.lastEntryID())) + raftLog.appliedTo(raftLog.committed, 0 /* size */) + + offset := uint64(500) + require.NoError(t, storage.Compact(offset)) + require.Equal(t, lastIndex, raftLog.lastIndex()) + + for j := offset; j <= raftLog.lastIndex(); j++ { + require.Equal(t, j, mustTerm(raftLog.term(j))) + } + + for j := offset; j <= raftLog.lastIndex(); j++ { + require.True(t, raftLog.matchTerm(entryID{term: j, index: j})) + } + + unstableEnts := raftLog.nextUnstableEnts() + require.Equal(t, 250, len(unstableEnts)) + require.Equal(t, uint64(751), unstableEnts[0].GetIndex()) + + prev := raftLog.lastIndex() + raftLog.append(&pb.Entry{Index: new(raftLog.lastIndex() + 1), Term: new(raftLog.lastIndex() + 1)}) + require.Equal(t, prev+1, raftLog.lastIndex()) + + ents, err := raftLog.entries(raftLog.lastIndex(), noLimit) + require.NoError(t, err) + require.Equal(t, 1, len(ents)) +} + +func TestHasNextCommittedEnts(t *testing.T) { + snap := &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{Term: new(uint64(1)), Index: new(uint64(3))}, + } + ents := index(4).terms(1, 1, 1) + tests := []struct { + applied uint64 + applying uint64 + allowUnstable bool + paused bool + snap bool + whasNext bool + }{ + {applied: 3, applying: 3, allowUnstable: true, whasNext: true}, + {applied: 3, applying: 4, allowUnstable: true, whasNext: true}, + {applied: 3, applying: 5, allowUnstable: true, whasNext: false}, + {applied: 4, applying: 4, allowUnstable: true, whasNext: true}, + {applied: 4, applying: 5, allowUnstable: true, whasNext: false}, + {applied: 5, applying: 5, allowUnstable: true, whasNext: false}, + // Don't allow unstable entries. + {applied: 3, applying: 3, allowUnstable: false, whasNext: true}, + {applied: 3, applying: 4, allowUnstable: false, whasNext: false}, + {applied: 3, applying: 5, allowUnstable: false, whasNext: false}, + {applied: 4, applying: 4, allowUnstable: false, whasNext: false}, + {applied: 4, applying: 5, allowUnstable: false, whasNext: false}, + {applied: 5, applying: 5, allowUnstable: false, whasNext: false}, + // Paused. + {applied: 3, applying: 3, allowUnstable: true, paused: true, whasNext: false}, + // With snapshot. + {applied: 3, applying: 3, allowUnstable: true, snap: true, whasNext: false}, + } + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + storage := NewMemoryStorage() + require.NoError(t, storage.ApplySnapshot(snap)) + require.NoError(t, storage.Append(ents[:1])) + + raftLog := newLog(storage, raftLogger) + raftLog.append(ents...) + raftLog.stableTo(entryID{term: 1, index: 4}) + raftLog.maybeCommit(entryID{term: 1, index: 5}) + raftLog.appliedTo(tt.applied, 0 /* size */) + raftLog.acceptApplying(tt.applying, 0 /* size */, tt.allowUnstable) + raftLog.applyingEntsPaused = tt.paused + if tt.snap { + newSnap := snap + newSnap.Metadata.Index = new(newSnap.GetMetadata().GetIndex() + 1) + raftLog.restore(newSnap) + } + require.Equal(t, tt.whasNext, raftLog.hasNextCommittedEnts(tt.allowUnstable)) + }) + } +} + +func TestNextCommittedEnts(t *testing.T) { + snap := &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{Term: new(uint64(1)), Index: new(uint64(3))}, + } + ents := index(4).terms(1, 1, 1) + tests := []struct { + applied uint64 + applying uint64 + allowUnstable bool + paused bool + snap bool + wents []*pb.Entry + }{ + {applied: 3, applying: 3, allowUnstable: true, wents: ents[:2]}, + {applied: 3, applying: 4, allowUnstable: true, wents: ents[1:2]}, + {applied: 3, applying: 5, allowUnstable: true, wents: nil}, + {applied: 4, applying: 4, allowUnstable: true, wents: ents[1:2]}, + {applied: 4, applying: 5, allowUnstable: true, wents: nil}, + {applied: 5, applying: 5, allowUnstable: true, wents: nil}, + // Don't allow unstable entries. + {applied: 3, applying: 3, allowUnstable: false, wents: ents[:1]}, + {applied: 3, applying: 4, allowUnstable: false, wents: nil}, + {applied: 3, applying: 5, allowUnstable: false, wents: nil}, + {applied: 4, applying: 4, allowUnstable: false, wents: nil}, + {applied: 4, applying: 5, allowUnstable: false, wents: nil}, + {applied: 5, applying: 5, allowUnstable: false, wents: nil}, + // Paused. + {applied: 3, applying: 3, allowUnstable: true, paused: true, wents: nil}, + // With snapshot. + {applied: 3, applying: 3, allowUnstable: true, snap: true, wents: nil}, + } + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + storage := NewMemoryStorage() + require.NoError(t, storage.ApplySnapshot(snap)) + require.NoError(t, storage.Append(ents[:1])) + + raftLog := newLog(storage, raftLogger) + raftLog.append(ents...) + raftLog.stableTo(entryID{term: 1, index: 4}) + raftLog.maybeCommit(entryID{term: 1, index: 5}) + raftLog.appliedTo(tt.applied, 0 /* size */) + raftLog.acceptApplying(tt.applying, 0 /* size */, tt.allowUnstable) + raftLog.applyingEntsPaused = tt.paused + if tt.snap { + newSnap := snap + newSnap.Metadata.Index = new(newSnap.GetMetadata().GetIndex() + 1) + raftLog.restore(newSnap) + } + require.Equal(t, tt.wents, raftLog.nextCommittedEnts(tt.allowUnstable)) + }) + } +} + +func TestAcceptApplying(t *testing.T) { + maxSize := entryEncodingSize(100) + snap := &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{Term: new(uint64(1)), Index: new(uint64(3))}, + } + ents := index(4).terms(1, 1, 1) + tests := []struct { + index uint64 + allowUnstable bool + size entryEncodingSize + wpaused bool + }{ + {index: 3, allowUnstable: true, size: maxSize - 1, wpaused: true}, + {index: 3, allowUnstable: true, size: maxSize, wpaused: true}, + {index: 3, allowUnstable: true, size: maxSize + 1, wpaused: true}, + {index: 4, allowUnstable: true, size: maxSize - 1, wpaused: true}, + {index: 4, allowUnstable: true, size: maxSize, wpaused: true}, + {index: 4, allowUnstable: true, size: maxSize + 1, wpaused: true}, + {index: 5, allowUnstable: true, size: maxSize - 1, wpaused: false}, + {index: 5, allowUnstable: true, size: maxSize, wpaused: true}, + {index: 5, allowUnstable: true, size: maxSize + 1, wpaused: true}, + // Don't allow unstable entries. + {index: 3, allowUnstable: false, size: maxSize - 1, wpaused: true}, + {index: 3, allowUnstable: false, size: maxSize, wpaused: true}, + {index: 3, allowUnstable: false, size: maxSize + 1, wpaused: true}, + {index: 4, allowUnstable: false, size: maxSize - 1, wpaused: false}, + {index: 4, allowUnstable: false, size: maxSize, wpaused: true}, + {index: 4, allowUnstable: false, size: maxSize + 1, wpaused: true}, + {index: 5, allowUnstable: false, size: maxSize - 1, wpaused: false}, + {index: 5, allowUnstable: false, size: maxSize, wpaused: true}, + {index: 5, allowUnstable: false, size: maxSize + 1, wpaused: true}, + } + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + storage := NewMemoryStorage() + require.NoError(t, storage.ApplySnapshot(snap)) + require.NoError(t, storage.Append(ents[:1])) + + raftLog := newLogWithSize(storage, raftLogger, maxSize) + raftLog.append(ents...) + raftLog.stableTo(entryID{term: 1, index: 4}) + raftLog.maybeCommit(entryID{term: 1, index: 5}) + raftLog.appliedTo(3, 0 /* size */) + + raftLog.acceptApplying(tt.index, tt.size, tt.allowUnstable) + require.Equal(t, tt.wpaused, raftLog.applyingEntsPaused) + }) + } +} + +func TestAppliedTo(t *testing.T) { + maxSize := entryEncodingSize(100) + overshoot := entryEncodingSize(5) + snap := &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{Term: new(uint64(1)), Index: new(uint64(3))}, + } + ents := index(4).terms(1, 1, 1) + tests := []struct { + index uint64 + size entryEncodingSize + wapplyingSize entryEncodingSize + wpaused bool + }{ + // Apply some of in-progress entries (applying = 5 below). + {index: 4, size: overshoot - 1, wapplyingSize: maxSize + 1, wpaused: true}, + {index: 4, size: overshoot, wapplyingSize: maxSize, wpaused: true}, + {index: 4, size: overshoot + 1, wapplyingSize: maxSize - 1, wpaused: false}, + // Apply all of in-progress entries. + {index: 5, size: overshoot - 1, wapplyingSize: maxSize + 1, wpaused: true}, + {index: 5, size: overshoot, wapplyingSize: maxSize, wpaused: true}, + {index: 5, size: overshoot + 1, wapplyingSize: maxSize - 1, wpaused: false}, + // Apply all of outstanding bytes. + {index: 4, size: maxSize + overshoot, wapplyingSize: 0, wpaused: false}, + // Apply more than outstanding bytes. + // Incorrect accounting doesn't underflow applyingSize. + {index: 4, size: maxSize + overshoot + 1, wapplyingSize: 0, wpaused: false}, + } + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + storage := NewMemoryStorage() + require.NoError(t, storage.ApplySnapshot(snap)) + require.NoError(t, storage.Append(ents[:1])) + + raftLog := newLogWithSize(storage, raftLogger, maxSize) + raftLog.append(ents...) + raftLog.stableTo(entryID{term: 1, index: 4}) + raftLog.maybeCommit(entryID{term: 1, index: 5}) + raftLog.appliedTo(3, 0 /* size */) + raftLog.acceptApplying(5, maxSize+overshoot, false /* allowUnstable */) + + raftLog.appliedTo(tt.index, tt.size) + require.Equal(t, tt.index, raftLog.applied) + require.Equal(t, uint64(5), raftLog.applying) + require.Equal(t, tt.wapplyingSize, raftLog.applyingEntsSize) + require.Equal(t, tt.wpaused, raftLog.applyingEntsPaused) + }) + } +} + +// TestNextUnstableEnts ensures unstableEntries returns the unstable part of the +// entries correctly. +func TestNextUnstableEnts(t *testing.T) { + previousEnts := index(1).terms(1, 2) + tests := []struct { + unstable uint64 + wents []*pb.Entry + }{ + {3, nil}, + {1, previousEnts}, + } + + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + // append stable entries to storage + storage := NewMemoryStorage() + require.NoError(t, storage.Append(previousEnts[:tt.unstable-1])) + + // append unstable entries to raftlog + raftLog := newLog(storage, raftLogger) + raftLog.append(previousEnts[tt.unstable-1:]...) + + ents := raftLog.nextUnstableEnts() + if l := len(ents); l > 0 { + raftLog.stableTo(pbEntryID(ents[l-1])) + } + require.Equal(t, tt.wents, ents) + require.Equal(t, previousEnts[len(previousEnts)-1].GetIndex()+1, raftLog.unstable.offset) + }) + } +} + +func TestCommitTo(t *testing.T) { + previousEnts := index(1).terms(1, 2, 3) + commit := uint64(2) + tests := []struct { + commit uint64 + wcommit uint64 + wpanic bool + }{ + {3, 3, false}, + {1, 2, false}, // never decrease + {4, 0, true}, // commit out of range -> panic + } + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + require.True(t, tt.wpanic) + } + }() + raftLog := newLog(NewMemoryStorage(), raftLogger) + raftLog.append(previousEnts...) + raftLog.committed = commit + raftLog.commitTo(tt.commit) + require.Equal(t, tt.wcommit, raftLog.committed) + }) + } +} + +func TestStableTo(t *testing.T) { + tests := []struct { + stablei uint64 + stablet uint64 + wunstable uint64 + }{ + {1, 1, 2}, + {2, 2, 3}, + {2, 1, 1}, // bad term + {3, 1, 1}, // bad index + } + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + raftLog := newLog(NewMemoryStorage(), raftLogger) + raftLog.append(index(1).terms(1, 2)...) + raftLog.stableTo(entryID{term: tt.stablet, index: tt.stablei}) + require.Equal(t, tt.wunstable, raftLog.unstable.offset) + }) + } +} + +func TestStableToWithSnap(t *testing.T) { + snapi, snapt := uint64(5), uint64(2) + tests := []struct { + stablei uint64 + stablet uint64 + newEnts []*pb.Entry + + wunstable uint64 + }{ + {snapi + 1, snapt, nil, snapi + 1}, + {snapi, snapt, nil, snapi + 1}, + {snapi - 1, snapt, nil, snapi + 1}, + + {snapi + 1, snapt + 1, nil, snapi + 1}, + {snapi, snapt + 1, nil, snapi + 1}, + {snapi - 1, snapt + 1, nil, snapi + 1}, + + {snapi + 1, snapt, index(snapi + 1).terms(snapt), snapi + 2}, + {snapi, snapt, index(snapi + 1).terms(snapt), snapi + 1}, + {snapi - 1, snapt, index(snapi + 1).terms(snapt), snapi + 1}, + + {snapi + 1, snapt + 1, index(snapi + 1).terms(snapt), snapi + 1}, + {snapi, snapt + 1, index(snapi + 1).terms(snapt), snapi + 1}, + {snapi - 1, snapt + 1, index(snapi + 1).terms(snapt), snapi + 1}, + } + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + s := NewMemoryStorage() + require.NoError(t, s.ApplySnapshot(&pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(snapi), Term: new(snapt)}})) + raftLog := newLog(s, raftLogger) + raftLog.append(tt.newEnts...) + raftLog.stableTo(entryID{term: tt.stablet, index: tt.stablei}) + require.Equal(t, tt.wunstable, raftLog.unstable.offset) + }) + } +} + +// TestCompaction ensures that the number of log entries is correct after compactions. +func TestCompaction(t *testing.T) { + tests := []struct { + lastIndex uint64 + compact []uint64 + wleft []int + wallow bool + }{ + // out of upper bound + {1000, []uint64{1001}, []int{-1}, false}, + {1000, []uint64{300, 500, 800, 900}, []int{700, 500, 200, 100}, true}, + // out of lower bound + {1000, []uint64{300, 299}, []int{700, -1}, false}, + } + + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + require.False(t, tt.wallow) + } + }() + storage := NewMemoryStorage() + require.NoError(t, storage.Append(index(1).termRange(1, tt.lastIndex+1))) + raftLog := newLog(storage, raftLogger) + raftLog.maybeCommit(entryID{term: 0, index: tt.lastIndex}) // TODO(pav-kv): this is a no-op + + raftLog.appliedTo(raftLog.committed, 0 /* size */) + for j := 0; j < len(tt.compact); j++ { + err := storage.Compact(tt.compact[j]) + if err != nil { + require.False(t, tt.wallow) + continue + } + require.Equal(t, tt.wleft[j], len(raftLog.allEntries())) + } + }) + } +} + +func TestLogRestore(t *testing.T) { + index := uint64(1000) + term := uint64(1000) + storage := NewMemoryStorage() + storage.ApplySnapshot(&pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(index), Term: new(term)}}) + raftLog := newLog(storage, raftLogger) + + require.Zero(t, len(raftLog.allEntries())) + require.Equal(t, index+1, raftLog.firstIndex()) + require.Equal(t, index, raftLog.committed) + require.Equal(t, index+1, raftLog.unstable.offset) + require.Equal(t, term, mustTerm(raftLog.term(index))) +} + +func TestIsOutOfBounds(t *testing.T) { + offset := uint64(100) + num := uint64(100) + storage := NewMemoryStorage() + storage.ApplySnapshot(&pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(offset)}}) + l := newLog(storage, raftLogger) + l.append(index(offset+1).termRange(offset+1, offset+num+1)...) + + first := offset + 1 + tests := []struct { + lo, hi uint64 + wpanic bool + wErrCompacted bool + }{ + { + first - 2, first + 1, + false, + true, + }, + { + first - 1, first + 1, + false, + true, + }, + { + first, first, + false, + false, + }, + { + first + num/2, first + num/2, + false, + false, + }, + { + first + num - 1, first + num - 1, + false, + false, + }, + { + first + num, first + num, + false, + false, + }, + { + first + num, first + num + 1, + true, + false, + }, + { + first + num + 1, first + num + 1, + true, + false, + }, + } + + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + require.True(t, tt.wpanic) + } + }() + err := l.mustCheckOutOfBounds(tt.lo, tt.hi) + require.False(t, tt.wpanic) + require.False(t, tt.wErrCompacted && err != ErrCompacted) + require.False(t, !tt.wErrCompacted && err != nil) + }) + } +} + +func TestTerm(t *testing.T) { + offset := uint64(100) + num := uint64(100) + + storage := NewMemoryStorage() + storage.ApplySnapshot(&pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(offset), Term: new(uint64(1))}}) + l := newLog(storage, raftLogger) + l.append(index(offset+1).termRange(1, num)...) + + for i, tt := range []struct { + idx uint64 + term uint64 + err error + }{ + {idx: offset - 1, err: ErrCompacted}, + {idx: offset, term: 1}, + {idx: offset + num/2, term: num / 2}, + {idx: offset + num - 1, term: num - 1}, + {idx: offset + num, err: ErrUnavailable}, + } { + t.Run(fmt.Sprint(i), func(t *testing.T) { + term, err := l.term(tt.idx) + require.Equal(t, tt.term, term) + require.Equal(t, tt.err, err) + }) + } +} + +func TestTermWithUnstableSnapshot(t *testing.T) { + storagesnapi := uint64(100) + unstablesnapi := storagesnapi + 5 + + storage := NewMemoryStorage() + storage.ApplySnapshot(&pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(storagesnapi), Term: new(uint64(1))}}) + l := newLog(storage, raftLogger) + l.restore(&pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(unstablesnapi), Term: new(uint64(1))}}) + + for i, tt := range []struct { + idx uint64 + term uint64 + err error + }{ + // cannot get term from storage + {idx: storagesnapi, err: ErrCompacted}, + // cannot get term from the gap between storage ents and unstable snapshot + {idx: storagesnapi + 1, err: ErrCompacted}, + {idx: unstablesnapi - 1, err: ErrCompacted}, + // get term from unstable snapshot index + {idx: unstablesnapi, term: 1}, + // the log beyond the unstable snapshot is empty + {idx: unstablesnapi + 1, err: ErrUnavailable}, + } { + t.Run(fmt.Sprint(i), func(t *testing.T) { + term, err := l.term(tt.idx) + require.Equal(t, tt.term, term) + require.Equal(t, tt.err, err) + }) + } +} + +func TestSlice(t *testing.T) { + offset := uint64(100) + num := uint64(100) + last := offset + num + half := offset + num/2 + halfe := &pb.Entry{Index: new(half), Term: new(half), Type: pb.EntryNormal.Enum()} + + entries := func(from, to uint64) []*pb.Entry { + return index(from).termRange(from, to) + } + + storage := NewMemoryStorage() + require.NoError(t, storage.ApplySnapshot(&pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{Index: new(offset)}})) + require.NoError(t, storage.Append(entries(offset+1, half))) + l := newLog(storage, raftLogger) + l.append(entries(half, last)...) + + for _, tt := range []struct { + lo uint64 + hi uint64 + lim uint64 + + w []*pb.Entry + wpanic bool + }{ + // ErrCompacted. + {lo: offset - 1, hi: offset + 1, lim: noLimit, w: nil}, + {lo: offset, hi: offset + 1, lim: noLimit, w: nil}, + // panics + {lo: half, hi: half - 1, lim: noLimit, wpanic: true}, // lo and hi inversion + {lo: last, hi: last + 1, lim: noLimit, wpanic: true}, // hi is out of bounds + + // No limit. + {lo: offset + 1, hi: offset + 1, lim: noLimit, w: nil}, + {lo: offset + 1, hi: half - 1, lim: noLimit, w: entries(offset+1, half-1)}, + {lo: offset + 1, hi: half, lim: noLimit, w: entries(offset+1, half)}, + {lo: offset + 1, hi: half + 1, lim: noLimit, w: entries(offset+1, half+1)}, + {lo: offset + 1, hi: last, lim: noLimit, w: entries(offset+1, last)}, + {lo: half - 1, hi: half, lim: noLimit, w: entries(half-1, half)}, + {lo: half - 1, hi: half + 1, lim: noLimit, w: entries(half-1, half+1)}, + {lo: half - 1, hi: last, lim: noLimit, w: entries(half-1, last)}, + {lo: half, hi: half + 1, lim: noLimit, w: entries(half, half+1)}, + {lo: half, hi: last, lim: noLimit, w: entries(half, last)}, + {lo: last - 1, hi: last, lim: noLimit, w: entries(last-1, last)}, + + // At least one entry is always returned. + {lo: offset + 1, hi: last, lim: 0, w: entries(offset+1, offset+2)}, + {lo: half - 1, hi: half + 1, lim: 0, w: entries(half-1, half)}, + {lo: half, hi: last, lim: 0, w: entries(half, half+1)}, + {lo: half + 1, hi: last, lim: 0, w: entries(half+1, half+2)}, + // Low limit. + {lo: offset + 1, hi: last, lim: uint64(proto.Size(halfe) - 1), w: entries(offset+1, offset+2)}, + {lo: half - 1, hi: half + 1, lim: uint64(proto.Size(halfe) - 1), w: entries(half-1, half)}, + {lo: half, hi: last, lim: uint64(proto.Size(halfe) - 1), w: entries(half, half+1)}, + // Just enough for one limit. + {lo: offset + 1, hi: last, lim: uint64(proto.Size(halfe)), w: entries(offset+1, offset+2)}, + {lo: half - 1, hi: half + 1, lim: uint64(proto.Size(halfe)), w: entries(half-1, half)}, + {lo: half, hi: last, lim: uint64(proto.Size(halfe)), w: entries(half, half+1)}, + // Not enough for two limit. + {lo: offset + 1, hi: last, lim: uint64(proto.Size(halfe) + 1), w: entries(offset+1, offset+2)}, + {lo: half - 1, hi: half + 1, lim: uint64(proto.Size(halfe) + 1), w: entries(half-1, half)}, + {lo: half, hi: last, lim: uint64(proto.Size(halfe) + 1), w: entries(half, half+1)}, + // Enough for two limit. + {lo: offset + 1, hi: last, lim: uint64(proto.Size(halfe) * 2), w: entries(offset+1, offset+3)}, + {lo: half - 2, hi: half + 1, lim: uint64(proto.Size(halfe) * 2), w: entries(half-2, half)}, + {lo: half - 1, hi: half + 1, lim: uint64(proto.Size(halfe) * 2), w: entries(half-1, half+1)}, + {lo: half, hi: last, lim: uint64(proto.Size(halfe) * 2), w: entries(half, half+2)}, + // Not enough for three. + {lo: half - 2, hi: half + 1, lim: uint64(proto.Size(halfe)*3 - 1), w: entries(half-2, half)}, + // Enough for three. + {lo: half - 1, hi: half + 2, lim: uint64(proto.Size(halfe) * 3), w: entries(half-1, half+2)}, + } { + t.Run("", func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + require.True(t, tt.wpanic) + } + }() + g, err := l.slice(tt.lo, tt.hi, entryEncodingSize(tt.lim)) + require.False(t, tt.lo <= offset && err != ErrCompacted) + require.False(t, tt.lo > offset && err != nil) + requireEqualEntries(t, tt.w, g) + }) + } +} + +func TestScan(t *testing.T) { + offset := uint64(47) + num := uint64(20) + last := offset + num + half := offset + num/2 + entries := func(from, to uint64) []*pb.Entry { + return index(from).termRange(from, to) + } + entrySize := entsSize(entries(half, half+1)) + + storage := NewMemoryStorage() + require.NoError(t, storage.ApplySnapshot(&pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{Index: new(offset)}})) + require.NoError(t, storage.Append(entries(offset+1, half))) + l := newLog(storage, raftLogger) + l.append(entries(half, last)...) + + // Test that scan() returns the same entries as slice(), on all inputs. + for _, pageSize := range []entryEncodingSize{0, 1, 10, 100, entrySize, entrySize + 1} { + for lo := offset + 1; lo < last; lo++ { + for hi := lo; hi <= last; hi++ { + var got []*pb.Entry + require.NoError(t, l.scan(lo, hi, pageSize, func(e []*pb.Entry) error { + got = append(got, e...) + require.True(t, len(e) == 1 || entsSize(e) <= pageSize) + return nil + })) + want, err := l.slice(lo, hi, noLimit) + require.NoError(t, err) + require.Equal(t, want, got, "scan() and slice() mismatch on [%d, %d) @ %d", lo, hi, pageSize) + } + } + } + + // Test that the callback error is propagated to the caller. + iters := 0 + require.ErrorIs(t, errBreak, l.scan(offset+1, half, 0, func([]*pb.Entry) error { + iters++ + if iters == 2 { + return errBreak + } + return nil + })) + require.Equal(t, 2, iters) + + // Test that we max out the limit, and not just always return a single entry. + // NB: this test works only because the requested range length is even. + require.NoError(t, l.scan(offset+1, offset+11, entrySize*2, func(ents []*pb.Entry) error { + require.Len(t, ents, 2) + require.Equal(t, entrySize*2, entsSize(ents)) + return nil + })) +} + +func mustTerm(term uint64, err error) uint64 { + if err != nil { + panic(err) + } + return term +} + +// index is a helper type for generating slices of pb.Entry. The value of index +// is the first entry index in the generated slices. +type index uint64 + +// terms generates a slice of entries at indices [index, index+len(terms)), with +// the given terms of each entry. Terms must be non-decreasing. +func (i index) terms(terms ...uint64) []*pb.Entry { + index := uint64(i) + entries := make([]*pb.Entry, 0, len(terms)) + for _, term := range terms { + entries = append(entries, &pb.Entry{Term: new(term), Index: new(index)}) + index++ + } + return entries +} + +// termRange generates a slice of to-from entries, at consecutive indices +// starting from i, and consecutive terms in [from, to). +func (i index) termRange(from, to uint64) []*pb.Entry { + index := uint64(i) + entries := make([]*pb.Entry, 0, to-from) + for term := from; term < to; term++ { + entries = append(entries, &pb.Entry{Term: new(term), Index: new(index), Type: pb.EntryNormal.Enum()}) + index++ + } + return entries +} diff --git a/third_party/raft/log_unstable.go b/third_party/raft/log_unstable.go new file mode 100644 index 0000000000..f637b61180 --- /dev/null +++ b/third_party/raft/log_unstable.go @@ -0,0 +1,249 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "google.golang.org/protobuf/proto" + + pb "go.etcd.io/raft/v3/raftpb" +) + +// unstable contains "unstable" log entries and snapshot state that has +// not yet been written to Storage. The type serves two roles. First, it +// holds on to new log entries and an optional snapshot until they are +// handed to a Ready struct for persistence. Second, it continues to +// hold on to this state after it has been handed off to provide raftLog +// with a view of the in-progress log entries and snapshot until their +// writes have been stabilized and are guaranteed to be reflected in +// queries of Storage. After this point, the corresponding log entries +// and/or snapshot can be cleared from unstable. +// +// unstable.entries[i] has raft log position i+unstable.offset. +// Note that unstable.offset may be less than the highest log +// position in storage; this means that the next write to storage +// might need to truncate the log before persisting unstable.entries. +type unstable struct { + // the incoming unstable snapshot, if any. + snapshot *pb.Snapshot + // all entries that have not yet been written to storage. + entries []*pb.Entry + // entries[i] has raft log position i+offset. + offset uint64 + + // if true, snapshot is being written to storage. + snapshotInProgress bool + // entries[:offsetInProgress-offset] are being written to storage. + // Like offset, offsetInProgress is exclusive, meaning that it + // contains the index following the largest in-progress entry. + // Invariant: offset <= offsetInProgress + offsetInProgress uint64 + + logger Logger +} + +// maybeFirstIndex returns the index of the first possible entry in entries +// if it has a snapshot. +func (u *unstable) maybeFirstIndex() (uint64, bool) { + if u.snapshot != nil { + return u.snapshot.GetMetadata().GetIndex() + 1, true + } + return 0, false +} + +// maybeLastIndex returns the last index if it has at least one +// unstable entry or snapshot. +func (u *unstable) maybeLastIndex() (uint64, bool) { + if l := len(u.entries); l != 0 { + return u.offset + uint64(l) - 1, true + } + if u.snapshot != nil { + return u.snapshot.GetMetadata().GetIndex(), true + } + return 0, false +} + +// maybeTerm returns the term of the entry at index i, if there +// is any. +func (u *unstable) maybeTerm(i uint64) (uint64, bool) { + if i < u.offset { + if u.snapshot != nil && u.snapshot.GetMetadata().GetIndex() == i { + return u.snapshot.GetMetadata().GetTerm(), true + } + return 0, false + } + + last, ok := u.maybeLastIndex() + if !ok { + return 0, false + } + if i > last { + return 0, false + } + + return u.entries[i-u.offset].GetTerm(), true +} + +// nextEntries returns the unstable entries that are not already in the process +// of being written to storage. +func (u *unstable) nextEntries() []*pb.Entry { + inProgress := int(u.offsetInProgress - u.offset) + if len(u.entries) == inProgress { + return nil + } + return u.entries[inProgress:] +} + +// nextSnapshot returns the unstable snapshot, if one exists that is not already +// in the process of being written to storage. +func (u *unstable) nextSnapshot() *pb.Snapshot { + if u.snapshot == nil || u.snapshotInProgress { + return nil + } + return proto.Clone(u.snapshot).(*pb.Snapshot) +} + +// acceptInProgress marks all entries and the snapshot, if any, in the unstable +// as having begun the process of being written to storage. The entries/snapshot +// will no longer be returned from nextEntries/nextSnapshot. However, new +// entries/snapshots added after a call to acceptInProgress will be returned +// from those methods, until the next call to acceptInProgress. +func (u *unstable) acceptInProgress() { + if len(u.entries) > 0 { + // NOTE: +1 because offsetInProgress is exclusive, like offset. + u.offsetInProgress = u.entries[len(u.entries)-1].GetIndex() + 1 + } + if u.snapshot != nil { + u.snapshotInProgress = true + } +} + +// stableTo marks entries up to the entry with the specified (index, term) as +// being successfully written to stable storage. +// +// The method should only be called when the caller can attest that the entries +// can not be overwritten by an in-progress log append. See the related comment +// in newStorageAppendRespMsg. +func (u *unstable) stableTo(id entryID) { + gt, ok := u.maybeTerm(id.index) + if !ok { + // Unstable entry missing. Ignore. + u.logger.Infof("entry at index %d missing from unstable log; ignoring", id.index) + return + } + if id.index < u.offset { + // Index matched unstable snapshot, not unstable entry. Ignore. + u.logger.Infof("entry at index %d matched unstable snapshot; ignoring", id.index) + return + } + if gt != id.term { + // Term mismatch between unstable entry and specified entry. Ignore. + // This is possible if part or all of the unstable log was replaced + // between that time that a set of entries started to be written to + // stable storage and when they finished. + u.logger.Infof("entry at (index,term)=(%d,%d) mismatched with "+ + "entry at (%d,%d) in unstable log; ignoring", id.index, id.term, id.index, gt) + return + } + num := int(id.index + 1 - u.offset) + u.entries = u.entries[num:] + u.offset = id.index + 1 + u.offsetInProgress = max(u.offsetInProgress, u.offset) + u.shrinkEntriesArray() +} + +// shrinkEntriesArray discards the underlying array used by the entries slice +// if most of it isn't being used. This avoids holding references to a bunch of +// potentially large entries that aren't needed anymore. Simply clearing the +// entries wouldn't be safe because clients might still be using them. +func (u *unstable) shrinkEntriesArray() { + // We replace the array if we're using less than half of the space in + // it. This number is fairly arbitrary, chosen as an attempt to balance + // memory usage vs number of allocations. It could probably be improved + // with some focused tuning. + const lenMultiple = 2 + if len(u.entries) == 0 { + u.entries = nil + } else if len(u.entries)*lenMultiple < cap(u.entries) { + newEntries := make([]*pb.Entry, len(u.entries)) + copy(newEntries, u.entries) + u.entries = newEntries + } +} + +func (u *unstable) stableSnapTo(i uint64) { + if u.snapshot != nil && u.snapshot.GetMetadata().GetIndex() == i { + u.snapshot = nil + u.snapshotInProgress = false + } +} + +func (u *unstable) restore(s *pb.Snapshot) { + u.offset = s.GetMetadata().GetIndex() + 1 + u.offsetInProgress = u.offset + u.entries = nil + u.snapshot = proto.Clone(s).(*pb.Snapshot) + u.snapshotInProgress = false +} + +func (u *unstable) truncateAndAppend(ents []*pb.Entry) { + fromIndex := ents[0].GetIndex() + switch { + case fromIndex == u.offset+uint64(len(u.entries)): + // fromIndex is the next index in the u.entries, so append directly. + u.entries = append(u.entries, ents...) + case fromIndex <= u.offset: + u.logger.Infof("replace the unstable entries from index %d", fromIndex) + // The log is being truncated to before our current offset + // portion, so set the offset and replace the entries. + u.entries = ents + u.offset = fromIndex + u.offsetInProgress = u.offset + default: + // Truncate to fromIndex (exclusive), and append the new entries. + u.logger.Infof("truncate the unstable entries before index %d", fromIndex) + keep := u.slice(u.offset, fromIndex) // NB: appending to this slice is safe, + u.entries = append(keep, ents...) // and will reallocate/copy it + // Only in-progress entries before fromIndex are still considered to be + // in-progress. + u.offsetInProgress = min(u.offsetInProgress, fromIndex) + } +} + +// slice returns the entries from the unstable log with indexes in the range +// [lo, hi). The entire range must be stored in the unstable log or the method +// will panic. The returned slice can be appended to, but the entries in it must +// not be changed because they are still shared with unstable. +// +// TODO(pavelkalinnikov): this, and similar []*pb.Entry slices, may bubble up all +// the way to the application code through Ready struct. Protect other slices +// similarly, and document how the client can use them. +func (u *unstable) slice(lo uint64, hi uint64) []*pb.Entry { + u.mustCheckOutOfBounds(lo, hi) + // NB: use the full slice expression to limit what the caller can do with the + // returned slice. For example, an append will reallocate and copy this slice + // instead of corrupting the neighbouring u.entries. + return u.entries[lo-u.offset : hi-u.offset : hi-u.offset] +} + +// u.offset <= lo <= hi <= u.offset+len(u.entries) +func (u *unstable) mustCheckOutOfBounds(lo, hi uint64) { + if lo > hi { + u.logger.Panicf("invalid unstable.slice %d > %d", lo, hi) + } + upper := u.offset + uint64(len(u.entries)) + if lo < u.offset || hi > upper { + u.logger.Panicf("unstable.slice[%d,%d) out of bound [%d,%d]", lo, hi, u.offset, upper) + } +} diff --git a/third_party/raft/log_unstable_test.go b/third_party/raft/log_unstable_test.go new file mode 100644 index 0000000000..d7128debaf --- /dev/null +++ b/third_party/raft/log_unstable_test.go @@ -0,0 +1,581 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + pb "go.etcd.io/raft/v3/raftpb" +) + +func TestUnstableMaybeFirstIndex(t *testing.T) { + tests := []struct { + entries []*pb.Entry + offset uint64 + snap *pb.Snapshot + + wok bool + windex uint64 + }{ + // no snapshot + { + index(5).terms(1), 5, nil, + false, 0, + }, + { + []*pb.Entry{}, 0, nil, + false, 0, + }, + // has snapshot + { + index(5).terms(1), 5, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + true, 5, + }, + { + []*pb.Entry{}, 5, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + true, 5, + }, + } + + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + u := unstable{ + entries: tt.entries, + offset: tt.offset, + snapshot: tt.snap, + logger: raftLogger, + } + index, ok := u.maybeFirstIndex() + require.Equal(t, tt.wok, ok) + require.Equal(t, tt.windex, index) + }) + } +} + +func TestMaybeLastIndex(t *testing.T) { + tests := []struct { + entries []*pb.Entry + offset uint64 + snap *pb.Snapshot + + wok bool + windex uint64 + }{ + // last in entries + { + index(5).terms(1), 5, nil, + true, 5, + }, + { + index(5).terms(1), 5, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + true, 5, + }, + // last in snapshot + { + []*pb.Entry{}, 5, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + true, 4, + }, + // empty unstable + { + []*pb.Entry{}, 0, nil, + false, 0, + }, + } + + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + u := unstable{ + entries: tt.entries, + offset: tt.offset, + snapshot: tt.snap, + logger: raftLogger, + } + index, ok := u.maybeLastIndex() + require.Equal(t, tt.wok, ok) + require.Equal(t, tt.windex, index) + }) + } +} + +func TestUnstableMaybeTerm(t *testing.T) { + tests := []struct { + entries []*pb.Entry + offset uint64 + snap *pb.Snapshot + index uint64 + + wok bool + wterm uint64 + }{ + // term from entries + { + index(5).terms(1), 5, nil, + 5, + true, 1, + }, + { + index(5).terms(1), 5, nil, + 6, + false, 0, + }, + { + index(5).terms(1), 5, nil, + 4, + false, 0, + }, + { + index(5).terms(1), 5, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 5, + true, 1, + }, + { + index(5).terms(1), 5, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 6, + false, 0, + }, + // term from snapshot + { + index(5).terms(1), 5, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 4, + true, 1, + }, + { + index(5).terms(1), 5, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 3, + false, 0, + }, + { + []*pb.Entry{}, 5, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 5, + false, 0, + }, + { + []*pb.Entry{}, 5, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 4, + true, 1, + }, + { + []*pb.Entry{}, 0, nil, + 5, + false, 0, + }, + } + + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + u := unstable{ + entries: tt.entries, + offset: tt.offset, + snapshot: tt.snap, + logger: raftLogger, + } + term, ok := u.maybeTerm(tt.index) + require.Equal(t, tt.wok, ok) + require.Equal(t, tt.wterm, term) + }) + } +} + +func TestUnstableRestore(t *testing.T) { + u := unstable{ + entries: index(5).terms(1), + offset: 5, + offsetInProgress: 6, + snapshot: &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + snapshotInProgress: true, + logger: raftLogger, + } + s := &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(6)), Term: new(uint64(2))}} + u.restore(s) + + require.Equal(t, s.GetMetadata().GetIndex()+1, u.offset) + require.Equal(t, s.GetMetadata().GetIndex()+1, u.offsetInProgress) + require.Zero(t, len(u.entries)) + require.Equal(t, s, u.snapshot) + require.False(t, u.snapshotInProgress) +} + +func TestUnstableNextEntries(t *testing.T) { + tests := []struct { + entries []*pb.Entry + offset uint64 + offsetInProgress uint64 + + wentries []*pb.Entry + }{ + // nothing in progress + { + index(5).terms(1, 1), 5, 5, + index(5).terms(1, 1), + }, + // partially in progress + { + index(5).terms(1, 1), 5, 6, + index(6).terms(1), + }, + // everything in progress + { + index(5).terms(1, 1), 5, 7, + nil, // nil, not empty slice + }, + } + + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + u := unstable{ + entries: tt.entries, + offset: tt.offset, + offsetInProgress: tt.offsetInProgress, + logger: raftLogger, + } + res := u.nextEntries() + require.Equal(t, tt.wentries, res) + }) + } +} + +func TestUnstableNextSnapshot(t *testing.T) { + s := &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}} + tests := []struct { + snapshot *pb.Snapshot + snapshotInProgress bool + + wsnapshot *pb.Snapshot + }{ + // snapshot not unstable + { + nil, false, + nil, + }, + // snapshot not in progress + { + s, false, + s, + }, + // snapshot in progress + { + s, true, + nil, + }, + } + + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + u := unstable{ + snapshot: tt.snapshot, + snapshotInProgress: tt.snapshotInProgress, + } + res := u.nextSnapshot() + require.Equal(t, tt.wsnapshot, res) + }) + } +} + +func TestUnstableAcceptInProgress(t *testing.T) { + tests := []struct { + entries []*pb.Entry + snapshot *pb.Snapshot + offsetInProgress uint64 + snapshotInProgress bool + + woffsetInProgress uint64 + wsnapshotInProgress bool + }{ + { + []*pb.Entry{}, nil, + 5, // no entries + false, // snapshot not already in progress + 5, false, + }, + { + index(5).terms(1), nil, + 5, // entries not in progress + false, // snapshot not already in progress + 6, false, + }, + { + index(5).terms(1, 1), nil, + 5, // entries not in progress + false, // snapshot not already in progress + 7, false, + }, + { + index(5).terms(1, 1), nil, + 6, // in-progress to the first entry + false, // snapshot not already in progress + 7, false, + }, + { + index(5).terms(1, 1), nil, + 7, // in-progress to the second entry + false, // snapshot not already in progress + 7, false, + }, + // with snapshot + { + []*pb.Entry{}, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 5, // no entries + false, // snapshot not already in progress + 5, true, + }, + { + index(5).terms(1), &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 5, // entries not in progress + false, // snapshot not already in progress + 6, true, + }, + { + index(5).terms(1, 1), &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 5, // entries not in progress + false, // snapshot not already in progress + 7, true, + }, + { + index(5).terms(1, 1), &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 6, // in-progress to the first entry + false, // snapshot not already in progress + 7, true, + }, + { + index(5).terms(1, 1), &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 7, // in-progress to the second entry + false, // snapshot not already in progress + 7, true, + }, + { + []*pb.Entry{}, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 5, // entries not in progress + true, // snapshot already in progress + 5, true, + }, + { + index(5).terms(1), &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 5, // entries not in progress + true, // snapshot already in progress + 6, true, + }, + { + index(5).terms(1, 1), &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 5, // entries not in progress + true, // snapshot already in progress + 7, true, + }, + { + index(5).terms(1, 1), &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 6, // in-progress to the first entry + true, // snapshot already in progress + 7, true, + }, + { + index(5).terms(1, 1), &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 7, // in-progress to the second entry + true, // snapshot already in progress + 7, true, + }, + } + + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + u := unstable{ + entries: tt.entries, + snapshot: tt.snapshot, + offsetInProgress: tt.offsetInProgress, + snapshotInProgress: tt.snapshotInProgress, + } + u.acceptInProgress() + require.Equal(t, tt.woffsetInProgress, u.offsetInProgress) + require.Equal(t, tt.wsnapshotInProgress, u.snapshotInProgress) + }) + } +} + +func TestUnstableStableTo(t *testing.T) { + tests := []struct { + entries []*pb.Entry + offset uint64 + offsetInProgress uint64 + snap *pb.Snapshot + index, term uint64 + + woffset uint64 + woffsetInProgress uint64 + wlen int + }{ + { + []*pb.Entry{}, 0, 0, nil, + 5, 1, + 0, 0, 0, + }, + { + index(5).terms(1), 5, 6, nil, + 5, 1, // stable to the first entry + 6, 6, 0, + }, + { + index(5).terms(1, 1), 5, 6, nil, + 5, 1, // stable to the first entry + 6, 6, 1, + }, + { + index(5).terms(1, 1), 5, 7, nil, + 5, 1, // stable to the first entry and in-progress ahead + 6, 7, 1, + }, + { + index(6).terms(2), 6, 7, nil, + 6, 1, // stable to the first entry and term mismatch + 6, 7, 1, + }, + { + index(5).terms(1), 5, 6, nil, + 4, 1, // stable to old entry + 5, 6, 1, + }, + { + index(5).terms(1), 5, 6, nil, + 4, 2, // stable to old entry + 5, 6, 1, + }, + // with snapshot + { + index(5).terms(1), 5, 6, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 5, 1, // stable to the first entry + 6, 6, 0, + }, + { + index(5).terms(1, 1), 5, 6, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 5, 1, // stable to the first entry + 6, 6, 1, + }, + { + index(5).terms(1, 1), 5, 7, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 5, 1, // stable to the first entry and in-progress ahead + 6, 7, 1, + }, + { + index(6).terms(2), 6, 7, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(5)), Term: new(uint64(1))}}, + 6, 1, // stable to the first entry and term mismatch + 6, 7, 1, + }, + { + index(5).terms(1), 5, 6, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(1))}}, + 4, 1, // stable to snapshot + 5, 6, 1, + }, + { + index(5).terms(2), 5, 6, &pb.Snapshot{Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(2))}}, + 4, 1, // stable to old entry + 5, 6, 1, + }, + } + + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + u := unstable{ + entries: tt.entries, + offset: tt.offset, + offsetInProgress: tt.offsetInProgress, + snapshot: tt.snap, + logger: raftLogger, + } + u.stableTo(entryID{term: tt.term, index: tt.index}) + require.Equal(t, tt.woffset, u.offset) + require.Equal(t, tt.woffsetInProgress, u.offsetInProgress) + require.Equal(t, tt.wlen, len(u.entries)) + }) + } +} + +func TestUnstableTruncateAndAppend(t *testing.T) { + tests := []struct { + entries []*pb.Entry + offset uint64 + offsetInProgress uint64 + snap *pb.Snapshot + toappend []*pb.Entry + + woffset uint64 + woffsetInProgress uint64 + wentries []*pb.Entry + }{ + // append to the end + { + index(5).terms(1), 5, 5, nil, + index(6).terms(1, 1), + 5, 5, index(5).terms(1, 1, 1), + }, + { + index(5).terms(1), 5, 6, nil, + index(6).terms(1, 1), + 5, 6, index(5).terms(1, 1, 1), + }, + // replace the unstable entries + { + index(5).terms(1), 5, 5, nil, + index(5).terms(2, 2), + 5, 5, index(5).terms(2, 2), + }, + { + index(5).terms(1), 5, 5, nil, + index(4).terms(2, 2, 2), + 4, 4, index(4).terms(2, 2, 2), + }, + { + index(5).terms(1), 5, 6, nil, + index(5).terms(2, 2), + 5, 5, index(5).terms(2, 2), + }, + // truncate the existing entries and append + { + index(5).terms(1, 1, 1), 5, 5, nil, + index(6).terms(2), + 5, 5, index(5).terms(1, 2), + }, + { + index(5).terms(1, 1, 1), 5, 5, nil, + index(7).terms(2, 2), + 5, 5, index(5).terms(1, 1, 2, 2), + }, + { + index(5).terms(1, 1, 1), 5, 6, nil, + index(6).terms(2), + 5, 6, index(5).terms(1, 2), + }, + { + index(5).terms(1, 1, 1), 5, 7, nil, + index(6).terms(2), + 5, 6, index(5).terms(1, 2), + }, + } + + for i, tt := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + u := unstable{ + entries: tt.entries, + offset: tt.offset, + offsetInProgress: tt.offsetInProgress, + snapshot: tt.snap, + logger: raftLogger, + } + u.truncateAndAppend(tt.toappend) + require.Equal(t, tt.woffset, u.offset) + require.Equal(t, tt.woffsetInProgress, u.offsetInProgress) + require.Equal(t, tt.wentries, u.entries) + }) + } +} diff --git a/third_party/raft/logger.go b/third_party/raft/logger.go new file mode 100644 index 0000000000..e3cb00cc9d --- /dev/null +++ b/third_party/raft/logger.go @@ -0,0 +1,142 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "fmt" + "io" + "log" + "os" + "sync" +) + +type Logger interface { + Debug(v ...interface{}) + Debugf(format string, v ...interface{}) + + Error(v ...interface{}) + Errorf(format string, v ...interface{}) + + Info(v ...interface{}) + Infof(format string, v ...interface{}) + + Warning(v ...interface{}) + Warningf(format string, v ...interface{}) + + Fatal(v ...interface{}) + Fatalf(format string, v ...interface{}) + + Panic(v ...interface{}) + Panicf(format string, v ...interface{}) +} + +func SetLogger(l Logger) { + raftLoggerMu.Lock() + raftLogger = l + raftLoggerMu.Unlock() +} + +func ResetDefaultLogger() { + SetLogger(defaultLogger) +} + +func getLogger() Logger { + raftLoggerMu.Lock() + defer raftLoggerMu.Unlock() + return raftLogger +} + +var ( + defaultLogger = &DefaultLogger{Logger: log.New(os.Stderr, "raft", log.LstdFlags)} + discardLogger = &DefaultLogger{Logger: log.New(io.Discard, "", 0)} + raftLoggerMu sync.Mutex + raftLogger = Logger(defaultLogger) +) + +const ( + calldepth = 2 +) + +// DefaultLogger is a default implementation of the Logger interface. +type DefaultLogger struct { + *log.Logger + debug bool +} + +func (l *DefaultLogger) EnableTimestamps() { + l.SetFlags(l.Flags() | log.Ldate | log.Ltime) +} + +func (l *DefaultLogger) EnableDebug() { + l.debug = true +} + +func (l *DefaultLogger) Debug(v ...interface{}) { + if l.debug { + l.Output(calldepth, header("DEBUG", fmt.Sprint(v...))) + } +} + +func (l *DefaultLogger) Debugf(format string, v ...interface{}) { + if l.debug { + l.Output(calldepth, header("DEBUG", fmt.Sprintf(format, v...))) + } +} + +func (l *DefaultLogger) Info(v ...interface{}) { + l.Output(calldepth, header("INFO", fmt.Sprint(v...))) +} + +func (l *DefaultLogger) Infof(format string, v ...interface{}) { + l.Output(calldepth, header("INFO", fmt.Sprintf(format, v...))) +} + +func (l *DefaultLogger) Error(v ...interface{}) { + l.Output(calldepth, header("ERROR", fmt.Sprint(v...))) +} + +func (l *DefaultLogger) Errorf(format string, v ...interface{}) { + l.Output(calldepth, header("ERROR", fmt.Sprintf(format, v...))) +} + +func (l *DefaultLogger) Warning(v ...interface{}) { + l.Output(calldepth, header("WARN", fmt.Sprint(v...))) +} + +func (l *DefaultLogger) Warningf(format string, v ...interface{}) { + l.Output(calldepth, header("WARN", fmt.Sprintf(format, v...))) +} + +func (l *DefaultLogger) Fatal(v ...interface{}) { + l.Output(calldepth, header("FATAL", fmt.Sprint(v...))) + os.Exit(1) +} + +func (l *DefaultLogger) Fatalf(format string, v ...interface{}) { + l.Output(calldepth, header("FATAL", fmt.Sprintf(format, v...))) + os.Exit(1) +} + +func (l *DefaultLogger) Panic(v ...interface{}) { + l.Logger.Panic(v...) +} + +func (l *DefaultLogger) Panicf(format string, v ...interface{}) { + l.Logger.Panicf(format, v...) +} + +func header(lvl, msg string) string { + return fmt.Sprintf("%s: %s", lvl, msg) +} diff --git a/third_party/raft/node.go b/third_party/raft/node.go new file mode 100644 index 0000000000..b65bf7a36e --- /dev/null +++ b/third_party/raft/node.go @@ -0,0 +1,616 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "context" + "errors" + + pb "go.etcd.io/raft/v3/raftpb" +) + +type SnapshotStatus int + +const ( + SnapshotFinish SnapshotStatus = 1 + SnapshotFailure SnapshotStatus = 2 +) + +var ( + emptyState = &pb.HardState{} + + // ErrStopped is returned by methods on Nodes that have been stopped. + ErrStopped = errors.New("raft: stopped") +) + +// SoftState provides state that is useful for logging and debugging. +// The state is volatile and does not need to be persisted to the WAL. +type SoftState struct { + Lead uint64 // must use atomic operations to access; keep 64-bit aligned. + RaftState StateType +} + +func (a *SoftState) equal(b *SoftState) bool { + return a.Lead == b.Lead && a.RaftState == b.RaftState +} + +// Ready encapsulates the entries and messages that are ready to read, +// be saved to stable storage, committed or sent to other peers. +// All fields in Ready are read-only. +type Ready struct { + // The current volatile state of a Node. + // SoftState will be nil if there is no update. + // It is not required to consume or store SoftState. + *SoftState + + // The current state of a Node to be saved to stable storage BEFORE + // Messages are sent. + // + // HardState will be nil if there is no update. + // + // If async storage writes are enabled, this field does not need to be acted + // on immediately. It will be reflected in a MsgStorageAppend message in the + // Messages slice. + *pb.HardState + + // ReadStates can be used for node to serve linearizable read requests locally + // when its applied index is greater than the index in ReadState. + // Note that the readState will be returned when raft receives msgReadIndex. + // The returned is only valid for the request that requested to read. + ReadStates []ReadState + + // Entries specifies entries to be saved to stable storage BEFORE + // Messages are sent. + // + // If async storage writes are enabled, this field does not need to be acted + // on immediately. It will be reflected in a MsgStorageAppend message in the + // Messages slice. + Entries []*pb.Entry + + // Snapshot specifies the snapshot to be saved to stable storage. + // + // If async storage writes are enabled, this field does not need to be acted + // on immediately. It will be reflected in a MsgStorageAppend message in the + // Messages slice. + Snapshot *pb.Snapshot + + // CommittedEntries specifies entries to be committed to a + // store/state-machine. These have previously been appended to stable + // storage. + // + // If async storage writes are enabled, this field does not need to be acted + // on immediately. It will be reflected in a MsgStorageApply message in the + // Messages slice. + CommittedEntries []*pb.Entry + + // Messages specifies outbound messages. + // + // If async storage writes are not enabled, these messages must be sent + // AFTER Entries are appended to stable storage. + // + // If async storage writes are enabled, these messages can be sent + // immediately as the messages that have the completion of the async writes + // as a precondition are attached to the individual MsgStorage{Append,Apply} + // messages instead. + // + // If it contains a MsgSnap message, the application MUST report back to raft + // when the snapshot has been received or has failed by calling ReportSnapshot. + Messages []*pb.Message + + // MustSync indicates whether the HardState and Entries must be durably + // written to disk or if a non-durable write is permissible. + MustSync bool +} + +func isHardStateEqual(a, b *pb.HardState) bool { + return a.GetTerm() == b.GetTerm() && a.GetVote() == b.GetVote() && a.GetCommit() == b.GetCommit() +} + +// IsEmptyHardState returns true if the given HardState is empty. +func IsEmptyHardState(st *pb.HardState) bool { + return st == nil || isHardStateEqual(st, emptyState) +} + +// IsEmptySnap returns true if the given Snapshot is empty. +func IsEmptySnap(sp *pb.Snapshot) bool { + return sp.GetMetadata().GetIndex() == 0 +} + +// Node represents a node in a raft cluster. +type Node interface { + // Tick increments the internal logical clock for the Node by a single tick. Election + // timeouts and heartbeat timeouts are in units of ticks. + Tick() + // Campaign causes the Node to transition to candidate state and start campaigning to become leader. + Campaign(ctx context.Context) error + // Propose proposes that data be appended to the log. Note that proposals can be lost without + // notice, therefore it is user's job to ensure proposal retries. + Propose(ctx context.Context, data []byte) error + // ProposeConfChange proposes a configuration change. Like any proposal, the + // configuration change may be dropped with or without an error being + // returned. In particular, configuration changes are dropped unless the + // leader has certainty that there is no prior unapplied configuration + // change in its log. + // + // The method accepts either a pb.ConfChange (deprecated) or pb.ConfChangeV2 + // message. The latter allows arbitrary configuration changes via joint + // consensus, notably including replacing a voter. Passing a ConfChangeV2 + // message is only allowed if all Nodes participating in the cluster run a + // version of this library aware of the V2 API. See pb.ConfChangeV2 for + // usage details and semantics. + ProposeConfChange(ctx context.Context, cc pb.ConfChangeI) error + + // Step advances the state machine using the given message. ctx.Err() will be returned, if any. + Step(ctx context.Context, msg *pb.Message) error + + // Ready returns a channel that returns the current point-in-time state. + // Users of the Node must call Advance after retrieving the state returned by Ready (unless + // async storage writes is enabled, in which case it should never be called). + // + // NOTE: No committed entries from the next Ready may be applied until all committed entries + // and snapshots from the previous one have finished. + Ready() <-chan Ready + + // Advance notifies the Node that the application has saved progress up to the last Ready. + // It prepares the node to return the next available Ready. + // + // The application should generally call Advance after it applies the entries in last Ready. + // + // However, as an optimization, the application may call Advance while it is applying the + // commands. For example. when the last Ready contains a snapshot, the application might take + // a long time to apply the snapshot data. To continue receiving Ready without blocking raft + // progress, it can call Advance before finishing applying the last ready. + // + // NOTE: Advance must not be called when using AsyncStorageWrites. Response messages from the + // local append and apply threads take its place. + Advance() + // ApplyConfChange applies a config change (previously passed to + // ProposeConfChange) to the node. This must be called whenever a config + // change is observed in Ready.CommittedEntries, except when the app decides + // to reject the configuration change (i.e. treats it as a noop instead), in + // which case it must not be called. + // + // Returns an opaque non-nil ConfState protobuf which must be recorded in + // snapshots. + ApplyConfChange(cc pb.ConfChangeI) *pb.ConfState + + // TransferLeadership attempts to transfer leadership to the given transferee. + TransferLeadership(ctx context.Context, lead, transferee uint64) + + // ForgetLeader forgets a follower's current leader, changing it to None. It + // remains a leaderless follower in the current term, without campaigning. + // + // This is useful with PreVote+CheckQuorum, where followers will normally not + // grant pre-votes if they've heard from the leader in the past election + // timeout interval. Leaderless followers can grant pre-votes immediately, so + // if a quorum of followers have strong reason to believe the leader is dead + // (for example via a side-channel or external failure detector) and forget it + // then they can elect a new leader immediately, without waiting out the + // election timeout. They will also revert to normal followers if they hear + // from the leader again, or transition to candidates on an election timeout. + // + // For example, consider a three-node cluster where 1 is the leader and 2+3 + // have just received a heartbeat from it. If 2 and 3 believe the leader has + // now died (maybe they know that an orchestration system shut down 1's VM), + // we can instruct 2 to forget the leader and 3 to campaign. 2 will then be + // able to grant 3's pre-vote and elect 3 as leader immediately (normally 2 + // would reject the vote until an election timeout passes because it has heard + // from the leader recently). However, 3 can not campaign unilaterally, a + // quorum have to agree that the leader is dead, which avoids disrupting the + // leader if individual nodes are wrong about it being dead. + // + // This does nothing with ReadOnlyLeaseBased, since it would allow a new + // leader to be elected without the old leader knowing. + ForgetLeader(ctx context.Context) error + + // ReadIndex request a read state. The read state will be set in the ready. + // Read state has a read index. Once the application advances further than the read + // index, any linearizable read requests issued before the read request can be + // processed safely. The read state will have the same rctx attached. + // Note that request can be lost without notice, therefore it is user's job + // to ensure read index retries. + ReadIndex(ctx context.Context, rctx []byte) error + + // Status returns the current status of the raft state machine. + Status() Status + // ReportUnreachable reports the given node is not reachable for the last send. + ReportUnreachable(id uint64) + // ReportSnapshot reports the status of the sent snapshot. The id is the raft ID of the follower + // who is meant to receive the snapshot, and the status is SnapshotFinish or SnapshotFailure. + // Calling ReportSnapshot with SnapshotFinish is a no-op. But, any failure in applying a + // snapshot (for e.g., while streaming it from leader to follower), should be reported to the + // leader with SnapshotFailure. When leader sends a snapshot to a follower, it pauses any raft + // log probes until the follower can apply the snapshot and advance its state. If the follower + // can't do that, for e.g., due to a crash, it could end up in a limbo, never getting any + // updates from the leader. Therefore, it is crucial that the application ensures that any + // failure in snapshot sending is caught and reported back to the leader; so it can resume raft + // log probing in the follower. + ReportSnapshot(id uint64, status SnapshotStatus) + // Stop performs any necessary termination of the Node. + Stop() +} + +type Peer struct { + ID uint64 + Context []byte +} + +func setupNode(c *Config, peers []Peer) *node { + if len(peers) == 0 { + panic("no peers given; use RestartNode instead") + } + rn, err := NewRawNode(c) + if err != nil { + panic(err) + } + err = rn.Bootstrap(peers) + if err != nil { + c.Logger.Warningf("error occurred during starting a new node: %v", err) + } + + n := newNode(rn) + return &n +} + +// StartNode returns a new Node given configuration and a list of raft peers. +// It appends a ConfChangeAddNode entry for each given peer to the initial log. +// +// Peers must not be zero length; call RestartNode in that case. +func StartNode(c *Config, peers []Peer) Node { + n := setupNode(c, peers) + go n.run() + return n +} + +// RestartNode is similar to StartNode but does not take a list of peers. +// The current membership of the cluster will be restored from the Storage. +// If the caller has an existing state machine, pass in the last log index that +// has been applied to it; otherwise use zero. +func RestartNode(c *Config) Node { + rn, err := NewRawNode(c) + if err != nil { + panic(err) + } + n := newNode(rn) + go n.run() + return &n +} + +type msgWithResult struct { + m *pb.Message + result chan error +} + +// node is the canonical implementation of the Node interface +type node struct { + propc chan msgWithResult + recvc chan *pb.Message + confc chan *pb.ConfChangeV2 + confstatec chan *pb.ConfState + readyc chan Ready + advancec chan struct{} + tickc chan struct{} + done chan struct{} + stop chan struct{} + status chan chan Status + + rn *RawNode +} + +func newNode(rn *RawNode) node { + return node{ + propc: make(chan msgWithResult), + recvc: make(chan *pb.Message), + confc: make(chan *pb.ConfChangeV2), + confstatec: make(chan *pb.ConfState), + readyc: make(chan Ready), + advancec: make(chan struct{}), + // make tickc a buffered chan, so raft node can buffer some ticks when the node + // is busy processing raft messages. Raft node will resume process buffered + // ticks when it becomes idle. + tickc: make(chan struct{}, 128), + done: make(chan struct{}), + stop: make(chan struct{}), + status: make(chan chan Status), + rn: rn, + } +} + +func (n *node) Stop() { + select { + case n.stop <- struct{}{}: + // Not already stopped, so trigger it + case <-n.done: + // Node has already been stopped - no need to do anything + return + } + // Block until the stop has been acknowledged by run() + <-n.done +} + +func (n *node) run() { + var propc chan msgWithResult + var readyc chan Ready + var advancec chan struct{} + var rd Ready + + r := n.rn.raft + + lead := None + + for { + if advancec == nil && n.rn.HasReady() { + // Populate a Ready. Note that this Ready is not guaranteed to + // actually be handled. We will arm readyc, but there's no guarantee + // that we will actually send on it. It's possible that we will + // service another channel instead, loop around, and then populate + // the Ready again. We could instead force the previous Ready to be + // handled first, but it's generally good to emit larger Readys plus + // it simplifies testing (by emitting less frequently and more + // predictably). + rd = n.rn.readyWithoutAccept() + readyc = n.readyc + } + + if lead != r.lead { + if r.hasLeader() { + if lead == None { + r.logger.Infof("raft.node: %x elected leader %x at term %d", r.id, r.lead, r.Term) + } else { + r.logger.Infof("raft.node: %x changed leader from %x to %x at term %d", r.id, lead, r.lead, r.Term) + } + propc = n.propc + } else { + r.logger.Infof("raft.node: %x lost leader %x at term %d", r.id, lead, r.Term) + propc = nil + } + lead = r.lead + } + + select { + // TODO: maybe buffer the config propose if there exists one (the way + // described in raft dissertation) + // Currently it is dropped in Step silently. + case pm := <-propc: + m := pm.m + m.From = new(r.id) + err := r.Step(m) + if pm.result != nil { + pm.result <- err + close(pm.result) + } + case m := <-n.recvc: + if IsResponseMsg(m.GetType()) && !IsLocalMsgTarget(m.GetFrom()) && r.trk.Progress[m.GetFrom()] == nil { + // Filter out response message from unknown From. + break + } + r.Step(m) + case cc := <-n.confc: + _, okBefore := r.trk.Progress[r.id] + cs := r.applyConfChange(cc) + // If the node was removed, block incoming proposals. Note that we + // only do this if the node was in the config before. Nodes may be + // a member of the group without knowing this (when they're catching + // up on the log and don't have the latest config) and we don't want + // to block the proposal channel in that case. + // + // NB: propc is reset when the leader changes, which, if we learn + // about it, sort of implies that we got readded, maybe? This isn't + // very sound and likely has bugs. + if _, okAfter := r.trk.Progress[r.id]; okBefore && !okAfter { + var found bool + for _, sl := range [][]uint64{cs.Voters, cs.VotersOutgoing} { + for _, id := range sl { + if id == r.id { + found = true + break + } + } + if found { + break + } + } + if !found { + propc = nil + } + } + select { + case n.confstatec <- cs: + case <-n.done: + } + case <-n.tickc: + n.rn.Tick() + case readyc <- rd: + n.rn.acceptReady(rd) + if !n.rn.asyncStorageWrites { + advancec = n.advancec + } else { + rd = Ready{} + } + readyc = nil + case <-advancec: + n.rn.Advance(rd) + rd = Ready{} + advancec = nil + case c := <-n.status: + c <- getStatus(r) + case <-n.stop: + close(n.done) + return + } + } +} + +// Tick increments the internal logical clock for this Node. Election timeouts +// and heartbeat timeouts are in units of ticks. +func (n *node) Tick() { + select { + case n.tickc <- struct{}{}: + case <-n.done: + default: + n.rn.raft.logger.Warningf("%x A tick missed to fire. Node blocks too long!", n.rn.raft.id) + } +} + +func (n *node) Campaign(ctx context.Context) error { + return n.step(ctx, &pb.Message{Type: pb.MsgHup.Enum()}) +} + +func (n *node) Propose(ctx context.Context, data []byte) error { + return n.stepWait(ctx, &pb.Message{Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: data}}}) +} + +func (n *node) Step(ctx context.Context, m *pb.Message) error { + // m should never be nil + if m == nil { + return errors.New("cannot step with nil message") + } + // Ignore unexpected local messages receiving over network. + if IsLocalMsg(m.GetType()) && !IsLocalMsgTarget(m.GetFrom()) { + // TODO: return an error? + return nil + } + return n.step(ctx, m) +} + +func confChangeToMsg(c pb.ConfChangeI) (*pb.Message, error) { + typ, data, err := pb.MarshalConfChange(c) + if err != nil { + return nil, err + } + return &pb.Message{Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Type: typ.Enum(), Data: data}}}, nil +} + +func (n *node) ProposeConfChange(ctx context.Context, cc pb.ConfChangeI) error { + msg, err := confChangeToMsg(cc) + if err != nil { + return err + } + return n.Step(ctx, msg) +} + +func (n *node) step(ctx context.Context, m *pb.Message) error { + return n.stepWithWaitOption(ctx, m, false) +} + +func (n *node) stepWait(ctx context.Context, m *pb.Message) error { + return n.stepWithWaitOption(ctx, m, true) +} + +// Step advances the state machine using msgs. The ctx.Err() will be returned, +// if any. +func (n *node) stepWithWaitOption(ctx context.Context, m *pb.Message, wait bool) error { + if m.GetType() != pb.MsgProp { + select { + case n.recvc <- m: + return nil + case <-ctx.Done(): + return ctx.Err() + case <-n.done: + return ErrStopped + } + } + ch := n.propc + pm := msgWithResult{m: m} + if wait { + pm.result = make(chan error, 1) + } + select { + case ch <- pm: + if !wait { + return nil + } + case <-ctx.Done(): + return ctx.Err() + case <-n.done: + return ErrStopped + } + select { + case err := <-pm.result: + if err != nil { + return err + } + case <-ctx.Done(): + return ctx.Err() + case <-n.done: + return ErrStopped + } + return nil +} + +func (n *node) Ready() <-chan Ready { return n.readyc } + +func (n *node) Advance() { + select { + case n.advancec <- struct{}{}: + case <-n.done: + } +} + +func (n *node) ApplyConfChange(cc pb.ConfChangeI) *pb.ConfState { + var cs *pb.ConfState + select { + case n.confc <- cc.AsV2(): + case <-n.done: + } + select { + case cs = <-n.confstatec: + case <-n.done: + } + return cs +} + +func (n *node) Status() Status { + c := make(chan Status) + select { + case n.status <- c: + return <-c + case <-n.done: + return Status{} + } +} + +func (n *node) ReportUnreachable(id uint64) { + select { + case n.recvc <- &pb.Message{Type: pb.MsgUnreachable.Enum(), From: new(id)}: + case <-n.done: + } +} + +func (n *node) ReportSnapshot(id uint64, status SnapshotStatus) { + rej := status == SnapshotFailure + + select { + case n.recvc <- &pb.Message{Type: pb.MsgSnapStatus.Enum(), From: new(id), Reject: new(rej)}: + case <-n.done: + } +} + +func (n *node) TransferLeadership(ctx context.Context, lead, transferee uint64) { + select { + // manually set 'from' and 'to', so that leader can voluntarily transfers its leadership + case n.recvc <- &pb.Message{Type: pb.MsgTransferLeader.Enum(), From: new(transferee), To: new(lead)}: + case <-n.done: + case <-ctx.Done(): + } +} + +func (n *node) ForgetLeader(ctx context.Context) error { + return n.step(ctx, &pb.Message{Type: pb.MsgForgetLeader.Enum()}) +} + +func (n *node) ReadIndex(ctx context.Context, rctx []byte) error { + return n.step(ctx, &pb.Message{Type: pb.MsgReadIndex.Enum(), Entries: []*pb.Entry{{Data: rctx}}}) +} diff --git a/third_party/raft/node_bench_test.go b/third_party/raft/node_bench_test.go new file mode 100644 index 0000000000..b3385283af --- /dev/null +++ b/third_party/raft/node_bench_test.go @@ -0,0 +1,51 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "context" + "testing" + "time" +) + +func BenchmarkOneNode(b *testing.B) { + ctx, cancel := context.WithCancel(b.Context()) + defer cancel() + + s := newTestMemoryStorage(withPeers(1)) + rn := newTestRawNode(1, 10, 1, s) + n := newNode(rn) + go n.run() + + defer n.Stop() + + n.Campaign(ctx) + go func() { + for i := 0; i < b.N; i++ { + n.Propose(ctx, []byte("foo")) + } + }() + + for { + rd := <-n.Ready() + s.Append(rd.Entries) + // a reasonable disk sync latency + time.Sleep(1 * time.Millisecond) + n.Advance() + if rd.HardState.GetCommit() == uint64(b.N+1) { + return + } + } +} diff --git a/third_party/raft/node_test.go b/third_party/raft/node_test.go new file mode 100644 index 0000000000..a1cd0a5e51 --- /dev/null +++ b/third_party/raft/node_test.go @@ -0,0 +1,1061 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "context" + "fmt" + "math" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "go.etcd.io/raft/v3/raftpb" +) + +// readyWithTimeout selects from n.Ready() with a 1-second timeout. It +// panics on timeout, which is better than the indefinite wait that +// would occur if this channel were read without being wrapped in a +// select. +func readyWithTimeout(n Node) Ready { + select { + case rd := <-n.Ready(): + if nn, ok := n.(*nodeTestHarness); ok { + n = nn.node + } + if nn, ok := n.(*node); ok { + nn.rn.raft.logger.Infof("emitted ready: %s", DescribeReady(rd, nil)) + } + return rd + case <-time.After(time.Second): + panic("timed out waiting for ready") + } +} + +// TestNodeStep ensures that node.Step sends msgProp to propc chan +// and other kinds of messages to recvc chan. +func TestNodeStep(t *testing.T) { + for i, msgn := range raftpb.MessageType_name { + n := &node{ + propc: make(chan msgWithResult, 1), + recvc: make(chan *raftpb.Message, 1), + } + msgt := raftpb.MessageType(i) + n.Step(t.Context(), &raftpb.Message{Type: msgt.Enum()}) + // Proposal goes to proc chan. Others go to recvc chan. + if msgt == raftpb.MsgProp { + select { + case <-n.propc: + default: + t.Errorf("%d: cannot receive %s on propc chan", msgt, msgn) + } + } else { + if IsLocalMsg(msgt) { + select { + case <-n.recvc: + t.Errorf("%d: step should ignore %s", msgt, msgn) + default: + } + } else { + select { + case <-n.recvc: + default: + t.Errorf("%d: cannot receive %s on recvc chan", msgt, msgn) + } + } + } + } +} + +// TestNodeStepUnblock should Cancel and Stop should unblock Step() +func TestNodeStepUnblock(t *testing.T) { + // a node without buffer to block step + n := &node{ + propc: make(chan msgWithResult), + done: make(chan struct{}), + } + + ctx, cancel := context.WithCancel(t.Context()) + stopFunc := func() { close(n.done) } + + tests := []struct { + unblock func() + werr error + }{ + {stopFunc, ErrStopped}, + {cancel, context.Canceled}, + } + + for i, tt := range tests { + errc := make(chan error, 1) + go func() { + err := n.Step(ctx, &raftpb.Message{Type: raftpb.MsgProp.Enum()}) + errc <- err + }() + tt.unblock() + select { + case err := <-errc: + assert.Equal(t, tt.werr, err, "#%d", i) + // clean up side-effect + if ctx.Err() != nil { + ctx = t.Context() + } + select { + case <-n.done: + n.done = make(chan struct{}) + default: + } + case <-time.After(1 * time.Second): + t.Fatalf("#%d: failed to unblock step", i) + } + } +} + +// TestNodePropose ensures that node.Propose sends the given proposal to the underlying raft. +func TestNodePropose(t *testing.T) { + var msgs []*raftpb.Message + appendStep := func(_ *raft, m *raftpb.Message) error { + t.Log(DescribeMessage(m, nil)) + if m.GetType() == raftpb.MsgAppResp { + return nil // injected by (*raft).advance + } + msgs = append(msgs, m) + return nil + } + + s := newTestMemoryStorage(withPeers(1)) + rn := newTestRawNode(1, 10, 1, s) + n := newNode(rn) + r := rn.raft + go n.run() + require.NoError(t, n.Campaign(t.Context())) + for { + rd := <-n.Ready() + s.Append(rd.Entries) + // change the step function to appendStep until this raft becomes leader + if rd.SoftState.Lead == r.id { + r.step = appendStep + n.Advance() + break + } + n.Advance() + } + n.Propose(t.Context(), []byte("somedata")) + n.Stop() + + require.Len(t, msgs, 1) + assert.Equal(t, raftpb.MsgProp, msgs[0].GetType()) + assert.Equal(t, []byte("somedata"), msgs[0].GetEntries()[0].GetData()) +} + +// TestDisableProposalForwarding ensures that proposals are not forwarded to +// the leader when DisableProposalForwarding is true. +func TestDisableProposalForwarding(t *testing.T) { + r1 := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + r2 := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + cfg3 := newTestConfig(3, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + cfg3.DisableProposalForwarding = true + r3 := newRaft(cfg3) + nt := newNetwork(r1, r2, r3) + + // elect r1 as leader + nt.send(&raftpb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: raftpb.MsgHup.Enum()}) + + var testEntries = []*raftpb.Entry{{Data: []byte("testdata")}} + + // send proposal to r2(follower) where DisableProposalForwarding is false + r2.Step(&raftpb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: raftpb.MsgProp.Enum(), Entries: testEntries}) + + // verify r2(follower) does forward the proposal when DisableProposalForwarding is false + require.Len(t, r2.msgs, 1) + + // send proposal to r3(follower) where DisableProposalForwarding is true + r3.Step(&raftpb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: raftpb.MsgProp.Enum(), Entries: testEntries}) + + // verify r3(follower) does not forward the proposal when DisableProposalForwarding is true + require.Empty(t, r3.msgs) +} + +// TestNodeReadIndexToOldLeader ensures that raftpb.MsgReadIndex to old leader +// gets forwarded to the new leader and 'send' method does not attach its term. +func TestNodeReadIndexToOldLeader(t *testing.T) { + r1 := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + r2 := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + r3 := newTestRaft(3, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + + nt := newNetwork(r1, r2, r3) + + // elect r1 as leader + nt.send(&raftpb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: raftpb.MsgHup.Enum()}) + + var testEntries = []*raftpb.Entry{{Data: []byte("testdata")}} + + // send readindex request to r2(follower) + r2.Step(&raftpb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: raftpb.MsgReadIndex.Enum(), Entries: testEntries}) + + // verify r2(follower) forwards this message to r1(leader) with term not set + require.Len(t, r2.msgs, 1) + readIndxMsg1 := &raftpb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: raftpb.MsgReadIndex.Enum(), Entries: testEntries} + require.Equal(t, readIndxMsg1, r2.msgs[0]) + + // send readindex request to r3(follower) + r3.Step(&raftpb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: raftpb.MsgReadIndex.Enum(), Entries: testEntries}) + + // verify r3(follower) forwards this message to r1(leader) with term not set as well. + require.Len(t, r3.msgs, 1) + readIndxMsg2 := &raftpb.Message{From: new(uint64(3)), To: new(uint64(1)), Type: raftpb.MsgReadIndex.Enum(), Entries: testEntries} + require.Equal(t, readIndxMsg2, r3.msgs[0]) + + // now elect r3 as leader + nt.send(&raftpb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: raftpb.MsgHup.Enum()}) + + // let r1 steps the two messages previously we got from r2, r3 + r1.Step(readIndxMsg1) + r1.Step(readIndxMsg2) + + // verify r1(follower) forwards these messages again to r3(new leader) + require.Len(t, r1.msgs, 2) + readIndxMsg3 := &raftpb.Message{From: new(uint64(2)), To: new(uint64(3)), Type: raftpb.MsgReadIndex.Enum(), Entries: testEntries} + require.Equal(t, readIndxMsg3, r1.msgs[0]) + readIndxMsg3 = &raftpb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: raftpb.MsgReadIndex.Enum(), Entries: testEntries} + require.Equal(t, readIndxMsg3, r1.msgs[1]) +} + +// TestNodeProposeConfig ensures that node.ProposeConfChange sends the given configuration proposal +// to the underlying raft. +func TestNodeProposeConfig(t *testing.T) { + var msgs []*raftpb.Message + appendStep := func(_ *raft, m *raftpb.Message) error { + if m.GetType() == raftpb.MsgAppResp { + return nil // injected by (*raft).advance + } + msgs = append(msgs, m) + return nil + } + + s := newTestMemoryStorage(withPeers(1)) + rn := newTestRawNode(1, 10, 1, s) + n := newNode(rn) + r := rn.raft + go n.run() + n.Campaign(t.Context()) + for { + rd := <-n.Ready() + s.Append(rd.Entries) + // change the step function to appendStep until this raft becomes leader + if rd.SoftState.Lead == r.id { + r.step = appendStep + n.Advance() + break + } + n.Advance() + } + cc := &raftpb.ConfChange{Type: raftpb.ConfChangeAddNode.Enum(), NodeId: new(uint64(1))} + ccdata, err := proto.Marshal(cc) + require.NoError(t, err) + n.ProposeConfChange(t.Context(), cc) + n.Stop() + + require.Len(t, msgs, 1) + assert.Equal(t, raftpb.MsgProp, msgs[0].GetType()) + assert.Equal(t, ccdata, msgs[0].GetEntries()[0].GetData()) +} + +// TestNodeProposeAddDuplicateNode ensures that two proposes to add the same node should +// not affect the later propose to add new node. +func TestNodeProposeAddDuplicateNode(t *testing.T) { + s := newTestMemoryStorage(withPeers(1)) + cfg := newTestConfig(1, 10, 1, s) + ctx, cancel, n := newNodeTestHarness(t.Context(), t, cfg) + defer cancel() + n.Campaign(ctx) + allCommittedEntries := make([]*raftpb.Entry, 0) + ticker := time.NewTicker(time.Millisecond * 100) + defer ticker.Stop() + goroutineStopped := make(chan struct{}) + applyConfChan := make(chan struct{}) + + rd := readyWithTimeout(n) + s.Append(rd.Entries) + n.Advance() + + go func() { + defer close(goroutineStopped) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + n.Tick() + case rd := <-n.Ready(): + t.Log(DescribeReady(rd, nil)) + s.Append(rd.Entries) + applied := false + for _, e := range rd.CommittedEntries { + allCommittedEntries = append(allCommittedEntries, e) + switch e.GetType() { + case raftpb.EntryNormal: + case raftpb.EntryConfChange: + cc := &raftpb.ConfChange{} + proto.Unmarshal(e.GetData(), cc) + n.ApplyConfChange(cc) + applied = true + } + } + n.Advance() + if applied { + applyConfChan <- struct{}{} + } + } + } + }() + + cc1 := &raftpb.ConfChange{Type: raftpb.ConfChangeAddNode.Enum(), NodeId: new(uint64(1))} + ccdata1, _ := proto.Marshal(cc1) + n.ProposeConfChange(ctx, cc1) + <-applyConfChan + + // try add the same node again + n.ProposeConfChange(ctx, cc1) + <-applyConfChan + + // the new node join should be ok + cc2 := &raftpb.ConfChange{Type: raftpb.ConfChangeAddNode.Enum(), NodeId: new(uint64(2))} + ccdata2, _ := proto.Marshal(cc2) + n.ProposeConfChange(ctx, cc2) + <-applyConfChan + + cancel() + <-goroutineStopped + + assert.Len(t, allCommittedEntries, 4) + assert.Equal(t, ccdata1, allCommittedEntries[1].GetData()) + assert.Equal(t, ccdata2, allCommittedEntries[3].GetData()) +} + +// TestBlockProposal ensures that node will block proposal when it does not +// know who is the current leader; node will accept proposal when it knows +// who is the current leader. +func TestBlockProposal(t *testing.T) { + s := newTestMemoryStorage(withPeers(1)) + rn := newTestRawNode(1, 10, 1, s) + n := newNode(rn) + go n.run() + defer n.Stop() + + errc := make(chan error, 1) + go func() { + errc <- n.Propose(t.Context(), []byte("somedata")) + }() + + time.Sleep(10 * time.Millisecond) + + select { + case err := <-errc: + t.Errorf("err = %v, want blocking", err) + default: + } + + n.Campaign(t.Context()) + rd := <-n.Ready() + s.Append(rd.Entries) + n.Advance() + select { + case err := <-errc: + assert.NoError(t, err) + case <-time.After(10 * time.Second): + t.Errorf("blocking proposal, want unblocking") + } +} + +func TestNodeProposeWaitDropped(t *testing.T) { + var msgs []*raftpb.Message + droppingMsg := []byte("test_dropping") + dropStep := func(_ *raft, m *raftpb.Message) error { + if m.GetType() == raftpb.MsgProp && strings.Contains(m.String(), string(droppingMsg)) { + t.Logf("dropping message: %v", m.String()) + return ErrProposalDropped + } + if m.GetType() == raftpb.MsgAppResp { + // This is produced by raft internally, see (*raft).advance. + return nil + } + msgs = append(msgs, m) + return nil + } + + s := newTestMemoryStorage(withPeers(1)) + rn := newTestRawNode(1, 10, 1, s) + n := newNode(rn) + r := rn.raft + go n.run() + n.Campaign(t.Context()) + for { + rd := <-n.Ready() + s.Append(rd.Entries) + // change the step function to dropStep until this raft becomes leader + if rd.SoftState.Lead == r.id { + r.step = dropStep + n.Advance() + break + } + n.Advance() + } + proposalTimeout := time.Millisecond * 100 + ctx, cancel := context.WithTimeout(t.Context(), proposalTimeout) + // propose with cancel should be cancelled earyly if dropped + assert.Equal(t, ErrProposalDropped, n.Propose(ctx, droppingMsg)) + cancel() + + n.Stop() + require.Empty(t, msgs) +} + +// TestNodeTick ensures that node.Tick() will increase the +// elapsed of the underlying raft state machine. +func TestNodeTick(t *testing.T) { + s := newTestMemoryStorage(withPeers(1)) + rn := newTestRawNode(1, 10, 1, s) + n := newNode(rn) + r := rn.raft + go n.run() + elapsed := r.electionElapsed + n.Tick() + + for len(n.tickc) != 0 { + time.Sleep(100 * time.Millisecond) + } + + n.Stop() + assert.Equal(t, elapsed+1, r.electionElapsed) +} + +// TestNodeStop ensures that node.Stop() blocks until the node has stopped +// processing, and that it is idempotent +func TestNodeStop(t *testing.T) { + rn := newTestRawNode(1, 10, 1, newTestMemoryStorage(withPeers(1))) + n := newNode(rn) + donec := make(chan struct{}) + + go func() { + n.run() + close(donec) + }() + + status := n.Status() + n.Stop() + + select { + case <-donec: + case <-time.After(time.Second): + t.Fatalf("timed out waiting for node to stop!") + } + + emptyStatus := Status{} + assert.NotEqual(t, emptyStatus, status) + + // Further status should return be empty, the node is stopped. + assert.Equal(t, emptyStatus, n.Status()) + + // Subsequent Stops should have no effect. + n.Stop() +} + +// TestNodeStart ensures that a node can be started correctly. The node should +// start with correct configuration change entries, and can accept and commit +// proposals. +func TestNodeStart(t *testing.T) { + cc := &raftpb.ConfChange{Type: raftpb.ConfChangeAddNode.Enum(), NodeId: new(uint64(1))} + ccdata, err := proto.Marshal(cc) + require.NoError(t, err) + wants := []Ready{ + { + HardState: &raftpb.HardState{Term: new(uint64(1)), Commit: new(uint64(1)), Vote: new(uint64(0))}, + Entries: []*raftpb.Entry{ + {Type: raftpb.EntryConfChange.Enum(), Term: new(uint64(1)), Index: new(uint64(1)), Data: ccdata}, + }, + CommittedEntries: []*raftpb.Entry{ + {Type: raftpb.EntryConfChange.Enum(), Term: new(uint64(1)), Index: new(uint64(1)), Data: ccdata}, + }, + MustSync: true, + }, + { + HardState: &raftpb.HardState{Term: new(uint64(2)), Commit: new(uint64(2)), Vote: new(uint64(1))}, + Entries: []*raftpb.Entry{{Term: new(uint64(2)), Index: new(uint64(3)), Data: []byte("foo")}}, + CommittedEntries: []*raftpb.Entry{{Term: new(uint64(2)), Index: new(uint64(2)), Data: nil}}, + MustSync: true, + }, + { + HardState: &raftpb.HardState{Term: new(uint64(2)), Commit: new(uint64(3)), Vote: new(uint64(1))}, + Entries: nil, + CommittedEntries: []*raftpb.Entry{{Term: new(uint64(2)), Index: new(uint64(3)), Data: []byte("foo")}}, + MustSync: false, + }, + } + storage := NewMemoryStorage() + c := &Config{ + ID: 1, + ElectionTick: 10, + HeartbeatTick: 1, + Storage: storage, + MaxSizePerMsg: noLimit, + MaxInflightMsgs: 256, + } + StartNode(c, []Peer{{ID: 1}}) + ctx, cancel, n := newNodeTestHarness(t.Context(), t, c, Peer{ID: 1}) + defer cancel() + + { + rd := <-n.Ready() + requireEqualReady(t, wants[0], rd) + storage.Append(rd.Entries) + n.Advance() + } + + require.NoError(t, n.Campaign(ctx)) + + { + // Persist vote. + rd := <-n.Ready() + storage.Append(rd.Entries) + n.Advance() + // Append empty entry. + rd = <-n.Ready() + storage.Append(rd.Entries) + n.Advance() + } + + n.Propose(ctx, []byte("foo")) + { + rd := <-n.Ready() + requireEqualReady(t, wants[1], rd) + storage.Append(rd.Entries) + n.Advance() + } + + { + rd := <-n.Ready() + requireEqualReady(t, wants[2], rd) + storage.Append(rd.Entries) + n.Advance() + } + + select { + case rd := <-n.Ready(): + t.Errorf("unexpected Ready: %+v", rd) + case <-time.After(time.Millisecond): + } +} + +func TestNodeRestart(t *testing.T) { + entries := []*raftpb.Entry{ + {Term: new(uint64(1)), Index: new(uint64(1))}, + {Term: new(uint64(1)), Index: new(uint64(2)), Data: []byte("foo")}, + } + st := &raftpb.HardState{Term: new(uint64(1)), Commit: new(uint64(1))} + + want := Ready{ + // No HardState is emitted because there was no change. + HardState: nil, + // commit up to index commit index in st + CommittedEntries: entries[:st.GetCommit()], + // MustSync is false because no HardState or new entries are provided. + MustSync: false, + } + + storage := NewMemoryStorage() + storage.SetHardState(st) + storage.Append(entries) + c := &Config{ + ID: 1, + ElectionTick: 10, + HeartbeatTick: 1, + Storage: storage, + MaxSizePerMsg: noLimit, + MaxInflightMsgs: 256, + } + n := RestartNode(c) + defer n.Stop() + assert.Equal(t, want, <-n.Ready()) + n.Advance() + + select { + case rd := <-n.Ready(): + t.Errorf("unexpected Ready: %+v", rd) + case <-time.After(time.Millisecond): + } +} + +func TestNodeRestartFromSnapshot(t *testing.T) { + snap := &raftpb.Snapshot{ + Metadata: &raftpb.SnapshotMetadata{ + ConfState: &raftpb.ConfState{Voters: []uint64{1, 2}}, + Index: new(uint64(2)), + Term: new(uint64(1)), + }, + } + entries := []*raftpb.Entry{ + {Term: new(uint64(1)), Index: new(uint64(3)), Data: []byte("foo")}, + } + st := &raftpb.HardState{Term: new(uint64(1)), Commit: new(uint64(3))} + + want := Ready{ + // No HardState is emitted because nothing changed relative to what is + // already persisted. + HardState: nil, + // commit up to index commit index in st + CommittedEntries: entries, + // MustSync is only true when there is a new HardState or new entries; + // neither is the case here. + MustSync: false, + } + + s := NewMemoryStorage() + s.SetHardState(st) + s.ApplySnapshot(snap) + s.Append(entries) + c := &Config{ + ID: 1, + ElectionTick: 10, + HeartbeatTick: 1, + Storage: s, + MaxSizePerMsg: noLimit, + MaxInflightMsgs: 256, + } + n := RestartNode(c) + defer n.Stop() + if assert.Equal(t, want, <-n.Ready()) { + n.Advance() + } + + select { + case rd := <-n.Ready(): + t.Errorf("unexpected Ready: %+v", rd) + case <-time.After(time.Millisecond): + } +} + +func TestNodeAdvance(t *testing.T) { + storage := newTestMemoryStorage(withPeers(1)) + c := &Config{ + ID: 1, + ElectionTick: 10, + HeartbeatTick: 1, + Storage: storage, + MaxSizePerMsg: noLimit, + MaxInflightMsgs: 256, + } + ctx, cancel, n := newNodeTestHarness(t.Context(), t, c) + defer cancel() + + n.Campaign(ctx) + // Persist vote. + rd := readyWithTimeout(n) + storage.Append(rd.Entries) + n.Advance() + // Append empty entry. + rd = readyWithTimeout(n) + storage.Append(rd.Entries) + n.Advance() + + n.Propose(ctx, []byte("foo")) + rd = readyWithTimeout(n) + storage.Append(rd.Entries) + n.Advance() + select { + case <-n.Ready(): + case <-time.After(100 * time.Millisecond): + t.Errorf("expect Ready after Advance, but there is no Ready available") + } +} + +func TestSoftStateEqual(t *testing.T) { + tests := []struct { + st *SoftState + we bool + }{ + {&SoftState{}, true}, + {&SoftState{Lead: 1}, false}, + {&SoftState{RaftState: StateLeader}, false}, + } + for i, tt := range tests { + assert.Equal(t, tt.we, tt.st.equal(&SoftState{}), "#%d", i) + } +} + +func TestIsHardStateEqual(t *testing.T) { + tests := []struct { + ht *raftpb.HardState + we bool + }{ + {nil, true}, + {&raftpb.HardState{Vote: new(uint64(1))}, false}, + {&raftpb.HardState{Commit: new(uint64(1))}, false}, + {&raftpb.HardState{Term: new(uint64(1))}, false}, + } + + for i, tt := range tests { + assert.Equal(t, tt.we, isHardStateEqual(tt.ht, nil), "#%d", i) + } +} + +func TestNodeProposeAddLearnerNode(t *testing.T) { + ticker := time.NewTicker(time.Millisecond * 100) + defer ticker.Stop() + s := newTestMemoryStorage(withPeers(1)) + rn := newTestRawNode(1, 10, 1, s) + n := newNode(rn) + go n.run() + n.Campaign(t.Context()) + stop := make(chan struct{}) + done := make(chan struct{}) + applyConfChan := make(chan struct{}) + go func() { + defer close(done) + for { + select { + case <-stop: + return + case <-ticker.C: + n.Tick() + case rd := <-n.Ready(): + s.Append(rd.Entries) + t.Logf("raft: %v", rd.Entries) + for _, ent := range rd.Entries { + if ent.GetType() != raftpb.EntryConfChange { + continue + } + cc := &raftpb.ConfChange{} + proto.Unmarshal(ent.GetData(), cc) + state := n.ApplyConfChange(cc) + assert.True(t, len(state.Learners) > 0 && state.Learners[0] == cc.GetNodeId() && cc.GetNodeId() == 2, + "apply conf change should return new added learner: %v", state.String()) + assert.Len(t, state.Voters, 1, + "add learner should not change the nodes: %v", state.String()) + + t.Logf("apply raft conf %v changed to: %v", cc, state.String()) + applyConfChan <- struct{}{} + } + n.Advance() + } + } + }() + cc := &raftpb.ConfChange{Type: raftpb.ConfChangeAddLearnerNode.Enum(), NodeId: new(uint64(2))} + n.ProposeConfChange(t.Context(), cc) + <-applyConfChan + close(stop) + <-done + n.Stop() +} + +func TestAppendPagination(t *testing.T) { + const maxSizePerMsg = 2048 + n := newNetworkWithConfig(func(c *Config) { + c.MaxSizePerMsg = maxSizePerMsg + }, nil, nil, nil) + + seenFullMessage := false + // Inspect all messages to see that we never exceed the limit, but + // we do see messages of larger than half the limit. + n.msgHook = func(m *raftpb.Message) bool { + if m.GetType() == raftpb.MsgApp { + size := 0 + for _, e := range m.GetEntries() { + size += len(e.GetData()) + } + assert.LessOrEqual(t, size, maxSizePerMsg, "sent MsgApp that is too large") + if size > maxSizePerMsg/2 { + seenFullMessage = true + } + } + return true + } + + n.send(&raftpb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: raftpb.MsgHup.Enum()}) + + // Partition the network while we make our proposals. This forces + // the entries to be batched into larger messages. + n.isolate(1) + blob := []byte(strings.Repeat("a", 1000)) + for i := 0; i < 5; i++ { + n.send(&raftpb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: raftpb.MsgProp.Enum(), Entries: []*raftpb.Entry{{Data: blob}}}) + } + n.recover() + + // After the partition recovers, tick the clock to wake everything + // back up and send the messages. + n.send(&raftpb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: raftpb.MsgBeat.Enum()}) + assert.True(t, seenFullMessage, "didn't see any messages more than half the max size; something is wrong with this test") +} + +func TestCommitPagination(t *testing.T) { + s := newTestMemoryStorage(withPeers(1)) + cfg := newTestConfig(1, 10, 1, s) + cfg.MaxCommittedSizePerReady = 2048 + ctx, cancel, n := newNodeTestHarness(t.Context(), t, cfg) + defer cancel() + n.Campaign(ctx) + + // Persist vote. + rd := readyWithTimeout(n) + s.Append(rd.Entries) + n.Advance() + // Append empty entry. + rd = readyWithTimeout(n) + s.Append(rd.Entries) + n.Advance() + // Apply empty entry. + rd = readyWithTimeout(n) + require.Len(t, rd.CommittedEntries, 1) + + s.Append(rd.Entries) + n.Advance() + + blob := []byte(strings.Repeat("a", 1000)) + for i := 0; i < 3; i++ { + require.NoError(t, n.Propose(ctx, blob), "#%d", i) + } + + // First the three proposals have to be appended. + rd = readyWithTimeout(n) + require.Len(t, rd.Entries, 3) + + s.Append(rd.Entries) + n.Advance() + + // The 3 proposals will commit in two batches. + rd = readyWithTimeout(n) + require.Len(t, rd.CommittedEntries, 2) + + s.Append(rd.Entries) + n.Advance() + rd = readyWithTimeout(n) + require.Len(t, rd.CommittedEntries, 1) + + s.Append(rd.Entries) + n.Advance() +} + +func TestCommitPaginationWithAsyncStorageWrites(t *testing.T) { + s := newTestMemoryStorage(withPeers(1)) + cfg := newTestConfig(1, 10, 1, s) + cfg.MaxCommittedSizePerReady = 2048 + cfg.AsyncStorageWrites = true + ctx, cancel, n := newNodeTestHarness(t.Context(), t, cfg) + defer cancel() + n.Campaign(ctx) + + // Persist vote. + rd := readyWithTimeout(n) + require.Len(t, rd.Messages, 1) + m := rd.Messages[0] + require.Equal(t, raftpb.MsgStorageAppend, m.GetType()) + require.NoError(t, s.Append(m.GetEntries())) + for _, resp := range m.GetResponses() { + require.NoError(t, n.Step(ctx, resp)) + } + // Append empty entry. + rd = readyWithTimeout(n) + require.Len(t, rd.Messages, 1) + m = rd.Messages[0] + require.Equal(t, raftpb.MsgStorageAppend, m.GetType()) + require.NoError(t, s.Append(m.GetEntries())) + for _, resp := range m.GetResponses() { + require.NoError(t, n.Step(ctx, resp)) + } + // Apply empty entry. + rd = readyWithTimeout(n) + require.Len(t, rd.Messages, 2) + for _, m := range rd.Messages { + switch m.GetType() { + case raftpb.MsgStorageAppend: + require.NoError(t, s.Append(m.GetEntries())) + for _, resp := range m.GetResponses() { + require.NoError(t, n.Step(ctx, resp)) + } + case raftpb.MsgStorageApply: + require.Len(t, m.GetEntries(), 1) + require.Len(t, m.GetResponses(), 1) + require.NoError(t, n.Step(ctx, m.GetResponses()[0])) + default: + t.Fatalf("unexpected: %v", m) + } + } + + // Propose first entry. + blob := []byte(strings.Repeat("a", 1024)) + require.NoError(t, n.Propose(ctx, blob)) + + // Append first entry. + rd = readyWithTimeout(n) + require.Len(t, rd.Messages, 1) + m = rd.Messages[0] + require.Equal(t, raftpb.MsgStorageAppend, m.GetType()) + require.Len(t, m.GetEntries(), 1) + require.NoError(t, s.Append(m.GetEntries())) + for _, resp := range m.GetResponses() { + require.NoError(t, n.Step(ctx, resp)) + } + + // Propose second entry. + require.NoError(t, n.Propose(ctx, blob)) + + // Append second entry. Don't apply first entry yet. + rd = readyWithTimeout(n) + require.Len(t, rd.Messages, 2) + var applyResps []*raftpb.Message + for _, m := range rd.Messages { + switch m.GetType() { + case raftpb.MsgStorageAppend: + require.NoError(t, s.Append(m.GetEntries())) + for _, resp := range m.GetResponses() { + require.NoError(t, n.Step(ctx, resp)) + } + case raftpb.MsgStorageApply: + require.Len(t, m.GetEntries(), 1) + require.Len(t, m.GetResponses(), 1) + applyResps = append(applyResps, m.GetResponses()[0]) + default: + t.Fatalf("unexpected: %v", m) + } + } + + // Propose third entry. + require.NoError(t, n.Propose(ctx, blob)) + + // Append third entry. Don't apply second entry yet. + rd = readyWithTimeout(n) + require.Len(t, rd.Messages, 2) + for _, m := range rd.Messages { + switch m.GetType() { + case raftpb.MsgStorageAppend: + require.NoError(t, s.Append(m.GetEntries())) + for _, resp := range m.GetResponses() { + require.NoError(t, n.Step(ctx, resp)) + } + case raftpb.MsgStorageApply: + require.Len(t, m.GetEntries(), 1) + require.Len(t, m.GetResponses(), 1) + applyResps = append(applyResps, m.GetResponses()[0]) + default: + t.Fatalf("unexpected: %v", m) + } + } + + // Third entry should not be returned to be applied until first entry's + // application is acknowledged. + drain := true + for drain { + select { + case rd := <-n.Ready(): + for _, m := range rd.Messages { + require.NotEqual(t, raftpb.MsgStorageApply, m.GetType(), "unexpected message: %v", m) + } + case <-time.After(10 * time.Millisecond): + drain = false + } + } + + // Acknowledged first entry application. + require.NoError(t, n.Step(ctx, applyResps[0])) + applyResps = applyResps[1:] + + // Third entry now returned for application. + rd = readyWithTimeout(n) + require.Len(t, rd.Messages, 1) + m = rd.Messages[0] + require.Equal(t, raftpb.MsgStorageApply, m.GetType()) + require.Len(t, m.GetEntries(), 1) + applyResps = append(applyResps, m.GetResponses()[0]) + + // Acknowledged second and third entry application. + for _, resp := range applyResps { + require.NoError(t, n.Step(ctx, resp)) + } + applyResps = nil +} + +type ignoreSizeHintMemStorage struct { + *MemoryStorage +} + +func (s *ignoreSizeHintMemStorage) Entries(lo, hi uint64, _ uint64) ([]*raftpb.Entry, error) { + return s.MemoryStorage.Entries(lo, hi, math.MaxUint64) +} + +// TestNodeCommitPaginationAfterRestart regression tests a scenario in which the +// Storage's Entries size limitation is slightly more permissive than Raft's +// internal one. The original bug was the following: +// +// - node learns that index 11 (or 100, doesn't matter) is committed +// - nextCommittedEnts returns index 1..10 in CommittedEntries due to size limiting. +// However, index 10 already exceeds maxBytes, due to a user-provided impl of Entries. +// - Commit index gets bumped to 10 +// - the node persists the HardState, but crashes before applying the entries +// - upon restart, the storage returns the same entries, but `slice` takes a different code path +// (since it is now called with an upper bound of 10) and removes the last entry. +// - Raft emits a HardState with a regressing commit index. +// +// A simpler version of this test would have the storage return a lot less entries than dictated +// by maxSize (for example, exactly one entry) after the restart, resulting in a larger regression. +// This wouldn't need to exploit anything about Raft-internal code paths to fail. +func TestNodeCommitPaginationAfterRestart(t *testing.T) { + s := &ignoreSizeHintMemStorage{ + MemoryStorage: newTestMemoryStorage(withPeers(1)), + } + persistedHardState := &raftpb.HardState{ + Term: new(uint64(1)), + Vote: new(uint64(1)), + Commit: new(uint64(10)), + } + + s.hardState = persistedHardState + s.ents = make([]*raftpb.Entry, 10) + var size uint64 + for i := range s.ents { + ent := &raftpb.Entry{ + Term: new(uint64(1)), + Index: new(uint64(i + 1)), + Type: raftpb.EntryNormal.Enum(), + Data: []byte("a"), + } + + s.ents[i] = ent + size += uint64(proto.Size(ent)) + } + + cfg := newTestConfig(1, 10, 1, s) + // Set a MaxSizePerMsg that would suggest to Raft that the last committed entry should + // not be included in the initial rd.CommittedEntries. However, our storage will ignore + // this and *will* return it (which is how the Commit index ended up being 10 initially). + cfg.MaxSizePerMsg = size - uint64(proto.Size(s.ents[len(s.ents)-1])) - 1 + + rn, err := NewRawNode(cfg) + require.NoError(t, err) + + n := newNode(rn) + go n.run() + defer n.Stop() + + rd := readyWithTimeout(&n) + assert.False(t, !IsEmptyHardState(rd.HardState) && rd.HardState.GetCommit() < persistedHardState.GetCommit(), + "HardState regressed: Commit %d -> %d\nCommitting:\n%+v", + persistedHardState.GetCommit(), rd.HardState.GetCommit(), + DescribeEntries(rd.CommittedEntries, func(data []byte) string { return fmt.Sprintf("%q", data) })) +} diff --git a/third_party/raft/node_util_test.go b/third_party/raft/node_util_test.go new file mode 100644 index 0000000000..49a477f130 --- /dev/null +++ b/third_party/raft/node_util_test.go @@ -0,0 +1,110 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "context" + "fmt" + "testing" + "time" +) + +type nodeTestHarness struct { + *node + t *testing.T +} + +func (l *nodeTestHarness) Debug(v ...interface{}) { + l.t.Log(v...) +} + +func (l *nodeTestHarness) Debugf(format string, v ...interface{}) { + l.t.Logf(format, v...) +} + +func (l *nodeTestHarness) Error(v ...interface{}) { + l.t.Error(v...) +} + +func (l *nodeTestHarness) Errorf(format string, v ...interface{}) { + l.t.Errorf(format, v...) +} + +func (l *nodeTestHarness) Info(v ...interface{}) { + l.t.Log(v...) +} + +func (l *nodeTestHarness) Infof(format string, v ...interface{}) { + l.t.Logf(format, v...) +} + +func (l *nodeTestHarness) Warning(v ...interface{}) { + l.t.Log(v...) +} + +func (l *nodeTestHarness) Warningf(format string, v ...interface{}) { + l.t.Logf(format, v...) +} + +func (l *nodeTestHarness) Fatal(v ...interface{}) { + l.t.Error(v...) + panic(fmt.Sprint(v...)) +} + +func (l *nodeTestHarness) Fatalf(format string, v ...interface{}) { + l.t.Errorf(format, v...) + panic(fmt.Sprintf(format, v...)) +} + +func (l *nodeTestHarness) Panic(v ...interface{}) { + l.t.Log(v...) + panic(fmt.Sprint(v...)) +} + +func (l *nodeTestHarness) Panicf(format string, v ...interface{}) { + l.t.Errorf(format, v...) + panic(fmt.Sprintf(format, v...)) +} + +func newNodeTestHarness(ctx context.Context, t *testing.T, cfg *Config, peers ...Peer) (_ context.Context, cancel func(), _ *nodeTestHarness) { + // Wrap context in a 10s timeout to make tests more robust. Otherwise, + // it's likely that deadlock will occur unless Node behaves exactly as + // expected - when you expect a Ready and start waiting on the channel + // but no Ready ever shows up, for example. + ctx, cancel = context.WithTimeout(ctx, 10*time.Second) + var n *node + if len(peers) > 0 { + n = setupNode(cfg, peers) + } else { + rn, err := NewRawNode(cfg) + if err != nil { + t.Fatal(err) + } + nn := newNode(rn) + n = &nn + } + go func() { + defer func() { + if r := recover(); r != nil { + t.Error(r) + } + }() + defer cancel() + defer n.Stop() + n.run() + }() + t.Cleanup(n.Stop) + return ctx, cancel, &nodeTestHarness{node: n, t: t} +} diff --git a/third_party/raft/quorum/bench_test.go b/third_party/raft/quorum/bench_test.go new file mode 100644 index 0000000000..5c7961ed6c --- /dev/null +++ b/third_party/raft/quorum/bench_test.go @@ -0,0 +1,40 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quorum + +import ( + "fmt" + "math" + "math/rand" + "testing" +) + +func BenchmarkMajorityConfig_CommittedIndex(b *testing.B) { + // go test -run - -bench . -benchmem ./raft/quorum + for _, n := range []int{1, 3, 5, 7, 9, 11} { + b.Run(fmt.Sprintf("voters=%d", n), func(b *testing.B) { + c := MajorityConfig{} + l := mapAckIndexer{} + for i := uint64(0); i < uint64(n); i++ { + c[i+1] = struct{}{} + l[i+1] = Index(rand.Int63n(math.MaxInt64)) + } + + for i := 0; i < b.N; i++ { + _ = c.CommittedIndex(l) + } + }) + } +} diff --git a/third_party/raft/quorum/datadriven_test.go b/third_party/raft/quorum/datadriven_test.go new file mode 100644 index 0000000000..a66e2633f6 --- /dev/null +++ b/third_party/raft/quorum/datadriven_test.go @@ -0,0 +1,247 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quorum + +import ( + "fmt" + "strings" + "testing" + + "github.com/cockroachdb/datadriven" + "github.com/stretchr/testify/require" +) + +// TestDataDriven parses and executes the test cases in ./testdata/*. An entry +// in such a file specifies the command, which is either of "committed" to check +// CommittedIndex or "vote" to verify a VoteResult. The underlying configuration +// and inputs are specified via the arguments 'cfg' and 'cfgj' (for the majority +// config and, optionally, majority config joint to the first one) and 'idx' +// (for CommittedIndex) and 'votes' (for VoteResult). +// +// Internally, the harness runs some additional checks on each test case for +// which it is known that the result shouldn't change. For example, +// interchanging the majority configurations of a joint quorum must not +// influence the result; if it does, this is noted in the test's output. +func TestDataDriven(t *testing.T) { + datadriven.Walk(t, "testdata", func(t *testing.T, path string) { + datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string { + // Two majority configs. The first one is always used (though it may + // be empty) and the second one is used iff joint is true. + var joint bool + var ids, idsj []uint64 + // The committed indexes for the nodes in the config in the order in + // which they appear in (ids,idsj), without repetition. An underscore + // denotes an omission (i.e. no information for this voter); this is + // different from 0. For example, + // + // cfg=(1,2) cfgj=(2,3,4) idxs=(_,5,_,7) initializes the idx for voter 2 + // to 5 and that for voter 4 to 7 (and no others). + // + // cfgj=zero is specified to instruct the test harness to treat cfgj + // as zero instead of not specified (i.e. it will trigger a joint + // quorum test instead of a majority quorum test for cfg only). + var idxs []Index + // Votes. These are initialized similar to idxs except the only values + // used are 1 (voted against) and 2 (voted for). This looks awkward, + // but is convenient because it allows sharing code between the two. + var votes []Index + + // Parse the args. + for _, arg := range d.CmdArgs { + for i := range arg.Vals { + switch arg.Key { + case "cfg": + var n uint64 + arg.Scan(t, i, &n) + ids = append(ids, n) + case "cfgj": + joint = true + if arg.Vals[i] == "zero" { + require.Len(t, arg.Vals, 1, "cannot mix 'zero' into configuration") + } else { + var n uint64 + arg.Scan(t, i, &n) + idsj = append(idsj, n) + } + case "idx": + var n uint64 + // Register placeholders as zeroes. + if arg.Vals[i] != "_" { + arg.Scan(t, i, &n) + // This is a restriction caused by the above + // special-casing for _. + require.NotZero(t, n, "cannot use 0 as idx") + } + idxs = append(idxs, Index(n)) + case "votes": + var s string + arg.Scan(t, i, &s) + switch s { + case "y": + votes = append(votes, 2) + case "n": + votes = append(votes, 1) + case "_": + votes = append(votes, 0) + default: + t.Fatalf("unknown vote: %s", s) + } + default: + t.Fatalf("unknown arg %s", arg.Key) + } + } + } + + // Build the two majority configs. + c := MajorityConfig{} + for _, id := range ids { + c[id] = struct{}{} + } + cj := MajorityConfig{} + for _, id := range idsj { + cj[id] = struct{}{} + } + + // Helper that returns an AckedIndexer which has the specified indexes + // mapped to the right IDs. + makeLookuper := func(idxs []Index, ids, idsj []uint64) mapAckIndexer { + l := mapAckIndexer{} + var p int // next to consume from idxs + for _, id := range append(append([]uint64(nil), ids...), idsj...) { + if _, ok := l[id]; ok { + continue + } + if p < len(idxs) { + // NB: this creates zero entries for placeholders that we remove later. + // The upshot of doing it that way is to avoid having to specify place- + // holders multiple times when omitting voters present in both halves of + // a joint config. + l[id] = idxs[p] + p++ + } + } + + for id := range l { + // Zero entries are created by _ placeholders; we don't want + // them in the lookuper because "no entry" is different from + // "zero entry". Note that we prevent tests from specifying + // zero commit indexes, so that there's no confusion between + // the two concepts. + if l[id] == 0 { + delete(l, id) + } + } + return l + } + + { + input := idxs + if d.Cmd == "vote" { + input = votes + } + if voters := JointConfig([2]MajorityConfig{c, cj}).IDs(); len(voters) != len(input) { + return fmt.Sprintf("error: mismatched input (explicit or _) for voters %v: %v", + voters, input) + } + } + + var buf strings.Builder + switch d.Cmd { + case "committed": + l := makeLookuper(idxs, ids, idsj) + + // Branch based on whether this is a majority or joint quorum + // test case. + if !joint { + idx := c.CommittedIndex(l) + fmt.Fprint(&buf, c.Describe(l)) + // These alternative computations should return the same + // result. If not, print to the output. + if aIdx := alternativeMajorityCommittedIndex(c, l); aIdx != idx { + fmt.Fprintf(&buf, "%s <-- via alternative computation\n", aIdx) + } + // Joining a majority with the empty majority should give same result. + if aIdx := JointConfig([2]MajorityConfig{c, {}}).CommittedIndex(l); aIdx != idx { + fmt.Fprintf(&buf, "%s <-- via zero-joint quorum\n", aIdx) + } + // Joining a majority with itself should give same result. + if aIdx := JointConfig([2]MajorityConfig{c, c}).CommittedIndex(l); aIdx != idx { + fmt.Fprintf(&buf, "%s <-- via self-joint quorum\n", aIdx) + } + overlay := func(c MajorityConfig, l AckedIndexer, id uint64, idx Index) AckedIndexer { + ll := mapAckIndexer{} + for iid := range c { + if iid == id { + ll[iid] = idx + } else if idx, ok := l.AckedIndex(iid); ok { + ll[iid] = idx + } + } + return ll + } + for id := range c { + iidx, _ := l.AckedIndex(id) + if idx > iidx && iidx > 0 { + // If the committed index was definitely above the currently + // inspected idx, the result shouldn't change if we lower it + // further. + lo := overlay(c, l, id, iidx-1) + if aIdx := c.CommittedIndex(lo); aIdx != idx { + fmt.Fprintf(&buf, "%s <-- overlaying %d->%d", aIdx, id, iidx) + } + lo = overlay(c, l, id, 0) + if aIdx := c.CommittedIndex(lo); aIdx != idx { + fmt.Fprintf(&buf, "%s <-- overlaying %d->0", aIdx, id) + } + } + } + fmt.Fprintf(&buf, "%s\n", idx) + } else { + cc := JointConfig([2]MajorityConfig{c, cj}) + fmt.Fprint(&buf, cc.Describe(l)) + idx := cc.CommittedIndex(l) + // Interchanging the majorities shouldn't make a difference. If it does, print. + if aIdx := JointConfig([2]MajorityConfig{cj, c}).CommittedIndex(l); aIdx != idx { + fmt.Fprintf(&buf, "%s <-- via symmetry\n", aIdx) + } + fmt.Fprintf(&buf, "%s\n", idx) + } + case "vote": + ll := makeLookuper(votes, ids, idsj) + l := map[uint64]bool{} + for id, v := range ll { + l[id] = v != 1 // NB: 1 == false, 2 == true + } + + if !joint { + // Test a majority quorum. + r := c.VoteResult(l) + fmt.Fprintf(&buf, "%v\n", r) + } else { + // Run a joint quorum test case. + r := JointConfig([2]MajorityConfig{c, cj}).VoteResult(l) + // Interchanging the majorities shouldn't make a difference. If it does, print. + if ar := JointConfig([2]MajorityConfig{cj, c}).VoteResult(l); ar != r { + fmt.Fprintf(&buf, "%v <-- via symmetry\n", ar) + } + fmt.Fprintf(&buf, "%v\n", r) + } + default: + t.Fatalf("unknown command: %s", d.Cmd) + } + return buf.String() + }) + }) +} diff --git a/third_party/raft/quorum/joint.go b/third_party/raft/quorum/joint.go new file mode 100644 index 0000000000..e3741e0b0a --- /dev/null +++ b/third_party/raft/quorum/joint.go @@ -0,0 +1,75 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quorum + +// JointConfig is a configuration of two groups of (possibly overlapping) +// majority configurations. Decisions require the support of both majorities. +type JointConfig [2]MajorityConfig + +func (c JointConfig) String() string { + if len(c[1]) > 0 { + return c[0].String() + "&&" + c[1].String() + } + return c[0].String() +} + +// IDs returns a newly initialized map representing the set of voters present +// in the joint configuration. +func (c JointConfig) IDs() map[uint64]struct{} { + m := map[uint64]struct{}{} + for _, cc := range c { + for id := range cc { + m[id] = struct{}{} + } + } + return m +} + +// Describe returns a (multi-line) representation of the commit indexes for the +// given lookuper. +func (c JointConfig) Describe(l AckedIndexer) string { + return MajorityConfig(c.IDs()).Describe(l) +} + +// CommittedIndex returns the largest committed index for the given joint +// quorum. An index is jointly committed if it is committed in both constituent +// majorities. +func (c JointConfig) CommittedIndex(l AckedIndexer) Index { + idx0 := c[0].CommittedIndex(l) + idx1 := c[1].CommittedIndex(l) + if idx0 < idx1 { + return idx0 + } + return idx1 +} + +// VoteResult takes a mapping of voters to yes/no (true/false) votes and returns +// a result indicating whether the vote is pending, lost, or won. A joint quorum +// requires both majority quorums to vote in favor. +func (c JointConfig) VoteResult(votes map[uint64]bool) VoteResult { + r1 := c[0].VoteResult(votes) + r2 := c[1].VoteResult(votes) + + if r1 == r2 { + // If they agree, return the agreed state. + return r1 + } + if r1 == VoteLost || r2 == VoteLost { + // If either config has lost, loss is the only possible outcome. + return VoteLost + } + // One side won, the other one is pending, so the whole outcome is. + return VotePending +} diff --git a/third_party/raft/quorum/majority.go b/third_party/raft/quorum/majority.go new file mode 100644 index 0000000000..2d656b5888 --- /dev/null +++ b/third_party/raft/quorum/majority.go @@ -0,0 +1,198 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quorum + +import ( + "cmp" + "fmt" + "math" + "slices" + "strings" +) + +// MajorityConfig is a set of IDs that uses majority quorums to make decisions. +type MajorityConfig map[uint64]struct{} + +func (c MajorityConfig) String() string { + sl := make([]uint64, 0, len(c)) + for id := range c { + sl = append(sl, id) + } + slices.Sort(sl) + var buf strings.Builder + buf.WriteByte('(') + for i := range sl { + if i > 0 { + buf.WriteByte(' ') + } + fmt.Fprintf(&buf, "%x", sl[i]) + } + buf.WriteByte(')') + return buf.String() +} + +// Describe returns a (multi-line) representation of the commit indexes for the +// given lookuper. +func (c MajorityConfig) Describe(l AckedIndexer) string { + if len(c) == 0 { + return "" + } + type tup struct { + id uint64 + idx Index + ok bool // idx found? + bar int // length of bar displayed for this tup + } + + // Below, populate .bar so that the i-th largest commit index has bar i (we + // plot this as sort of a progress bar). The actual code is a bit more + // complicated and also makes sure that equal index => equal bar. + + n := len(c) + info := make([]tup, 0, n) + for id := range c { + idx, ok := l.AckedIndex(id) + info = append(info, tup{id: id, idx: idx, ok: ok}) + } + + // Sort by index + slices.SortFunc(info, func(a, b tup) int { + if n := cmp.Compare(a.idx, b.idx); n != 0 { + return n + } + return cmp.Compare(a.id, b.id) + }) + + // Populate .bar. + for i := 1; i < len(info); i++ { + if info[i-1].idx < info[i].idx { + info[i].bar = i + } else { + info[i].bar = info[i-1].bar + } + } + + // Sort by ID. + slices.SortFunc(info, func(a, b tup) int { + return cmp.Compare(a.id, b.id) + }) + + var buf strings.Builder + + // Print. + fmt.Fprint(&buf, strings.Repeat(" ", n)+" idx\n") + for i := range info { + bar := info[i].bar + if !info[i].ok { + fmt.Fprint(&buf, "?"+strings.Repeat(" ", n)) + } else { + fmt.Fprint(&buf, strings.Repeat("x", bar)+">"+strings.Repeat(" ", n-bar)) + } + fmt.Fprintf(&buf, " %5d (id=%x)\n", info[i].idx, info[i].id) + } + return buf.String() +} + +// Slice returns the MajorityConfig as a sorted slice. +func (c MajorityConfig) Slice() []uint64 { + var sl []uint64 + for id := range c { + sl = append(sl, id) + } + slices.Sort(sl) + return sl +} + +// CommittedIndex computes the committed index from those supplied via the +// provided AckedIndexer (for the active config). +func (c MajorityConfig) CommittedIndex(l AckedIndexer) Index { + n := len(c) + if n == 0 { + // This plays well with joint quorums which, when one half is the zero + // MajorityConfig, should behave like the other half. + return math.MaxUint64 + } + + // Use an on-stack slice to collect the committed indexes when n <= 7 + // (otherwise we alloc). The alternative is to stash a slice on + // MajorityConfig, but this impairs usability (as is, MajorityConfig is just + // a map, and that's nice). The assumption is that running with a + // replication factor of >7 is rare, and in cases in which it happens + // performance is a lesser concern (additionally the performance + // implications of an allocation here are far from drastic). + var stk [7]uint64 + var srt []uint64 + if len(stk) >= n { + srt = stk[:n] + } else { + srt = make([]uint64, n) + } + + { + // Fill the slice with the indexes observed. Any unused slots will be + // left as zero; these correspond to voters that may report in, but + // haven't yet. We fill from the right (since the zeroes will end up on + // the left after sorting below anyway). + i := n - 1 + for id := range c { + if idx, ok := l.AckedIndex(id); ok { + srt[i] = uint64(idx) + i-- + } + } + } + slices.Sort(srt) + + // The smallest index into the array for which the value is acked by a + // quorum. In other words, from the end of the slice, move n/2+1 to the + // left (accounting for zero-indexing). + pos := n - (n/2 + 1) + return Index(srt[pos]) +} + +// VoteResult takes a mapping of voters to yes/no (true/false) votes and returns +// a result indicating whether the vote is pending (i.e. neither a quorum of +// yes/no has been reached), won (a quorum of yes has been reached), or lost (a +// quorum of no has been reached). +func (c MajorityConfig) VoteResult(votes map[uint64]bool) VoteResult { + if len(c) == 0 { + // By convention, the elections on an empty config win. This comes in + // handy with joint quorums because it'll make a half-populated joint + // quorum behave like a majority quorum. + return VoteWon + } + + var votedCnt int //vote counts for yes. + var missing int + for id := range c { + v, ok := votes[id] + if !ok { + missing++ + continue + } + if v { + votedCnt++ + } + } + + q := len(c)/2 + 1 + if votedCnt >= q { + return VoteWon + } + if votedCnt+missing >= q { + return VotePending + } + return VoteLost +} diff --git a/third_party/raft/quorum/majority_test.go b/third_party/raft/quorum/majority_test.go new file mode 100644 index 0000000000..665d382f79 --- /dev/null +++ b/third_party/raft/quorum/majority_test.go @@ -0,0 +1,110 @@ +// Copyright 2025 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quorum + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDescribe(t *testing.T) { + testCases := []struct { + name string + c MajorityConfig + indexer mapAckIndexer + expectedResult string + }{ + { + name: "different commit index for each voter", + c: MajorityConfig{ + 1: struct{}{}, + 2: struct{}{}, + 3: struct{}{}, + }, + indexer: mapAckIndexer{ + 1: 101, + 2: 102, + 3: 103, + }, + expectedResult: ` idx +> 101 (id=1) +x> 102 (id=2) +xx> 103 (id=3) +`, + }, + { + name: "same commit index for all voters", + c: MajorityConfig{ + 1: struct{}{}, + 2: struct{}{}, + 3: struct{}{}, + }, + indexer: mapAckIndexer{ + 1: 100, + 2: 100, + 3: 100, + }, + expectedResult: ` idx +> 100 (id=1) +> 100 (id=2) +> 100 (id=3) +`, + }, + { + name: "same commit index for part of the voters", + c: MajorityConfig{ + 1: struct{}{}, + 2: struct{}{}, + 3: struct{}{}, + }, + indexer: mapAckIndexer{ + 1: 100, + 2: 102, + 3: 102, + }, + expectedResult: ` idx +> 100 (id=1) +x> 102 (id=2) +x> 102 (id=3) +`, + }, + { + name: "missing some commit indexes", + c: MajorityConfig{ + 1: struct{}{}, + 2: struct{}{}, + 3: struct{}{}, + }, + indexer: mapAckIndexer{ + 1: 101, + 3: 103, + }, + expectedResult: ` idx +x> 101 (id=1) +? 0 (id=2) +xx> 103 (id=3) +`, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualResult := tc.c.Describe(tc.indexer) + require.Equal(t, tc.expectedResult, actualResult) + }) + } + +} diff --git a/third_party/raft/quorum/quick_test.go b/third_party/raft/quorum/quick_test.go new file mode 100644 index 0000000000..940ce16350 --- /dev/null +++ b/third_party/raft/quorum/quick_test.go @@ -0,0 +1,122 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quorum + +import ( + "math" + "math/rand" + "reflect" + "testing" + "testing/quick" + + "github.com/stretchr/testify/require" +) + +// TestQuick uses quickcheck to heuristically assert that the main +// implementation of (MajorityConfig).CommittedIndex agrees with a "dumb" +// alternative version. +func TestQuick(t *testing.T) { + cfg := &quick.Config{ + MaxCount: 50000, + } + + t.Run("majority_commit", func(t *testing.T) { + fn1 := func(c memberMap, l idxMap) uint64 { + return uint64(MajorityConfig(c).CommittedIndex(mapAckIndexer(l))) + } + fn2 := func(c memberMap, l idxMap) uint64 { + return uint64(alternativeMajorityCommittedIndex(MajorityConfig(c), mapAckIndexer(l))) + } + require.NoError(t, quick.CheckEqual(fn1, fn2, cfg)) + }) +} + +// smallRandIdxMap returns a reasonably sized map of ids to commit indexes. +func smallRandIdxMap(rand *rand.Rand, _ int) map[uint64]Index { + // Hard-code a reasonably small size here (quick will hard-code 50, which + // is not useful here). + size := 10 + + n := rand.Intn(size) + ids := rand.Perm(2 * n)[:n] + idxs := make([]int, len(ids)) + for i := range idxs { + idxs[i] = rand.Intn(n) + } + + m := map[uint64]Index{} + for i := range ids { + m[uint64(ids[i])] = Index(idxs[i]) + } + return m +} + +type idxMap map[uint64]Index + +func (idxMap) Generate(rand *rand.Rand, size int) reflect.Value { + m := smallRandIdxMap(rand, size) + return reflect.ValueOf(m) +} + +type memberMap map[uint64]struct{} + +func (memberMap) Generate(rand *rand.Rand, size int) reflect.Value { + m := smallRandIdxMap(rand, size) + mm := map[uint64]struct{}{} + for id := range m { + mm[id] = struct{}{} + } + return reflect.ValueOf(mm) +} + +// This is an alternative implementation of (MajorityConfig).CommittedIndex(l). +func alternativeMajorityCommittedIndex(c MajorityConfig, l AckedIndexer) Index { + if len(c) == 0 { + return math.MaxUint64 + } + + idToIdx := map[uint64]Index{} + for id := range c { + if idx, ok := l.AckedIndex(id); ok { + idToIdx[id] = idx + } + } + + // Build a map from index to voters who have acked that or any higher index. + idxToVotes := map[Index]int{} + for _, idx := range idToIdx { + idxToVotes[idx] = 0 + } + + for _, idx := range idToIdx { + for idy := range idxToVotes { + if idy > idx { + continue + } + idxToVotes[idy]++ + } + } + + // Find the maximum index that has achieved quorum. + q := len(c)/2 + 1 + var maxQuorumIdx Index + for idx, n := range idxToVotes { + if n >= q && idx > maxQuorumIdx { + maxQuorumIdx = idx + } + } + + return maxQuorumIdx +} diff --git a/third_party/raft/quorum/quorum.go b/third_party/raft/quorum/quorum.go new file mode 100644 index 0000000000..2899e46c96 --- /dev/null +++ b/third_party/raft/quorum/quorum.go @@ -0,0 +1,58 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quorum + +import ( + "math" + "strconv" +) + +// Index is a Raft log position. +type Index uint64 + +func (i Index) String() string { + if i == math.MaxUint64 { + return "∞" + } + return strconv.FormatUint(uint64(i), 10) +} + +// AckedIndexer allows looking up a commit index for a given ID of a voter +// from a corresponding MajorityConfig. +type AckedIndexer interface { + AckedIndex(voterID uint64) (idx Index, found bool) +} + +type mapAckIndexer map[uint64]Index + +func (m mapAckIndexer) AckedIndex(id uint64) (Index, bool) { + idx, ok := m[id] + return idx, ok +} + +// VoteResult indicates the outcome of a vote. +// +//go:generate stringer -type=VoteResult +type VoteResult uint8 + +const ( + // VotePending indicates that the decision of the vote depends on future + // votes, i.e. neither "yes" or "no" has reached quorum yet. + VotePending VoteResult = 1 + iota + // VoteLost indicates that the quorum has voted "no". + VoteLost + // VoteWon indicates that the quorum has voted "yes". + VoteWon +) diff --git a/third_party/raft/quorum/testdata/joint_commit.txt b/third_party/raft/quorum/testdata/joint_commit.txt new file mode 100644 index 0000000000..97e041cc3c --- /dev/null +++ b/third_party/raft/quorum/testdata/joint_commit.txt @@ -0,0 +1,481 @@ +# No difference between a simple majority quorum and a simple majority quorum +# joint with an empty majority quorum. (This is asserted for all datadriven tests +# by the framework, so we don't dwell on it more). +# +# Note that by specifying cfgj explicitly we tell the test harness to treat the +# input as a joint quorum and not a majority quorum. If we didn't specify +# cfgj=zero the test would pass just the same, but it wouldn't be exercising the +# joint quorum path. +committed cfg=(1,2,3) cfgj=zero idx=(100,101,99) +---- + idx +x> 100 (id=1) +xx> 101 (id=2) +> 99 (id=3) +100 + +# Joint nonoverlapping singleton quorums. + +committed cfg=(1) cfgj=(2) idx=(_,_) +---- + idx +? 0 (id=1) +? 0 (id=2) +0 + +# Voter 1 has 100 committed, 2 nothing. This means we definitely won't commit +# past 100. +committed cfg=(1) cfgj=(2) idx=(100,_) +---- + idx +x> 100 (id=1) +? 0 (id=2) +0 + +# Committed index collapses once both majorities do, to the lower index. +committed cfg=(1) cfgj=(2) idx=(13, 100) +---- + idx +> 13 (id=1) +x> 100 (id=2) +13 + +# Joint overlapping (i.e. identical) singleton quorum. + +committed cfg=(1) cfgj=(1) idx=(_) +---- + idx +? 0 (id=1) +0 + +committed cfg=(1) cfgj=(1) idx=(100) +---- + idx +> 100 (id=1) +100 + + + +# Two-node config joint with non-overlapping single node config +committed cfg=(1,3) cfgj=(2) idx=(_,_,_) +---- + idx +? 0 (id=1) +? 0 (id=2) +? 0 (id=3) +0 + +committed cfg=(1,3) cfgj=(2) idx=(100,_,_) +---- + idx +xx> 100 (id=1) +? 0 (id=2) +? 0 (id=3) +0 + +# 1 has 100 committed, 2 has 50 (collapsing half of the joint quorum to 50). +committed cfg=(1,3) cfgj=(2) idx=(100,_,50) +---- + idx +xx> 100 (id=1) +x> 50 (id=2) +? 0 (id=3) +0 + +# 2 reports 45, collapsing the other half (to 45). +committed cfg=(1,3) cfgj=(2) idx=(100,45,50) +---- + idx +xx> 100 (id=1) +x> 50 (id=2) +> 45 (id=3) +45 + +# Two-node config with overlapping single-node config. + +committed cfg=(1,2) cfgj=(2) idx=(_,_) +---- + idx +? 0 (id=1) +? 0 (id=2) +0 + +# 1 reports 100. +committed cfg=(1,2) cfgj=(2) idx=(100,_) +---- + idx +x> 100 (id=1) +? 0 (id=2) +0 + +# 2 reports 100. +committed cfg=(1,2) cfgj=(2) idx=(_,100) +---- + idx +? 0 (id=1) +x> 100 (id=2) +0 + +committed cfg=(1,2) cfgj=(2) idx=(50,100) +---- + idx +> 50 (id=1) +x> 100 (id=2) +50 + +committed cfg=(1,2) cfgj=(2) idx=(100,50) +---- + idx +x> 100 (id=1) +> 50 (id=2) +50 + + + +# Joint non-overlapping two-node configs. + +committed cfg=(1,2) cfgj=(3,4) idx=(50,_,_,_) +---- + idx +xxx> 50 (id=1) +? 0 (id=2) +? 0 (id=3) +? 0 (id=4) +0 + +committed cfg=(1,2) cfgj=(3,4) idx=(50,_,49,_) +---- + idx +xxx> 50 (id=1) +? 0 (id=2) +xx> 49 (id=3) +? 0 (id=4) +0 + +committed cfg=(1,2) cfgj=(3,4) idx=(50,48,49,_) +---- + idx +xxx> 50 (id=1) +x> 48 (id=2) +xx> 49 (id=3) +? 0 (id=4) +0 + +committed cfg=(1,2) cfgj=(3,4) idx=(50,48,49,47) +---- + idx +xxx> 50 (id=1) +x> 48 (id=2) +xx> 49 (id=3) +> 47 (id=4) +47 + +# Joint overlapping two-node configs. +committed cfg=(1,2) cfgj=(2,3) idx=(_,_,_) +---- + idx +? 0 (id=1) +? 0 (id=2) +? 0 (id=3) +0 + +committed cfg=(1,2) cfgj=(2,3) idx=(100,_,_) +---- + idx +xx> 100 (id=1) +? 0 (id=2) +? 0 (id=3) +0 + +committed cfg=(1,2) cfgj=(2,3) idx=(_,100,_) +---- + idx +? 0 (id=1) +xx> 100 (id=2) +? 0 (id=3) +0 + +committed cfg=(1,2) cfgj=(2,3) idx=(_,100,99) +---- + idx +? 0 (id=1) +xx> 100 (id=2) +x> 99 (id=3) +0 + +committed cfg=(1,2) cfgj=(2,3) idx=(101,100,99) +---- + idx +xx> 101 (id=1) +x> 100 (id=2) +> 99 (id=3) +99 + +# Joint identical two-node configs. +committed cfg=(1,2) cfgj=(1,2) idx=(_,_) +---- + idx +? 0 (id=1) +? 0 (id=2) +0 + +committed cfg=(1,2) cfgj=(1,2) idx=(_,40) +---- + idx +? 0 (id=1) +x> 40 (id=2) +0 + +committed cfg=(1,2) cfgj=(1,2) idx=(41,40) +---- + idx +x> 41 (id=1) +> 40 (id=2) +40 + + + +# Joint disjoint three-node configs. + +committed cfg=(1,2,3) cfgj=(4,5,6) idx=(_,_,_,_,_,_) +---- + idx +? 0 (id=1) +? 0 (id=2) +? 0 (id=3) +? 0 (id=4) +? 0 (id=5) +? 0 (id=6) +0 + +committed cfg=(1,2,3) cfgj=(4,5,6) idx=(100,_,_,_,_,_) +---- + idx +xxxxx> 100 (id=1) +? 0 (id=2) +? 0 (id=3) +? 0 (id=4) +? 0 (id=5) +? 0 (id=6) +0 + +committed cfg=(1,2,3) cfgj=(4,5,6) idx=(100,_,_,90,_,_) +---- + idx +xxxxx> 100 (id=1) +? 0 (id=2) +? 0 (id=3) +xxxx> 90 (id=4) +? 0 (id=5) +? 0 (id=6) +0 + +committed cfg=(1,2,3) cfgj=(4,5,6) idx=(100,99,_,_,_,_) +---- + idx +xxxxx> 100 (id=1) +xxxx> 99 (id=2) +? 0 (id=3) +? 0 (id=4) +? 0 (id=5) +? 0 (id=6) +0 + +# First quorum <= 99, second one <= 97. Both quorums guarantee that 90 is +# committed. +committed cfg=(1,2,3) cfgj=(4,5,6) idx=(_,99,90,97,95,_) +---- + idx +? 0 (id=1) +xxxxx> 99 (id=2) +xx> 90 (id=3) +xxxx> 97 (id=4) +xxx> 95 (id=5) +? 0 (id=6) +90 + +# First quorum collapsed to 92. Second one already had at least 95 committed, +# so the result also collapses. +committed cfg=(1,2,3) cfgj=(4,5,6) idx=(92,99,90,97,95,_) +---- + idx +xx> 92 (id=1) +xxxxx> 99 (id=2) +x> 90 (id=3) +xxxx> 97 (id=4) +xxx> 95 (id=5) +? 0 (id=6) +92 + +# Second quorum collapses, but nothing changes in the output. +committed cfg=(1,2,3) cfgj=(4,5,6) idx=(92,99,90,97,95,77) +---- + idx +xx> 92 (id=1) +xxxxx> 99 (id=2) +x> 90 (id=3) +xxxx> 97 (id=4) +xxx> 95 (id=5) +> 77 (id=6) +92 + + +# Joint overlapping three-node configs. + +committed cfg=(1,2,3) cfgj=(1,4,5) idx=(_,_,_,_,_) +---- + idx +? 0 (id=1) +? 0 (id=2) +? 0 (id=3) +? 0 (id=4) +? 0 (id=5) +0 + +committed cfg=(1,2,3) cfgj=(1,4,5) idx=(100,_,_,_,_) +---- + idx +xxxx> 100 (id=1) +? 0 (id=2) +? 0 (id=3) +? 0 (id=4) +? 0 (id=5) +0 + +committed cfg=(1,2,3) cfgj=(1,4,5) idx=(100,101,_,_,_) +---- + idx +xxx> 100 (id=1) +xxxx> 101 (id=2) +? 0 (id=3) +? 0 (id=4) +? 0 (id=5) +0 + +committed cfg=(1,2,3) cfgj=(1,4,5) idx=(100,101,100,_,_) +---- + idx +xx> 100 (id=1) +xxxx> 101 (id=2) +xx> 100 (id=3) +? 0 (id=4) +? 0 (id=5) +0 + +# Second quorum could commit either 98 or 99, but first quorum is open. +committed cfg=(1,2,3) cfgj=(1,4,5) idx=(_,100,_,99,98) +---- + idx +? 0 (id=1) +xxxx> 100 (id=2) +? 0 (id=3) +xxx> 99 (id=4) +xx> 98 (id=5) +0 + +# Additionally, first quorum can commit either 100 or 99 +committed cfg=(1,2,3) cfgj=(1,4,5) idx=(_,100,99,99,98) +---- + idx +? 0 (id=1) +xxxx> 100 (id=2) +xx> 99 (id=3) +xx> 99 (id=4) +x> 98 (id=5) +98 + +committed cfg=(1,2,3) cfgj=(1,4,5) idx=(1,100,99,99,98) +---- + idx +> 1 (id=1) +xxxx> 100 (id=2) +xx> 99 (id=3) +xx> 99 (id=4) +x> 98 (id=5) +98 + +committed cfg=(1,2,3) cfgj=(1,4,5) idx=(100,100,99,99,98) +---- + idx +xxx> 100 (id=1) +xxx> 100 (id=2) +x> 99 (id=3) +x> 99 (id=4) +> 98 (id=5) +99 + + +# More overlap. + +committed cfg=(1,2,3) cfgj=(2,3,4) idx=(_,_,_,_) +---- + idx +? 0 (id=1) +? 0 (id=2) +? 0 (id=3) +? 0 (id=4) +0 + +committed cfg=(1,2,3) cfgj=(2,3,4) idx=(_,100,99,_) +---- + idx +? 0 (id=1) +xxx> 100 (id=2) +xx> 99 (id=3) +? 0 (id=4) +99 + +committed cfg=(1,2,3) cfgj=(2,3,4) idx=(98,100,99,_) +---- + idx +x> 98 (id=1) +xxx> 100 (id=2) +xx> 99 (id=3) +? 0 (id=4) +99 + +committed cfg=(1,2,3) cfgj=(2,3,4) idx=(100,100,99,_) +---- + idx +xx> 100 (id=1) +xx> 100 (id=2) +x> 99 (id=3) +? 0 (id=4) +99 + +committed cfg=(1,2,3) cfgj=(2,3,4) idx=(100,100,99,98) +---- + idx +xx> 100 (id=1) +xx> 100 (id=2) +x> 99 (id=3) +> 98 (id=4) +99 + +committed cfg=(1,2,3) cfgj=(2,3,4) idx=(100,_,_,101) +---- + idx +xx> 100 (id=1) +? 0 (id=2) +? 0 (id=3) +xxx> 101 (id=4) +0 + +committed cfg=(1,2,3) cfgj=(2,3,4) idx=(100,99,_,101) +---- + idx +xx> 100 (id=1) +x> 99 (id=2) +? 0 (id=3) +xxx> 101 (id=4) +99 + +# Identical. This is also exercised in the test harness, so it's listed here +# only briefly. +committed cfg=(1,2,3) cfgj=(1,2,3) idx=(50,45,_) +---- + idx +xx> 50 (id=1) +x> 45 (id=2) +? 0 (id=3) +45 diff --git a/third_party/raft/quorum/testdata/joint_vote.txt b/third_party/raft/quorum/testdata/joint_vote.txt new file mode 100644 index 0000000000..36cd0cabcf --- /dev/null +++ b/third_party/raft/quorum/testdata/joint_vote.txt @@ -0,0 +1,165 @@ +# Empty joint config wins all votes. This isn't used in production. Note that +# by specifying cfgj explicitly we tell the test harness to treat the input as +# a joint quorum and not a majority quorum. +vote cfgj=zero +---- +VoteWon + +# More examples with close to trivial configs. + +vote cfg=(1) cfgj=zero votes=(_) +---- +VotePending + +vote cfg=(1) cfgj=zero votes=(y) +---- +VoteWon + +vote cfg=(1) cfgj=zero votes=(n) +---- +VoteLost + +vote cfg=(1) cfgj=(1) votes=(_) +---- +VotePending + +vote cfg=(1) cfgj=(1) votes=(y) +---- +VoteWon + +vote cfg=(1) cfgj=(1) votes=(n) +---- +VoteLost + +vote cfg=(1) cfgj=(2) votes=(_,_) +---- +VotePending + +vote cfg=(1) cfgj=(2) votes=(y,_) +---- +VotePending + +vote cfg=(1) cfgj=(2) votes=(y,y) +---- +VoteWon + +vote cfg=(1) cfgj=(2) votes=(y,n) +---- +VoteLost + +vote cfg=(1) cfgj=(2) votes=(n,_) +---- +VoteLost + +vote cfg=(1) cfgj=(2) votes=(n,n) +---- +VoteLost + +vote cfg=(1) cfgj=(2) votes=(n,y) +---- +VoteLost + +# Two node configs. + +vote cfg=(1,2) cfgj=(3,4) votes=(_,_,_,_) +---- +VotePending + +vote cfg=(1,2) cfgj=(3,4) votes=(y,_,_,_) +---- +VotePending + +vote cfg=(1,2) cfgj=(3,4) votes=(y,y,_,_) +---- +VotePending + +vote cfg=(1,2) cfgj=(3,4) votes=(y,y,n,_) +---- +VoteLost + +vote cfg=(1,2) cfgj=(3,4) votes=(y,y,n,n) +---- +VoteLost + +vote cfg=(1,2) cfgj=(3,4) votes=(y,y,y,n) +---- +VoteLost + +vote cfg=(1,2) cfgj=(3,4) votes=(y,y,y,y) +---- +VoteWon + +vote cfg=(1,2) cfgj=(2,3) votes=(_,_,_) +---- +VotePending + +vote cfg=(1,2) cfgj=(2,3) votes=(_,n,_) +---- +VoteLost + +vote cfg=(1,2) cfgj=(2,3) votes=(y,y,_) +---- +VotePending + +vote cfg=(1,2) cfgj=(2,3) votes=(y,y,n) +---- +VoteLost + +vote cfg=(1,2) cfgj=(2,3) votes=(y,y,y) +---- +VoteWon + +vote cfg=(1,2) cfgj=(1,2) votes=(_,_) +---- +VotePending + +vote cfg=(1,2) cfgj=(1,2) votes=(y,_) +---- +VotePending + +vote cfg=(1,2) cfgj=(1,2) votes=(y,n) +---- +VoteLost + +vote cfg=(1,2) cfgj=(1,2) votes=(n,_) +---- +VoteLost + +vote cfg=(1,2) cfgj=(1,2) votes=(n,n) +---- +VoteLost + + +# Simple example for overlapping three node configs. + +vote cfg=(1,2,3) cfgj=(2,3,4) votes=(_,_,_,_) +---- +VotePending + +vote cfg=(1,2,3) cfgj=(2,3,4) votes=(_,n,_,_) +---- +VotePending + +vote cfg=(1,2,3) cfgj=(2,3,4) votes=(_,n,n,_) +---- +VoteLost + +vote cfg=(1,2,3) cfgj=(2,3,4) votes=(_,y,y,_) +---- +VoteWon + +vote cfg=(1,2,3) cfgj=(2,3,4) votes=(y,y,_,_) +---- +VotePending + +vote cfg=(1,2,3) cfgj=(2,3,4) votes=(y,y,n,_) +---- +VotePending + +vote cfg=(1,2,3) cfgj=(2,3,4) votes=(y,y,n,n) +---- +VoteLost + +vote cfg=(1,2,3) cfgj=(2,3,4) votes=(y,y,n,y) +---- +VoteWon diff --git a/third_party/raft/quorum/testdata/majority_commit.txt b/third_party/raft/quorum/testdata/majority_commit.txt new file mode 100644 index 0000000000..38289963ea --- /dev/null +++ b/third_party/raft/quorum/testdata/majority_commit.txt @@ -0,0 +1,153 @@ +# The empty quorum commits "everything". This is useful for its use in joint +# quorums. +committed +---- +∞ + + + +# A single voter quorum is not final when no index is known. +committed cfg=(1) idx=(_) +---- + idx +? 0 (id=1) +0 + +# When an index is known, that's the committed index, and that's final. +committed cfg=(1) idx=(12) +---- + idx +> 12 (id=1) +12 + + + + +# With two nodes, start out similarly. +committed cfg=(1, 2) idx=(_,_) +---- + idx +? 0 (id=1) +? 0 (id=2) +0 + +# The first committed index becomes known (for n1). Nothing changes in the +# output because idx=12 is not known to be on a quorum (which is both nodes). +committed cfg=(1, 2) idx=(12,_) +---- + idx +x> 12 (id=1) +? 0 (id=2) +0 + +# The second index comes in and finalize the decision. The result will be the +# smaller of the two indexes. +committed cfg=(1,2) idx=(12,5) +---- + idx +x> 12 (id=1) +> 5 (id=2) +5 + + + + +# No surprises for three nodes. +committed cfg=(1,2,3) idx=(_,_,_) +---- + idx +? 0 (id=1) +? 0 (id=2) +? 0 (id=3) +0 + +committed cfg=(1,2,3) idx=(12,_,_) +---- + idx +xx> 12 (id=1) +? 0 (id=2) +? 0 (id=3) +0 + +# We see a committed index, but a higher committed index for the last pending +# votes could change (increment) the outcome, so not final yet. +committed cfg=(1,2,3) idx=(12,5,_) +---- + idx +xx> 12 (id=1) +x> 5 (id=2) +? 0 (id=3) +5 + +# a) the case in which it does: +committed cfg=(1,2,3) idx=(12,5,6) +---- + idx +xx> 12 (id=1) +> 5 (id=2) +x> 6 (id=3) +6 + +# b) the case in which it does not: +committed cfg=(1,2,3) idx=(12,5,4) +---- + idx +xx> 12 (id=1) +x> 5 (id=2) +> 4 (id=3) +5 + +# c) a different case in which the last index is pending but it has no chance of +# swaying the outcome (because nobody in the current quorum agrees on anything +# higher than the candidate): +committed cfg=(1,2,3) idx=(5,5,_) +---- + idx +x> 5 (id=1) +x> 5 (id=2) +? 0 (id=3) +5 + +# c) continued: Doesn't matter what shows up last. The result is final. +committed cfg=(1,2,3) idx=(5,5,12) +---- + idx +> 5 (id=1) +> 5 (id=2) +xx> 12 (id=3) +5 + +# With all committed idx known, the result is final. +committed cfg=(1, 2, 3) idx=(100, 101, 103) +---- + idx +> 100 (id=1) +x> 101 (id=2) +xx> 103 (id=3) +101 + + + +# Some more complicated examples. Similar to case c) above. The result is +# already final because no index higher than 103 is one short of quorum. +committed cfg=(1, 2, 3, 4, 5) idx=(101, 104, 103, 103,_) +---- + idx +x> 101 (id=1) +xxxx> 104 (id=2) +xx> 103 (id=3) +xx> 103 (id=4) +? 0 (id=5) +103 + +# A similar case which is not final because another vote for >= 103 would change +# the outcome. +committed cfg=(1, 2, 3, 4, 5) idx=(101, 102, 103, 103,_) +---- + idx +x> 101 (id=1) +xx> 102 (id=2) +xxx> 103 (id=3) +xxx> 103 (id=4) +? 0 (id=5) +102 diff --git a/third_party/raft/quorum/testdata/majority_vote.txt b/third_party/raft/quorum/testdata/majority_vote.txt new file mode 100644 index 0000000000..5f9564b4f5 --- /dev/null +++ b/third_party/raft/quorum/testdata/majority_vote.txt @@ -0,0 +1,97 @@ +# The empty config always announces a won vote. +vote +---- +VoteWon + +vote cfg=(1) votes=(_) +---- +VotePending + +vote cfg=(1) votes=(n) +---- +VoteLost + +vote cfg=(123) votes=(y) +---- +VoteWon + + + + +vote cfg=(4,8) votes=(_,_) +---- +VotePending + +# With two voters, a single rejection loses the vote. +vote cfg=(4,8) votes=(n,_) +---- +VoteLost + +vote cfg=(4,8) votes=(y,_) +---- +VotePending + +vote cfg=(4,8) votes=(n,y) +---- +VoteLost + +vote cfg=(4,8) votes=(y,y) +---- +VoteWon + + + +vote cfg=(2,4,7) votes=(_,_,_) +---- +VotePending + +vote cfg=(2,4,7) votes=(n,_,_) +---- +VotePending + +vote cfg=(2,4,7) votes=(y,_,_) +---- +VotePending + +vote cfg=(2,4,7) votes=(n,n,_) +---- +VoteLost + +vote cfg=(2,4,7) votes=(y,n,_) +---- +VotePending + +vote cfg=(2,4,7) votes=(y,y,_) +---- +VoteWon + +vote cfg=(2,4,7) votes=(y,y,n) +---- +VoteWon + +vote cfg=(2,4,7) votes=(n,y,n) +---- +VoteLost + + + +# Test some random example with seven nodes (why not). +vote cfg=(1,2,3,4,5,6,7) votes=(y,y,n,y,_,_,_) +---- +VotePending + +vote cfg=(1,2,3,4,5,6,7) votes=(_,y,y,_,n,y,n) +---- +VotePending + +vote cfg=(1,2,3,4,5,6,7) votes=(y,y,n,y,_,n,y) +---- +VoteWon + +vote cfg=(1,2,3,4,5,6,7) votes=(y,y,_,n,y,n,n) +---- +VotePending + +vote cfg=(1,2,3,4,5,6,7) votes=(y,y,n,y,n,n,n) +---- +VoteLost diff --git a/third_party/raft/quorum/voteresult_string.go b/third_party/raft/quorum/voteresult_string.go new file mode 100644 index 0000000000..9eca8fd0c9 --- /dev/null +++ b/third_party/raft/quorum/voteresult_string.go @@ -0,0 +1,26 @@ +// Code generated by "stringer -type=VoteResult"; DO NOT EDIT. + +package quorum + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[VotePending-1] + _ = x[VoteLost-2] + _ = x[VoteWon-3] +} + +const _VoteResult_name = "VotePendingVoteLostVoteWon" + +var _VoteResult_index = [...]uint8{0, 11, 19, 26} + +func (i VoteResult) String() string { + i -= 1 + if i >= VoteResult(len(_VoteResult_index)-1) { + return "VoteResult(" + strconv.FormatInt(int64(i+1), 10) + ")" + } + return _VoteResult_name[_VoteResult_index[i]:_VoteResult_index[i+1]] +} diff --git a/third_party/raft/raft.go b/third_party/raft/raft.go new file mode 100644 index 0000000000..8f94a11d96 --- /dev/null +++ b/third_party/raft/raft.go @@ -0,0 +1,2177 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "bytes" + "crypto/rand" + "errors" + "fmt" + "math" + "math/big" + "slices" + "strings" + "sync" + + "google.golang.org/protobuf/proto" + + "go.etcd.io/raft/v3/confchange" + "go.etcd.io/raft/v3/quorum" + pb "go.etcd.io/raft/v3/raftpb" + "go.etcd.io/raft/v3/tracker" +) + +const ( + // None is a placeholder node ID used when there is no leader. + None uint64 = 0 + // LocalAppendThread is a reference to a local thread that saves unstable + // log entries and snapshots to stable storage. The identifier is used as a + // target for MsgStorageAppend messages when AsyncStorageWrites is enabled. + LocalAppendThread uint64 = math.MaxUint64 + // LocalApplyThread is a reference to a local thread that applies committed + // log entries to the local state machine. The identifier is used as a + // target for MsgStorageApply messages when AsyncStorageWrites is enabled. + LocalApplyThread uint64 = math.MaxUint64 - 1 +) + +// Possible values for StateType. +const ( + StateFollower StateType = iota + StateCandidate + StateLeader + StatePreCandidate + numStates +) + +type ReadOnlyOption int + +const ( + // ReadOnlySafe guarantees the linearizability of the read only request by + // communicating with the quorum. It is the default and suggested option. + ReadOnlySafe ReadOnlyOption = iota + // ReadOnlyLeaseBased ensures linearizability of the read only request by + // relying on the leader lease. It can be affected by clock drift. + // If the clock drift is unbounded, leader might keep the lease longer than it + // should (clock can move backward/pause without any bound). ReadIndex is not safe + // in that case. + ReadOnlyLeaseBased +) + +// Possible values for CampaignType +const ( + // campaignPreElection represents the first phase of a normal election when + // Config.PreVote is true. + campaignPreElection CampaignType = "CampaignPreElection" + // campaignElection represents a normal (time-based) election (the second phase + // of the election when Config.PreVote is true). + campaignElection CampaignType = "CampaignElection" + // campaignTransfer represents the type of leader transfer + campaignTransfer CampaignType = "CampaignTransfer" +) + +const noLimit = math.MaxUint64 + +// ErrProposalDropped is returned when the proposal is ignored by some cases, +// so that the proposer can be notified and fail fast. +var ErrProposalDropped = errors.New("raft proposal dropped") + +// lockedRand is a small wrapper around rand.Rand to provide +// synchronization among multiple raft groups. Only the methods needed +// by the code are exposed (e.g. Intn). +type lockedRand struct { + mu sync.Mutex +} + +func (r *lockedRand) Intn(n int) int { + r.mu.Lock() + v, _ := rand.Int(rand.Reader, big.NewInt(int64(n))) + r.mu.Unlock() + return int(v.Int64()) +} + +var globalRand = &lockedRand{} + +// CampaignType represents the type of campaigning +// the reason we use the type of string instead of uint64 +// is because it's simpler to compare and fill in raft entries +type CampaignType string + +// StateType represents the role of a node in a cluster. +type StateType uint64 + +var stmap = [...]string{ + "StateFollower", + "StateCandidate", + "StateLeader", + "StatePreCandidate", +} + +func (st StateType) String() string { + return stmap[st] +} + +// Config contains the parameters to start a raft. +type Config struct { + // ID is the identity of the local raft. ID cannot be 0. + ID uint64 + + // ElectionRandom optionally supplies per-node bounded election randomness. + // It must return a value in [0, n) and must not be shared between nodes. + // A nil provider preserves the upstream default for existing callers. + ElectionRandom func(n int) int + + // ElectionTick is the number of Node.Tick invocations that must pass between + // elections. That is, if a follower does not receive any message from the + // leader of current term before ElectionTick has elapsed, it will become + // candidate and start an election. ElectionTick must be greater than + // HeartbeatTick. We suggest ElectionTick = 10 * HeartbeatTick to avoid + // unnecessary leader switching. + ElectionTick int + // HeartbeatTick is the number of Node.Tick invocations that must pass between + // heartbeats. That is, a leader sends heartbeat messages to maintain its + // leadership every HeartbeatTick ticks. + HeartbeatTick int + + // Storage is the storage for raft. raft generates entries and states to be + // stored in storage. raft reads the persisted entries and states out of + // Storage when it needs. raft reads out the previous state and configuration + // out of storage when restarting. + Storage Storage + // Applied is the last applied index. It should only be set when restarting + // raft. raft will not return entries to the application smaller or equal to + // Applied. If Applied is unset when restarting, raft might return previous + // applied entries. This is a very application dependent configuration. + Applied uint64 + + // AsyncStorageWrites configures the raft node to write to its local storage + // (raft log and state machine) using a request/response message passing + // interface instead of the default Ready/Advance function call interface. + // Local storage messages can be pipelined and processed asynchronously + // (with respect to Ready iteration), facilitating reduced interference + // between Raft proposals and increased batching of log appends and state + // machine application. As a result, use of asynchronous storage writes can + // reduce end-to-end commit latency and increase maximum throughput. + // + // When true, the Ready.Message slice will include MsgStorageAppend and + // MsgStorageApply messages. The messages will target a LocalAppendThread + // and a LocalApplyThread, respectively. Messages to the same target must be + // reliably processed in order. In other words, they can't be dropped (like + // messages over the network) and those targeted at the same thread can't be + // reordered. Messages to different targets can be processed in any order. + // + // MsgStorageAppend carries Raft log entries to append, election votes / + // term changes / updated commit indexes to persist, and snapshots to apply. + // All writes performed in service of a MsgStorageAppend must be durable + // before response messages are delivered. However, if the MsgStorageAppend + // carries no response messages, durability is not required. The message + // assumes the role of the Entries, HardState, and Snapshot fields in Ready. + // + // MsgStorageApply carries committed entries to apply. Writes performed in + // service of a MsgStorageApply need not be durable before response messages + // are delivered. The message assumes the role of the CommittedEntries field + // in Ready. + // + // Local messages each carry one or more response messages which should be + // delivered after the corresponding storage write has been completed. These + // responses may target the same node or may target other nodes. The storage + // threads are not responsible for understanding the response messages, only + // for delivering them to the correct target after performing the storage + // write. + AsyncStorageWrites bool + + // MaxSizePerMsg limits the max byte size of each append message. Smaller + // value lowers the raft recovery cost(initial probing and message lost + // during normal operation). On the other side, it might affect the + // throughput during normal replication. Note: math.MaxUint64 for unlimited, + // 0 for at most one entry per message. + MaxSizePerMsg uint64 + // MaxCommittedSizePerReady limits the size of the committed entries which + // can be applying at the same time. + // + // Despite its name (preserved for compatibility), this quota applies across + // Ready structs to encompass all outstanding entries in unacknowledged + // MsgStorageApply messages when AsyncStorageWrites is enabled. + MaxCommittedSizePerReady uint64 + // MaxUncommittedEntriesSize limits the aggregate byte size of the + // uncommitted entries that may be appended to a leader's log. Once this + // limit is exceeded, proposals will begin to return ErrProposalDropped + // errors. Note: 0 for no limit. + MaxUncommittedEntriesSize uint64 + // MaxInflightMsgs limits the max number of in-flight append messages during + // optimistic replication phase. The application transportation layer usually + // has its own sending buffer over TCP/UDP. Setting MaxInflightMsgs to avoid + // overflowing that sending buffer. TODO (xiangli): feedback to application to + // limit the proposal rate? + MaxInflightMsgs int + // MaxInflightBytes limits the number of in-flight bytes in append messages. + // Complements MaxInflightMsgs. Ignored if zero. + // + // This effectively bounds the bandwidth-delay product. Note that especially + // in high-latency deployments setting this too low can lead to a dramatic + // reduction in throughput. For example, with a peer that has a round-trip + // latency of 100ms to the leader and this setting is set to 1 MB, there is a + // throughput limit of 10 MB/s for this group. With RTT of 400ms, this drops + // to 2.5 MB/s. See Little's law to understand the maths behind. + MaxInflightBytes uint64 + + // CheckQuorum specifies if the leader should check quorum activity. Leader + // steps down when quorum is not active for an electionTimeout. + CheckQuorum bool + + // PreVote enables the Pre-Vote algorithm described in raft thesis section + // 9.6. This prevents disruption when a node that has been partitioned away + // rejoins the cluster. + PreVote bool + + // ReadOnlyOption specifies how the read only request is processed. + // + // ReadOnlySafe guarantees the linearizability of the read only request by + // communicating with the quorum. It is the default and suggested option. + // + // ReadOnlyLeaseBased ensures linearizability of the read only request by + // relying on the leader lease. It can be affected by clock drift. + // If the clock drift is unbounded, leader might keep the lease longer than it + // should (clock can move backward/pause without any bound). ReadIndex is not safe + // in that case. + // CheckQuorum MUST be enabled if ReadOnlyOption is ReadOnlyLeaseBased. + ReadOnlyOption ReadOnlyOption + + // Logger is the logger used for raft log. For multinode which can host + // multiple raft group, each raft group can have its own logger + Logger Logger + + // DisableProposalForwarding set to true means that followers will drop + // proposals, rather than forwarding them to the leader. One use case for + // this feature would be in a situation where the Raft leader is used to + // compute the data of a proposal, for example, adding a timestamp from a + // hybrid logical clock to data in a monotonically increasing way. Forwarding + // should be disabled to prevent a follower with an inaccurate hybrid + // logical clock from assigning the timestamp and then forwarding the data + // to the leader. + DisableProposalForwarding bool + + // DisableConfChangeValidation turns off propose-time verification of + // configuration changes against the currently active configuration of the + // raft instance. These checks are generally sensible (cannot leave a joint + // config unless in a joint config, et cetera) but they have false positives + // because the active configuration may not be the most recent + // configuration. This is because configurations are activated during log + // application, and even the leader can trail log application by an + // unbounded number of entries. + // Symmetrically, the mechanism has false negatives - because the check may + // not run against the "actual" config that will be the predecessor of the + // newly proposed one, the check may pass but the new config may be invalid + // when it is being applied. In other words, the checks are best-effort. + // + // Users should *not* use this option unless they have a reliable mechanism + // (above raft) that serializes and verifies configuration changes. If an + // invalid configuration change enters the log and gets applied, a panic + // will result. + // + // This option may be removed once false positives are no longer possible. + // See: https://github.com/etcd-io/raft/issues/80 + DisableConfChangeValidation bool + + // StepDownOnRemoval makes the leader step down when it is removed from the + // group or demoted to a learner. + // + // This behavior will become unconditional in the future. See: + // https://github.com/etcd-io/raft/issues/83 + StepDownOnRemoval bool + + // raft state tracer + TraceLogger TraceLogger +} + +func (c *Config) validate() error { + if c.ID == None { + return errors.New("cannot use none as id") + } + if IsLocalMsgTarget(c.ID) { + return errors.New("cannot use local target as id") + } + + if c.HeartbeatTick <= 0 { + return errors.New("heartbeat tick must be greater than 0") + } + + if c.ElectionTick <= c.HeartbeatTick { + return errors.New("election tick must be greater than heartbeat tick") + } + + if c.Storage == nil { + return errors.New("storage cannot be nil") + } + + if c.MaxUncommittedEntriesSize == 0 { + c.MaxUncommittedEntriesSize = noLimit + } + + // default MaxCommittedSizePerReady to MaxSizePerMsg because they were + // previously the same parameter. + if c.MaxCommittedSizePerReady == 0 { + c.MaxCommittedSizePerReady = c.MaxSizePerMsg + } + + if c.MaxInflightMsgs <= 0 { + return errors.New("max inflight messages must be greater than 0") + } + if c.MaxInflightBytes == 0 { + c.MaxInflightBytes = noLimit + } else if c.MaxInflightBytes < c.MaxSizePerMsg { + return errors.New("max inflight bytes must be >= max message size") + } + + if c.Logger == nil { + c.Logger = getLogger() + } + + if c.ReadOnlyOption == ReadOnlyLeaseBased && !c.CheckQuorum { + return errors.New("CheckQuorum must be enabled when ReadOnlyOption is ReadOnlyLeaseBased") + } + + return nil +} + +type raft struct { + id uint64 + + Term uint64 + Vote uint64 + + readStates []ReadState + + // the log + raftLog *raftLog + + maxMsgSize entryEncodingSize + maxUncommittedSize entryPayloadSize + + trk tracker.ProgressTracker + + state StateType + + // isLearner is true if the local raft node is a learner. + isLearner bool + + // msgs contains the list of messages that should be sent out immediately to + // other nodes. + // + // Messages in this list must target other nodes. + msgs []*pb.Message + // msgsAfterAppend contains the list of messages that should be sent after + // the accumulated unstable state (e.g. term, vote, []entry, and snapshot) + // has been persisted to durable storage. This includes waiting for any + // unstable state that is already in the process of being persisted (i.e. + // has already been handed out in a prior Ready struct) to complete. + // + // Messages in this list may target other nodes or may target this node. + // + // Messages in this list have the type MsgAppResp, MsgVoteResp, or + // MsgPreVoteResp. See the comment in raft.send for details. + msgsAfterAppend []*pb.Message + + // the leader id + lead uint64 + // leadTransferee is id of the leader transfer target when its value is not zero. + // Follow the procedure defined in raft thesis 3.10. + leadTransferee uint64 + // Only one conf change may be pending (in the log, but not yet + // applied) at a time. This is enforced via pendingConfIndex, which + // is set to a value >= the log index of the latest pending + // configuration change (if any). Config changes are only allowed to + // be proposed if the leader's applied index is greater than this + // value. + pendingConfIndex uint64 + // disableConfChangeValidation is Config.DisableConfChangeValidation, + // see there for details. + disableConfChangeValidation bool + // an estimate of the size of the uncommitted tail of the Raft log. Used to + // prevent unbounded log growth. Only maintained by the leader. Reset on + // term changes. + uncommittedSize entryPayloadSize + + readOnly *readOnly + + // number of ticks since it reached last electionTimeout when it is leader + // or candidate. + // number of ticks since it reached last electionTimeout or received a + // valid message from current leader when it is a follower. + electionElapsed int + + // number of ticks since it reached last heartbeatTimeout. + // only leader keeps heartbeatElapsed. + heartbeatElapsed int + + checkQuorum bool + preVote bool + + heartbeatTimeout int + electionTimeout int + // randomizedElectionTimeout is a random number between + // [electiontimeout, 2 * electiontimeout - 1]. It gets reset + // when raft changes its state to follower or candidate. + randomizedElectionTimeout int + electionRandom func(int) int + disableProposalForwarding bool + stepDownOnRemoval bool + + tick func() + step stepFunc + + logger Logger + + // pendingReadIndexMessages is used to store messages of type MsgReadIndex + // that can't be answered as new leader didn't committed any log in + // current term. Those will be handled as fast as first log is committed in + // current term. + pendingReadIndexMessages []*pb.Message + + traceLogger TraceLogger +} + +func newRaft(c *Config) *raft { + if err := c.validate(); err != nil { + panic(err.Error()) + } + raftlog := newLogWithSize(c.Storage, c.Logger, entryEncodingSize(c.MaxCommittedSizePerReady)) + hs, cs, err := c.Storage.InitialState() + if err != nil { + panic(err) // TODO(bdarnell) + } + + r := &raft{ + id: c.ID, + electionRandom: c.ElectionRandom, + lead: None, + isLearner: false, + raftLog: raftlog, + maxMsgSize: entryEncodingSize(c.MaxSizePerMsg), + maxUncommittedSize: entryPayloadSize(c.MaxUncommittedEntriesSize), + trk: tracker.MakeProgressTracker(c.MaxInflightMsgs, c.MaxInflightBytes), + electionTimeout: c.ElectionTick, + heartbeatTimeout: c.HeartbeatTick, + logger: c.Logger, + checkQuorum: c.CheckQuorum, + preVote: c.PreVote, + readOnly: newReadOnly(c.ReadOnlyOption), + disableProposalForwarding: c.DisableProposalForwarding, + disableConfChangeValidation: c.DisableConfChangeValidation, + stepDownOnRemoval: c.StepDownOnRemoval, + traceLogger: c.TraceLogger, + } + + traceInitState(r) + + lastID := r.raftLog.lastEntryID() + cfg, trk, err := confchange.Restore(confchange.Changer{ + Tracker: r.trk, + LastIndex: lastID.index, + }, cs) + if err != nil { + panic(err) + } + assertConfStatesEquivalent(r.logger, cs, r.switchToConfig(cfg, trk)) + + if !IsEmptyHardState(hs) { + r.loadState(hs) + } + if c.Applied > 0 { + raftlog.appliedTo(c.Applied, 0 /* size */) + } + r.becomeFollower(r.Term, None) + + var nodesStrs []string + for _, n := range r.trk.VoterNodes() { + nodesStrs = append(nodesStrs, fmt.Sprintf("%x", n)) + } + + // TODO(pav-kv): it should be ok to simply print %+v for lastID. + r.logger.Infof("newRaft %x [peers: [%s], term: %d, commit: %d, applied: %d, lastindex: %d, lastterm: %d]", + r.id, strings.Join(nodesStrs, ","), r.Term, r.raftLog.committed, r.raftLog.applied, lastID.index, lastID.term) + return r +} + +func (r *raft) hasLeader() bool { return r.lead != None } + +func (r *raft) softState() SoftState { return SoftState{Lead: r.lead, RaftState: r.state} } + +func (r *raft) hardState() *pb.HardState { + return &pb.HardState{ + Term: new(r.Term), + Vote: new(r.Vote), + Commit: new(r.raftLog.committed), + } +} + +// send schedules persisting state to a stable storage and AFTER that +// sending the message (as part of next Ready message processing). +func (r *raft) send(m *pb.Message) { + if m.GetFrom() == None { + m.From = new(r.id) + } + if m.GetType() == pb.MsgVote || m.GetType() == pb.MsgVoteResp || m.GetType() == pb.MsgPreVote || m.GetType() == pb.MsgPreVoteResp { + if m.GetTerm() == 0 { + // All {pre-,}campaign messages need to have the term set when + // sending. + // - MsgVote: m.Term is the term the node is campaigning for, + // non-zero as we increment the term when campaigning. + // - MsgVoteResp: m.Term is the new r.Term if the MsgVote was + // granted, non-zero for the same reason MsgVote is + // - MsgPreVote: m.Term is the term the node will campaign, + // non-zero as we use m.Term to indicate the next term we'll be + // campaigning for + // - MsgPreVoteResp: m.Term is the term received in the original + // MsgPreVote if the pre-vote was granted, non-zero for the + // same reasons MsgPreVote is + r.logger.Panicf("term should be set when sending %s", m.GetType()) + } + } else { + if m.GetTerm() != 0 { + r.logger.Panicf("term should not be set when sending %s (was %d)", m.GetType(), m.GetTerm()) + } + // do not attach term to MsgProp, MsgReadIndex + // proposals are a way to forward to the leader and + // should be treated as local message. + // MsgReadIndex is also forwarded to leader. + if m.GetType() != pb.MsgProp && m.GetType() != pb.MsgReadIndex { + m.Term = new(r.Term) + } + } + if m.GetType() == pb.MsgAppResp || m.GetType() == pb.MsgVoteResp || m.GetType() == pb.MsgPreVoteResp { + // If async storage writes are enabled, messages added to the msgs slice + // are allowed to be sent out before unstable state (e.g. log entry + // writes and election votes) have been durably synced to the local + // disk. + // + // For most message types, this is not an issue. However, response + // messages that relate to "voting" on either leader election or log + // appends require durability before they can be sent. It would be + // incorrect to publish a vote in an election before that vote has been + // synced to stable storage locally. Similarly, it would be incorrect to + // acknowledge a log append to the leader before that entry has been + // synced to stable storage locally. + // + // Per the Raft thesis, section 3.8 Persisted state and server restarts: + // + // > Raft servers must persist enough information to stable storage to + // > survive server restarts safely. In particular, each server persists + // > its current term and vote; this is necessary to prevent the server + // > from voting twice in the same term or replacing log entries from a + // > newer leader with those from a deposed leader. Each server also + // > persists new log entries before they are counted towards the entries’ + // > commitment; this prevents committed entries from being lost or + // > “uncommitted” when servers restart + // + // To enforce this durability requirement, these response messages are + // queued to be sent out as soon as the current collection of unstable + // state (the state that the response message was predicated upon) has + // been durably persisted. This unstable state may have already been + // passed to a Ready struct whose persistence is in progress or may be + // waiting for the next Ready struct to begin being written to Storage. + // These messages must wait for all of this state to be durable before + // being published. + // + // Rejected responses (m.Reject == true) present an interesting case + // where the durability requirement is less unambiguous. A rejection may + // be predicated upon unstable state. For instance, a node may reject a + // vote for one peer because it has already begun syncing its vote for + // another peer. Or it may reject a vote from one peer because it has + // unstable log entries that indicate that the peer is behind on its + // log. In these cases, it is likely safe to send out the rejection + // response immediately without compromising safety in the presence of a + // server restart. However, because these rejections are rare and + // because the safety of such behavior has not been formally verified, + // we err on the side of safety and omit a `&& !m.Reject` condition + // above. + r.msgsAfterAppend = append(r.msgsAfterAppend, m) + traceSendMessage(r, m) + } else { + if m.GetTo() == r.id { + r.logger.Panicf("message should not be self-addressed when sending %s", m.GetType()) + } + r.msgs = append(r.msgs, m) + traceSendMessage(r, m) + } +} + +// sendAppend sends an append RPC with new entries (if any) and the +// current commit index to the given peer. +func (r *raft) sendAppend(to uint64) { + r.maybeSendAppend(to, true) +} + +// maybeSendAppend sends an append RPC with new entries to the given peer, +// if necessary. Returns true if a message was sent. The sendIfEmpty +// argument controls whether messages with no entries will be sent +// ("empty" messages are useful to convey updated Commit indexes, but +// are undesirable when we're sending multiple messages in a batch). +// +// TODO(pav-kv): make invocation of maybeSendAppend stateless. The Progress +// struct contains all the state necessary for deciding whether to send a +// message. +func (r *raft) maybeSendAppend(to uint64, sendIfEmpty bool) bool { + pr := r.trk.Progress[to] + if pr.IsPaused() { + return false + } + + prevIndex := pr.Next - 1 + prevTerm, err := r.raftLog.term(prevIndex) + if err != nil { + // The log probably got truncated at >= pr.Next, so we can't catch up the + // follower log anymore. Send a snapshot instead. + return r.maybeSendSnapshot(to, pr) + } + + var ents []*pb.Entry + // In a throttled StateReplicate only send empty MsgApp, to ensure progress. + // Otherwise, if we had a full Inflights and all inflight messages were in + // fact dropped, replication to that follower would stall. Instead, an empty + // MsgApp will eventually reach the follower (heartbeats responses prompt the + // leader to send an append), allowing it to be acked or rejected, both of + // which will clear out Inflights. + if pr.State != tracker.StateReplicate || !pr.Inflights.Full() { + ents, err = r.raftLog.entries(pr.Next, r.maxMsgSize) + } + if len(ents) == 0 && !sendIfEmpty { + return false + } + // TODO(pav-kv): move this check up to where err is returned. + if err != nil { // send a snapshot if we failed to get the entries + return r.maybeSendSnapshot(to, pr) + } + + // Send the actual MsgApp otherwise, and update the progress accordingly. + r.send(&pb.Message{ + To: new(to), + Type: pb.MsgApp.Enum(), + Index: new(prevIndex), + LogTerm: new(prevTerm), + Entries: ents, + Commit: new(r.raftLog.committed), + }) + pr.SentEntries(len(ents), uint64(payloadsSize(ents))) + pr.SentCommit(r.raftLog.committed) + return true +} + +// maybeSendSnapshot fetches a snapshot from Storage, and sends it to the given +// node. Returns true iff the snapshot message has been emitted successfully. +func (r *raft) maybeSendSnapshot(to uint64, pr *tracker.Progress) bool { + if !pr.RecentActive { + r.logger.Debugf("ignore sending snapshot to %x since it is not recently active", to) + return false + } + + snapshot, err := r.raftLog.snapshot() + if err != nil { + if err == ErrSnapshotTemporarilyUnavailable { + r.logger.Debugf("%x failed to send snapshot to %x because snapshot is temporarily unavailable", r.id, to) + return false + } + panic(err) // TODO(bdarnell) + } + if IsEmptySnap(snapshot) { + panic("need non-empty snapshot") + } + sindex, sterm := snapshot.GetMetadata().GetIndex(), snapshot.GetMetadata().GetTerm() + r.logger.Debugf("%x [firstindex: %d, commit: %d] sent snapshot[index: %d, term: %d] to %x [%s]", + r.id, r.raftLog.firstIndex(), r.raftLog.committed, sindex, sterm, to, pr) + pr.BecomeSnapshot(sindex) + r.logger.Debugf("%x paused sending replication messages to %x [%s]", r.id, to, pr) + + r.send(&pb.Message{To: new(to), Type: pb.MsgSnap.Enum(), Snapshot: snapshot}) + return true +} + +// sendHeartbeat sends a heartbeat RPC to the given peer. +func (r *raft) sendHeartbeat(to uint64, ctx []byte) { + pr := r.trk.Progress[to] + // Attach the commit as min(to.matched, r.committed). + // When the leader sends out heartbeat message, + // the receiver(follower) might not be matched with the leader + // or it might not have all the committed entries. + // The leader MUST NOT forward the follower's commit to + // an unmatched index. + commit := min(pr.Match, r.raftLog.committed) + r.send(&pb.Message{ + To: new(to), + Type: pb.MsgHeartbeat.Enum(), + Commit: new(commit), + Context: ctx, + }) + pr.SentCommit(commit) +} + +// bcastAppend sends RPC, with entries to all peers that are not up-to-date +// according to the progress recorded in r.trk. +func (r *raft) bcastAppend() { + r.trk.Visit(func(id uint64, _ *tracker.Progress) { + if id == r.id { + return + } + r.sendAppend(id) + }) +} + +// bcastHeartbeat sends RPC, without entries to all the peers. +func (r *raft) bcastHeartbeat() { + r.bcastHeartbeatWithCtx(r.readOnly.heartbeatCtx()) +} + +func (r *raft) bcastHeartbeatWithCtx(ctx []byte) { + r.trk.Visit(func(id uint64, _ *tracker.Progress) { + if id == r.id { + return + } + r.sendHeartbeat(id, ctx) + }) +} + +func (r *raft) appliedTo(index uint64, size entryEncodingSize) { + oldApplied := r.raftLog.applied + newApplied := max(index, oldApplied) + r.raftLog.appliedTo(newApplied, size) + + if r.trk.Config.AutoLeave && newApplied >= r.pendingConfIndex && r.state == StateLeader { + // If the current (and most recent, at least for this leader's term) + // configuration should be auto-left, initiate that now. We use a + // nil Data which unmarshals into an empty ConfChangeV2 and has the + // benefit that appendEntry can never refuse it based on its size + // (which registers as zero). + m, err := confChangeToMsg(nil) + if err != nil { + panic(err) + } + // NB: this proposal can't be dropped due to size, but can be + // dropped if a leadership transfer is in progress. We'll keep + // checking this condition on each applied entry, so either the + // leadership transfer will succeed and the new leader will leave + // the joint configuration, or the leadership transfer will fail, + // and we will propose the config change on the next advance. + if err := r.Step(m); err != nil { + r.logger.Debugf("not initiating automatic transition out of joint configuration %s: %v", r.trk.Config, err) + } else { + r.logger.Infof("initiating automatic transition out of joint configuration %s", r.trk.Config) + } + } +} + +func (r *raft) appliedSnap(snap *pb.Snapshot) { + index := snap.GetMetadata().GetIndex() + r.raftLog.stableSnapTo(index) + r.appliedTo(index, 0 /* size */) +} + +// maybeCommit attempts to advance the commit index. Returns true if the commit +// index changed (in which case the caller should call r.bcastAppend). This can +// only be called in StateLeader. +func (r *raft) maybeCommit() bool { + defer traceCommit(r) + + return r.raftLog.maybeCommit(entryID{term: r.Term, index: r.trk.Committed()}) +} + +func (r *raft) reset(term uint64) { + if r.Term != term { + r.Term = term + r.Vote = None + } + r.lead = None + + r.electionElapsed = 0 + r.heartbeatElapsed = 0 + r.resetRandomizedElectionTimeout() + + r.abortLeaderTransfer() + + r.trk.ResetVotes() + r.trk.Visit(func(id uint64, pr *tracker.Progress) { + *pr = tracker.Progress{ + Match: 0, + Next: r.raftLog.lastIndex() + 1, + Inflights: tracker.NewInflights(r.trk.MaxInflight, r.trk.MaxInflightBytes), + IsLearner: pr.IsLearner, + } + if id == r.id { + pr.Match = r.raftLog.lastIndex() + } + }) + + r.pendingConfIndex = 0 + r.uncommittedSize = 0 + r.readOnly = newReadOnly(r.readOnly.option) +} + +func (r *raft) appendEntry(es ...*pb.Entry) (accepted bool) { + li := r.raftLog.lastIndex() + cloned := make([]*pb.Entry, len(es)) + for i := range es { + // TODO: Consider whether cloning/copying is necessary at all, and aim to eliminate it if possible. + cloned[i] = proto.Clone(es[i]).(*pb.Entry) + cloned[i].Term = new(r.Term) + cloned[i].Index = new(li + 1 + uint64(i)) + } + // Track the size of this uncommitted proposal. + if !r.increaseUncommittedSize(cloned) { + r.logger.Warningf( + "%x appending new entries to log would exceed uncommitted entry size limit; dropping proposal", + r.id, + ) + // Drop the proposal. + return false + } + + traceReplicate(r, cloned...) + + // use latest "last" index after truncate/append + li = r.raftLog.append(cloned...) + // The leader needs to self-ack the entries just appended once they have + // been durably persisted (since it doesn't send an MsgApp to itself). This + // response message will be added to msgsAfterAppend and delivered back to + // this node after these entries have been written to stable storage. When + // handled, this is roughly equivalent to: + // + // r.trk.Progress[r.id].MaybeUpdate(e.Index) + // if r.maybeCommit() { + // r.bcastAppend() + // } + r.send(&pb.Message{To: new(r.id), Type: pb.MsgAppResp.Enum(), Index: new(li)}) + return true +} + +// tickElection is run by followers and candidates after r.electionTimeout. +func (r *raft) tickElection() { + r.electionElapsed++ + + if r.promotable() && r.pastElectionTimeout() { + r.electionElapsed = 0 + if err := r.Step(&pb.Message{From: new(r.id), Type: pb.MsgHup.Enum()}); err != nil { + r.logger.Debugf("error occurred during election: %v", err) + } + } +} + +// tickHeartbeat is run by leaders to send a MsgBeat after r.heartbeatTimeout. +func (r *raft) tickHeartbeat() { + r.heartbeatElapsed++ + r.electionElapsed++ + + if r.electionElapsed >= r.electionTimeout { + r.electionElapsed = 0 + if r.checkQuorum { + if err := r.Step(&pb.Message{From: new(r.id), Type: pb.MsgCheckQuorum.Enum()}); err != nil { + r.logger.Debugf("error occurred during checking sending heartbeat: %v", err) + } + } + // If current leader cannot transfer leadership in electionTimeout, it becomes leader again. + if r.state == StateLeader && r.leadTransferee != None { + r.abortLeaderTransfer() + } + } + + if r.state != StateLeader { + return + } + + if r.heartbeatElapsed >= r.heartbeatTimeout { + r.heartbeatElapsed = 0 + if err := r.Step(&pb.Message{From: new(r.id), Type: pb.MsgBeat.Enum()}); err != nil { + r.logger.Debugf("error occurred during checking sending heartbeat: %v", err) + } + } +} + +func (r *raft) becomeFollower(term uint64, lead uint64) { + r.step = stepFollower + r.reset(term) + r.tick = r.tickElection + r.lead = lead + r.state = StateFollower + r.logger.Infof("%x became follower at term %d", r.id, r.Term) + + traceBecomeFollower(r) +} + +func (r *raft) becomeCandidate() { + // TODO(xiangli) remove the panic when the raft implementation is stable + if r.state == StateLeader { + panic("invalid transition [leader -> candidate]") + } + r.step = stepCandidate + r.reset(r.Term + 1) + r.tick = r.tickElection + r.Vote = r.id + r.state = StateCandidate + r.logger.Infof("%x became candidate at term %d", r.id, r.Term) + + traceBecomeCandidate(r) +} + +func (r *raft) becomePreCandidate() { + // TODO(xiangli) remove the panic when the raft implementation is stable + if r.state == StateLeader { + panic("invalid transition [leader -> pre-candidate]") + } + // Becoming a pre-candidate changes our step functions and state, + // but doesn't change anything else. In particular it does not increase + // r.Term or change r.Vote. + r.step = stepCandidate + r.trk.ResetVotes() + r.tick = r.tickElection + r.lead = None + r.state = StatePreCandidate + r.logger.Infof("%x became pre-candidate at term %d", r.id, r.Term) +} + +func (r *raft) becomeLeader() { + // TODO(xiangli) remove the panic when the raft implementation is stable + if r.state == StateFollower { + panic("invalid transition [follower -> leader]") + } + r.step = stepLeader + r.reset(r.Term) + r.tick = r.tickHeartbeat + r.lead = r.id + r.state = StateLeader + // Followers enter replicate mode when they've been successfully probed + // (perhaps after having received a snapshot as a result). The leader is + // trivially in this state. Note that r.reset() has initialized this + // progress with the last index already. + pr := r.trk.Progress[r.id] + pr.BecomeReplicate() + // The leader always has RecentActive == true; MsgCheckQuorum makes sure to + // preserve this. + pr.RecentActive = true + + // Conservatively set the pendingConfIndex to the last index in the + // log. There may or may not be a pending config change, but it's + // safe to delay any future proposals until we commit all our + // pending log entries, and scanning the entire tail of the log + // could be expensive. + r.pendingConfIndex = r.raftLog.lastIndex() + + traceBecomeLeader(r) + emptyEnt := &pb.Entry{Data: nil} + if !r.appendEntry(emptyEnt) { + // This won't happen because we just called reset() above. + r.logger.Panic("empty entry was dropped") + } + // The payloadSize of an empty entry is 0 (see TestPayloadSizeOfEmptyEntry), + // so the preceding log append does not count against the uncommitted log + // quota of the new leader. In other words, after the call to appendEntry, + // r.uncommittedSize is still 0. + r.logger.Infof("%x became leader at term %d", r.id, r.Term) +} + +func (r *raft) hup(t CampaignType) { + if r.state == StateLeader { + r.logger.Debugf("%x ignoring MsgHup because already leader", r.id) + return + } + + if !r.promotable() { + r.logger.Warningf("%x is unpromotable and can not campaign", r.id) + return + } + if r.hasUnappliedConfChanges() { + r.logger.Warningf("%x cannot campaign at term %d since there are still pending configuration changes to apply", r.id, r.Term) + return + } + + r.logger.Infof("%x is starting a new election at term %d", r.id, r.Term) + r.campaign(t) +} + +// errBreak is a sentinel error used to break a callback-based loop. +var errBreak = errors.New("break") + +func (r *raft) hasUnappliedConfChanges() bool { + if r.raftLog.applied >= r.raftLog.committed { // in fact applied == committed + return false + } + found := false + // Scan all unapplied committed entries to find a config change. Paginate the + // scan, to avoid a potentially unlimited memory spike. + lo, hi := r.raftLog.applied+1, r.raftLog.committed+1 + // Reuse the maxApplyingEntsSize limit because it is used for similar purposes + // (limiting the read of unapplied committed entries) when raft sends entries + // via the Ready struct for application. + // TODO(pavelkalinnikov): find a way to budget memory/bandwidth for this scan + // outside the raft package. + pageSize := r.raftLog.maxApplyingEntsSize + if err := r.raftLog.scan(lo, hi, pageSize, func(ents []*pb.Entry) error { + for i := range ents { + if ents[i].GetType() == pb.EntryConfChange || ents[i].GetType() == pb.EntryConfChangeV2 { + found = true + return errBreak + } + } + return nil + }); err != nil && err != errBreak { + r.logger.Panicf("error scanning unapplied entries [%d, %d): %v", lo, hi, err) + } + return found +} + +// campaign transitions the raft instance to candidate state. This must only be +// called after verifying that this is a legitimate transition. +func (r *raft) campaign(t CampaignType) { + if !r.promotable() { + // This path should not be hit (callers are supposed to check), but + // better safe than sorry. + r.logger.Warningf("%x is unpromotable; campaign() should have been called", r.id) + } + var term uint64 + var voteMsg pb.MessageType + if t == campaignPreElection { + r.becomePreCandidate() + voteMsg = pb.MsgPreVote + // PreVote RPCs are sent for the next term before we've incremented r.Term. + term = r.Term + 1 + } else { + r.becomeCandidate() + voteMsg = pb.MsgVote + term = r.Term + } + var ids []uint64 + { + idMap := r.trk.Voters.IDs() + ids = make([]uint64, 0, len(idMap)) + for id := range idMap { + ids = append(ids, id) + } + slices.Sort(ids) + } + for _, id := range ids { + if id == r.id { + // The candidate votes for itself and should account for this self + // vote once the vote has been durably persisted (since it doesn't + // send a MsgVote to itself). This response message will be added to + // msgsAfterAppend and delivered back to this node after the vote + // has been written to stable storage. + r.send(&pb.Message{To: new(id), Term: new(term), Type: voteRespMsgType(voteMsg).Enum()}) + continue + } + // TODO(pav-kv): it should be ok to simply print %+v for the lastEntryID. + last := r.raftLog.lastEntryID() + r.logger.Infof("%x [logterm: %d, index: %d] sent %s request to %x at term %d", + r.id, last.term, last.index, voteMsg, id, r.Term) + + var ctx []byte + if t == campaignTransfer { + ctx = []byte(t) + } + r.send(&pb.Message{To: new(id), Term: new(term), Type: voteMsg.Enum(), Index: new(last.index), LogTerm: new(last.term), Context: ctx}) + } +} + +func (r *raft) poll(id uint64, t pb.MessageType, v bool) (granted int, rejected int, result quorum.VoteResult) { + if v { + r.logger.Infof("%x received %s from %x at term %d", r.id, t, id, r.Term) + } else { + r.logger.Infof("%x received %s rejection from %x at term %d", r.id, t, id, r.Term) + } + r.trk.RecordVote(id, v) + return r.trk.TallyVotes() +} + +// Step advances the state machine using the given message. +// +// Callers must treat m as immutable after passing it to Step. Mutating it +// concurrently can lead to unexpected behavior. +func (r *raft) Step(m *pb.Message) error { + // m should never be nil + if m == nil { + return errors.New("nil message") + } + traceReceiveMessage(r, m) + + // Handle the message term, which may result in our stepping down to a follower. + switch { + case m.GetTerm() == 0: + // local message + case m.GetTerm() > r.Term: + if m.GetType() == pb.MsgVote || m.GetType() == pb.MsgPreVote { + force := bytes.Equal(m.GetContext(), []byte(campaignTransfer)) + inLease := r.checkQuorum && r.lead != None && r.electionElapsed < r.electionTimeout + if !force && inLease { + // If a server receives a RequestVote request within the minimum election timeout + // of hearing from a current leader, it does not update its term or grant its vote + last := r.raftLog.lastEntryID() + // TODO(pav-kv): it should be ok to simply print the %+v of the lastEntryID. + r.logger.Infof("%x [logterm: %d, index: %d, vote: %x] ignored %s from %x [logterm: %d, index: %d] at term %d: lease is not expired (remaining ticks: %d)", + r.id, last.term, last.index, r.Vote, m.GetType(), m.GetFrom(), m.GetLogTerm(), m.GetIndex(), r.Term, r.electionTimeout-r.electionElapsed) + return nil + } + } + switch { + case m.GetType() == pb.MsgPreVote: + // Never change our term in response to a PreVote + case m.GetType() == pb.MsgPreVoteResp && !m.GetReject(): + // We send pre-vote requests with a term in our future. If the + // pre-vote is granted, we will increment our term when we get a + // quorum. If it is not, the term comes from the node that + // rejected our vote so we should become a follower at the new + // term. + default: + r.logger.Infof("%x [term: %d] received a %s message with higher term from %x [term: %d]", + r.id, r.Term, m.GetType(), m.GetFrom(), m.GetTerm()) + if m.GetType() == pb.MsgApp || m.GetType() == pb.MsgHeartbeat || m.GetType() == pb.MsgSnap { + r.becomeFollower(m.GetTerm(), m.GetFrom()) + } else { + r.becomeFollower(m.GetTerm(), None) + } + } + + case m.GetTerm() < r.Term: + if (r.checkQuorum || r.preVote) && (m.GetType() == pb.MsgHeartbeat || m.GetType() == pb.MsgApp) { + // We have received messages from a leader at a lower term. It is possible + // that these messages were simply delayed in the network, but this could + // also mean that this node has advanced its term number during a network + // partition, and it is now unable to either win an election or to rejoin + // the majority on the old term. If checkQuorum is false, this will be + // handled by incrementing term numbers in response to MsgVote with a + // higher term, but if checkQuorum is true we may not advance the term on + // MsgVote and must generate other messages to advance the term. The net + // result of these two features is to minimize the disruption caused by + // nodes that have been removed from the cluster's configuration: a + // removed node will send MsgVotes (or MsgPreVotes) which will be ignored, + // but it will not receive MsgApp or MsgHeartbeat, so it will not create + // disruptive term increases, by notifying leader of this node's activeness. + // The above comments also true for Pre-Vote + // + // When follower gets isolated, it soon starts an election ending + // up with a higher term than leader, although it won't receive enough + // votes to win the election. When it regains connectivity, this response + // with "pb.MsgAppResp" of higher term would force leader to step down. + // However, this disruption is inevitable to free this stuck node with + // fresh election. This can be prevented with Pre-Vote phase. + r.send(&pb.Message{To: m.From, Type: pb.MsgAppResp.Enum()}) + } else if m.GetType() == pb.MsgPreVote { + // Before Pre-Vote enable, there may have candidate with higher term, + // but less log. After update to Pre-Vote, the cluster may deadlock if + // we drop messages with a lower term. + last := r.raftLog.lastEntryID() + // TODO(pav-kv): it should be ok to simply print %+v of the lastEntryID. + r.logger.Infof("%x [logterm: %d, index: %d, vote: %x] rejected %s from %x [logterm: %d, index: %d] at term %d", + r.id, last.term, last.index, r.Vote, m.GetType(), m.GetFrom(), m.GetLogTerm(), m.GetIndex(), r.Term) + r.send(&pb.Message{To: m.From, Term: new(r.Term), Type: pb.MsgPreVoteResp.Enum(), Reject: new(true)}) + } else if m.GetType() == pb.MsgStorageAppendResp { + if m.GetIndex() != 0 { + // Don't consider the appended log entries to be stable because + // they may have been overwritten in the unstable log during a + // later term. See the comment in newStorageAppendResp for more + // about this race. + r.logger.Infof("%x [term: %d] ignored entry appends from a %s message with lower term [term: %d]", + r.id, r.Term, m.GetType(), m.GetTerm()) + } + if m.GetSnapshot() != nil { + // Even if the snapshot applied under a different term, its + // application is still valid. Snapshots carry committed + // (term-independent) state. + r.appliedSnap(m.GetSnapshot()) + } + } else { + // ignore other cases + r.logger.Infof("%x [term: %d] ignored a %s message with lower term from %x [term: %d]", + r.id, r.Term, m.GetType(), m.GetFrom(), m.GetTerm()) + } + return nil + } + + switch m.GetType() { + case pb.MsgHup: + if r.preVote { + r.hup(campaignPreElection) + } else { + r.hup(campaignElection) + } + + case pb.MsgStorageAppendResp: + if m.GetIndex() != 0 { + r.raftLog.stableTo(entryID{term: m.GetLogTerm(), index: m.GetIndex()}) + } + if m.GetSnapshot() != nil { + r.appliedSnap(m.GetSnapshot()) + } + + case pb.MsgStorageApplyResp: + if len(m.GetEntries()) > 0 { + index := m.GetEntries()[len(m.GetEntries())-1].GetIndex() + r.appliedTo(index, entsSize(m.GetEntries())) + r.reduceUncommittedSize(payloadsSize(m.GetEntries())) + } + + case pb.MsgVote, pb.MsgPreVote: + // We can vote if this is a repeat of a vote we've already cast... + canVote := r.Vote == m.GetFrom() || + // ...we haven't voted and we don't think there's a leader yet in this term... + (r.Vote == None && r.lead == None) || + // ...or this is a PreVote for a future term... + (m.GetType() == pb.MsgPreVote && m.GetTerm() > r.Term) + // ...and we believe the candidate is up to date. + lastID := r.raftLog.lastEntryID() + candLastID := entryID{term: m.GetLogTerm(), index: m.GetIndex()} + if canVote && r.raftLog.isUpToDate(candLastID) { + // Note: it turns out that that learners must be allowed to cast votes. + // This seems counter- intuitive but is necessary in the situation in which + // a learner has been promoted (i.e. is now a voter) but has not learned + // about this yet. + // For example, consider a group in which id=1 is a learner and id=2 and + // id=3 are voters. A configuration change promoting 1 can be committed on + // the quorum `{2,3}` without the config change being appended to the + // learner's log. If the leader (say 2) fails, there are de facto two + // voters remaining. Only 3 can win an election (due to its log containing + // all committed entries), but to do so it will need 1 to vote. But 1 + // considers itself a learner and will continue to do so until 3 has + // stepped up as leader, replicates the conf change to 1, and 1 applies it. + // Ultimately, by receiving a request to vote, the learner realizes that + // the candidate believes it to be a voter, and that it should act + // accordingly. The candidate's config may be stale, too; but in that case + // it won't win the election, at least in the absence of the bug discussed + // in: + // https://github.com/etcd-io/etcd/issues/7625#issuecomment-488798263. + r.logger.Infof("%x [logterm: %d, index: %d, vote: %x] cast %s for %x [logterm: %d, index: %d] at term %d", + r.id, lastID.term, lastID.index, r.Vote, m.GetType(), m.GetFrom(), candLastID.term, candLastID.index, r.Term) + // When responding to Msg{Pre,}Vote messages we include the term + // from the message, not the local term. To see why, consider the + // case where a single node was previously partitioned away and + // it's local term is now out of date. If we include the local term + // (recall that for pre-votes we don't update the local term), the + // (pre-)campaigning node on the other end will proceed to ignore + // the message (it ignores all out of date messages). + // The term in the original message and current local term are the + // same in the case of regular votes, but different for pre-votes. + r.send(&pb.Message{To: m.From, Term: m.Term, Type: voteRespMsgType(m.GetType()).Enum()}) + if m.GetType() == pb.MsgVote { + // Only record real votes. + r.electionElapsed = 0 + r.Vote = m.GetFrom() + } + } else { + r.logger.Infof("%x [logterm: %d, index: %d, vote: %x] rejected %s from %x [logterm: %d, index: %d] at term %d", + r.id, lastID.term, lastID.index, r.Vote, m.GetType(), m.GetFrom(), candLastID.term, candLastID.index, r.Term) + r.send(&pb.Message{To: m.From, Term: new(r.Term), Type: voteRespMsgType(m.GetType()).Enum(), Reject: new(true)}) + } + + default: + err := r.step(r, m) + if err != nil { + return err + } + } + return nil +} + +type stepFunc func(r *raft, m *pb.Message) error + +func stepLeader(r *raft, m *pb.Message) error { + // These message types do not require any progress for m.From. + switch m.GetType() { + case pb.MsgBeat: + r.bcastHeartbeat() + return nil + case pb.MsgCheckQuorum: + if !r.trk.QuorumActive() { + r.logger.Warningf("%x stepped down to follower since quorum is not active", r.id) + r.becomeFollower(r.Term, None) + } + // Mark everyone (but ourselves) as inactive in preparation for the next + // CheckQuorum. + r.trk.Visit(func(id uint64, pr *tracker.Progress) { + if id != r.id { + pr.RecentActive = false + } + }) + return nil + case pb.MsgProp: + if len(m.GetEntries()) == 0 { + r.logger.Panicf("%x stepped empty MsgProp", r.id) + } + if r.trk.Progress[r.id] == nil { + // If we are not currently a member of the range (i.e. this node + // was removed from the configuration while serving as leader), + // drop any new proposals. + return ErrProposalDropped + } + if r.leadTransferee != None { + r.logger.Debugf("%x [term %d] transfer leadership to %x is in progress; dropping proposal", r.id, r.Term, r.leadTransferee) + return ErrProposalDropped + } + + for i := range m.GetEntries() { + e := m.GetEntries()[i] + var cc pb.ConfChangeI + if e.GetType() == pb.EntryConfChange { + ccc := &pb.ConfChange{} + if err := proto.Unmarshal(e.GetData(), ccc); err != nil { + panic(err) + } + cc = ccc + } else if e.GetType() == pb.EntryConfChangeV2 { + ccc := &pb.ConfChangeV2{} + if err := proto.Unmarshal(e.GetData(), ccc); err != nil { + panic(err) + } + cc = ccc + } + if cc != nil { + alreadyPending := r.pendingConfIndex > r.raftLog.applied + alreadyJoint := len(r.trk.Config.Voters[1]) > 0 + wantsLeaveJoint := len(cc.AsV2().Changes) == 0 + + var failedCheck string + if alreadyPending { + failedCheck = fmt.Sprintf("possible unapplied conf change at index %d (applied to %d)", r.pendingConfIndex, r.raftLog.applied) + } else if alreadyJoint && !wantsLeaveJoint { + failedCheck = "must transition out of joint config first" + } else if !alreadyJoint && wantsLeaveJoint { + failedCheck = "not in joint state; refusing empty conf change" + } + + if failedCheck != "" && !r.disableConfChangeValidation { + r.logger.Infof("%x ignoring conf change %s at config %s: %s", r.id, DescribeConfChange(cc), r.trk.Config, failedCheck) + m.GetEntries()[i] = &pb.Entry{Type: pb.EntryNormal.Enum()} + } else { + r.pendingConfIndex = r.raftLog.lastIndex() + uint64(i) + 1 + traceChangeConfEvent(cc, r) + } + } + } + + if !r.appendEntry(m.GetEntries()...) { + return ErrProposalDropped + } + r.bcastAppend() + return nil + case pb.MsgReadIndex: + // only one voting member (the leader) in the cluster + if r.trk.IsSingleton() { + if resp := r.responseToReadIndexReq(m, r.raftLog.committed); resp.GetTo() != None { + r.send(resp) + } + return nil + } + + // Postpone read only request when this leader has not committed + // any log entry at its term. + if !r.committedEntryInCurrentTerm() { + r.pendingReadIndexMessages = append(r.pendingReadIndexMessages, m) + return nil + } + + sendMsgReadIndexResponse(r, m) + + return nil + case pb.MsgForgetLeader: + return nil // noop on leader + } + + // All other message types require a progress for m.From (pr). + pr := r.trk.Progress[m.GetFrom()] + if pr == nil { + r.logger.Debugf("%x no progress available for %x", r.id, m.GetFrom()) + return nil + } + switch m.GetType() { + case pb.MsgAppResp: + // NB: this code path is also hit from (*raft).advance, where the leader steps + // an MsgAppResp to acknowledge the appended entries in the last Ready. + + pr.RecentActive = true + + if m.GetReject() { + // RejectHint is the suggested next base entry for appending (i.e. + // we try to append entry RejectHint+1 next), and LogTerm is the + // term that the follower has at index RejectHint. Older versions + // of this library did not populate LogTerm for rejections and it + // is zero for followers with an empty log. + // + // Under normal circumstances, the leader's log is longer than the + // follower's and the follower's log is a prefix of the leader's + // (i.e. there is no divergent uncommitted suffix of the log on the + // follower). In that case, the first probe reveals where the + // follower's log ends (RejectHint=follower's last index) and the + // subsequent probe succeeds. + // + // However, when networks are partitioned or systems overloaded, + // large divergent log tails can occur. The naive attempt, probing + // entry by entry in decreasing order, will be the product of the + // length of the diverging tails and the network round-trip latency, + // which can easily result in hours of time spent probing and can + // even cause outright outages. The probes are thus optimized as + // described below. + r.logger.Debugf("%x received MsgAppResp(rejected, hint: (index %d, term %d)) from %x for index %d", + r.id, m.GetRejectHint(), m.GetLogTerm(), m.GetFrom(), m.GetIndex()) + nextProbeIdx := m.GetRejectHint() + if m.GetLogTerm() > 0 { + // If the follower has an uncommitted log tail, we would end up + // probing one by one until we hit the common prefix. + // + // For example, if the leader has: + // + // idx 1 2 3 4 5 6 7 8 9 + // ----------------- + // term (L) 1 3 3 3 5 5 5 5 5 + // term (F) 1 1 1 1 2 2 + // + // Then, after sending an append anchored at (idx=9,term=5) we + // would receive a RejectHint of 6 and LogTerm of 2. Without the + // code below, we would try an append at index 6, which would + // fail again. + // + // However, looking only at what the leader knows about its own + // log and the rejection hint, it is clear that a probe at index + // 6, 5, 4, 3, and 2 must fail as well: + // + // For all of these indexes, the leader's log term is larger than + // the rejection's log term. If a probe at one of these indexes + // succeeded, its log term at that index would match the leader's, + // i.e. 3 or 5 in this example. But the follower already told the + // leader that it is still at term 2 at index 6, and since the + // log term only ever goes up (within a log), this is a contradiction. + // + // At index 1, however, the leader can draw no such conclusion, + // as its term 1 is not larger than the term 2 from the + // follower's rejection. We thus probe at 1, which will succeed + // in this example. In general, with this approach we probe at + // most once per term found in the leader's log. + // + // There is a similar mechanism on the follower (implemented in + // handleAppendEntries via a call to findConflictByTerm) that is + // useful if the follower has a large divergent uncommitted log + // tail[1], as in this example: + // + // idx 1 2 3 4 5 6 7 8 9 + // ----------------- + // term (L) 1 3 3 3 3 3 3 3 7 + // term (F) 1 3 3 4 4 5 5 5 6 + // + // Naively, the leader would probe at idx=9, receive a rejection + // revealing the log term of 6 at the follower. Since the leader's + // term at the previous index is already smaller than 6, the leader- + // side optimization discussed above is ineffective. The leader thus + // probes at index 8 and, naively, receives a rejection for the same + // index and log term 5. Again, the leader optimization does not improve + // over linear probing as term 5 is above the leader's term 3 for that + // and many preceding indexes; the leader would have to probe linearly + // until it would finally hit index 3, where the probe would succeed. + // + // Instead, we apply a similar optimization on the follower. When the + // follower receives the probe at index 8 (log term 3), it concludes + // that all of the leader's log preceding that index has log terms of + // 3 or below. The largest index in the follower's log with a log term + // of 3 or below is index 3. The follower will thus return a rejection + // for index=3, log term=3 instead. The leader's next probe will then + // succeed at that index. + // + // [1]: more precisely, if the log terms in the large uncommitted + // tail on the follower are larger than the leader's. At first, + // it may seem unintuitive that a follower could even have such + // a large tail, but it can happen: + // + // 1. Leader appends (but does not commit) entries 2 and 3, crashes. + // idx 1 2 3 4 5 6 7 8 9 + // ----------------- + // term (L) 1 2 2 [crashes] + // term (F) 1 + // term (F) 1 + // + // 2. a follower becomes leader and appends entries at term 3. + // ----------------- + // term (x) 1 2 2 [down] + // term (F) 1 3 3 3 3 + // term (F) 1 + // + // 3. term 3 leader goes down, term 2 leader returns as term 4 + // leader. It commits the log & entries at term 4. + // + // ----------------- + // term (L) 1 2 2 2 + // term (x) 1 3 3 3 3 [down] + // term (F) 1 + // ----------------- + // term (L) 1 2 2 2 4 4 4 + // term (F) 1 3 3 3 3 [gets probed] + // term (F) 1 2 2 2 4 4 4 + // + // 4. the leader will now probe the returning follower at index + // 7, the rejection points it at the end of the follower's log + // which is at a higher log term than the actually committed + // log. + nextProbeIdx, _ = r.raftLog.findConflictByTerm(m.GetRejectHint(), m.GetLogTerm()) + } + if pr.MaybeDecrTo(m.GetIndex(), nextProbeIdx) { + r.logger.Debugf("%x decreased progress of %x to [%s]", r.id, m.GetFrom(), pr) + if pr.State == tracker.StateReplicate { + pr.BecomeProbe() + } + r.sendAppend(m.GetFrom()) + } + } else { + // We want to update our tracking if the response updates our + // matched index or if the response can move a probing peer back + // into StateReplicate (see heartbeat_rep_recovers_from_probing.txt + // for an example of the latter case). + // NB: the same does not make sense for StateSnapshot - if `m.Index` + // equals pr.Match we know we don't m.Index+1 in our log, so moving + // back to replicating state is not useful; besides pr.PendingSnapshot + // would prevent it. + if pr.MaybeUpdate(m.GetIndex()) || (pr.Match == m.GetIndex() && pr.State == tracker.StateProbe) { + switch { + case pr.State == tracker.StateProbe: + pr.BecomeReplicate() + case pr.State == tracker.StateSnapshot && pr.Match+1 >= r.raftLog.firstIndex(): + // Note that we don't take into account PendingSnapshot to + // enter this branch. No matter at which index a snapshot + // was actually applied, as long as this allows catching up + // the follower from the log, we will accept it. This gives + // systems more flexibility in how they implement snapshots; + // see the comments on PendingSnapshot. + r.logger.Debugf("%x recovered from needing snapshot, resumed sending replication messages to %x [%s]", r.id, m.GetFrom(), pr) + // Transition back to replicating state via probing state + // (which takes the snapshot into account). If we didn't + // move to replicating state, that would only happen with + // the next round of appends (but there may not be a next + // round for a while, exposing an inconsistent RaftStatus). + pr.BecomeProbe() + pr.BecomeReplicate() + case pr.State == tracker.StateReplicate: + pr.Inflights.FreeLE(m.GetIndex()) + } + + if r.maybeCommit() { + // committed index has progressed for the term, so it is safe + // to respond to pending read index requests + releasePendingReadIndexMessages(r) + r.bcastAppend() + } else if r.id != m.GetFrom() && pr.CanBumpCommit(r.raftLog.committed) { + // This node may be missing the latest commit index, so send it. + // NB: this is not strictly necessary because the periodic heartbeat + // messages deliver commit indices too. However, a message sent now + // may arrive earlier than the next heartbeat fires. + r.sendAppend(m.GetFrom()) + } + // We've updated flow control information above, which may + // allow us to send multiple (size-limited) in-flight messages + // at once (such as when transitioning from probe to + // replicate, or when freeTo() covers multiple messages). If + // we have more entries to send, send as many messages as we + // can (without sending empty messages for the commit index) + if r.id != m.GetFrom() { + for r.maybeSendAppend(m.GetFrom(), false /* sendIfEmpty */) { + } + } + // Transfer leadership is in progress. + if m.GetFrom() == r.leadTransferee && pr.Match == r.raftLog.lastIndex() { + r.logger.Infof("%x sent MsgTimeoutNow to %x after received MsgAppResp", r.id, m.GetFrom()) + r.sendTimeoutNow(m.GetFrom()) + } + } + } + case pb.MsgHeartbeatResp: + pr.RecentActive = true + pr.MsgAppFlowPaused = false + + // NB: if the follower is paused (full Inflights), this will still send an + // empty append, allowing it to recover from situations in which all the + // messages that filled up Inflights in the first place were dropped. Note + // also that the outgoing heartbeat already communicated the commit index. + // + // If the follower is fully caught up but also in StateProbe (as can happen + // if ReportUnreachable was called), we also want to send an append (it will + // be empty) to allow the follower to transition back to StateReplicate once + // it responds. + // + // Note that StateSnapshot typically satisfies pr.Match < lastIndex, but + // `pr.Paused()` is always true for StateSnapshot, so sendAppend is a + // no-op. + if pr.Match < r.raftLog.lastIndex() || pr.State == tracker.StateProbe { + r.sendAppend(m.GetFrom()) + } + + if r.readOnly.option != ReadOnlySafe || len(m.GetContext()) == 0 { + return nil + } + + r.readOnly.recvAck(m.GetFrom(), m.GetContext()) + rss := r.readOnly.maybeAdvance(r.trk.Voters) + for _, rs := range rss { + if resp := r.responseToReadIndexReq(rs.req, rs.index); resp.GetTo() != None { + r.send(resp) + } + } + case pb.MsgSnapStatus: + if pr.State != tracker.StateSnapshot { + return nil + } + if !m.GetReject() { + pr.BecomeProbe() + r.logger.Debugf("%x snapshot succeeded, resumed sending replication messages to %x [%s]", r.id, m.GetFrom(), pr) + } else { + // NB: the order here matters or we'll be probing erroneously from + // the snapshot index, but the snapshot never applied. + pr.PendingSnapshot = 0 + pr.BecomeProbe() + r.logger.Debugf("%x snapshot failed, resumed sending replication messages to %x [%s]", r.id, m.GetFrom(), pr) + } + // If snapshot finish, wait for the MsgAppResp from the remote node before sending + // out the next MsgApp. + // If snapshot failure, wait for a heartbeat interval before next try + pr.MsgAppFlowPaused = true + case pb.MsgUnreachable: + // During optimistic replication, if the remote becomes unreachable, + // there is huge probability that a MsgApp is lost. + if pr.State == tracker.StateReplicate { + pr.BecomeProbe() + } + r.logger.Debugf("%x failed to send message to %x because it is unreachable [%s]", r.id, m.GetFrom(), pr) + case pb.MsgTransferLeader: + if pr.IsLearner { + r.logger.Debugf("%x is learner. Ignored transferring leadership", r.id) + return nil + } + leadTransferee := m.GetFrom() + lastLeadTransferee := r.leadTransferee + if lastLeadTransferee != None { + if lastLeadTransferee == leadTransferee { + r.logger.Infof("%x [term %d] transfer leadership to %x is in progress, ignores request to same node %x", + r.id, r.Term, leadTransferee, leadTransferee) + return nil + } + r.abortLeaderTransfer() + r.logger.Infof("%x [term %d] abort previous transferring leadership to %x", r.id, r.Term, lastLeadTransferee) + } + if leadTransferee == r.id { + r.logger.Debugf("%x is already leader. Ignored transferring leadership to self", r.id) + return nil + } + // Transfer leadership to third party. + r.logger.Infof("%x [term %d] starts to transfer leadership to %x", r.id, r.Term, leadTransferee) + // Transfer leadership should be finished in one electionTimeout, so reset r.electionElapsed. + r.electionElapsed = 0 + r.leadTransferee = leadTransferee + if pr.Match == r.raftLog.lastIndex() { + r.sendTimeoutNow(leadTransferee) + r.logger.Infof("%x sends MsgTimeoutNow to %x immediately as %x already has up-to-date log", r.id, leadTransferee, leadTransferee) + } else { + r.sendAppend(leadTransferee) + } + } + return nil +} + +// stepCandidate is shared by StateCandidate and StatePreCandidate; the difference is +// whether they respond to MsgVoteResp or MsgPreVoteResp. +func stepCandidate(r *raft, m *pb.Message) error { + // Only handle vote responses corresponding to our candidacy (while in + // StateCandidate, we may get stale MsgPreVoteResp messages in this term from + // our pre-candidate state). + var myVoteRespType pb.MessageType + if r.state == StatePreCandidate { + myVoteRespType = pb.MsgPreVoteResp + } else { + myVoteRespType = pb.MsgVoteResp + } + switch m.GetType() { + case pb.MsgProp: + r.logger.Infof("%x no leader at term %d; dropping proposal", r.id, r.Term) + return ErrProposalDropped + case pb.MsgApp: + r.becomeFollower(m.GetTerm(), m.GetFrom()) // always m.Term == r.Term + r.handleAppendEntries(m) + case pb.MsgHeartbeat: + r.becomeFollower(m.GetTerm(), m.GetFrom()) // always m.Term == r.Term + r.handleHeartbeat(m) + case pb.MsgSnap: + r.becomeFollower(m.GetTerm(), m.GetFrom()) // always m.Term == r.Term + r.handleSnapshot(m) + case myVoteRespType: + gr, rj, res := r.poll(m.GetFrom(), m.GetType(), !m.GetReject()) + r.logger.Infof("%x has received %d %s votes and %d vote rejections", r.id, gr, m.GetType(), rj) + switch res { + case quorum.VoteWon: + if r.state == StatePreCandidate { + r.campaign(campaignElection) + } else { + r.becomeLeader() + r.bcastAppend() + } + case quorum.VoteLost: + // pb.MsgPreVoteResp contains future term of pre-candidate + // m.Term > r.Term; reuse r.Term + r.becomeFollower(r.Term, None) + } + case pb.MsgTimeoutNow: + r.logger.Debugf("%x [term %d state %v] ignored MsgTimeoutNow from %x", r.id, r.Term, r.state, m.GetFrom()) + } + return nil +} + +func stepFollower(r *raft, m *pb.Message) error { + switch m.GetType() { + case pb.MsgProp: + if r.lead == None { + r.logger.Infof("%x no leader at term %d; dropping proposal", r.id, r.Term) + return ErrProposalDropped + } else if r.disableProposalForwarding { + r.logger.Infof("%x not forwarding to leader %x at term %d; dropping proposal", r.id, r.lead, r.Term) + return ErrProposalDropped + } + m.To = new(r.lead) + r.send(m) + case pb.MsgApp: + r.electionElapsed = 0 + r.lead = m.GetFrom() + r.handleAppendEntries(m) + case pb.MsgHeartbeat: + r.electionElapsed = 0 + r.lead = m.GetFrom() + r.handleHeartbeat(m) + case pb.MsgSnap: + r.electionElapsed = 0 + r.lead = m.GetFrom() + r.handleSnapshot(m) + case pb.MsgTransferLeader: + if r.lead == None { + r.logger.Infof("%x no leader at term %d; dropping leader transfer msg", r.id, r.Term) + return nil + } + m.To = new(r.lead) + r.send(m) + case pb.MsgForgetLeader: + if r.readOnly.option == ReadOnlyLeaseBased { + r.logger.Error("ignoring MsgForgetLeader due to ReadOnlyLeaseBased") + return nil + } + if r.lead != None { + r.logger.Infof("%x forgetting leader %x at term %d", r.id, r.lead, r.Term) + r.lead = None + } + case pb.MsgTimeoutNow: + r.logger.Infof("%x [term %d] received MsgTimeoutNow from %x and starts an election to get leadership.", r.id, r.Term, m.GetFrom()) + // Leadership transfers never use pre-vote even if r.preVote is true; we + // know we are not recovering from a partition so there is no need for the + // extra round trip. + r.hup(campaignTransfer) + case pb.MsgReadIndex: + if r.lead == None { + r.logger.Infof("%x no leader at term %d; dropping index reading msg", r.id, r.Term) + return nil + } + m.To = new(r.lead) + r.send(m) + case pb.MsgReadIndexResp: + if len(m.GetEntries()) != 1 { + r.logger.Errorf("%x invalid format of MsgReadIndexResp from %x, entries count: %d", r.id, m.GetFrom(), len(m.GetEntries())) + return nil + } + r.readStates = append(r.readStates, ReadState{Index: m.GetIndex(), RequestCtx: m.GetEntries()[0].GetData()}) + } + return nil +} + +// logSliceFromMsgApp extracts the appended logSlice from a MsgApp message. +func logSliceFromMsgApp(m *pb.Message) logSlice { + // TODO(pav-kv): consider also validating the logSlice here. + return logSlice{ + term: m.GetTerm(), + prev: entryID{term: m.GetLogTerm(), index: m.GetIndex()}, + entries: m.GetEntries(), + } +} + +func (r *raft) handleAppendEntries(m *pb.Message) { + // TODO(pav-kv): construct logSlice up the stack next to receiving the + // message, and validate it before taking any action (e.g. bumping term). + a := logSliceFromMsgApp(m) + + if a.prev.index < r.raftLog.committed { + r.send(&pb.Message{To: m.From, Type: pb.MsgAppResp.Enum(), Index: new(r.raftLog.committed)}) + return + } + if mlastIndex, ok := r.raftLog.maybeAppend(a, m.GetCommit()); ok { + r.send(&pb.Message{To: m.From, Type: pb.MsgAppResp.Enum(), Index: new(mlastIndex)}) + return + } + r.logger.Debugf("%x [logterm: %d, index: %d] rejected MsgApp [logterm: %d, index: %d] from %x", + r.id, r.raftLog.zeroTermOnOutOfBounds(r.raftLog.term(m.GetIndex())), m.GetIndex(), m.GetLogTerm(), m.GetIndex(), m.GetFrom()) + + // Our log does not match the leader's at index m.Index. Return a hint to the + // leader - a guess on the maximal (index, term) at which the logs match. Do + // this by searching through the follower's log for the maximum (index, term) + // pair with a term <= the MsgApp's LogTerm and an index <= the MsgApp's + // Index. This can help skip all indexes in the follower's uncommitted tail + // with terms greater than the MsgApp's LogTerm. + // + // See the other caller for findConflictByTerm (in stepLeader) for a much more + // detailed explanation of this mechanism. + + // NB: m.Index >= raftLog.committed by now (see the early return above), and + // raftLog.lastIndex() >= raftLog.committed by invariant, so min of the two is + // also >= raftLog.committed. Hence, the findConflictByTerm argument is within + // the valid interval, which then will return a valid (index, term) pair with + // a non-zero term (unless the log is empty). However, it is safe to send a zero + // LogTerm in this response in any case, so we don't verify it here. + hintIndex := min(m.GetIndex(), r.raftLog.lastIndex()) + hintIndex, hintTerm := r.raftLog.findConflictByTerm(hintIndex, m.GetLogTerm()) + r.send(&pb.Message{ + To: m.From, + Type: pb.MsgAppResp.Enum(), + Index: new(m.GetIndex()), + Reject: new(true), + RejectHint: new(hintIndex), + LogTerm: new(hintTerm), + }) +} + +func (r *raft) handleHeartbeat(m *pb.Message) { + r.raftLog.commitTo(m.GetCommit()) + r.send(&pb.Message{To: m.From, Type: pb.MsgHeartbeatResp.Enum(), Context: m.GetContext()}) +} + +func (r *raft) handleSnapshot(m *pb.Message) { + // MsgSnap messages should always carry a non-nil Snapshot, but err on the + // side of safety and treat a nil Snapshot as a zero-valued Snapshot. + s := m.GetSnapshot() + s = pb.EnsureSnapshot(s) + sindex, sterm := s.GetMetadata().GetIndex(), s.GetMetadata().GetTerm() + if r.restore(s) { + r.logger.Infof("%x [commit: %d] restored snapshot [index: %d, term: %d]", + r.id, r.raftLog.committed, sindex, sterm) + r.send(&pb.Message{To: m.From, Type: pb.MsgAppResp.Enum(), Index: new(r.raftLog.lastIndex())}) + } else { + r.logger.Infof("%x [commit: %d] ignored snapshot [index: %d, term: %d]", + r.id, r.raftLog.committed, sindex, sterm) + r.send(&pb.Message{To: m.From, Type: pb.MsgAppResp.Enum(), Index: new(r.raftLog.committed)}) + } +} + +// restore recovers the state machine from a snapshot. It restores the log and the +// configuration of state machine. If this method returns false, the snapshot was +// ignored, either because it was obsolete or because of an error. +func (r *raft) restore(s *pb.Snapshot) bool { + if s.GetMetadata().GetIndex() <= r.raftLog.committed { + return false + } + if r.state != StateFollower { + // This is defense-in-depth: if the leader somehow ended up applying a + // snapshot, it could move into a new term without moving into a + // follower state. This should never fire, but if it did, we'd have + // prevented damage by returning early, so log only a loud warning. + // + // At the time of writing, the instance is guaranteed to be in follower + // state when this method is called. + r.logger.Warningf("%x attempted to restore snapshot as leader; should never happen", r.id) + r.becomeFollower(r.Term+1, None) + return false + } + + // More defense-in-depth: throw away snapshot if recipient is not in the + // config. This shouldn't ever happen (at the time of writing) but lots of + // code here and there assumes that r.id is in the progress tracker. + found := false + cs := s.GetMetadata().GetConfState() + cs = pb.EnsureConfState(cs) + + for _, set := range [][]uint64{ + cs.Voters, + cs.Learners, + cs.VotersOutgoing, + // `LearnersNext` doesn't need to be checked. According to the rules, if a peer in + // `LearnersNext`, it has to be in `VotersOutgoing`. + } { + for _, id := range set { + if id == r.id { + found = true + break + } + } + if found { + break + } + } + if !found { + r.logger.Warningf( + "%x attempted to restore snapshot but it is not in the ConfState %v; should never happen", + r.id, cs, + ) + return false + } + + // Now go ahead and actually restore. + + id := entryID{term: s.GetMetadata().GetTerm(), index: s.GetMetadata().GetIndex()} + if r.raftLog.matchTerm(id) { + // TODO(pav-kv): can print %+v of the id, but it will change the format. + last := r.raftLog.lastEntryID() + r.logger.Infof("%x [commit: %d, lastindex: %d, lastterm: %d] fast-forwarded commit to snapshot [index: %d, term: %d]", + r.id, r.raftLog.committed, last.index, last.term, id.index, id.term) + r.raftLog.commitTo(s.GetMetadata().GetIndex()) + return false + } + + r.raftLog.restore(s) + + // Reset the configuration and add the (potentially updated) peers in anew. + r.trk = tracker.MakeProgressTracker(r.trk.MaxInflight, r.trk.MaxInflightBytes) + cfg, trk, err := confchange.Restore(confchange.Changer{ + Tracker: r.trk, + LastIndex: r.raftLog.lastIndex(), + }, cs) + + if err != nil { + // This should never happen. Either there's a bug in our config change + // handling or the client corrupted the conf change. + panic(fmt.Sprintf("unable to restore config %+v: %s", cs, err)) + } + + assertConfStatesEquivalent(r.logger, cs, r.switchToConfig(cfg, trk)) + + last := r.raftLog.lastEntryID() + r.logger.Infof("%x [commit: %d, lastindex: %d, lastterm: %d] restored snapshot [index: %d, term: %d]", + r.id, r.raftLog.committed, last.index, last.term, id.index, id.term) + return true +} + +// promotable indicates whether state machine can be promoted to leader, +// which is true when its own id is in progress list. +func (r *raft) promotable() bool { + pr := r.trk.Progress[r.id] + return pr != nil && !pr.IsLearner && !r.raftLog.hasNextOrInProgressSnapshot() +} + +func (r *raft) applyConfChange(cc *pb.ConfChangeV2) *pb.ConfState { + cfg, trk, err := func() (tracker.Config, tracker.ProgressMap, error) { + changer := confchange.Changer{ + Tracker: r.trk, + LastIndex: r.raftLog.lastIndex(), + } + if cc.LeaveJoint() { + return changer.LeaveJoint() + } else if autoLeave, ok := cc.EnterJoint(); ok { + return changer.EnterJoint(autoLeave, cc.Changes...) + } + return changer.Simple(cc.Changes...) + }() + + if err != nil { + // TODO(tbg): return the error to the caller. + panic(err) + } + + return r.switchToConfig(cfg, trk) +} + +// switchToConfig reconfigures this node to use the provided configuration. It +// updates the in-memory state and, when necessary, carries out additional +// actions such as reacting to the removal of nodes or changed quorum +// requirements. +// +// The inputs usually result from restoring a ConfState or applying a ConfChange. +func (r *raft) switchToConfig(cfg tracker.Config, trk tracker.ProgressMap) *pb.ConfState { + traceConfChangeEvent(cfg, r) + + r.trk.Config = cfg + r.trk.Progress = trk + + r.logger.Infof("%x switched to configuration %s", r.id, r.trk.Config) + cs := r.trk.ConfState() + pr, ok := r.trk.Progress[r.id] + + // Update whether the node itself is a learner, resetting to false when the + // node is removed. + r.isLearner = ok && pr.IsLearner + + if (!ok || r.isLearner) && r.state == StateLeader { + // This node is leader and was removed or demoted, step down if requested. + // + // We prevent demotions at the time writing but hypothetically we handle + // them the same way as removing the leader. + // + // TODO(tbg): ask follower with largest Match to TimeoutNow (to avoid + // interruption). This might still drop some proposals but it's better than + // nothing. + if r.stepDownOnRemoval { + r.becomeFollower(r.Term, None) + } + return cs + } + + // The remaining steps only make sense if this node is the leader and there + // are other nodes. + if r.state != StateLeader || len(cs.Voters) == 0 { + return cs + } + + if r.maybeCommit() { + // If the configuration change means that more entries are committed now, + // broadcast/append to everyone in the updated config. + r.bcastAppend() + } else { + // Otherwise, still probe the newly added replicas; there's no reason to + // let them wait out a heartbeat interval (or the next incoming + // proposal). + r.trk.Visit(func(id uint64, _ *tracker.Progress) { + if id == r.id { + return + } + r.maybeSendAppend(id, false /* sendIfEmpty */) + }) + } + // If the leadTransferee was removed or demoted, abort the leadership transfer. + if _, tOK := r.trk.Config.Voters.IDs()[r.leadTransferee]; !tOK && r.leadTransferee != 0 { + r.abortLeaderTransfer() + } + + return cs +} + +func (r *raft) loadState(state *pb.HardState) { + if state.GetCommit() < r.raftLog.committed || state.GetCommit() > r.raftLog.lastIndex() { + r.logger.Panicf("%x state.commit %d is out of range [%d, %d]", r.id, state.GetCommit(), r.raftLog.committed, r.raftLog.lastIndex()) + } + r.raftLog.committed = state.GetCommit() + r.Term = state.GetTerm() + r.Vote = state.GetVote() +} + +// pastElectionTimeout returns true if r.electionElapsed is greater +// than or equal to the randomized election timeout in +// [electiontimeout, 2 * electiontimeout - 1]. +func (r *raft) pastElectionTimeout() bool { + return r.electionElapsed >= r.randomizedElectionTimeout +} + +func (r *raft) resetRandomizedElectionTimeout() { + random := r.electionRandom + if random == nil { + random = globalRand.Intn + } + v := random(r.electionTimeout) + if v < 0 || v >= r.electionTimeout { + panic("raft: election random value outside bound") + } + r.randomizedElectionTimeout = r.electionTimeout + v +} + +func (r *raft) sendTimeoutNow(to uint64) { + r.send(&pb.Message{To: new(to), Type: pb.MsgTimeoutNow.Enum()}) +} + +func (r *raft) abortLeaderTransfer() { + r.leadTransferee = None +} + +// committedEntryInCurrentTerm return true if the peer has committed an entry in its term. +func (r *raft) committedEntryInCurrentTerm() bool { + // NB: r.Term is never 0 on a leader, so if zeroTermOnOutOfBounds returns 0, + // we won't see it as a match with r.Term. + return r.raftLog.zeroTermOnOutOfBounds(r.raftLog.term(r.raftLog.committed)) == r.Term +} + +// responseToReadIndexReq constructs a response for `req`. If `req` comes from the peer +// itself, a blank value will be returned. +func (r *raft) responseToReadIndexReq(req *pb.Message, readIndex uint64) *pb.Message { + if req.GetFrom() == None || req.GetFrom() == r.id { + r.readStates = append(r.readStates, ReadState{ + Index: readIndex, + RequestCtx: req.GetEntries()[0].GetData(), + }) + return &pb.Message{} + } + return &pb.Message{ + Type: pb.MsgReadIndexResp.Enum(), + To: req.From, + Index: new(readIndex), + Entries: req.GetEntries(), + } +} + +// increaseUncommittedSize computes the size of the proposed entries and +// determines whether they would push leader over its maxUncommittedSize limit. +// If the new entries would exceed the limit, the method returns false. If not, +// the increase in uncommitted entry size is recorded and the method returns +// true. +// +// Empty payloads are never refused. This is used both for appending an empty +// entry at a new leader's term, as well as leaving a joint configuration. +func (r *raft) increaseUncommittedSize(ents []*pb.Entry) bool { + s := payloadsSize(ents) + if r.uncommittedSize > 0 && s > 0 && r.uncommittedSize+s > r.maxUncommittedSize { + // If the uncommitted tail of the Raft log is empty, allow any size + // proposal. Otherwise, limit the size of the uncommitted tail of the + // log and drop any proposal that would push the size over the limit. + // Note the added requirement s>0 which is used to make sure that + // appending single empty entries to the log always succeeds, used both + // for replicating a new leader's initial empty entry, and for + // auto-leaving joint configurations. + return false + } + r.uncommittedSize += s + return true +} + +// reduceUncommittedSize accounts for the newly committed entries by decreasing +// the uncommitted entry size limit. +func (r *raft) reduceUncommittedSize(s entryPayloadSize) { + if s > r.uncommittedSize { + // uncommittedSize may underestimate the size of the uncommitted Raft + // log tail but will never overestimate it. Saturate at 0 instead of + // allowing overflow. + r.uncommittedSize = 0 + } else { + r.uncommittedSize -= s + } +} + +func releasePendingReadIndexMessages(r *raft) { + if len(r.pendingReadIndexMessages) == 0 { + // Fast path for the common case to avoid a call to storage.LastIndex() + // via committedEntryInCurrentTerm. + return + } + if !r.committedEntryInCurrentTerm() { + r.logger.Error("pending MsgReadIndex should be released only after first commit in current term") + return + } + + msgs := r.pendingReadIndexMessages + r.pendingReadIndexMessages = nil + + for _, m := range msgs { + sendMsgReadIndexResponse(r, m) + } +} + +func sendMsgReadIndexResponse(r *raft, m *pb.Message) { + // thinking: use an internally defined context instead of the user given context. + // We can express this in terms of the term and index instead of a user-supplied value. + // This would allow multiple reads to piggyback on the same message. + switch r.readOnly.option { + // If more than the local vote is needed, go through a full broadcast. + case ReadOnlySafe: + r.readOnly.addRequest(r.raftLog.committed, m) + // The local node automatically acks the request. + r.readOnly.recvAck(r.id, r.readOnly.heartbeatCtx()) + r.bcastHeartbeat() + case ReadOnlyLeaseBased: + if resp := r.responseToReadIndexReq(m, r.raftLog.committed); resp.GetTo() != None { + r.send(resp) + } + } +} diff --git a/third_party/raft/raft_flow_control_test.go b/third_party/raft/raft_flow_control_test.go new file mode 100644 index 0000000000..c97bd3d26a --- /dev/null +++ b/third_party/raft/raft_flow_control_test.go @@ -0,0 +1,138 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "testing" + + "github.com/stretchr/testify/require" + + pb "go.etcd.io/raft/v3/raftpb" +) + +// TestMsgAppFlowControlFull ensures: +// 1. msgApp can fill the sending window until full +// 2. when the window is full, no more msgApp can be sent. + +func TestMsgAppFlowControlFull(t *testing.T) { + r := newTestRaft(1, 5, 1, newTestMemoryStorage(withPeers(1, 2))) + r.becomeCandidate() + r.becomeLeader() + + pr2 := r.trk.Progress[2] + // force the progress to be in replicate state + pr2.BecomeReplicate() + // fill in the inflights window + for i := 0; i < r.trk.MaxInflight; i++ { + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + ms := r.readMessages() + require.Len(t, ms, 1) + require.Equal(t, pb.MsgApp, ms[0].GetType()) + } + + // ensure 1 + require.True(t, pr2.IsPaused()) + + // ensure 2 + for i := 0; i < 10; i++ { + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + ms := r.readMessages() + require.Empty(t, ms) + } +} + +// TestMsgAppFlowControlMoveForward ensures msgAppResp can move +// forward the sending window correctly: +// 1. valid msgAppResp.index moves the windows to pass all smaller or equal index. +// 2. out-of-dated msgAppResp has no effect on the sliding window. +func TestMsgAppFlowControlMoveForward(t *testing.T) { + r := newTestRaft(1, 5, 1, newTestMemoryStorage(withPeers(1, 2))) + r.becomeCandidate() + r.becomeLeader() + + pr2 := r.trk.Progress[2] + // force the progress to be in replicate state + pr2.BecomeReplicate() + // fill in the inflights window + for i := 0; i < r.trk.MaxInflight; i++ { + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + r.readMessages() + } + + // 1 is noop, 2 is the first proposal we just sent. + // so we start with 2. + for tt := 2; tt < r.trk.MaxInflight; tt++ { + // move forward the window + r.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgAppResp.Enum(), Index: new(uint64(tt))}) + r.readMessages() + + // fill in the inflights window again + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + ms := r.readMessages() + require.Len(t, ms, 1) + require.Equal(t, pb.MsgApp, ms[0].GetType()) + + // ensure 1 + require.True(t, pr2.IsPaused()) + + // ensure 2 + for i := 0; i < tt; i++ { + r.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgAppResp.Enum(), Index: new(uint64(i))}) + require.True(t, pr2.IsPaused()) + } + } +} + +// TestMsgAppFlowControlRecvHeartbeat ensures a heartbeat response +// frees one slot if the window is full. +func TestMsgAppFlowControlRecvHeartbeat(t *testing.T) { + r := newTestRaft(1, 5, 1, newTestMemoryStorage(withPeers(1, 2))) + r.becomeCandidate() + r.becomeLeader() + + pr2 := r.trk.Progress[2] + // force the progress to be in replicate state + pr2.BecomeReplicate() + // fill in the inflights window + for i := 0; i < r.trk.MaxInflight; i++ { + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + r.readMessages() + } + + for tt := 1; tt < 5; tt++ { + // recv tt msgHeartbeatResp and expect one free slot + for i := 0; i < tt; i++ { + require.True(t, pr2.IsPaused()) + // Unpauses the progress, sends an empty MsgApp, and pauses it again. + r.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgHeartbeatResp.Enum()}) + ms := r.readMessages() + require.Len(t, ms, 1) + require.Equal(t, pb.MsgApp, ms[0].GetType()) + require.Empty(t, ms[0].GetEntries()) + } + + // No more appends are sent if there are no heartbeats. + for i := 0; i < 10; i++ { + require.True(t, pr2.IsPaused()) + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + ms := r.readMessages() + require.Empty(t, ms) + } + + // clear all pending messages. + r.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgHeartbeatResp.Enum()}) + r.readMessages() + } +} diff --git a/third_party/raft/raft_paper_test.go b/third_party/raft/raft_paper_test.go new file mode 100644 index 0000000000..f2fb2fddd5 --- /dev/null +++ b/third_party/raft/raft_paper_test.go @@ -0,0 +1,829 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +This file contains tests which verify that the scenarios described +in the raft paper (https://raft.github.io/raft.pdf) are +handled by the raft implementation correctly. Each test focuses on +several sentences written in the paper. This could help us to prevent +most implementation bugs. + +Each test is composed of three parts: init, test and check. +Init part uses simple and understandable way to simulate the init state. +Test part uses Step function to generate the scenario. Check part checks +outgoing messages and state. +*/ +package raft + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + pb "go.etcd.io/raft/v3/raftpb" +) + +func TestFollowerUpdateTermFromMessage(t *testing.T) { + testUpdateTermFromMessage(t, StateFollower) +} +func TestCandidateUpdateTermFromMessage(t *testing.T) { + testUpdateTermFromMessage(t, StateCandidate) +} +func TestLeaderUpdateTermFromMessage(t *testing.T) { + testUpdateTermFromMessage(t, StateLeader) +} + +// testUpdateTermFromMessage tests that if one server’s current term is +// smaller than the other’s, then it updates its current term to the larger +// value. If a candidate or leader discovers that its term is out of date, +// it immediately reverts to follower state. +// Reference: section 5.1 +func testUpdateTermFromMessage(t *testing.T, state StateType) { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + switch state { + case StateFollower: + r.becomeFollower(1, 2) + case StateCandidate: + r.becomeCandidate() + case StateLeader: + r.becomeCandidate() + r.becomeLeader() + } + + r.Step(&pb.Message{Type: pb.MsgApp.Enum(), Term: new(uint64(2))}) + + assert.Equal(t, uint64(2), r.Term) + assert.Equal(t, StateFollower, r.state) +} + +// TestRejectStaleTermMessage tests that if a server receives a request with +// a stale term number, it rejects the request. +// Our implementation ignores the request instead. +// Reference: section 5.1 +func TestRejectStaleTermMessage(t *testing.T) { + called := false + fakeStep := func(_ *raft, _ *pb.Message) error { + called = true + return nil + } + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + r.step = fakeStep + r.loadState(&pb.HardState{Term: new(uint64(2))}) + + r.Step(&pb.Message{Type: pb.MsgApp.Enum(), Term: new(r.Term - 1)}) + + assert.False(t, called) +} + +// TestStartAsFollower tests that when servers start up, they begin as followers. +// Reference: section 5.2 +func TestStartAsFollower(t *testing.T) { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + assert.Equal(t, StateFollower, r.state) +} + +// TestLeaderBcastBeat tests that if the leader receives a heartbeat tick, +// it will send a MsgHeartbeat with m.Index = 0, m.LogTerm=0 and empty entries +// as heartbeat to all followers. +// Reference: section 5.2 +func TestLeaderBcastBeat(t *testing.T) { + // heartbeat interval + hi := 1 + r := newTestRaft(1, 10, hi, newTestMemoryStorage(withPeers(1, 2, 3))) + r.becomeCandidate() + r.becomeLeader() + for i := 0; i < 10; i++ { + mustAppendEntry(r, &pb.Entry{Index: new(uint64(i) + 1)}) + } + + for i := 0; i < hi; i++ { + r.tick() + } + + msgs := r.readMessages() + sort.Sort(messageSlice(msgs)) + assert.Equal(t, []*pb.Message{ + {From: new(uint64(1)), To: new(uint64(2)), Term: new(uint64(1)), Type: pb.MsgHeartbeat.Enum(), Commit: new(uint64(0))}, + {From: new(uint64(1)), To: new(uint64(3)), Term: new(uint64(1)), Type: pb.MsgHeartbeat.Enum(), Commit: new(uint64(0))}, + }, msgs) +} + +func TestFollowerStartElection(t *testing.T) { + testNonleaderStartElection(t, StateFollower) +} +func TestCandidateStartNewElection(t *testing.T) { + testNonleaderStartElection(t, StateCandidate) +} + +// testNonleaderStartElection tests that if a follower receives no communication +// over election timeout, it begins an election to choose a new leader. It +// increments its current term and transitions to candidate state. It then +// votes for itself and issues RequestVote RPCs in parallel to each of the +// other servers in the cluster. +// Reference: section 5.2 +// Also if a candidate fails to obtain a majority, it will time out and +// start a new election by incrementing its term and initiating another +// round of RequestVote RPCs. +// Reference: section 5.2 +func testNonleaderStartElection(t *testing.T, state StateType) { + // election timeout + et := 10 + r := newTestRaft(1, et, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + switch state { + case StateFollower: + r.becomeFollower(1, 2) + case StateCandidate: + r.becomeCandidate() + } + + for i := 1; i < 2*et; i++ { + r.tick() + } + r.advanceMessagesAfterAppend() + + assert.Equal(t, uint64(2), r.Term) + assert.Equal(t, StateCandidate, r.state) + assert.True(t, r.trk.Votes[r.id]) + + msgs := r.readMessages() + sort.Sort(messageSlice(msgs)) + assert.Equal(t, []*pb.Message{ + {From: new(uint64(1)), To: new(uint64(2)), Term: new(uint64(2)), Type: pb.MsgVote.Enum(), Index: new(uint64(0)), LogTerm: new(uint64(0))}, + {From: new(uint64(1)), To: new(uint64(3)), Term: new(uint64(2)), Type: pb.MsgVote.Enum(), Index: new(uint64(0)), LogTerm: new(uint64(0))}, + }, msgs) +} + +// TestLeaderElectionInOneRoundRPC tests all cases that may happen in +// leader election during one round of RequestVote RPC: +// a) it wins the election +// b) it loses the election +// c) it is unclear about the result +// Reference: section 5.2 +func TestLeaderElectionInOneRoundRPC(t *testing.T) { + tests := []struct { + size int + votes map[uint64]bool + state StateType + }{ + // win the election when receiving votes from a majority of the servers + {1, map[uint64]bool{}, StateLeader}, + {3, map[uint64]bool{2: true, 3: true}, StateLeader}, + {3, map[uint64]bool{2: true}, StateLeader}, + {5, map[uint64]bool{2: true, 3: true, 4: true, 5: true}, StateLeader}, + {5, map[uint64]bool{2: true, 3: true, 4: true}, StateLeader}, + {5, map[uint64]bool{2: true, 3: true}, StateLeader}, + + // return to follower state if it receives vote denial from a majority + {3, map[uint64]bool{2: false, 3: false}, StateFollower}, + {5, map[uint64]bool{2: false, 3: false, 4: false, 5: false}, StateFollower}, + {5, map[uint64]bool{2: true, 3: false, 4: false, 5: false}, StateFollower}, + + // stay in candidate if it does not obtain the majority + {3, map[uint64]bool{}, StateCandidate}, + {5, map[uint64]bool{2: true}, StateCandidate}, + {5, map[uint64]bool{2: false, 3: false}, StateCandidate}, + {5, map[uint64]bool{}, StateCandidate}, + } + for i, tt := range tests { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(idsBySize(tt.size)...))) + + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + r.advanceMessagesAfterAppend() + for id, vote := range tt.votes { + r.Step(&pb.Message{From: new(id), To: new(uint64(1)), Term: new(r.Term), Type: pb.MsgVoteResp.Enum(), Reject: new(!vote)}) + } + + assert.Equal(t, tt.state, r.state, "#%d", i) + assert.Equal(t, uint64(1), r.Term, "#%d", i) + } +} + +// TestFollowerVote tests that each follower will vote for at most one +// candidate in a given term, on a first-come-first-served basis. +// Reference: section 5.2 +func TestFollowerVote(t *testing.T) { + tests := []struct { + vote uint64 + nvote uint64 + wreject bool + }{ + {None, 2, false}, + {None, 3, false}, + {2, 2, false}, + {3, 3, false}, + {2, 3, true}, + {3, 2, true}, + } + for i, tt := range tests { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + r.loadState(&pb.HardState{Term: new(uint64(1)), Vote: new(tt.vote)}) + + r.Step(&pb.Message{From: new(tt.nvote), To: new(uint64(1)), Term: new(uint64(1)), Type: pb.MsgVote.Enum()}) + + expected := &pb.Message{From: new(uint64(1)), To: new(tt.nvote), Term: new(uint64(1)), Type: pb.MessageType_MsgVoteResp.Enum()} + if tt.wreject { + expected.Reject = new(true) + } + assert.Equal(t, []*pb.Message{expected}, r.msgsAfterAppend, "#%d", i) + } +} + +// TestCandidateFallback tests that while waiting for votes, +// if a candidate receives an AppendEntries RPC from another server claiming +// to be leader whose term is at least as large as the candidate's current term, +// it recognizes the leader as legitimate and returns to follower state. +// Reference: section 5.2 +func TestCandidateFallback(t *testing.T) { + tests := []*pb.Message{ + {From: new(uint64(2)), To: new(uint64(1)), Term: new(uint64(1)), Type: pb.MsgApp.Enum()}, + {From: new(uint64(2)), To: new(uint64(1)), Term: new(uint64(2)), Type: pb.MsgApp.Enum()}, + } + for i, tt := range tests { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + require.Equal(t, StateCandidate, r.state, "#%d", i) + + r.Step(tt) + + assert.Equal(t, StateFollower, r.state, "#%d", i) + assert.Equal(t, tt.GetTerm(), r.Term, "#%d", i) + } +} + +func TestFollowerElectionTimeoutRandomized(t *testing.T) { + SetLogger(discardLogger) + defer SetLogger(defaultLogger) + testNonleaderElectionTimeoutRandomized(t, StateFollower) +} +func TestCandidateElectionTimeoutRandomized(t *testing.T) { + SetLogger(discardLogger) + defer SetLogger(defaultLogger) + testNonleaderElectionTimeoutRandomized(t, StateCandidate) +} + +// testNonleaderElectionTimeoutRandomized tests that election timeout for +// follower or candidate is randomized. +// Reference: section 5.2 +func testNonleaderElectionTimeoutRandomized(t *testing.T, state StateType) { + et := 10 + r := newTestRaft(1, et, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + timeouts := make(map[int]bool) + for round := 0; round < 50*et; round++ { + switch state { + case StateFollower: + r.becomeFollower(r.Term+1, 2) + case StateCandidate: + r.becomeCandidate() + } + + time := 0 + for len(r.readMessages()) == 0 { + r.tick() + time++ + } + timeouts[time] = true + } + + for d := et; d < 2*et; d++ { + assert.True(t, timeouts[d], "timeout in %d ticks should happen", d) + } +} + +func TestFollowersElectionTimeoutNonconflict(t *testing.T) { + SetLogger(discardLogger) + defer SetLogger(defaultLogger) + testNonleadersElectionTimeoutNonconflict(t, StateFollower) +} +func TestCandidatesElectionTimeoutNonconflict(t *testing.T) { + SetLogger(discardLogger) + defer SetLogger(defaultLogger) + testNonleadersElectionTimeoutNonconflict(t, StateCandidate) +} + +// testNonleadersElectionTimeoutNonconflict tests that in most cases only a +// single server(follower or candidate) will time out, which reduces the +// likelihood of split vote in the new election. +// Reference: section 5.2 +func testNonleadersElectionTimeoutNonconflict(t *testing.T, state StateType) { + et := 10 + size := 5 + rs := make([]*raft, size) + ids := idsBySize(size) + for k := range rs { + rs[k] = newTestRaft(ids[k], et, 1, newTestMemoryStorage(withPeers(ids...))) + } + conflicts := 0 + for round := 0; round < 1000; round++ { + for _, r := range rs { + switch state { + case StateFollower: + r.becomeFollower(r.Term+1, None) + case StateCandidate: + r.becomeCandidate() + } + } + + timeoutNum := 0 + for timeoutNum == 0 { + for _, r := range rs { + r.tick() + if len(r.readMessages()) > 0 { + timeoutNum++ + } + } + } + // several rafts time out at the same tick + if timeoutNum > 1 { + conflicts++ + } + } + + assert.LessOrEqual(t, float64(conflicts)/1000, 0.3) +} + +// TestLeaderStartReplication tests that when receiving client proposals, +// the leader appends the proposal to its log as a new entry, then issues +// AppendEntries RPCs in parallel to each of the other servers to replicate +// the entry. Also, when sending an AppendEntries RPC, the leader includes +// the index and term of the entry in its log that immediately precedes +// the new entries. +// Also, it writes the new entry into stable storage. +// Reference: section 5.3 +func TestLeaderStartReplication(t *testing.T) { + s := newTestMemoryStorage(withPeers(1, 2, 3)) + r := newTestRaft(1, 10, 1, s) + r.becomeCandidate() + r.becomeLeader() + commitNoopEntry(r, s) + li := r.raftLog.lastIndex() + + ents := []*pb.Entry{{Data: []byte("some data")}} + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: ents}) + + assert.Equal(t, li+1, r.raftLog.lastIndex()) + assert.Equal(t, li, r.raftLog.committed) + msgs := r.readMessages() + sort.Sort(messageSlice(msgs)) + wents := []*pb.Entry{{Index: new(li + 1), Term: new(uint64(1)), Data: []byte("some data")}} + assertEqualMessages(t, []*pb.Message{ + {From: new(uint64(1)), To: new(uint64(2)), Term: new(uint64(1)), Type: pb.MsgApp.Enum(), Index: new(li), LogTerm: new(uint64(1)), Entries: wents, Commit: new(li)}, + {From: new(uint64(1)), To: new(uint64(3)), Term: new(uint64(1)), Type: pb.MsgApp.Enum(), Index: new(li), LogTerm: new(uint64(1)), Entries: wents, Commit: new(li)}, + }, msgs) + requireEqualEntries(t, []*pb.Entry{ + {Index: new(li + 1), Term: new(uint64(1)), Data: []byte("some data")}, + }, r.raftLog.nextUnstableEnts()) +} + +// TestLeaderCommitEntry tests that when the entry has been safely replicated, +// the leader gives out the applied entries, which can be applied to its state +// machine. +// Also, the leader keeps track of the highest index it knows to be committed, +// and it includes that index in future AppendEntries RPCs so that the other +// servers eventually find out. +// Reference: section 5.3 +func TestLeaderCommitEntry(t *testing.T) { + s := newTestMemoryStorage(withPeers(1, 2, 3)) + r := newTestRaft(1, 10, 1, s) + r.becomeCandidate() + r.becomeLeader() + commitNoopEntry(r, s) + li := r.raftLog.lastIndex() + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("some data")}}}) + + for _, m := range r.readMessages() { + r.Step(acceptAndReply(m)) + } + + assert.Equal(t, li+1, r.raftLog.committed) + requireEqualEntries(t, []*pb.Entry{ + {Index: new(li + 1), Term: new(uint64(1)), Data: []byte("some data")}, + }, r.raftLog.nextCommittedEnts(true)) + msgs := r.readMessages() + sort.Sort(messageSlice(msgs)) + for i, m := range msgs { + assert.Equal(t, uint64(i+2), m.GetTo()) + assert.Equal(t, pb.MsgApp, m.GetType()) + assert.Equal(t, li+1, m.GetCommit()) + } +} + +// TestLeaderAcknowledgeCommit tests that a log entry is committed once the +// leader that created the entry has replicated it on a majority of the servers. +// Reference: section 5.3 +func TestLeaderAcknowledgeCommit(t *testing.T) { + tests := []struct { + size int + nonLeaderAcceptors map[uint64]bool + wack bool + }{ + {1, nil, true}, + {3, nil, false}, + {3, map[uint64]bool{2: true}, true}, + {3, map[uint64]bool{2: true, 3: true}, true}, + {5, nil, false}, + {5, map[uint64]bool{2: true}, false}, + {5, map[uint64]bool{2: true, 3: true}, true}, + {5, map[uint64]bool{2: true, 3: true, 4: true}, true}, + {5, map[uint64]bool{2: true, 3: true, 4: true, 5: true}, true}, + } + for i, tt := range tests { + s := newTestMemoryStorage(withPeers(idsBySize(tt.size)...)) + r := newTestRaft(1, 10, 1, s) + r.becomeCandidate() + r.becomeLeader() + commitNoopEntry(r, s) + li := r.raftLog.lastIndex() + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("some data")}}}) + r.advanceMessagesAfterAppend() + for _, m := range r.msgs { + if tt.nonLeaderAcceptors[m.GetTo()] { + r.Step(acceptAndReply(m)) + } + } + + assert.Equal(t, tt.wack, r.raftLog.committed > li, "#%d", i) + } +} + +// TestLeaderCommitPrecedingEntries tests that when leader commits a log entry, +// it also commits all preceding entries in the leader’s log, including +// entries created by previous leaders. +// Also, it applies the entry to its local state machine (in log order). +// Reference: section 5.3 +func TestLeaderCommitPrecedingEntries(t *testing.T) { + tests := [][]*pb.Entry{ + {}, + {{Term: new(uint64(2)), Index: new(uint64(1))}}, + {{Term: new(uint64(1)), Index: new(uint64(1))}, {Term: new(uint64(2)), Index: new(uint64(2))}}, + {{Term: new(uint64(1)), Index: new(uint64(1))}}, + } + for i, tt := range tests { + storage := newTestMemoryStorage(withPeers(1, 2, 3)) + storage.Append(tt) + r := newTestRaft(1, 10, 1, storage) + r.loadState(&pb.HardState{Term: new(uint64(2))}) + r.becomeCandidate() + r.becomeLeader() + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("some data")}}}) + + for _, m := range r.readMessages() { + r.Step(acceptAndReply(m)) + } + + li := uint64(len(tt)) + requireEqualEntries(t, append(tt, + &pb.Entry{Term: new(uint64(3)), Index: new(li + 1)}, + &pb.Entry{Term: new(uint64(3)), Index: new(li + 2), Data: []byte("some data")}, + ), r.raftLog.nextCommittedEnts(true), "#%d", i) + } +} + +// TestFollowerCommitEntry tests that once a follower learns that a log entry +// is committed, it applies the entry to its local state machine (in log order). +// Reference: section 5.3 +func TestFollowerCommitEntry(t *testing.T) { + tests := []struct { + ents []*pb.Entry + commit uint64 + }{ + { + []*pb.Entry{ + {Term: new(uint64(1)), Index: new(uint64(1)), Data: []byte("some data")}, + }, + 1, + }, + { + []*pb.Entry{ + {Term: new(uint64(1)), Index: new(uint64(1)), Data: []byte("some data")}, + {Term: new(uint64(1)), Index: new(uint64(2)), Data: []byte("some data2")}, + }, + 2, + }, + { + []*pb.Entry{ + {Term: new(uint64(1)), Index: new(uint64(1)), Data: []byte("some data2")}, + {Term: new(uint64(1)), Index: new(uint64(2)), Data: []byte("some data")}, + }, + 2, + }, + { + []*pb.Entry{ + {Term: new(uint64(1)), Index: new(uint64(1)), Data: []byte("some data")}, + {Term: new(uint64(1)), Index: new(uint64(2)), Data: []byte("some data2")}, + }, + 1, + }, + } + for i, tt := range tests { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + r.becomeFollower(1, 2) + + r.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgApp.Enum(), Term: new(uint64(1)), Entries: tt.ents, Commit: new(tt.commit)}) + + assert.Equal(t, tt.commit, r.raftLog.committed, "#%d", i) + assert.Equal(t, tt.ents[:int(tt.commit)], r.raftLog.nextCommittedEnts(true), "#%d", i) + } +} + +// TestFollowerCheckMsgApp tests that if the follower does not find an +// entry in its log with the same index and term as the one in AppendEntries RPC, +// then it refuses the new entries. Otherwise it replies that it accepts the +// append entries. +// Reference: section 5.3 +func TestFollowerCheckMsgApp(t *testing.T) { + ents := []*pb.Entry{{Term: new(uint64(1)), Index: new(uint64(1))}, {Term: new(uint64(2)), Index: new(uint64(2))}} + tests := []struct { + term uint64 + index uint64 + windex uint64 + wreject bool + wrejectHint uint64 + wlogterm uint64 + }{ + // match with committed entries + {0, 0, 1, false, 0, 0}, + {ents[0].GetTerm(), ents[0].GetIndex(), 1, false, 0, 0}, + // match with uncommitted entries + {ents[1].GetTerm(), ents[1].GetIndex(), 2, false, 0, 0}, + + // unmatch with existing entry + {ents[0].GetTerm(), ents[1].GetIndex(), ents[1].GetIndex(), true, 1, 1}, + // unexisting entry + {ents[1].GetTerm() + 1, ents[1].GetIndex() + 1, ents[1].GetIndex() + 1, true, 2, 2}, + } + for i, tt := range tests { + storage := newTestMemoryStorage(withPeers(1, 2, 3)) + storage.Append(ents) + r := newTestRaft(1, 10, 1, storage) + r.loadState(&pb.HardState{Commit: new(uint64(1))}) + r.becomeFollower(2, 2) + + r.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgApp.Enum(), Term: new(uint64(2)), LogTerm: new(tt.term), Index: new(tt.index)}) + + expected := &pb.Message{From: new(uint64(1)), To: new(uint64(2)), Type: new(pb.MessageType_MsgAppResp), Term: new(uint64(2)), Index: new(tt.windex)} + if tt.wreject { + expected.Reject = new(true) + expected.RejectHint = new(tt.wrejectHint) + expected.LogTerm = new(tt.wlogterm) + } + assert.Equal(t, []*pb.Message{expected}, r.readMessages(), "#%d", i) + } +} + +// TestFollowerAppendEntries tests that when AppendEntries RPC is valid, +// the follower will delete the existing conflict entry and all that follow it, +// and append any new entries not already in the log. +// Also, it writes the new entry into stable storage. +// Reference: section 5.3 +func TestFollowerAppendEntries(t *testing.T) { + tests := []struct { + index, term uint64 + ents []*pb.Entry + wents []*pb.Entry + wunstable []*pb.Entry + }{ + { + 2, 2, + index(3).terms(3), + index(1).terms(1, 2, 3), + index(3).terms(3), + }, + { + 1, 1, + index(2).terms(3, 4), + index(1).terms(1, 3, 4), + index(2).terms(3, 4), + }, + { + 0, 0, + index(1).terms(1), + index(1).terms(1, 2), + nil, + }, + { + 0, 0, + index(1).terms(3), + index(1).terms(3), + index(1).terms(3), + }, + } + for i, tt := range tests { + storage := newTestMemoryStorage(withPeers(1, 2, 3)) + storage.Append([]*pb.Entry{{Term: new(uint64(1)), Index: new(uint64(1))}, {Term: new(uint64(2)), Index: new(uint64(2))}}) + r := newTestRaft(1, 10, 1, storage) + r.becomeFollower(2, 2) + + r.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgApp.Enum(), Term: new(uint64(2)), LogTerm: new(tt.term), Index: new(tt.index), Entries: tt.ents}) + + requireEqualEntries(t, tt.wents, r.raftLog.allEntries(), "#%d", i) + requireEqualEntries(t, tt.wunstable, r.raftLog.nextUnstableEnts(), "#%d", i) + } +} + +// TestLeaderSyncFollowerLog tests that the leader could bring a follower's log +// into consistency with its own. +// Reference: section 5.3, figure 7 +func TestLeaderSyncFollowerLog(t *testing.T) { + ents := index(0).terms(0, 1, 1, 1, 4, 4, 5, 5, 6, 6, 6) + term := uint64(8) + for i, tt := range [][]*pb.Entry{ + index(0).terms(0, 1, 1, 1, 4, 4, 5, 5, 6, 6), + index(0).terms(0, 1, 1, 1, 4, 4), + index(0).terms(0, 1, 1, 1, 4, 4, 5, 5, 6, 6, 6, 6), + index(0).terms(0, 1, 1, 1, 4, 4, 5, 5, 6, 6, 6, 7, 7), + index(0).terms(0, 1, 1, 1, 4, 4, 4, 4), + index(0).terms(0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3), + } { + leadStorage := newTestMemoryStorage(withPeers(1, 2, 3)) + leadStorage.Append(ents) + lead := newTestRaft(1, 10, 1, leadStorage) + lead.loadState(&pb.HardState{Commit: new(lead.raftLog.lastIndex()), Term: new(term)}) + followerStorage := newTestMemoryStorage(withPeers(1, 2, 3)) + followerStorage.Append(tt) + follower := newTestRaft(2, 10, 1, followerStorage) + follower.loadState(&pb.HardState{Term: new(term - 1)}) + // It is necessary to have a three-node cluster. + // The second may have more up-to-date log than the first one, so the + // first node needs the vote from the third node to become the leader. + n := newNetwork(lead, follower, nopStepper) + n.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + // The election occurs in the term after the one we loaded with + // lead.loadState above. + n.send(&pb.Message{From: new(uint64(3)), To: new(uint64(1)), Type: pb.MsgVoteResp.Enum(), Term: new(term + 1)}) + + n.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}}) + + assert.Empty(t, diffu(ltoa(lead.raftLog), ltoa(follower.raftLog)), "#%d", i) + } +} + +// TestVoteRequest tests that the vote request includes information about the candidate’s log +// and are sent to all of the other nodes. +// Reference: section 5.4.1 +func TestVoteRequest(t *testing.T) { + tests := []struct { + ents []*pb.Entry + wterm uint64 + }{ + {index(1).terms(1), 2}, + {index(1).terms(1, 2), 3}, + } + for j, tt := range tests { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + r.Step(&pb.Message{ + From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgApp.Enum(), Term: new(tt.wterm - 1), LogTerm: new(uint64(0)), Index: new(uint64(0)), Entries: tt.ents, + }) + r.readMessages() + + for i := 1; i < r.electionTimeout*2; i++ { + r.tickElection() + } + + msgs := r.readMessages() + sort.Sort(messageSlice(msgs)) + require.Len(t, msgs, 2, "#%d", j) + for i, m := range msgs { + assert.Equal(t, pb.MsgVote, m.GetType(), "#%d.%d", j, i) + assert.Equal(t, uint64(i+2), m.GetTo(), "#%d.%d", j, i) + assert.Equal(t, tt.wterm, m.GetTerm(), "#%d.%d", j, i) + + assert.Equal(t, tt.ents[len(tt.ents)-1].GetIndex(), m.GetIndex(), "#%d.%d", j, i) + assert.Equal(t, tt.ents[len(tt.ents)-1].GetTerm(), m.GetLogTerm(), "#%d.%d", j, i) + } + } +} + +// TestVoter tests the voter denies its vote if its own log is more up-to-date +// than that of the candidate. +// Reference: section 5.4.1 +func TestVoter(t *testing.T) { + tests := []struct { + ents []*pb.Entry + logterm uint64 + index uint64 + + wreject bool + }{ + // same logterm + {index(1).terms(1), 1, 1, false}, + {index(1).terms(1), 1, 2, false}, + {index(1).terms(1, 1), 1, 1, true}, + // candidate higher logterm + {index(1).terms(1), 2, 1, false}, + {index(1).terms(1), 2, 2, false}, + {[]*pb.Entry{{Term: new(uint64(1)), Index: new(uint64(1))}, {Term: new(uint64(1)), Index: new(uint64(2))}}, 2, 1, false}, + // voter higher logterm + {index(1).terms(2), 1, 1, true}, + {index(1).terms(2), 1, 2, true}, + {index(1).terms(2, 2), 1, 1, true}, + {index(1).terms(1, 1), 1, 1, true}, + } + for i, tt := range tests { + storage := newTestMemoryStorage(withPeers(1, 2)) + storage.Append(tt.ents) + r := newTestRaft(1, 10, 1, storage) + + r.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgVote.Enum(), Term: new(uint64(3)), LogTerm: new(tt.logterm), Index: new(tt.index)}) + + msgs := r.readMessages() + require.Len(t, msgs, 1, "#%d", i) + m := msgs[0] + assert.Equal(t, pb.MsgVoteResp, m.GetType(), "#%d", i) + assert.Equal(t, tt.wreject, m.GetReject(), "#%d", i) + } +} + +// TestLeaderOnlyCommitsLogFromCurrentTerm tests that only log entries from the leader’s +// current term are committed by counting replicas. +// Reference: section 5.4.2 +func TestLeaderOnlyCommitsLogFromCurrentTerm(t *testing.T) { + ents := []*pb.Entry{{Term: new(uint64(1)), Index: new(uint64(1))}, {Term: new(uint64(2)), Index: new(uint64(2))}} + tests := []struct { + index uint64 + wcommit uint64 + }{ + // do not commit log entries in previous terms + {1, 0}, + {2, 0}, + // commit log in current term + {3, 3}, + } + for i, tt := range tests { + storage := newTestMemoryStorage(withPeers(1, 2)) + storage.Append(ents) + r := newTestRaft(1, 10, 1, storage) + r.loadState(&pb.HardState{Term: new(uint64(2))}) + // become leader at term 3 + r.becomeCandidate() + r.becomeLeader() + r.readMessages() + // propose a entry to current term + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}}) + + r.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgAppResp.Enum(), Term: new(r.Term), Index: new(tt.index)}) + r.advanceMessagesAfterAppend() + assert.Equal(t, tt.wcommit, r.raftLog.committed, "#%d", i) + } +} + +type messageSlice []*pb.Message + +func (s messageSlice) Len() int { return len(s) } +func (s messageSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s messageSlice) Less(i, j int) bool { + ai, bi := s[i].GetTo(), s[j].GetTo() + if ai != bi { + return ai < bi + } + at, bt := s[i].GetType(), s[j].GetType() + if at != bt { + return at < bt + } + return s[i].GetFrom() < s[j].GetFrom() +} + +func commitNoopEntry(r *raft, s *MemoryStorage) { + if r.state != StateLeader { + panic("it should only be used when it is the leader") + } + r.bcastAppend() + // simulate the response of MsgApp + msgs := r.readMessages() + for _, m := range msgs { + if m.GetType() != pb.MsgApp || len(m.GetEntries()) != 1 || m.GetEntries()[0].GetData() != nil { + panic("not a message to append noop entry") + } + r.Step(acceptAndReply(m)) + } + // ignore further messages to refresh followers' commit index + r.readMessages() + s.Append(r.raftLog.nextUnstableEnts()) + r.raftLog.appliedTo(r.raftLog.committed, 0 /* size */) + r.raftLog.stableTo(r.raftLog.lastEntryID()) +} + +func acceptAndReply(m *pb.Message) *pb.Message { + if m.GetType() != pb.MsgApp { + panic("type should be MsgApp") + } + return &pb.Message{ + From: m.To, + To: m.From, + Term: m.Term, + Type: pb.MsgAppResp.Enum(), + Index: new(m.GetIndex() + uint64(len(m.GetEntries()))), + } +} diff --git a/third_party/raft/raft_snap_test.go b/third_party/raft/raft_snap_test.go new file mode 100644 index 0000000000..396290f10f --- /dev/null +++ b/third_party/raft/raft_snap_test.go @@ -0,0 +1,122 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + pb "go.etcd.io/raft/v3/raftpb" +) + +var ( + testingSnap = &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{ + Index: new(uint64(11)), // magic number + Term: new(uint64(11)), // magic number + ConfState: &pb.ConfState{Voters: []uint64{1, 2}}, + }, + } +) + +func TestSendingSnapshotSetPendingSnapshot(t *testing.T) { + storage := newTestMemoryStorage(withPeers(1)) + sm := newTestRaft(1, 10, 1, storage) + sm.restore(testingSnap) + + sm.becomeCandidate() + sm.becomeLeader() + + // force set the next of node 2, so that + // node 2 needs a snapshot + sm.trk.Progress[2].Next = sm.raftLog.firstIndex() + + sm.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgAppResp.Enum(), Index: new(sm.trk.Progress[2].Next - 1), Reject: new(true)}) + require.Equal(t, uint64(11), sm.trk.Progress[2].PendingSnapshot) +} + +func TestPendingSnapshotPauseReplication(t *testing.T) { + storage := newTestMemoryStorage(withPeers(1, 2)) + sm := newTestRaft(1, 10, 1, storage) + sm.restore(testingSnap) + + sm.becomeCandidate() + sm.becomeLeader() + + sm.trk.Progress[2].BecomeSnapshot(11) + + sm.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + msgs := sm.readMessages() + require.Empty(t, msgs) +} + +func TestSnapshotFailure(t *testing.T) { + storage := newTestMemoryStorage(withPeers(1, 2)) + sm := newTestRaft(1, 10, 1, storage) + sm.restore(testingSnap) + + sm.becomeCandidate() + sm.becomeLeader() + + sm.trk.Progress[2].Next = 1 + sm.trk.Progress[2].BecomeSnapshot(11) + + sm.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgSnapStatus.Enum(), Reject: new(true)}) + require.Zero(t, sm.trk.Progress[2].PendingSnapshot) + require.Equal(t, uint64(1), sm.trk.Progress[2].Next) + assert.True(t, sm.trk.Progress[2].MsgAppFlowPaused) +} + +func TestSnapshotSucceed(t *testing.T) { + storage := newTestMemoryStorage(withPeers(1, 2)) + sm := newTestRaft(1, 10, 1, storage) + sm.restore(testingSnap) + + sm.becomeCandidate() + sm.becomeLeader() + + sm.trk.Progress[2].Next = 1 + sm.trk.Progress[2].BecomeSnapshot(11) + + sm.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgSnapStatus.Enum(), Reject: new(false)}) + require.Zero(t, sm.trk.Progress[2].PendingSnapshot) + require.Equal(t, uint64(12), sm.trk.Progress[2].Next) + assert.True(t, sm.trk.Progress[2].MsgAppFlowPaused) +} + +func TestSnapshotAbort(t *testing.T) { + storage := newTestMemoryStorage(withPeers(1, 2)) + sm := newTestRaft(1, 10, 1, storage) + sm.restore(testingSnap) + + sm.becomeCandidate() + sm.becomeLeader() + + sm.trk.Progress[2].Next = 1 + sm.trk.Progress[2].BecomeSnapshot(11) + + // A successful msgAppResp that has a higher/equal index than the + // pending snapshot should abort the pending snapshot. + sm.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgAppResp.Enum(), Index: new(uint64(11))}) + require.Zero(t, sm.trk.Progress[2].PendingSnapshot) + // The follower entered StateReplicate and the leader send an append + // and optimistically updated the progress (so we see 13 instead of 12). + // There is something to append because the leader appended an empty entry + // to the log at index 12 when it assumed leadership. + require.Equal(t, uint64(13), sm.trk.Progress[2].Next) + require.Equal(t, 1, sm.trk.Progress[2].Inflights.Count()) +} diff --git a/third_party/raft/raft_test.go b/third_party/raft/raft_test.go new file mode 100644 index 0000000000..8d82cf8428 --- /dev/null +++ b/third_party/raft/raft_test.go @@ -0,0 +1,4261 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "fmt" + "math" + "math/rand" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + pb "go.etcd.io/raft/v3/raftpb" + "go.etcd.io/raft/v3/tracker" +) + +// requireEqualEntries asserts that two entry slices are semantically equal, +// comparing field values via proto.Equal to avoid false failures from internal +// protobuf metadata fields (e.g. atomicMessageInfo) that differ between +// entries created as struct literals and those returned by proto operations. +func requireEqualEntries(t testing.TB, expected, actual []*pb.Entry, msgAndArgs ...interface{}) { + t.Helper() + require.Len(t, actual, len(expected), msgAndArgs...) + for i := range expected { + require.True(t, proto.Equal(expected[i], actual[i]), "entry %d mismatch: expected %v, got %v", i, expected[i], actual[i]) + } +} + +// assertEqualMessages asserts that two message slices are semantically equal, +// using proto.Equal to avoid false failures from atomicMessageInfo in embedded +// entry fields. +func assertEqualMessages(t testing.TB, expected, actual []*pb.Message, msgAndArgs ...interface{}) { + t.Helper() + assert.Len(t, actual, len(expected), msgAndArgs...) + n := len(expected) + if len(actual) < n { + n = len(actual) + } + for i := 0; i < n; i++ { + assert.True(t, proto.Equal(expected[i], actual[i]), "message %d mismatch: expected %v, got %v", i, expected[i], actual[i]) + } +} + +// requireEqualReady asserts that two Ready values are semantically equal, +// comparing protobuf message fields (Entries, CommittedEntries, Messages) +// via proto.Equal to avoid atomicMessageInfo false failures. +func requireEqualReady(t testing.TB, expected, actual Ready) { + t.Helper() + requireEqualEntries(t, expected.Entries, actual.Entries) + requireEqualEntries(t, expected.CommittedEntries, actual.CommittedEntries) + assertEqualMessages(t, expected.Messages, actual.Messages) + exp := expected + exp.Entries = nil + exp.CommittedEntries = nil + exp.Messages = nil + act := actual + act.Entries = nil + act.CommittedEntries = nil + act.Messages = nil + require.Equal(t, exp, act) +} + +// nextEnts returns the appliable entries and updates the applied index. +func nextEnts(r *raft, s *MemoryStorage) (ents []*pb.Entry) { + // Append unstable entries. + s.Append(r.raftLog.nextUnstableEnts()) + r.raftLog.stableTo(r.raftLog.lastEntryID()) + + // Run post-append steps. + r.advanceMessagesAfterAppend() + + // Return committed entries. + ents = r.raftLog.nextCommittedEnts(true) + r.raftLog.appliedTo(r.raftLog.committed, 0 /* size */) + return ents +} + +func mustAppendEntry(r *raft, ents ...*pb.Entry) { + if !r.appendEntry(ents...) { + panic("entry unexpectedly dropped") + } +} + +type stateMachine interface { + Step(m *pb.Message) error + readMessages() []*pb.Message + advanceMessagesAfterAppend() +} + +func (r *raft) readMessages() []*pb.Message { + r.advanceMessagesAfterAppend() + msgs := r.msgs + r.msgs = nil + return msgs +} + +func (r *raft) advanceMessagesAfterAppend() { + for { + msgs := r.takeMessagesAfterAppend() + if len(msgs) == 0 { + break + } + r.stepOrSend(msgs) + } +} + +func (r *raft) takeMessagesAfterAppend() []*pb.Message { + msgs := r.msgsAfterAppend + r.msgsAfterAppend = nil + return msgs +} + +func (r *raft) stepOrSend(msgs []*pb.Message) error { + for _, m := range msgs { + if m.GetTo() == r.id { + if err := r.Step(m); err != nil { + return err + } + } else { + r.msgs = append(r.msgs, m) + } + } + return nil +} + +func TestProgressLeader(t *testing.T) { + s := newTestMemoryStorage(withPeers(1, 2)) + r := newTestRaft(1, 5, 1, s) + r.becomeCandidate() + r.becomeLeader() + r.trk.Progress[2].BecomeReplicate() + + // Send proposals to r1. The first 5 entries should be queued in the unstable log. + propMsg := &pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("foo")}}} + for i := 0; i < 5; i++ { + require.NoError(t, r.Step(propMsg), "#%d", i) + } + + require.Zero(t, r.trk.Progress[1].Match) + + ents := r.raftLog.nextUnstableEnts() + require.Len(t, ents, 6) + require.Len(t, ents[0].GetData(), 0) + require.Equal(t, "foo", string(ents[5].GetData())) + + r.advanceMessagesAfterAppend() + + require.Equal(t, uint64(6), r.trk.Progress[1].Match) + require.Equal(t, uint64(7), r.trk.Progress[1].Next) +} + +// TestProgressResumeByHeartbeatResp ensures raft.heartbeat reset progress.paused by heartbeat response. +func TestProgressResumeByHeartbeatResp(t *testing.T) { + r := newTestRaft(1, 5, 1, newTestMemoryStorage(withPeers(1, 2))) + r.becomeCandidate() + r.becomeLeader() + + r.trk.Progress[2].MsgAppFlowPaused = true + + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgBeat.Enum()}) + assert.True(t, r.trk.Progress[2].MsgAppFlowPaused) + + r.trk.Progress[2].BecomeReplicate() + assert.False(t, r.trk.Progress[2].MsgAppFlowPaused) + r.trk.Progress[2].MsgAppFlowPaused = true + r.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgHeartbeatResp.Enum()}) + assert.False(t, r.trk.Progress[2].MsgAppFlowPaused) +} + +func TestProgressPaused(t *testing.T) { + r := newTestRaft(1, 5, 1, newTestMemoryStorage(withPeers(1, 2))) + r.becomeCandidate() + r.becomeLeader() + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + + ms := r.readMessages() + assert.Len(t, ms, 1) +} + +func TestProgressFlowControl(t *testing.T) { + cfg := newTestConfig(1, 5, 1, newTestMemoryStorage(withPeers(1, 2))) + cfg.MaxInflightMsgs = 3 + cfg.MaxSizePerMsg = 2048 + cfg.MaxInflightBytes = 9000 // A little over MaxInflightMsgs * MaxSizePerMsg. + r := newRaft(cfg) + r.becomeCandidate() + r.becomeLeader() + + // Throw away all the messages relating to the initial election. + r.readMessages() + + // While node 2 is in probe state, propose a bunch of entries. + r.trk.Progress[2].BecomeProbe() + blob := []byte(strings.Repeat("a", 1000)) + large := []byte(strings.Repeat("b", 5000)) + for i := 0; i < 22; i++ { + blob := blob + if i >= 10 && i < 16 { // Temporarily send large messages. + blob = large + } + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: blob}}}) + } + + ms := r.readMessages() + // First append has two entries: the empty entry to confirm the + // election, and the first proposal (only one proposal gets sent + // because we're in probe state). + require.Len(t, ms, 1) + require.Equal(t, pb.MsgApp, ms[0].GetType()) + + require.Len(t, ms[0].GetEntries(), 2) + + require.Empty(t, ms[0].GetEntries()[0].GetData()) + require.Len(t, ms[0].GetEntries()[1].GetData(), 1000) + + ackAndVerify := func(index uint64, expEntries ...int) uint64 { + r.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgAppResp.Enum(), Index: new(index)}) + ms := r.readMessages() + require.Equal(t, len(expEntries), len(ms)) + + for i, m := range ms { + assert.Equal(t, pb.MsgApp, m.GetType(), "#%d", i) + assert.Len(t, m.GetEntries(), expEntries[i], "#%d", i) + } + last := ms[len(ms)-1].GetEntries() + if len(last) == 0 { + return index + } + return last[len(last)-1].GetIndex() + } + + // When this append is acked, we change to replicate state and can + // send multiple messages at once. + index := ackAndVerify(ms[0].GetEntries()[1].GetIndex(), 2, 2, 2) + // Ack all three of those messages together and get another 3 messages. The + // third message contains a single large entry, in contrast to 2 before. + index = ackAndVerify(index, 2, 1, 1) + // All subsequent messages contain one large entry, and we cap at 2 messages + // because it overflows MaxInflightBytes. + index = ackAndVerify(index, 1, 1) + index = ackAndVerify(index, 1, 1) + // Start getting small messages again. + index = ackAndVerify(index, 1, 2, 2) + ackAndVerify(index, 2) +} + +func TestUncommittedEntryLimit(t *testing.T) { + // Use a relatively large number of entries here to prevent regression of a + // bug which computed the size before it was fixed. This test would fail + // with the bug, either because we'd get dropped proposals earlier than we + // expect them, or because the final tally ends up nonzero. (At the time of + // writing, the former). + const maxEntries = 1024 + testEntry := &pb.Entry{Data: []byte("testdata")} + maxEntrySize := maxEntries * payloadSize(testEntry) + + require.Zero(t, payloadSize(&pb.Entry{Data: nil})) + + cfg := newTestConfig(1, 5, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + cfg.MaxUncommittedEntriesSize = uint64(maxEntrySize) + cfg.MaxInflightMsgs = 2 * 1024 // avoid interference + r := newRaft(cfg) + r.becomeCandidate() + r.becomeLeader() + require.Zero(t, r.uncommittedSize) + + // Set the two followers to the replicate state. Commit to tail of log. + const numFollowers = 2 + r.trk.Progress[2].BecomeReplicate() + r.trk.Progress[3].BecomeReplicate() + r.uncommittedSize = 0 + + // Send proposals to r1. The first 5 entries should be appended to the log. + propMsg := &pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{testEntry}} + propEnts := make([]*pb.Entry, maxEntries) + for i := 0; i < maxEntries; i++ { + require.NoError(t, r.Step(propMsg), "#%d", i) + propEnts[i] = testEntry + } + + // Send one more proposal to r1. It should be rejected. + require.Equal(t, ErrProposalDropped, r.Step(propMsg)) + + // Read messages and reduce the uncommitted size as if we had committed + // these entries. + ms := r.readMessages() + require.Len(t, ms, maxEntries*numFollowers) + r.reduceUncommittedSize(payloadsSize(propEnts)) + require.Zero(t, r.uncommittedSize) + + // Send a single large proposal to r1. Should be accepted even though it + // pushes us above the limit because we were beneath it before the proposal. + propEnts = make([]*pb.Entry, 2*maxEntries) + for i := range propEnts { + propEnts[i] = testEntry + } + propMsgLarge := &pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: propEnts} + require.NoError(t, r.Step(propMsgLarge)) + + // Send one more proposal to r1. It should be rejected, again. + require.Equal(t, ErrProposalDropped, r.Step(propMsg)) + + // But we can always append an entry with no Data. This is used both for the + // leader's first empty entry and for auto-transitioning out of joint config + // states. + require.NoError(t, r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}})) + + // Read messages and reduce the uncommitted size as if we had committed + // these entries. + ms = r.readMessages() + require.Len(t, ms, 2*numFollowers) + r.reduceUncommittedSize(payloadsSize(propEnts)) + require.Zero(t, r.uncommittedSize) +} + +func TestLeaderElection(t *testing.T) { + testLeaderElection(t, false) +} + +func TestLeaderElectionPreVote(t *testing.T) { + testLeaderElection(t, true) +} + +func testLeaderElection(t *testing.T, preVote bool) { + var cfg func(*Config) + candState := StateCandidate + candTerm := uint64(1) + if preVote { + cfg = preVoteConfig + // In pre-vote mode, an election that fails to complete + // leaves the node in pre-candidate state without advancing + // the term. + candState = StatePreCandidate + candTerm = 0 + } + tests := []struct { + *network + state StateType + expTerm uint64 + }{ + {newNetworkWithConfig(cfg, nil, nil, nil), StateLeader, 1}, + {newNetworkWithConfig(cfg, nil, nil, nopStepper), StateLeader, 1}, + {newNetworkWithConfig(cfg, nil, nopStepper, nopStepper), candState, candTerm}, + {newNetworkWithConfig(cfg, nil, nopStepper, nopStepper, nil), candState, candTerm}, + {newNetworkWithConfig(cfg, nil, nopStepper, nopStepper, nil, nil), StateLeader, 1}, + + // three logs further along than 0, but in the same term so rejections + // are returned instead of the votes being ignored. + {newNetworkWithConfig(cfg, + nil, entsWithConfig(cfg, 1), entsWithConfig(cfg, 1), entsWithConfig(cfg, 1, 1), nil), + StateFollower, 1}, + } + + for i, tt := range tests { + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + sm := tt.network.peers[1].(*raft) + assert.Equal(t, tt.state, sm.state, "#%d", i) + assert.Equal(t, tt.expTerm, sm.Term, "#%d", i) + } +} + +// TestLearnerElectionTimeout verfies that the leader should not start election even +// when times out. +func TestLearnerElectionTimeout(t *testing.T) { + n1 := newTestLearnerRaft(1, 10, 1, newTestMemoryStorage(withPeers(1), withLearners(2))) + n2 := newTestLearnerRaft(2, 10, 1, newTestMemoryStorage(withPeers(1), withLearners(2))) + + n1.becomeFollower(1, None) + n2.becomeFollower(1, None) + + // n2 is learner. Learner should not start election even when times out. + setRandomizedElectionTimeout(n2, n2.electionTimeout) + for i := 0; i < n2.electionTimeout; i++ { + n2.tick() + } + + assert.Equal(t, StateFollower, n2.state) +} + +// TestLearnerPromotion verifies that the learner should not election until +// it is promoted to a normal peer. +func TestLearnerPromotion(t *testing.T) { + n1 := newTestLearnerRaft(1, 10, 1, newTestMemoryStorage(withPeers(1), withLearners(2))) + n2 := newTestLearnerRaft(2, 10, 1, newTestMemoryStorage(withPeers(1), withLearners(2))) + + n1.becomeFollower(1, None) + n2.becomeFollower(1, None) + + nt := newNetwork(n1, n2) + + assert.NotEqual(t, StateLeader, n1.state) + + // n1 should become leader + setRandomizedElectionTimeout(n1, n1.electionTimeout) + for i := 0; i < n1.electionTimeout; i++ { + n1.tick() + } + n1.advanceMessagesAfterAppend() + + assert.Equal(t, StateLeader, n1.state) + assert.Equal(t, StateFollower, n2.state) + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgBeat.Enum()}) + + n1.applyConfChange((&pb.ConfChange{NodeId: new(uint64(2)), Type: pb.ConfChangeAddNode.Enum()}).AsV2()) + n2.applyConfChange((&pb.ConfChange{NodeId: new(uint64(2)), Type: pb.ConfChangeAddNode.Enum()}).AsV2()) + assert.False(t, n2.isLearner) + + // n2 start election, should become leader + setRandomizedElectionTimeout(n2, n2.electionTimeout) + for i := 0; i < n2.electionTimeout; i++ { + n2.tick() + } + n2.advanceMessagesAfterAppend() + + nt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgBeat.Enum()}) + + assert.Equal(t, StateFollower, n1.state) + assert.Equal(t, StateLeader, n2.state) +} + +// TestLearnerCanVote checks that a learner can vote when it receives a valid Vote request. +// See (*raft).Step for why this is necessary and correct behavior. +func TestLearnerCanVote(t *testing.T) { + n2 := newTestLearnerRaft(2, 10, 1, newTestMemoryStorage(withPeers(1), withLearners(2))) + + n2.becomeFollower(1, None) + + n2.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(2)), Term: new(uint64(2)), Type: pb.MsgVote.Enum(), LogTerm: new(uint64(11)), Index: new(uint64(11))}) + + msgs := n2.readMessages() + require.Len(t, msgs, 1) + require.Equal(t, msgs[0].GetType(), pb.MsgVoteResp) + require.False(t, msgs[0].GetReject(), "expected learner to not reject vote") +} + +func TestLeaderCycle(t *testing.T) { + testLeaderCycle(t, false) +} + +func TestLeaderCyclePreVote(t *testing.T) { + testLeaderCycle(t, true) +} + +// testLeaderCycle verifies that each node in a cluster can campaign +// and be elected in turn. This ensures that elections (including +// pre-vote) work when not starting from a clean slate (as they do in +// TestLeaderElection) +func testLeaderCycle(t *testing.T, preVote bool) { + var cfg func(*Config) + if preVote { + cfg = preVoteConfig + } + n := newNetworkWithConfig(cfg, nil, nil, nil) + for campaignerID := uint64(1); campaignerID <= 3; campaignerID++ { + n.send(&pb.Message{From: new(campaignerID), To: new(campaignerID), Type: pb.MsgHup.Enum()}) + + for _, peer := range n.peers { + sm := peer.(*raft) + if sm.id == campaignerID { + assert.Equal(t, StateLeader, sm.state, "preVote=%v: campaigning node %d", preVote, sm.id) + } else { + assert.Equal(t, StateFollower, sm.state, "preVote=%v: campaigning node %d, current node %d", preVote, campaignerID, sm.id) + } + } + } +} + +// TestLeaderElectionOverwriteNewerLogs tests a scenario in which a +// newly-elected leader does *not* have the newest (i.e. highest term) +// log entries, and must overwrite higher-term log entries with +// lower-term ones. +func TestLeaderElectionOverwriteNewerLogs(t *testing.T) { + testLeaderElectionOverwriteNewerLogs(t, false) +} + +func TestLeaderElectionOverwriteNewerLogsPreVote(t *testing.T) { + testLeaderElectionOverwriteNewerLogs(t, true) +} + +func testLeaderElectionOverwriteNewerLogs(t *testing.T, preVote bool) { + var cfg func(*Config) + if preVote { + cfg = preVoteConfig + } + // This network represents the results of the following sequence of + // events: + // - Node 1 won the election in term 1. + // - Node 1 replicated a log entry to node 2 but died before sending + // it to other nodes. + // - Node 3 won the second election in term 2. + // - Node 3 wrote an entry to its logs but died without sending it + // to any other nodes. + // + // At this point, nodes 1, 2, and 3 all have uncommitted entries in + // their logs and could win an election at term 3. The winner's log + // entry overwrites the losers'. (TestLeaderSyncFollowerLog tests + // the case where older log entries are overwritten, so this test + // focuses on the case where the newer entries are lost). + n := newNetworkWithConfig(cfg, + entsWithConfig(cfg, 1), // Node 1: Won first election + entsWithConfig(cfg, 1), // Node 2: Got logs from node 1 + entsWithConfig(cfg, 2), // Node 3: Won second election + votedWithConfig(cfg, 3, 2), // Node 4: Voted but didn't get logs + votedWithConfig(cfg, 3, 2)) // Node 5: Voted but didn't get logs + + // Node 1 campaigns. The election fails because a quorum of nodes + // know about the election that already happened at term 2. Node 1's + // term is pushed ahead to 2. + n.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + sm1 := n.peers[1].(*raft) + assert.Equal(t, StateFollower, sm1.state) + assert.Equal(t, uint64(2), sm1.Term) + + // Node 1 campaigns again with a higher term. This time it succeeds. + n.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + assert.Equal(t, StateLeader, sm1.state) + assert.Equal(t, uint64(3), sm1.Term) + + // Now all nodes agree on a log entry with term 1 at index 1 (and + // term 3 at index 2). + for i := range n.peers { + sm := n.peers[i].(*raft) + entries := sm.raftLog.allEntries() + require.Len(t, entries, 2) + assert.Equal(t, uint64(1), entries[0].GetTerm()) + assert.Equal(t, uint64(3), entries[1].GetTerm()) + } +} + +func TestVoteFromAnyState(t *testing.T) { + testVoteFromAnyState(t, pb.MsgVote) +} + +func TestPreVoteFromAnyState(t *testing.T) { + testVoteFromAnyState(t, pb.MsgPreVote) +} + +func testVoteFromAnyState(t *testing.T, vt pb.MessageType) { + for st := StateType(0); st < numStates; st++ { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + r.Term = 1 + + switch st { + case StateFollower: + r.becomeFollower(r.Term, 3) + case StatePreCandidate: + r.becomePreCandidate() + case StateCandidate: + r.becomeCandidate() + case StateLeader: + r.becomeCandidate() + r.becomeLeader() + } + + // Note that setting our state above may have advanced r.Term + // past its initial value. + origTerm := r.Term + newTerm := r.Term + 1 + + msg := &pb.Message{ + From: new(uint64(2)), + To: new(uint64(1)), + Type: vt.Enum(), + Term: new(newTerm), + LogTerm: new(newTerm), + Index: new(uint64(42)), + } + assert.NoError(t, r.Step(msg), "%s,%s", vt, st) + msgs := r.readMessages() + if assert.Len(t, msgs, 1, "%s,%s", vt, st) { + resp := msgs[0] + assert.Equal(t, voteRespMsgType(vt), resp.GetType(), "%s,%s", vt, st) + assert.False(t, resp.GetReject(), "%s,%s", vt, st) + } + + // If this was a real vote, we reset our state and term. + if vt == pb.MsgVote { + assert.Equal(t, StateFollower, r.state, "%s,%s", vt, st) + assert.Equal(t, newTerm, r.Term, "%s,%s", vt, st) + assert.Equal(t, uint64(2), r.Vote, "%s,%s", vt, st) + } else { + // In a prevote, nothing changes. + assert.Equal(t, st, r.state, "%s,%s", vt, st) + assert.Equal(t, origTerm, r.Term, "%s,%s", vt, st) + // if st == StateFollower or StatePreCandidate, r hasn't voted yet. + // In StateCandidate or StateLeader, it's voted for itself. + assert.True(t, r.Vote == None || r.Vote == 1, "%s,%s: vote %d, want %d or 1", vt, st, r.Vote, None) + } + } +} + +func TestLogReplication(t *testing.T) { + tests := []struct { + *network + msgs []*pb.Message + wcommitted uint64 + }{ + { + newNetwork(nil, nil, nil), + []*pb.Message{ + {From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}, + }, + 2, + }, + { + newNetwork(nil, nil, nil), + []*pb.Message{ + {From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}, + {From: new(uint64(1)), To: new(uint64(2)), Type: pb.MsgHup.Enum()}, + {From: new(uint64(1)), To: new(uint64(2)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}, + }, + 4, + }, + } + + for i, tt := range tests { + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + for _, m := range tt.msgs { + tt.send(m) + } + + for j, x := range tt.network.peers { + sm := x.(*raft) + + assert.Equal(t, tt.wcommitted, sm.raftLog.committed, "#%d.%d", i, j) + + var ents []*pb.Entry + for _, e := range nextEnts(sm, tt.network.storage[j]) { + if e.GetData() != nil { + ents = append(ents, e) + } + } + var props []*pb.Message + for _, m := range tt.msgs { + if m.GetType() == pb.MsgProp { + props = append(props, m) + } + } + for k, m := range props { + assert.Equal(t, m.GetEntries()[0].GetData(), ents[k].GetData(), "#%d.%d", i, j) + } + } + } +} + +// TestLearnerLogReplication tests that a learner can receive entries from the leader. +func TestLearnerLogReplication(t *testing.T) { + s1 := newTestMemoryStorage(withPeers(1), withLearners(2)) + n1 := newTestLearnerRaft(1, 10, 1, s1) + n2 := newTestLearnerRaft(2, 10, 1, newTestMemoryStorage(withPeers(1), withLearners(2))) + + nt := newNetwork(n1, n2) + nt.t = t + + n1.becomeFollower(1, None) + n2.becomeFollower(1, None) + + setRandomizedElectionTimeout(n1, n1.electionTimeout) + for i := 0; i < n1.electionTimeout; i++ { + n1.tick() + } + n1.advanceMessagesAfterAppend() + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgBeat.Enum()}) + + // n1 is leader and n2 is learner + assert.Equal(t, StateLeader, n1.state) + assert.True(t, n2.isLearner) + + nextCommitted := uint64(2) + { + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + } + + assert.Equal(t, nextCommitted, n1.raftLog.committed) + assert.Equal(t, n1.raftLog.committed, n2.raftLog.committed) + + match := n1.trk.Progress[2].Match + assert.Equal(t, n2.raftLog.committed, match) +} + +func TestSingleNodeCommit(t *testing.T) { + s := newTestMemoryStorage(withPeers(1)) + cfg := newTestConfig(1, 10, 1, s) + r := newRaft(cfg) + tt := newNetwork(r) + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("some data")}}}) + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("some data")}}}) + + sm := tt.peers[1].(*raft) + assert.Equal(t, uint64(3), sm.raftLog.committed) +} + +// TestCannotCommitWithoutNewTermEntry tests the entries cannot be committed +// when leader changes, no new proposal comes in and ChangeTerm proposal is +// filtered. +func TestCannotCommitWithoutNewTermEntry(t *testing.T) { + tt := newNetwork(nil, nil, nil, nil, nil) + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + // 0 cannot reach 2,3,4 + tt.cut(1, 3) + tt.cut(1, 4) + tt.cut(1, 5) + + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("some data")}}}) + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("some data")}}}) + + sm := tt.peers[1].(*raft) + assert.Equal(t, uint64(1), sm.raftLog.committed) + + // network recovery + tt.recover() + // avoid committing ChangeTerm proposal + tt.ignore(pb.MsgApp) + + // elect 2 as the new leader with term 2 + tt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgHup.Enum()}) + + // no log entries from previous term should be committed + sm = tt.peers[2].(*raft) + assert.Equal(t, uint64(1), sm.raftLog.committed) + + tt.recover() + // send heartbeat; reset wait + tt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgBeat.Enum()}) + // append an entry at current term + tt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("some data")}}}) + // expect the committed to be advanced + assert.Equal(t, uint64(5), sm.raftLog.committed) +} + +// TestCommitWithoutNewTermEntry tests the entries could be committed +// when leader changes, no new proposal comes in. +func TestCommitWithoutNewTermEntry(t *testing.T) { + tt := newNetwork(nil, nil, nil, nil, nil) + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + // 0 cannot reach 3,4,5 + tt.cut(1, 3) + tt.cut(1, 4) + tt.cut(1, 5) + + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("some data")}}}) + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("some data")}}}) + + sm := tt.peers[1].(*raft) + assert.Equal(t, uint64(1), sm.raftLog.committed) + + // network recovery + tt.recover() + + // elect 2 as the new leader with term 2 + // after append a ChangeTerm entry from the current term, all entries + // should be committed + tt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgHup.Enum()}) + + assert.Equal(t, uint64(4), sm.raftLog.committed) +} + +func TestDuelingCandidates(t *testing.T) { + s1 := newTestMemoryStorage(withPeers(1, 2, 3)) + s2 := newTestMemoryStorage(withPeers(1, 2, 3)) + s3 := newTestMemoryStorage(withPeers(1, 2, 3)) + a := newTestRaft(1, 10, 1, s1) + b := newTestRaft(2, 10, 1, s2) + c := newTestRaft(3, 10, 1, s3) + + nt := newNetwork(a, b, c) + nt.cut(1, 3) + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + + // 1 becomes leader since it receives votes from 1 and 2 + sm := nt.peers[1].(*raft) + assert.Equal(t, StateLeader, sm.state) + + // 3 stays as candidate since it receives a vote from 3 and a rejection from 2 + sm = nt.peers[3].(*raft) + assert.Equal(t, StateCandidate, sm.state) + + nt.recover() + + // candidate 3 now increases its term and tries to vote again + // we expect it to disrupt the leader 1 since it has a higher term + // 3 will be follower again since both 1 and 2 rejects its vote request since 3 does not have a long enough log + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + assert.Equal(t, StateFollower, sm.state) + + tests := []struct { + sm *raft + state StateType + term uint64 + lastIndex uint64 + }{ + {a, StateFollower, 2, 1}, + {b, StateFollower, 2, 1}, + {c, StateFollower, 2, 0}, + } + + for i, tt := range tests { + assert.Equal(t, tt.state, tt.sm.state, "#%d", i) + assert.Equal(t, tt.term, tt.sm.Term, "#%d", i) + assert.Equal(t, tt.lastIndex, tt.sm.raftLog.lastIndex(), "#%d", i) + } +} + +func TestDuelingPreCandidates(t *testing.T) { + cfgA := newTestConfig(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + cfgB := newTestConfig(2, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + cfgC := newTestConfig(3, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + cfgA.PreVote = true + cfgB.PreVote = true + cfgC.PreVote = true + a := newRaft(cfgA) + b := newRaft(cfgB) + c := newRaft(cfgC) + + nt := newNetwork(a, b, c) + nt.t = t + nt.cut(1, 3) + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + + // 1 becomes leader since it receives votes from 1 and 2 + sm := nt.peers[1].(*raft) + assert.Equal(t, StateLeader, sm.state) + + // 3 campaigns then reverts to follower when its PreVote is rejected + sm = nt.peers[3].(*raft) + assert.Equal(t, StateFollower, sm.state) + + nt.recover() + + // Candidate 3 now increases its term and tries to vote again. + // With PreVote, it does not disrupt the leader. + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + + tests := []struct { + sm *raft + state StateType + term uint64 + lastIndex uint64 + }{ + {a, StateLeader, 1, 1}, + {b, StateFollower, 1, 1}, + {c, StateFollower, 1, 0}, + } + + for i, tt := range tests { + assert.Equal(t, tt.state, tt.sm.state, "#%d", i) + assert.Equal(t, tt.term, tt.sm.Term, "#%d", i) + assert.Equal(t, tt.lastIndex, tt.sm.raftLog.lastIndex(), "#%d", i) + } +} + +func TestCandidateConcede(t *testing.T) { + tt := newNetwork(nil, nil, nil) + tt.isolate(1) + + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + tt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + + // heal the partition + tt.recover() + // send heartbeat; reset wait + tt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgBeat.Enum()}) + + data := []byte("force follower") + // send a proposal to 3 to flush out a MsgApp to 1 + tt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: data}}}) + // send heartbeat; flush out commit + tt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgBeat.Enum()}) + + a := tt.peers[1].(*raft) + assert.Equal(t, StateFollower, a.state) + assert.Equal(t, uint64(1), a.Term) + + wantLog := ltoa(newLog(&MemoryStorage{ + ents: []*pb.Entry{{}, {Data: nil, Term: new(uint64(1)), Index: new(uint64(1))}, {Term: new(uint64(1)), Index: new(uint64(2)), Data: data}}, + }, nil)) + for i, p := range tt.peers { + if sm, ok := p.(*raft); ok { + l := ltoa(sm.raftLog) + assert.Empty(t, diffu(wantLog, l), "#%d", i) + } else { + t.Logf("#%d: empty log", i) + } + } +} + +func TestSingleNodeCandidate(t *testing.T) { + tt := newNetwork(nil) + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + sm := tt.peers[1].(*raft) + assert.Equal(t, StateLeader, sm.state) +} + +func TestSingleNodePreCandidate(t *testing.T) { + tt := newNetworkWithConfig(preVoteConfig, nil) + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + sm := tt.peers[1].(*raft) + assert.Equal(t, StateLeader, sm.state) +} + +func TestOldMessages(t *testing.T) { + tt := newNetwork(nil, nil, nil) + // make 0 leader @ term 3 + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + tt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgHup.Enum()}) + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + // pretend we're an old leader trying to make progress; this entry is expected to be ignored. + tt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgApp.Enum(), Term: new(uint64(2)), Entries: index(3).terms(2)}) + // commit a new entry + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + + ents := index(0).terms(0, 1, 2, 3, 3) + ents[4].Data = []byte("somedata") + ilog := newLog(&MemoryStorage{ents: ents}, nil) + base := ltoa(ilog) + for i, p := range tt.peers { + if sm, ok := p.(*raft); ok { + l := ltoa(sm.raftLog) + assert.Empty(t, diffu(base, l), "#%d", i) + } else { + t.Logf("#%d: empty log", i) + } + } +} + +// TestOldMessagesReply - optimization - reply with new term. + +func TestProposal(t *testing.T) { + tests := []struct { + *network + success bool + }{ + {newNetwork(nil, nil, nil), true}, + {newNetwork(nil, nil, nopStepper), true}, + {newNetwork(nil, nopStepper, nopStepper), false}, + {newNetwork(nil, nopStepper, nopStepper, nil), false}, + {newNetwork(nil, nopStepper, nopStepper, nil, nil), true}, + } + + for j, tt := range tests { + send := func(m *pb.Message) { + defer func() { + // only recover if we expect it to panic (success==false) + if !tt.success { + e := recover() + if e != nil { + t.Logf("#%d: err: %s", j, e) + } + } + }() + tt.send(m) + } + + data := []byte("somedata") + + // promote 1 to become leader + send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: data}}}) + r := tt.network.peers[1].(*raft) + + wantLog := newLog(NewMemoryStorage(), raftLogger) + if tt.success { + wantLog = newLog(&MemoryStorage{ + ents: []*pb.Entry{{}, {Data: nil, Term: new(uint64(1)), Index: new(uint64(1))}, {Term: new(uint64(1)), Index: new(uint64(2)), Data: data}}, + }, nil) + } + base := ltoa(wantLog) + for i, p := range tt.peers { + if sm, ok := p.(*raft); ok { + l := ltoa(sm.raftLog) + assert.Empty(t, diffu(base, l), "#%d, peer %d", j, i) + } else { + t.Logf("#%d: peer %d empty log", j, i) + } + } + assert.Equal(t, uint64(1), r.Term, "#%d", j) + } +} + +func TestProposalByProxy(t *testing.T) { + data := []byte("somedata") + tests := []*network{ + newNetwork(nil, nil, nil), + newNetwork(nil, nil, nopStepper), + } + + for j, tt := range tests { + // promote 0 the leader + tt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + // propose via follower + tt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + + wantLog := newLog(&MemoryStorage{ + ents: []*pb.Entry{{}, {Data: nil, Term: new(uint64(1)), Index: new(uint64(1))}, {Term: new(uint64(1)), Data: data, Index: new(uint64(2))}}, + }, nil) + base := ltoa(wantLog) + for i, p := range tt.peers { + if sm, ok := p.(*raft); ok { + l := ltoa(sm.raftLog) + assert.Empty(t, diffu(base, l), "#%d.%d", j, i) + } else { + t.Logf("#%d: peer %d empty log", j, i) + } + } + sm := tt.peers[1].(*raft) + assert.Equal(t, uint64(1), sm.Term, "#%d", j) + } +} + +func TestCommit(t *testing.T) { + tests := []struct { + matches []uint64 + logs []*pb.Entry + smTerm uint64 + w uint64 + }{ + // single + {[]uint64{1}, index(1).terms(1), 1, 1}, + {[]uint64{1}, index(1).terms(1), 2, 0}, + {[]uint64{2}, index(1).terms(1, 2), 2, 2}, + {[]uint64{1}, index(1).terms(2), 2, 1}, + + // odd + {[]uint64{2, 1, 1}, index(1).terms(1, 2), 1, 1}, + {[]uint64{2, 1, 1}, index(1).terms(1, 1), 2, 0}, + {[]uint64{2, 1, 2}, index(1).terms(1, 2), 2, 2}, + {[]uint64{2, 1, 2}, index(1).terms(1, 1), 2, 0}, + + // even + {[]uint64{2, 1, 1, 1}, index(1).terms(1, 2), 1, 1}, + {[]uint64{2, 1, 1, 1}, index(1).terms(1, 1), 2, 0}, + {[]uint64{2, 1, 1, 2}, index(1).terms(1, 2), 1, 1}, + {[]uint64{2, 1, 1, 2}, index(1).terms(1, 1), 2, 0}, + {[]uint64{2, 1, 2, 2}, index(1).terms(1, 2), 2, 2}, + {[]uint64{2, 1, 2, 2}, index(1).terms(1, 1), 2, 0}, + } + + for i, tt := range tests { + storage := newTestMemoryStorage(withPeers(1)) + storage.Append(tt.logs) + storage.hardState = &pb.HardState{Term: new(tt.smTerm)} + + sm := newTestRaft(1, 10, 2, storage) + for j := 0; j < len(tt.matches); j++ { + id := uint64(j) + 1 + if id > 1 { + sm.applyConfChange((&pb.ConfChange{Type: pb.ConfChangeAddNode.Enum(), NodeId: new(id)}).AsV2()) + } + pr := sm.trk.Progress[id] + pr.Match, pr.Next = tt.matches[j], tt.matches[j]+1 + } + sm.maybeCommit() + assert.Equal(t, tt.w, sm.raftLog.committed, "#%d", i) + } +} + +func TestPastElectionTimeout(t *testing.T) { + tests := []struct { + elapse int + wprobability float64 + round bool + }{ + {5, 0, false}, + {10, 0.1, true}, + {13, 0.4, true}, + {15, 0.6, true}, + {18, 0.9, true}, + {20, 1, false}, + } + + for i, tt := range tests { + sm := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1))) + sm.electionElapsed = tt.elapse + c := 0 + for j := 0; j < 10000; j++ { + sm.resetRandomizedElectionTimeout() + if sm.pastElectionTimeout() { + c++ + } + } + got := float64(c) / 10000.0 + if tt.round { + got = math.Floor(got*10+0.5) / 10.0 + } + assert.Equal(t, tt.wprobability, got, "#%d", i) + } +} + +// TestStepIgnoreOldTermMsg to ensure that the Step function ignores the message +// from old term and does not pass it to the actual stepX function. +func TestStepIgnoreOldTermMsg(t *testing.T) { + called := false + fakeStep := func(_ *raft, _ *pb.Message) error { + called = true + return nil + } + sm := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1))) + sm.step = fakeStep + sm.Term = 2 + sm.Step(&pb.Message{Type: pb.MsgApp.Enum(), Term: new(sm.Term - 1)}) + assert.False(t, called) +} + +// TestHandleMsgApp ensures: +// 1. Reply false if log doesn’t contain an entry at prevLogIndex whose term matches prevLogTerm. +// 2. If an existing entry conflicts with a new one (same index but different terms), +// delete the existing entry and all that follow it; append any new entries not already in the log. +// 3. If leaderCommit > commitIndex, set commitIndex = min(leaderCommit, index of last new entry). +func TestHandleMsgApp(t *testing.T) { + tests := []struct { + m *pb.Message + wIndex uint64 + wCommit uint64 + wReject bool + }{ + // Ensure 1 + {&pb.Message{Type: pb.MsgApp.Enum(), Term: new(uint64(2)), LogTerm: new(uint64(3)), Index: new(uint64(2)), Commit: new(uint64(3))}, 2, 0, true}, // previous log mismatch + {&pb.Message{Type: pb.MsgApp.Enum(), Term: new(uint64(2)), LogTerm: new(uint64(3)), Index: new(uint64(3)), Commit: new(uint64(3))}, 2, 0, true}, // previous log non-exist + + // Ensure 2 + {&pb.Message{Type: pb.MsgApp.Enum(), Term: new(uint64(2)), LogTerm: new(uint64(1)), Index: new(uint64(1)), Commit: new(uint64(1))}, 2, 1, false}, + {&pb.Message{Type: pb.MsgApp.Enum(), Term: new(uint64(2)), LogTerm: new(uint64(0)), Index: new(uint64(0)), Commit: new(uint64(1)), Entries: []*pb.Entry{{Index: new(uint64(1)), Term: new(uint64(2))}}}, 1, 1, false}, + {&pb.Message{Type: pb.MsgApp.Enum(), Term: new(uint64(2)), LogTerm: new(uint64(2)), Index: new(uint64(2)), Commit: new(uint64(3)), Entries: []*pb.Entry{{Index: new(uint64(3)), Term: new(uint64(2))}, {Index: new(uint64(4)), Term: new(uint64(2))}}}, 4, 3, false}, + {&pb.Message{Type: pb.MsgApp.Enum(), Term: new(uint64(2)), LogTerm: new(uint64(2)), Index: new(uint64(2)), Commit: new(uint64(4)), Entries: []*pb.Entry{{Index: new(uint64(3)), Term: new(uint64(2))}}}, 3, 3, false}, + {&pb.Message{Type: pb.MsgApp.Enum(), Term: new(uint64(2)), LogTerm: new(uint64(1)), Index: new(uint64(1)), Commit: new(uint64(4)), Entries: []*pb.Entry{{Index: new(uint64(2)), Term: new(uint64(2))}}}, 2, 2, false}, + + // Ensure 3 + {&pb.Message{Type: pb.MsgApp.Enum(), Term: new(uint64(1)), LogTerm: new(uint64(1)), Index: new(uint64(1)), Commit: new(uint64(3))}, 2, 1, false}, // match entry 1, commit up to last new entry 1 + {&pb.Message{Type: pb.MsgApp.Enum(), Term: new(uint64(1)), LogTerm: new(uint64(1)), Index: new(uint64(1)), Commit: new(uint64(3)), Entries: []*pb.Entry{{Index: new(uint64(2)), Term: new(uint64(2))}}}, 2, 2, false}, // match entry 1, commit up to last new entry 2 + {&pb.Message{Type: pb.MsgApp.Enum(), Term: new(uint64(2)), LogTerm: new(uint64(2)), Index: new(uint64(2)), Commit: new(uint64(3))}, 2, 2, false}, // match entry 2, commit up to last new entry 2 + {&pb.Message{Type: pb.MsgApp.Enum(), Term: new(uint64(2)), LogTerm: new(uint64(2)), Index: new(uint64(2)), Commit: new(uint64(4))}, 2, 2, false}, // commit up to log.last() + } + + for i, tt := range tests { + storage := newTestMemoryStorage(withPeers(1)) + require.NoError(t, storage.Append(index(1).terms(1, 2))) + sm := newTestRaft(1, 10, 1, storage) + sm.becomeFollower(2, None) + + sm.handleAppendEntries(tt.m) + assert.Equal(t, tt.wIndex, sm.raftLog.lastIndex(), "#%d", i) + assert.Equal(t, tt.wCommit, sm.raftLog.committed, "#%d", i) + m := sm.readMessages() + require.Len(t, m, 1, "#%d", i) + assert.Equal(t, tt.wReject, m[0].GetReject(), "#%d", i) + } +} + +// TestHandleHeartbeat ensures that the follower commits to the commit in the message. +func TestHandleHeartbeat(t *testing.T) { + commit := uint64(2) + tests := []struct { + m *pb.Message + wCommit uint64 + }{ + {&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgHeartbeat.Enum(), Term: new(uint64(2)), Commit: new(commit + 1)}, commit + 1}, + {&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgHeartbeat.Enum(), Term: new(uint64(2)), Commit: new(commit - 1)}, commit}, // do not decrease commit + } + + for i, tt := range tests { + storage := newTestMemoryStorage(withPeers(1, 2)) + require.NoError(t, storage.Append(index(1).terms(1, 2, 3))) + sm := newTestRaft(1, 5, 1, storage) + sm.becomeFollower(2, 2) + sm.raftLog.commitTo(commit) + sm.handleHeartbeat(tt.m) + assert.Equal(t, tt.wCommit, sm.raftLog.committed, "#%d", i) + m := sm.readMessages() + require.Len(t, m, 1, "#%d", i) + assert.Equal(t, pb.MsgHeartbeatResp, m[0].GetType(), "#%d", i) + } +} + +// TestHandleHeartbeatResp ensures that we re-send log entries when we get a heartbeat response. +func TestHandleHeartbeatResp(t *testing.T) { + storage := newTestMemoryStorage(withPeers(1, 2)) + require.NoError(t, storage.Append(index(1).terms(1, 2, 3))) + sm := newTestRaft(1, 5, 1, storage) + sm.becomeCandidate() + sm.becomeLeader() + sm.raftLog.commitTo(sm.raftLog.lastIndex()) + + // A heartbeat response from a node that is behind; re-send MsgApp + sm.Step(&pb.Message{From: new(uint64(2)), Type: pb.MsgHeartbeatResp.Enum()}) + msgs := sm.readMessages() + require.Len(t, msgs, 1) + assert.Equal(t, pb.MsgApp, msgs[0].GetType()) + + // A second heartbeat response generates another MsgApp re-send + sm.Step(&pb.Message{From: new(uint64(2)), Type: pb.MsgHeartbeatResp.Enum()}) + msgs = sm.readMessages() + require.Len(t, msgs, 1) + assert.Equal(t, pb.MsgApp, msgs[0].GetType()) + + // Once we have an MsgAppResp, heartbeats no longer send MsgApp. + sm.Step(&pb.Message{ + From: new(uint64(2)), + Type: pb.MsgAppResp.Enum(), + Index: new(msgs[0].GetIndex() + uint64(len(msgs[0].GetEntries()))), + }) + // Consume the message sent in response to MsgAppResp + sm.readMessages() + + sm.Step(&pb.Message{From: new(uint64(2)), Type: pb.MsgHeartbeatResp.Enum()}) + msgs = sm.readMessages() + require.Empty(t, msgs) +} + +// TestRaftFreesReadOnlyMem ensures raft will free read requests from +// readOnly unconfirmedReads once they are confirmed. +// related issue: https://github.com/etcd-io/etcd/issues/7571 +func TestRaftFreesReadOnlyMem(t *testing.T) { + sm := newTestRaft(1, 5, 1, newTestMemoryStorage(withPeers(1, 2))) + sm.becomeCandidate() + sm.becomeLeader() + sm.raftLog.commitTo(sm.raftLog.lastIndex()) + + reqCtx := []byte("ctx") + + // leader starts linearizable read request. + // more info: raft dissertation 6.4, step 2. + sm.Step(&pb.Message{From: new(uint64(2)), Type: pb.MsgReadIndex.Enum(), Entries: []*pb.Entry{{Data: reqCtx}}}) + msgs := sm.readMessages() + require.Len(t, msgs, 1) + require.Equal(t, pb.MsgHeartbeat, msgs[0].GetType()) + require.Len(t, sm.readOnly.unconfirmedReads, 1) + + // heartbeat responses from majority of followers (1 in this case) + // acknowledge the authority of the leader. + // more info: raft dissertation 6.4, step 3. + sm.Step(&pb.Message{From: new(uint64(2)), Type: pb.MsgHeartbeatResp.Enum(), Context: msgs[0].GetContext()}) + require.Empty(t, sm.readOnly.unconfirmedReads) +} + +// TestMsgAppRespWaitReset verifies the resume behavior of a leader +// MsgAppResp. +func TestMsgAppRespWaitReset(t *testing.T) { + s := newTestMemoryStorage(withPeers(1, 2, 3)) + sm := newTestRaft(1, 5, 1, s) + sm.becomeCandidate() + sm.becomeLeader() + + // Run n1 which includes sending a message like the below + // one to n2, but also appending to its own log. + nextEnts(sm, s) + + // Node 2 acks the first entry, making it committed. + sm.Step(&pb.Message{ + From: new(uint64(2)), + Type: pb.MsgAppResp.Enum(), + Index: new(uint64(1)), + }) + require.Equal(t, uint64(1), sm.raftLog.committed) + + // Also consume the MsgApp messages that update Commit on the followers. + sm.readMessages() + + // A new command is now proposed on node 1. + sm.Step(&pb.Message{ + From: new(uint64(1)), + Type: pb.MsgProp.Enum(), + Entries: []*pb.Entry{{}}, + }) + + // The command is broadcast to all nodes not in the wait state. + // Node 2 left the wait state due to its MsgAppResp, but node 3 is still waiting. + msgs := sm.readMessages() + require.Len(t, msgs, 1) + assert.Equal(t, pb.MsgApp, msgs[0].GetType()) + assert.Equal(t, uint64(2), msgs[0].GetTo()) + assert.Len(t, msgs[0].GetEntries(), 1) + assert.Equal(t, uint64(2), msgs[0].GetEntries()[0].GetIndex()) + + // Now Node 3 acks the first entry. This releases the wait and entry 2 is sent. + sm.Step(&pb.Message{ + From: new(uint64(3)), + Type: pb.MsgAppResp.Enum(), + Index: new(uint64(1)), + }) + msgs = sm.readMessages() + require.Len(t, msgs, 1) + assert.Equal(t, pb.MsgApp, msgs[0].GetType()) + assert.Equal(t, uint64(3), msgs[0].GetTo()) + assert.Len(t, msgs[0].GetEntries(), 1) + assert.Equal(t, uint64(2), msgs[0].GetEntries()[0].GetIndex()) +} + +func TestRecvMsgVote(t *testing.T) { + testRecvMsgVote(t, pb.MsgVote) +} + +func TestRecvMsgPreVote(t *testing.T) { + testRecvMsgVote(t, pb.MsgPreVote) +} + +func testRecvMsgVote(t *testing.T, msgType pb.MessageType) { + tests := []struct { + state StateType + index, logTerm uint64 + voteFor uint64 + wreject bool + }{ + {StateFollower, 0, 0, None, true}, + {StateFollower, 0, 1, None, true}, + {StateFollower, 0, 2, None, true}, + {StateFollower, 0, 3, None, false}, + + {StateFollower, 1, 0, None, true}, + {StateFollower, 1, 1, None, true}, + {StateFollower, 1, 2, None, true}, + {StateFollower, 1, 3, None, false}, + + {StateFollower, 2, 0, None, true}, + {StateFollower, 2, 1, None, true}, + {StateFollower, 2, 2, None, false}, + {StateFollower, 2, 3, None, false}, + + {StateFollower, 3, 0, None, true}, + {StateFollower, 3, 1, None, true}, + {StateFollower, 3, 2, None, false}, + {StateFollower, 3, 3, None, false}, + + {StateFollower, 3, 2, 2, false}, + {StateFollower, 3, 2, 1, true}, + + {StateLeader, 3, 3, 1, true}, + {StatePreCandidate, 3, 3, 1, true}, + {StateCandidate, 3, 3, 1, true}, + } + + for i, tt := range tests { + sm := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1))) + sm.state = tt.state + switch tt.state { + case StateFollower: + sm.step = stepFollower + case StateCandidate, StatePreCandidate: + sm.step = stepCandidate + case StateLeader: + sm.step = stepLeader + } + sm.Vote = tt.voteFor + sm.raftLog = newLog(&MemoryStorage{ents: index(0).terms(0, 2, 2)}, nil) + + // raft.Term is greater than or equal to raft.raftLog.lastTerm. In this + // test we're only testing MsgVote responses when the campaigning node + // has a different raft log compared to the recipient node. + // Additionally we're verifying behaviour when the recipient node has + // already given out its vote for its current term. We're not testing + // what the recipient node does when receiving a message with a + // different term number, so we simply initialize both term numbers to + // be the same. + term := max(sm.raftLog.lastEntryID().term, tt.logTerm) + sm.Term = term + sm.Step(&pb.Message{Type: msgType.Enum(), Term: new(term), From: new(uint64(2)), Index: new(tt.index), LogTerm: new(tt.logTerm)}) + + msgs := sm.readMessages() + require.Len(t, msgs, 1, "#%d", i) + assert.Equal(t, voteRespMsgType(msgType), msgs[0].GetType(), "#%d", i) + assert.Equal(t, tt.wreject, msgs[0].GetReject(), "#%d", i) + } +} + +func TestStateTransition(t *testing.T) { + tests := []struct { + from StateType + to StateType + wallow bool + wterm uint64 + wlead uint64 + }{ + {StateFollower, StateFollower, true, 1, None}, + {StateFollower, StatePreCandidate, true, 0, None}, + {StateFollower, StateCandidate, true, 1, None}, + {StateFollower, StateLeader, false, 0, None}, + + {StatePreCandidate, StateFollower, true, 0, None}, + {StatePreCandidate, StatePreCandidate, true, 0, None}, + {StatePreCandidate, StateCandidate, true, 1, None}, + {StatePreCandidate, StateLeader, true, 0, 1}, + + {StateCandidate, StateFollower, true, 0, None}, + {StateCandidate, StatePreCandidate, true, 0, None}, + {StateCandidate, StateCandidate, true, 1, None}, + {StateCandidate, StateLeader, true, 0, 1}, + + {StateLeader, StateFollower, true, 1, None}, + {StateLeader, StatePreCandidate, false, 0, None}, + {StateLeader, StateCandidate, false, 1, None}, + {StateLeader, StateLeader, true, 0, 1}, + } + + for i, tt := range tests { + func() { + defer func() { + if r := recover(); r != nil { + assert.False(t, tt.wallow, "#%d", i) + } + }() + + sm := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1))) + sm.state = tt.from + + switch tt.to { + case StateFollower: + sm.becomeFollower(tt.wterm, tt.wlead) + case StatePreCandidate: + sm.becomePreCandidate() + case StateCandidate: + sm.becomeCandidate() + case StateLeader: + sm.becomeLeader() + } + + assert.Equal(t, tt.wterm, sm.Term, "#%d", i) + assert.Equal(t, tt.wlead, sm.lead, "#%d", i) + }() + } +} + +func TestAllServerStepdown(t *testing.T) { + tests := []struct { + state StateType + + wstate StateType + wterm uint64 + windex uint64 + }{ + {StateFollower, StateFollower, 3, 0}, + {StatePreCandidate, StateFollower, 3, 0}, + {StateCandidate, StateFollower, 3, 0}, + {StateLeader, StateFollower, 3, 1}, + } + + tmsgTypes := [...]pb.MessageType{pb.MsgVote, pb.MsgApp} + tterm := uint64(3) + + for i, tt := range tests { + sm := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + switch tt.state { + case StateFollower: + sm.becomeFollower(1, None) + case StatePreCandidate: + sm.becomePreCandidate() + case StateCandidate: + sm.becomeCandidate() + case StateLeader: + sm.becomeCandidate() + sm.becomeLeader() + } + + for j, msgType := range tmsgTypes { + sm.Step(&pb.Message{From: new(uint64(2)), Type: msgType.Enum(), Term: new(tterm), LogTerm: new(tterm)}) + + assert.Equal(t, tt.wstate, sm.state, "#%d.%d", i, j) + assert.Equal(t, tt.wterm, sm.Term, "#%d.%d", i, j) + assert.Equal(t, tt.windex, sm.raftLog.lastIndex(), "#%d.%d", i, j) + assert.Len(t, sm.raftLog.allEntries(), int(tt.windex), "#%d.%d", i, j) + + wlead := uint64(2) + if msgType == pb.MsgVote { + wlead = None + } + assert.Equal(t, wlead, sm.lead, "#%d.%d", i, j) + } + } +} + +func TestCandidateResetTermMsgHeartbeat(t *testing.T) { + testCandidateResetTerm(t, pb.MsgHeartbeat) +} + +func TestCandidateResetTermMsgApp(t *testing.T) { + testCandidateResetTerm(t, pb.MsgApp) +} + +// testCandidateResetTerm tests when a candidate receives a +// MsgHeartbeat or MsgApp from leader, "Step" resets the term +// with leader's and reverts back to follower. +func testCandidateResetTerm(t *testing.T, mt pb.MessageType) { + a := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + b := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + c := newTestRaft(3, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + + nt := newNetwork(a, b, c) + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + assert.Equal(t, StateLeader, a.state) + assert.Equal(t, StateFollower, b.state) + assert.Equal(t, StateFollower, c.state) + + // isolate 3 and increase term in rest + nt.isolate(3) + + nt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgHup.Enum()}) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + assert.Equal(t, StateLeader, a.state) + assert.Equal(t, StateFollower, b.state) + + // trigger campaign in isolated c + c.resetRandomizedElectionTimeout() + for i := 0; i < c.randomizedElectionTimeout; i++ { + c.tick() + } + c.advanceMessagesAfterAppend() + + assert.Equal(t, StateCandidate, c.state) + + nt.recover() + + // leader sends to isolated candidate + // and expects candidate to revert to follower + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(3)), Term: new(a.Term), Type: mt.Enum()}) + + assert.Equal(t, StateFollower, c.state) + // follower c term is reset with leader's + assert.Equal(t, a.Term, c.Term) + +} + +// The following three tests exercise the behavior of a (pre-)candidate when its +// own self-vote is delivered back to itself after the peer has already learned +// that it has lost the election. The self-vote should be ignored in these cases. + +func TestCandidateSelfVoteAfterLostElection(t *testing.T) { + testCandidateSelfVoteAfterLostElection(t, false) +} + +func TestCandidateSelfVoteAfterLostElectionPreVote(t *testing.T) { + testCandidateSelfVoteAfterLostElection(t, true) +} + +func testCandidateSelfVoteAfterLostElection(t *testing.T, preVote bool) { + sm := newTestRaft(1, 5, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + sm.preVote = preVote + + // n1 calls an election. + sm.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + steps := sm.takeMessagesAfterAppend() + + // n1 hears that n2 already won the election before it has had a + // change to sync its vote to disk and account for its self-vote. + // Becomes a follower. + sm.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Term: new(sm.Term), Type: pb.MsgHeartbeat.Enum()}) + assert.Equal(t, StateFollower, sm.state) + + // n1 remains a follower even after its self-vote is delivered. + sm.stepOrSend(steps) + assert.Equal(t, StateFollower, sm.state) + + // Its self-vote does not make its way to its ProgressTracker. + granted, _, _ := sm.trk.TallyVotes() + assert.Zero(t, granted) + +} + +func TestCandidateDeliversPreCandidateSelfVoteAfterBecomingCandidate(t *testing.T) { + sm := newTestRaft(1, 5, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + sm.preVote = true + + // n1 calls an election. + sm.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + assert.Equal(t, StatePreCandidate, sm.state) + steps := sm.takeMessagesAfterAppend() + + // n1 receives pre-candidate votes from both other peers before + // voting for itself. n1 becomes a candidate. + // NB: pre-vote messages carry the local term + 1. + sm.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Term: new(sm.Term + 1), Type: pb.MsgPreVoteResp.Enum()}) + sm.Step(&pb.Message{From: new(uint64(3)), To: new(uint64(1)), Term: new(sm.Term + 1), Type: pb.MsgPreVoteResp.Enum()}) + assert.Equal(t, StateCandidate, sm.state) + + // n1 remains a candidate even after its delayed pre-vote self-vote is + // delivered. + sm.stepOrSend(steps) + assert.Equal(t, StateCandidate, sm.state) + + steps = sm.takeMessagesAfterAppend() + + // Its pre-vote self-vote does not make its way to its ProgressTracker. + granted, _, _ := sm.trk.TallyVotes() + assert.Zero(t, granted) + + // A single vote from n2 does not move n1 to the leader. + sm.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Term: new(sm.Term), Type: pb.MsgVoteResp.Enum()}) + assert.Equal(t, StateCandidate, sm.state) + + // n1 becomes the leader once its self-vote is received because now + // quorum is reached. + sm.stepOrSend(steps) + assert.Equal(t, StateLeader, sm.state) +} + +func TestLeaderMsgAppSelfAckAfterTermChange(t *testing.T) { + sm := newTestRaft(1, 5, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + sm.becomeCandidate() + sm.becomeLeader() + + // n1 proposes a write. + sm.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + steps := sm.takeMessagesAfterAppend() + + // n1 hears that n2 is the new leader. + sm.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Term: new(sm.Term + 1), Type: pb.MsgHeartbeat.Enum()}) + assert.Equal(t, StateFollower, sm.state) + + // n1 advances, ignoring its earlier self-ack of its MsgApp. The + // corresponding MsgAppResp is ignored because it carries an earlier term. + sm.stepOrSend(steps) + assert.Equal(t, StateFollower, sm.state) +} + +func TestLeaderStepdownWhenQuorumActive(t *testing.T) { + sm := newTestRaft(1, 5, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + + sm.checkQuorum = true + + sm.becomeCandidate() + sm.becomeLeader() + + for i := 0; i < sm.electionTimeout+1; i++ { + sm.Step(&pb.Message{From: new(uint64(2)), Type: pb.MsgHeartbeatResp.Enum(), Term: new(sm.Term)}) + sm.tick() + } + + assert.Equal(t, StateLeader, sm.state) +} + +func TestLeaderStepdownWhenQuorumLost(t *testing.T) { + sm := newTestRaft(1, 5, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + + sm.checkQuorum = true + + sm.becomeCandidate() + sm.becomeLeader() + + for i := 0; i < sm.electionTimeout+1; i++ { + sm.tick() + } + + assert.Equal(t, StateFollower, sm.state) +} + +func TestLeaderSupersedingWithCheckQuorum(t *testing.T) { + a := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + b := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + c := newTestRaft(3, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + + a.checkQuorum = true + b.checkQuorum = true + c.checkQuorum = true + + nt := newNetwork(a, b, c) + setRandomizedElectionTimeout(b, b.electionTimeout+1) + + for i := 0; i < b.electionTimeout; i++ { + b.tick() + } + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + assert.Equal(t, StateLeader, a.state) + assert.Equal(t, StateFollower, c.state) + + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + + // Peer b rejected c's vote since its electionElapsed had not reached to electionTimeout + assert.Equal(t, StateCandidate, c.state) + + // Letting b's electionElapsed reach to electionTimeout + for i := 0; i < b.electionTimeout; i++ { + b.tick() + } + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + + assert.Equal(t, StateLeader, c.state) +} + +func TestLeaderElectionWithCheckQuorum(t *testing.T) { + a := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + b := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + c := newTestRaft(3, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + + a.checkQuorum = true + b.checkQuorum = true + c.checkQuorum = true + + nt := newNetwork(a, b, c) + setRandomizedElectionTimeout(a, a.electionTimeout+1) + setRandomizedElectionTimeout(b, b.electionTimeout+2) + + // Immediately after creation, votes are cast regardless of the + // election timeout. + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + assert.Equal(t, StateLeader, a.state) + assert.Equal(t, StateFollower, c.state) + + // need to reset randomizedElectionTimeout larger than electionTimeout again, + // because the value might be reset to electionTimeout since the last state changes + setRandomizedElectionTimeout(a, a.electionTimeout+1) + setRandomizedElectionTimeout(b, b.electionTimeout+2) + for i := 0; i < a.electionTimeout; i++ { + a.tick() + } + for i := 0; i < b.electionTimeout; i++ { + b.tick() + } + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + + assert.Equal(t, StateFollower, a.state) + assert.Equal(t, StateLeader, c.state) +} + +// TestFreeStuckCandidateWithCheckQuorum ensures that a candidate with a higher term +// can disrupt the leader even if the leader still "officially" holds the lease, The +// leader is expected to step down and adopt the candidate's term +func TestFreeStuckCandidateWithCheckQuorum(t *testing.T) { + a := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + b := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + c := newTestRaft(3, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + + a.checkQuorum = true + b.checkQuorum = true + c.checkQuorum = true + + nt := newNetwork(a, b, c) + setRandomizedElectionTimeout(b, b.electionTimeout+1) + + for i := 0; i < b.electionTimeout; i++ { + b.tick() + } + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + nt.isolate(1) + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + + assert.Equal(t, StateFollower, b.state) + assert.Equal(t, StateCandidate, c.state) + assert.Equal(t, b.Term+1, c.Term) + + // Vote again for safety + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + + assert.Equal(t, StateFollower, b.state) + assert.Equal(t, StateCandidate, c.state) + assert.Equal(t, b.Term+2, c.Term) + + nt.recover() + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(3)), Type: pb.MsgHeartbeat.Enum(), Term: new(a.Term)}) + + // Disrupt the leader so that the stuck peer is freed + assert.Equal(t, StateFollower, a.state) + assert.Equal(t, a.Term, c.Term) + + // Vote again, should become leader this time + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + + assert.Equal(t, StateLeader, c.state) +} + +func TestNonPromotableVoterWithCheckQuorum(t *testing.T) { + a := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2))) + b := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1))) + + a.checkQuorum = true + b.checkQuorum = true + + nt := newNetwork(a, b) + setRandomizedElectionTimeout(b, b.electionTimeout+1) + // Need to remove 2 again to make it a non-promotable node since newNetwork overwritten some internal states + b.applyConfChange((&pb.ConfChange{Type: pb.ConfChangeRemoveNode.Enum(), NodeId: new(uint64(2))}).AsV2()) + + require.False(t, b.promotable()) + + for i := 0; i < b.electionTimeout; i++ { + b.tick() + } + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + assert.Equal(t, StateLeader, a.state) + assert.Equal(t, StateFollower, b.state) + assert.Equal(t, uint64(1), b.lead) +} + +// TestDisruptiveFollower tests isolated follower, +// with slow network incoming from leader, election times out +// to become a candidate with an increased term. Then, the +// candiate's response to late leader heartbeat forces the leader +// to step down. +func TestDisruptiveFollower(t *testing.T) { + n1 := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + n2 := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + n3 := newTestRaft(3, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + + n1.checkQuorum = true + n2.checkQuorum = true + n3.checkQuorum = true + + n1.becomeFollower(1, None) + n2.becomeFollower(1, None) + n3.becomeFollower(1, None) + + nt := newNetwork(n1, n2, n3) + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + // check state + require.Equal(t, StateLeader, n1.state) + require.Equal(t, StateFollower, n2.state) + require.Equal(t, StateFollower, n3.state) + + // etcd server "advanceTicksForElection" on restart; + // this is to expedite campaign trigger when given larger + // election timeouts (e.g. multi-datacenter deploy) + // Or leader messages are being delayed while ticks elapse + setRandomizedElectionTimeout(n3, n3.electionTimeout+2) + for i := 0; i < n3.randomizedElectionTimeout-1; i++ { + n3.tick() + } + + // ideally, before last election tick elapses, + // the follower n3 receives "pb.MsgApp" or "pb.MsgHeartbeat" + // from leader n1, and then resets its "electionElapsed" + // however, last tick may elapse before receiving any + // messages from leader, thus triggering campaign + n3.tick() + + // n1 is still leader yet + // while its heartbeat to candidate n3 is being delayed + + // check state + require.Equal(t, StateLeader, n1.state) + require.Equal(t, StateFollower, n2.state) + require.Equal(t, StateCandidate, n3.state) + + // check term + require.Equal(t, uint64(2), n1.Term) + require.Equal(t, uint64(2), n2.Term) + require.Equal(t, uint64(3), n3.Term) + + // while outgoing vote requests are still queued in n3, + // leader heartbeat finally arrives at candidate n3 + // however, due to delayed network from leader, leader + // heartbeat was sent with lower term than candidate's + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(3)), Term: new(n1.Term), Type: pb.MsgHeartbeat.Enum()}) + + // then candidate n3 responds with "pb.MsgAppResp" of higher term + // and leader steps down from a message with higher term + // this is to disrupt the current leader, so that candidate + // with higher term can be freed with following election + + // check state + require.Equal(t, StateFollower, n1.state) + require.Equal(t, StateFollower, n2.state) + require.Equal(t, StateCandidate, n3.state) + + // check term + require.Equal(t, uint64(3), n1.Term) + require.Equal(t, uint64(2), n2.Term) + require.Equal(t, uint64(3), n3.Term) +} + +// TestDisruptiveFollowerPreVote tests isolated follower, +// with slow network incoming from leader, election times out +// to become a pre-candidate with less log than current leader. +// Then pre-vote phase prevents this isolated node from forcing +// current leader to step down, thus less disruptions. +func TestDisruptiveFollowerPreVote(t *testing.T) { + n1 := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + n2 := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + n3 := newTestRaft(3, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + + n1.checkQuorum = true + n2.checkQuorum = true + n3.checkQuorum = true + + n1.becomeFollower(1, None) + n2.becomeFollower(1, None) + n3.becomeFollower(1, None) + + nt := newNetwork(n1, n2, n3) + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + // check state + require.Equal(t, StateLeader, n1.state) + require.Equal(t, StateFollower, n2.state) + require.Equal(t, StateFollower, n3.state) + + nt.isolate(3) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + n1.preVote = true + n2.preVote = true + n3.preVote = true + nt.recover() + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + + // check state + require.Equal(t, StateLeader, n1.state) + require.Equal(t, StateFollower, n2.state) + require.Equal(t, StatePreCandidate, n3.state) + + // check term + require.Equal(t, uint64(2), n1.Term) + require.Equal(t, uint64(2), n2.Term) + require.Equal(t, uint64(2), n3.Term) + + // delayed leader heartbeat does not force current leader to step down + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(3)), Term: new(n1.Term), Type: pb.MsgHeartbeat.Enum()}) + require.Equal(t, StateLeader, n1.state) +} + +func TestReadOnlyOptionSafe(t *testing.T) { + a := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + b := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + c := newTestRaft(3, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + + nt := newNetwork(a, b, c) + setRandomizedElectionTimeout(b, b.electionTimeout+1) + + for i := 0; i < b.electionTimeout; i++ { + b.tick() + } + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + require.Equal(t, StateLeader, a.state) + + tests := []struct { + sm *raft + proposals int + wri uint64 + wctx []byte + }{ + {a, 10, 11, []byte("ctx1")}, + {b, 10, 21, []byte("ctx2")}, + {c, 10, 31, []byte("ctx3")}, + {a, 10, 41, []byte("ctx4")}, + {b, 10, 51, []byte("ctx5")}, + {c, 10, 61, []byte("ctx6")}, + } + + for i, tt := range tests { + for j := 0; j < tt.proposals; j++ { + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}}) + } + + nt.send(&pb.Message{From: new(tt.sm.id), To: new(tt.sm.id), Type: pb.MsgReadIndex.Enum(), Entries: []*pb.Entry{{Data: tt.wctx}}}) + + r := tt.sm + assert.NotEmpty(t, r.readStates, "#%d", i) + rs := r.readStates[0] + assert.Equal(t, tt.wri, rs.Index, "#%d", i) + assert.Equal(t, tt.wctx, rs.RequestCtx, "#%d", i) + r.readStates = nil + } +} + +func TestReadOnlyWithLearner(t *testing.T) { + s := newTestMemoryStorage(withPeers(1), withLearners(2)) + a := newTestLearnerRaft(1, 10, 1, s) + b := newTestLearnerRaft(2, 10, 1, newTestMemoryStorage(withPeers(1), withLearners(2))) + + nt := newNetwork(a, b) + setRandomizedElectionTimeout(b, b.electionTimeout+1) + + for i := 0; i < b.electionTimeout; i++ { + b.tick() + } + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + require.Equal(t, StateLeader, a.state) + + tests := []struct { + sm *raft + proposals int + wri uint64 + wctx []byte + }{ + {a, 10, 11, []byte("ctx1")}, + {b, 10, 21, []byte("ctx2")}, + {a, 10, 31, []byte("ctx3")}, + {b, 10, 41, []byte("ctx4")}, + } + + for i, tt := range tests { + for j := 0; j < tt.proposals; j++ { + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}}) + nextEnts(a, s) // append the entries on the leader + } + + nt.send(&pb.Message{From: new(tt.sm.id), To: new(tt.sm.id), Type: pb.MsgReadIndex.Enum(), Entries: []*pb.Entry{{Data: tt.wctx}}}) + + r := tt.sm + require.NotEmpty(t, r.readStates, "#%d", i) + rs := r.readStates[0] + assert.Equal(t, tt.wri, rs.Index, "#%d", i) + assert.Equal(t, tt.wctx, rs.RequestCtx, "#%d", i) + r.readStates = nil + } +} + +func TestReadOnlyOptionLease(t *testing.T) { + a := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + b := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + c := newTestRaft(3, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + a.readOnly.option = ReadOnlyLeaseBased + b.readOnly.option = ReadOnlyLeaseBased + c.readOnly.option = ReadOnlyLeaseBased + a.checkQuorum = true + b.checkQuorum = true + c.checkQuorum = true + + nt := newNetwork(a, b, c) + setRandomizedElectionTimeout(b, b.electionTimeout+1) + + for i := 0; i < b.electionTimeout; i++ { + b.tick() + } + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + require.Equal(t, StateLeader, a.state) + + tests := []struct { + sm *raft + proposals int + wri uint64 + wctx []byte + }{ + {a, 10, 11, []byte("ctx1")}, + {b, 10, 21, []byte("ctx2")}, + {c, 10, 31, []byte("ctx3")}, + {a, 10, 41, []byte("ctx4")}, + {b, 10, 51, []byte("ctx5")}, + {c, 10, 61, []byte("ctx6")}, + } + + for i, tt := range tests { + for j := 0; j < tt.proposals; j++ { + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}}) + } + + nt.send(&pb.Message{From: new(tt.sm.id), To: new(tt.sm.id), Type: pb.MsgReadIndex.Enum(), Entries: []*pb.Entry{{Data: tt.wctx}}}) + + r := tt.sm + rs := r.readStates[0] + assert.Equal(t, tt.wri, rs.Index, "#%d", i) + assert.Equal(t, tt.wctx, rs.RequestCtx, "#%d", i) + r.readStates = nil + } +} + +// TestReadOnlyForNewLeader ensures that a leader only accepts MsgReadIndex message +// when it commits at least one log entry at it term. +func TestReadOnlyForNewLeader(t *testing.T) { + nodeConfigs := []struct { + id uint64 + committed uint64 + applied uint64 + compactIndex uint64 + }{ + {1, 1, 1, 0}, + {2, 2, 2, 2}, + {3, 2, 2, 2}, + } + peers := make([]stateMachine, 0) + for _, c := range nodeConfigs { + storage := newTestMemoryStorage(withPeers(1, 2, 3)) + require.NoError(t, storage.Append(index(1).terms(1, 1))) + storage.SetHardState(&pb.HardState{Term: new(uint64(1)), Commit: new(c.committed)}) + if c.compactIndex != 0 { + storage.Compact(c.compactIndex) + } + cfg := newTestConfig(c.id, 10, 1, storage) + cfg.Applied = c.applied + raft := newRaft(cfg) + peers = append(peers, raft) + } + nt := newNetwork(peers...) + + // Drop MsgApp to forbid peer a to commit any log entry at its term after it becomes leader. + nt.ignore(pb.MsgApp) + // Force peer a to become leader. + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + sm := nt.peers[1].(*raft) + require.Equal(t, StateLeader, sm.state) + + // Ensure peer a drops read only request. + var windex uint64 = 4 + wctx := []byte("ctx") + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgReadIndex.Enum(), Entries: []*pb.Entry{{Data: wctx}}}) + require.Empty(t, sm.readStates) + + nt.recover() + + // Force peer a to commit a log entry at its term + for i := 0; i < sm.heartbeatTimeout; i++ { + sm.tick() + } + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}}) + require.Equal(t, uint64(4), sm.raftLog.committed) + lastLogTerm := sm.raftLog.zeroTermOnOutOfBounds(sm.raftLog.term(sm.raftLog.committed)) + require.Equal(t, sm.Term, lastLogTerm) + + // Ensure peer a processed postponed read only request after it committed an entry at its term. + require.Len(t, sm.readStates, 1) + rs := sm.readStates[0] + require.Equal(t, windex, rs.Index) + require.Equal(t, wctx, rs.RequestCtx) + + // Ensure peer a accepts read only request after it committed an entry at its term. + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgReadIndex.Enum(), Entries: []*pb.Entry{{Data: wctx}}}) + require.Len(t, sm.readStates, 2) + rs = sm.readStates[1] + require.Equal(t, windex, rs.Index) + require.Equal(t, wctx, rs.RequestCtx) +} + +// TestReadOnlyDuplicateRequest ensures that sending multiple ReadIndex requests +// with the same RequestCtx does not cause invalid readStates to be returned. +func TestReadOnlyDuplicateRequest(t *testing.T) { + r1 := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + r2 := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + r3 := newTestRaft(3, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + net := newNetwork(r1, r2, r3) + + // These will be delayed until a new leader gets elected and commits a new entry + var delayedMsgs []*pb.Message + + // net hook that delays (but doesn't duplicate) only heartbeat responses + delayHeartbeatResps := func(m *pb.Message) bool { + if m.GetType() == pb.MsgHeartbeatResp { + delayedMsgs = append(delayedMsgs, m) + return false + } + return true + } + + // elect r1 as leader + net.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + // Create readIndexMsgA; its eventual readIndex must be least readIndexMinimumA + readIndexMsgA := &pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgReadIndex.Enum(), + Entries: []*pb.Entry{{Data: []byte("A")}}} + readIndexMinimumA := r1.raftLog.committed + + // Explicitly duplicate readIndexMsgA for later; this would be done either by + // the network itself, or by users of the raft library retrying `ReadIndex()`. + delayedMsgs = append(delayedMsgs, readIndexMsgA) + + // Process readIndex request, but delay heartbeats until the end of this test + net.msgHook = delayHeartbeatResps + net.send(readIndexMsgA) + net.msgHook = nil // stop delaying + + // tick and send whatever raft wants to send; this will finish processing readIndexA + r1.tick() + r1.advanceMessagesAfterAppend() + msgs := r1.msgs + r1.msgs = nil + net.send(msgs...) + + net.isolate(r1.id) + + // elect r2 as leader, and commit a new entry + net.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgHup.Enum()}) + net.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), + Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("someOp")}}}) + + // Try reading from the disconnected and stale leader r1 + // Create readIndexMsgB; its eventual readIndex must be least readIndexMinimumB + readIndexMsgB := &pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgReadIndex.Enum(), + Entries: []*pb.Entry{{Data: []byte("B")}}} + readIndexMinimumB := r2.raftLog.committed + + net.send(readIndexMsgB) + net.recover() + // First send the delayed readIndexMsgA, then the delayed heartbeats. This + // should not cause B to be confirmed. + net.send(delayedMsgs...) + + // make sure the readIndexes aren't too small (which would violate linearizability) + for _, rd := range slices.Concat(r1.readStates, r2.readStates, r3.readStates) { + if string(rd.RequestCtx) == "A" { + assert.True(t, rd.Index >= readIndexMinimumA, "readIndex for A is %v (should be >= %v)", rd.Index, readIndexMinimumA) + } else if string(rd.RequestCtx) == "B" { + assert.True(t, rd.Index >= readIndexMinimumB, "readIndex for B is %v (should be >= %v)", rd.Index, readIndexMinimumB) + } + } +} + +func TestLeaderAppResp(t *testing.T) { + // initial progress: match = 0; next = 3 + tests := []struct { + index uint64 + reject bool + // progress + wmatch uint64 + wnext uint64 + // message + wmsgNum int + windex uint64 + wcommitted uint64 + }{ + {3, true, 0, 3, 0, 0, 0}, // stale resp; no replies + {2, true, 0, 2, 1, 1, 0}, // denied resp; leader does not commit; decrease next and send probing msg + {2, false, 2, 4, 2, 2, 2}, // accept resp; leader commits; broadcast with commit index + // Follower is StateProbing at 0, it sends MsgAppResp for 0 (which + // matches the pr.Match) so it is moved to StateReplicate and as many + // entries as possible are sent to it (1, 2, and 3). Correspondingly the + // Next is then 4 (an Entry at 4 does not exist, indicating the follower + // will be up to date should it process the emitted MsgApp). + {0, false, 0, 4, 1, 0, 0}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + // sm term is 1 after it becomes the leader. + // thus the last log term must be 1 to be committed. + sm := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + sm.raftLog = newLog(&MemoryStorage{ents: index(0).terms(0, 1, 1)}, nil) + sm.becomeCandidate() + sm.becomeLeader() + sm.readMessages() + require.NoError(t, sm.Step( + &pb.Message{ + From: new(uint64(2)), + Type: pb.MsgAppResp.Enum(), + Index: new(tt.index), + Term: new(sm.Term), + Reject: new(tt.reject), + RejectHint: new(tt.index), + }, + )) + + p := sm.trk.Progress[2] + require.Equal(t, tt.wmatch, p.Match) + require.Equal(t, tt.wnext, p.Next) + + msgs := sm.readMessages() + + require.Len(t, msgs, tt.wmsgNum) + for _, msg := range msgs { + require.Equal(t, tt.windex, msg.GetIndex(), "%v", DescribeMessage(msg, nil)) + require.Equal(t, tt.wcommitted, msg.GetCommit(), "%v", DescribeMessage(msg, nil)) + } + }) + } +} + +// TestBcastBeat is when the leader receives a heartbeat tick, it should +// send a MsgHeartbeat with m.Index = 0, m.LogTerm=0 and empty entries. +func TestBcastBeat(t *testing.T) { + offset := uint64(1000) + // make a state machine with log.offset = 1000 + s := &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{ + Index: new(offset), + Term: new(uint64(1)), + ConfState: &pb.ConfState{Voters: []uint64{1, 2, 3}}, + }, + } + storage := NewMemoryStorage() + storage.ApplySnapshot(s) + sm := newTestRaft(1, 10, 1, storage) + sm.Term = 1 + + sm.becomeCandidate() + sm.becomeLeader() + for i := 0; i < 10; i++ { + mustAppendEntry(sm, &pb.Entry{Index: new(uint64(i) + 1)}) + } + sm.advanceMessagesAfterAppend() + + // slow follower + sm.trk.Progress[2].Match, sm.trk.Progress[2].Next = 5, 6 + // normal follower + sm.trk.Progress[3].Match, sm.trk.Progress[3].Next = sm.raftLog.lastIndex(), sm.raftLog.lastIndex()+1 + + sm.Step(&pb.Message{Type: pb.MsgBeat.Enum()}) + msgs := sm.readMessages() + require.Len(t, msgs, 2) + + wantCommitMap := map[uint64]uint64{ + 2: min(sm.raftLog.committed, sm.trk.Progress[2].Match), + 3: min(sm.raftLog.committed, sm.trk.Progress[3].Match), + } + for i, m := range msgs { + require.Equal(t, pb.MsgHeartbeat, m.GetType(), "#%d", i) + require.Zero(t, m.GetIndex(), "#%d", i) + require.Zero(t, m.GetLogTerm(), "#%d", i) + + commit, ok := wantCommitMap[m.GetTo()] + require.True(t, ok, "#%d", i) + require.Equal(t, commit, m.GetCommit(), "#%d", i) + delete(wantCommitMap, m.GetTo()) + + require.Empty(t, m.GetEntries(), "#%d", i) + } +} + +// TestRecvMsgBeat tests the output of the state machine when receiving MsgBeat +func TestRecvMsgBeat(t *testing.T) { + tests := []struct { + state StateType + wMsg int + }{ + {StateLeader, 2}, + // candidate and follower should ignore MsgBeat + {StateCandidate, 0}, + {StateFollower, 0}, + } + + for i, tt := range tests { + sm := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + sm.raftLog = newLog(&MemoryStorage{ents: index(0).terms(0, 1, 1)}, nil) + sm.Term = 1 + sm.state = tt.state + switch tt.state { + case StateFollower: + sm.step = stepFollower + case StateCandidate: + sm.step = stepCandidate + case StateLeader: + sm.step = stepLeader + } + sm.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgBeat.Enum()}) + + msgs := sm.readMessages() + assert.Len(t, msgs, tt.wMsg, "#%d", i) + for _, m := range msgs { + assert.Equal(t, pb.MsgHeartbeat, m.GetType(), "#%d", i) + } + } +} + +func TestLeaderIncreaseNext(t *testing.T) { + previousEnts := index(1).terms(1, 2, 3) + tests := []struct { + // progress + state tracker.StateType + next uint64 + + wnext uint64 + }{ + // state replicate, optimistically increase next + // previous entries + noop entry + propose + 1 + {tracker.StateReplicate, 2, uint64(len(previousEnts) + 1 + 1 + 1)}, + // state probe, not optimistically increase next + {tracker.StateProbe, 2, 2}, + } + + for i, tt := range tests { + sm := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2))) + sm.raftLog.append(previousEnts...) + sm.becomeCandidate() + sm.becomeLeader() + sm.trk.Progress[2].State = tt.state + sm.trk.Progress[2].Next = tt.next + sm.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + + p := sm.trk.Progress[2] + assert.Equal(t, tt.wnext, p.Next, "#%d", i) + } +} + +func TestSendAppendForProgressProbe(t *testing.T) { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2))) + r.becomeCandidate() + r.becomeLeader() + r.readMessages() + r.trk.Progress[2].BecomeProbe() + + // each round is a heartbeat + for i := 0; i < 3; i++ { + if i == 0 { + // we expect that raft will only send out one msgAPP on the first + // loop. After that, the follower is paused until a heartbeat response is + // received. + mustAppendEntry(r, &pb.Entry{Data: []byte("somedata")}) + r.sendAppend(2) + msg := r.readMessages() + assert.Len(t, msg, 1) + assert.Zero(t, msg[0].GetIndex()) + } + + assert.True(t, r.trk.Progress[2].MsgAppFlowPaused) + for j := 0; j < 10; j++ { + mustAppendEntry(r, &pb.Entry{Data: []byte("somedata")}) + r.sendAppend(2) + assert.Empty(t, r.readMessages()) + } + + // do a heartbeat + for j := 0; j < r.heartbeatTimeout; j++ { + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgBeat.Enum()}) + } + assert.True(t, r.trk.Progress[2].MsgAppFlowPaused) + + // consume the heartbeat + msg := r.readMessages() + assert.Len(t, msg, 1) + assert.Equal(t, pb.MsgHeartbeat, msg[0].GetType()) + } + + // a heartbeat response will allow another message to be sent + r.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgHeartbeatResp.Enum()}) + msg := r.readMessages() + assert.Len(t, msg, 1) + assert.Zero(t, msg[0].GetIndex()) + assert.True(t, r.trk.Progress[2].MsgAppFlowPaused) +} + +func TestSendAppendForProgressReplicate(t *testing.T) { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2))) + r.becomeCandidate() + r.becomeLeader() + r.readMessages() + r.trk.Progress[2].BecomeReplicate() + + for i := 0; i < 10; i++ { + mustAppendEntry(r, &pb.Entry{Data: []byte("somedata")}) + r.sendAppend(2) + msgs := r.readMessages() + assert.Len(t, msgs, 1, "#%d", i) + } +} + +func TestSendAppendForProgressSnapshot(t *testing.T) { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2))) + r.becomeCandidate() + r.becomeLeader() + r.readMessages() + r.trk.Progress[2].BecomeSnapshot(10) + + for i := 0; i < 10; i++ { + mustAppendEntry(r, &pb.Entry{Data: []byte("somedata")}) + r.sendAppend(2) + msgs := r.readMessages() + assert.Empty(t, msgs, "#%d", i) + } +} + +func TestRecvMsgUnreachable(t *testing.T) { + previousEnts := index(1).terms(1, 2, 3) + s := newTestMemoryStorage(withPeers(1, 2)) + s.Append(previousEnts) + r := newTestRaft(1, 10, 1, s) + r.becomeCandidate() + r.becomeLeader() + r.readMessages() + // set node 2 to state replicate + r.trk.Progress[2].Match = 3 + r.trk.Progress[2].BecomeReplicate() + r.trk.Progress[2].Next = 6 + + r.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgUnreachable.Enum()}) + + assert.Equal(t, tracker.StateProbe, r.trk.Progress[2].State) + wnext := r.trk.Progress[2].Match + 1 + assert.Equal(t, wnext, r.trk.Progress[2].Next) +} + +func TestRestore(t *testing.T) { + s := &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{ + Index: new(uint64(11)), // magic number + Term: new(uint64(11)), // magic number + ConfState: &pb.ConfState{Voters: []uint64{1, 2, 3}}, + }, + } + + storage := newTestMemoryStorage(withPeers(1, 2)) + sm := newTestRaft(1, 10, 1, storage) + require.True(t, sm.restore(s)) + + assert.Equal(t, s.GetMetadata().GetIndex(), sm.raftLog.lastIndex()) + assert.Equal(t, s.GetMetadata().GetTerm(), mustTerm(sm.raftLog.term(s.GetMetadata().GetIndex()))) + assert.Equal(t, s.GetMetadata().GetConfState().Voters, sm.trk.VoterNodes()) + + require.False(t, sm.restore(s)) + for i := 0; i < sm.randomizedElectionTimeout; i++ { + sm.tick() + } + assert.Equal(t, StateFollower, sm.state) +} + +// TestRestoreWithLearner restores a snapshot which contains learners. +func TestRestoreWithLearner(t *testing.T) { + s := &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{ + Index: new(uint64(11)), // magic number + Term: new(uint64(11)), // magic number + ConfState: &pb.ConfState{Voters: []uint64{1, 2}, Learners: []uint64{3}}, + }, + } + + storage := newTestMemoryStorage(withPeers(1, 2), withLearners(3)) + sm := newTestLearnerRaft(3, 8, 2, storage) + assert.True(t, sm.restore(s)) + + assert.Equal(t, s.GetMetadata().GetIndex(), sm.raftLog.lastIndex()) + assert.Equal(t, s.GetMetadata().GetTerm(), mustTerm(sm.raftLog.term(s.GetMetadata().GetIndex()))) + + sg := sm.trk.VoterNodes() + assert.Len(t, sg, len(s.GetMetadata().GetConfState().Voters)) + + lns := sm.trk.LearnerNodes() + assert.Len(t, lns, len(s.GetMetadata().GetConfState().Learners)) + + for _, n := range s.GetMetadata().GetConfState().Voters { + assert.False(t, sm.trk.Progress[n].IsLearner) + } + for _, n := range s.GetMetadata().GetConfState().Learners { + assert.True(t, sm.trk.Progress[n].IsLearner) + } + + assert.False(t, sm.restore(s)) +} + +// TestRestoreWithVotersOutgoing tests if outgoing voter can receive and apply snapshot correctly. +func TestRestoreWithVotersOutgoing(t *testing.T) { + s := &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{ + Index: new(uint64(11)), // magic number + Term: new(uint64(11)), // magic number + ConfState: &pb.ConfState{Voters: []uint64{2, 3, 4}, VotersOutgoing: []uint64{1, 2, 3}}, + }, + } + + storage := newTestMemoryStorage(withPeers(1, 2)) + sm := newTestRaft(1, 10, 1, storage) + require.True(t, sm.restore(s)) + + assert.Equal(t, s.GetMetadata().GetIndex(), sm.raftLog.lastIndex()) + assert.Equal(t, mustTerm(sm.raftLog.term(s.GetMetadata().GetIndex())), s.GetMetadata().GetTerm()) + + sg := sm.trk.VoterNodes() + assert.Equal(t, []uint64{1, 2, 3, 4}, sg) + + require.False(t, sm.restore(s)) + + // It should not campaign before actually applying data. + for i := 0; i < sm.randomizedElectionTimeout; i++ { + sm.tick() + } + assert.Equal(t, StateFollower, sm.state) +} + +// TestRestoreVoterToLearner verifies that a normal peer can be downgraded to a +// learner through a snapshot. At the time of writing, we don't allow +// configuration changes to do this directly, but note that the snapshot may +// compress multiple changes to the configuration into one: the voter could have +// been removed, then readded as a learner and the snapshot reflects both +// changes. In that case, a voter receives a snapshot telling it that it is now +// a learner. In fact, the node has to accept that snapshot, or it is +// permanently cut off from the Raft log. +func TestRestoreVoterToLearner(t *testing.T) { + s := &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{ + Index: new(uint64(11)), // magic number + Term: new(uint64(11)), // magic number + ConfState: &pb.ConfState{Voters: []uint64{1, 2}, Learners: []uint64{3}}, + }, + } + + storage := newTestMemoryStorage(withPeers(1, 2, 3)) + sm := newTestRaft(3, 10, 1, storage) + + assert.False(t, sm.isLearner) + assert.True(t, sm.restore(s)) +} + +// TestRestoreLearnerPromotion checks that a learner can become to a follower after +// restoring snapshot. +func TestRestoreLearnerPromotion(t *testing.T) { + s := &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{ + Index: new(uint64(11)), // magic number + Term: new(uint64(11)), // magic number + ConfState: &pb.ConfState{Voters: []uint64{1, 2, 3}}, + }, + } + + storage := newTestMemoryStorage(withPeers(1, 2), withLearners(3)) + sm := newTestLearnerRaft(3, 10, 1, storage) + + assert.True(t, sm.isLearner) + assert.True(t, sm.restore(s)) + assert.False(t, sm.isLearner) +} + +// TestLearnerReceiveSnapshot tests that a learner can receive a snpahost from leader +func TestLearnerReceiveSnapshot(t *testing.T) { + // restore the state machine from a snapshot so it has a compacted log and a snapshot + s := &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{ + Index: new(uint64(11)), // magic number + Term: new(uint64(11)), // magic number + ConfState: &pb.ConfState{Voters: []uint64{1}, Learners: []uint64{2}}, + }, + } + + store := newTestMemoryStorage(withPeers(1), withLearners(2)) + n1 := newTestLearnerRaft(1, 10, 1, store) + n2 := newTestLearnerRaft(2, 10, 1, newTestMemoryStorage(withPeers(1), withLearners(2))) + + n1.restore(s) + snap := n1.raftLog.nextUnstableSnapshot() + store.ApplySnapshot(snap) + n1.appliedSnap(snap) + + nt := newNetwork(n1, n2) + + setRandomizedElectionTimeout(n1, n1.electionTimeout) + for i := 0; i < n1.electionTimeout; i++ { + n1.tick() + } + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgBeat.Enum()}) + + assert.Equal(t, n1.raftLog.committed, n2.raftLog.committed) +} + +func TestRestoreIgnoreSnapshot(t *testing.T) { + previousEnts := index(1).terms(1, 1, 1) + commit := uint64(1) + storage := newTestMemoryStorage(withPeers(1, 2)) + sm := newTestRaft(1, 10, 1, storage) + sm.raftLog.append(previousEnts...) + sm.raftLog.commitTo(commit) + + s := &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{ + Index: new(commit), + Term: new(uint64(1)), + ConfState: &pb.ConfState{Voters: []uint64{1, 2}}, + }, + } + + // ignore snapshot + assert.False(t, sm.restore(s)) + assert.Equal(t, sm.raftLog.committed, commit) + + // ignore snapshot and fast forward commit + s.Metadata.Index = new(commit + 1) + assert.False(t, sm.restore(s)) + assert.Equal(t, sm.raftLog.committed, commit+1) +} + +func TestProvideSnap(t *testing.T) { + // restore the state machine from a snapshot so it has a compacted log and a snapshot + s := &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{ + Index: new(uint64(11)), // magic number + Term: new(uint64(11)), // magic number + ConfState: &pb.ConfState{Voters: []uint64{1, 2}}, + }, + } + storage := newTestMemoryStorage(withPeers(1)) + sm := newTestRaft(1, 10, 1, storage) + sm.restore(s) + + sm.becomeCandidate() + sm.becomeLeader() + + // force set the next of node 2, so that node 2 needs a snapshot + sm.trk.Progress[2].Next = sm.raftLog.firstIndex() + sm.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgAppResp.Enum(), Index: new(sm.trk.Progress[2].Next - 1), Reject: new(true)}) + + msgs := sm.readMessages() + require.Len(t, msgs, 1) + m := msgs[0] + assert.Equal(t, m.GetType(), pb.MsgSnap) +} + +func TestIgnoreProvidingSnap(t *testing.T) { + // restore the state machine from a snapshot so it has a compacted log and a snapshot + s := &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{ + Index: new(uint64(11)), // magic number + Term: new(uint64(11)), // magic number + ConfState: &pb.ConfState{Voters: []uint64{1, 2}}, + }, + } + storage := newTestMemoryStorage(withPeers(1)) + sm := newTestRaft(1, 10, 1, storage) + sm.restore(s) + + sm.becomeCandidate() + sm.becomeLeader() + + // force set the next of node 2, so that node 2 needs a snapshot + // change node 2 to be inactive, expect node 1 ignore sending snapshot to 2 + sm.trk.Progress[2].Next = sm.raftLog.firstIndex() - 1 + sm.trk.Progress[2].RecentActive = false + + sm.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("somedata")}}}) + + msgs := sm.readMessages() + assert.Empty(t, msgs) +} + +func TestRestoreFromSnapMsg(t *testing.T) { + s := &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{ + Index: new(uint64(11)), // magic number + Term: new(uint64(11)), // magic number + ConfState: &pb.ConfState{Voters: []uint64{1, 2}}, + }, + } + m := &pb.Message{Type: pb.MsgSnap.Enum(), From: new(uint64(1)), Term: new(uint64(2)), Snapshot: s} + + sm := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2))) + sm.Step(m) + + assert.Equal(t, uint64(1), sm.lead) + // TODO(bdarnell): what should this test? +} + +func TestSlowNodeRestore(t *testing.T) { + nt := newNetwork(nil, nil, nil) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + nt.isolate(3) + for j := 0; j <= 100; j++ { + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}}) + } + lead := nt.peers[1].(*raft) + nextEnts(lead, nt.storage[1]) + nt.storage[1].CreateSnapshot(lead.raftLog.applied, &pb.ConfState{Voters: lead.trk.VoterNodes()}, nil) + nt.storage[1].Compact(lead.raftLog.applied) + + nt.recover() + // send heartbeats so that the leader can learn everyone is active. + // node 3 will only be considered as active when node 1 receives a reply from it. + for { + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgBeat.Enum()}) + if lead.trk.Progress[3].RecentActive { + break + } + } + + // trigger a snapshot + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}}) + + follower := nt.peers[3].(*raft) + + // trigger a commit + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}}) + assert.Equal(t, lead.raftLog.committed, follower.raftLog.committed) +} + +// TestStepConfig tests that when raft step msgProp in EntryConfChange type, +// it appends the entry to log and sets pendingConf to be true. +func TestStepConfig(t *testing.T) { + // a raft that cannot make progress + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2))) + r.becomeCandidate() + r.becomeLeader() + index := r.raftLog.lastIndex() + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Type: pb.EntryConfChange.Enum()}}}) + assert.Equal(t, index+1, r.raftLog.lastIndex()) + assert.Equal(t, index+1, r.pendingConfIndex) +} + +// TestStepIgnoreConfig tests that if raft step the second msgProp in +// EntryConfChange type when the first one is uncommitted, the node will set +// the proposal to noop and keep its original state. +func TestStepIgnoreConfig(t *testing.T) { + // a raft that cannot make progress + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2))) + r.becomeCandidate() + r.becomeLeader() + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Type: pb.EntryConfChange.Enum()}}}) + index := r.raftLog.lastIndex() + pendingConfIndex := r.pendingConfIndex + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Type: pb.EntryConfChange.Enum()}}}) + wents := []*pb.Entry{{Type: pb.EntryNormal.Enum(), Term: new(uint64(1)), Index: new(uint64(3)), Data: nil}} + ents, err := r.raftLog.entries(index+1, noLimit) + require.NoError(t, err) + requireEqualEntries(t, wents, ents) + assert.Equal(t, pendingConfIndex, r.pendingConfIndex) +} + +// TestNewLeaderPendingConfig tests that new leader sets its pendingConfigIndex +// based on uncommitted entries. +func TestNewLeaderPendingConfig(t *testing.T) { + tests := []struct { + addEntry bool + wpendingIndex uint64 + }{ + {false, 0}, + {true, 1}, + } + for i, tt := range tests { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2))) + if tt.addEntry { + mustAppendEntry(r, &pb.Entry{Type: pb.EntryNormal.Enum()}) + } + r.becomeCandidate() + r.becomeLeader() + assert.Equal(t, tt.wpendingIndex, r.pendingConfIndex, "#%d", i) + } +} + +// TestAddNode tests that addNode could update nodes correctly. +func TestAddNode(t *testing.T) { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1))) + r.applyConfChange((&pb.ConfChange{NodeId: new(uint64(2)), Type: pb.ConfChangeAddNode.Enum()}).AsV2()) + nodes := r.trk.VoterNodes() + assert.Equal(t, []uint64{1, 2}, nodes) +} + +// TestAddLearner tests that addLearner could update nodes correctly. +func TestAddLearner(t *testing.T) { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1))) + // Add new learner peer. + r.applyConfChange((&pb.ConfChange{NodeId: new(uint64(2)), Type: pb.ConfChangeAddLearnerNode.Enum()}).AsV2()) + require.False(t, r.isLearner) + nodes := r.trk.LearnerNodes() + assert.Equal(t, []uint64{2}, nodes) + require.True(t, r.trk.Progress[2].IsLearner) + + // Promote peer to voter. + r.applyConfChange((&pb.ConfChange{NodeId: new(uint64(2)), Type: pb.ConfChangeAddNode.Enum()}).AsV2()) + require.False(t, r.trk.Progress[2].IsLearner) + + // Demote r. + r.applyConfChange((&pb.ConfChange{NodeId: new(uint64(1)), Type: pb.ConfChangeAddLearnerNode.Enum()}).AsV2()) + require.True(t, r.trk.Progress[1].IsLearner) + require.True(t, r.isLearner) + + // Promote r again. + r.applyConfChange((&pb.ConfChange{NodeId: new(uint64(1)), Type: pb.ConfChangeAddNode.Enum()}).AsV2()) + require.False(t, r.trk.Progress[1].IsLearner) + require.False(t, r.isLearner) +} + +// TestAddNodeCheckQuorum tests that addNode does not trigger a leader election +// immediately when checkQuorum is set. +func TestAddNodeCheckQuorum(t *testing.T) { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1))) + r.checkQuorum = true + + r.becomeCandidate() + r.becomeLeader() + + for i := 0; i < r.electionTimeout-1; i++ { + r.tick() + } + + r.applyConfChange((&pb.ConfChange{NodeId: new(uint64(2)), Type: pb.ConfChangeAddNode.Enum()}).AsV2()) + + // This tick will reach electionTimeout, which triggers a quorum check. + r.tick() + + // Node 1 should still be the leader after a single tick. + assert.Equal(t, StateLeader, r.state) + + // After another electionTimeout ticks without hearing from node 2, + // node 1 should step down. + for i := 0; i < r.electionTimeout; i++ { + r.tick() + } + + assert.Equal(t, StateFollower, r.state) +} + +// TestRemoveNode tests that removeNode could update nodes and +// removed list correctly. +func TestRemoveNode(t *testing.T) { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2))) + r.applyConfChange((&pb.ConfChange{NodeId: new(uint64(2)), Type: pb.ConfChangeRemoveNode.Enum()}).AsV2()) + w := []uint64{1} + assert.Equal(t, w, r.trk.VoterNodes()) + + // Removing the remaining voter will panic. + defer func() { + assert.NotNil(t, recover(), "did not panic") + }() + r.applyConfChange((&pb.ConfChange{NodeId: new(uint64(1)), Type: pb.ConfChangeRemoveNode.Enum()}).AsV2()) +} + +// TestRemoveLearner tests that removeNode could update nodes and +// removed list correctly. +func TestRemoveLearner(t *testing.T) { + r := newTestLearnerRaft(1, 10, 1, newTestMemoryStorage(withPeers(1), withLearners(2))) + r.applyConfChange((&pb.ConfChange{NodeId: new(uint64(2)), Type: pb.ConfChangeRemoveNode.Enum()}).AsV2()) + w := []uint64{1} + assert.Equal(t, w, r.trk.VoterNodes()) + + w = nil + assert.Equal(t, w, r.trk.LearnerNodes()) + + // Removing the remaining voter will panic. + defer func() { + assert.NotNil(t, recover(), "did not panic") + }() + r.applyConfChange((&pb.ConfChange{NodeId: new(uint64(1)), Type: pb.ConfChangeRemoveNode.Enum()}).AsV2()) +} + +func TestPromotable(t *testing.T) { + id := uint64(1) + tests := []struct { + peers []uint64 + wp bool + }{ + {[]uint64{1}, true}, + {[]uint64{1, 2, 3}, true}, + {[]uint64{}, false}, + {[]uint64{2, 3}, false}, + } + for i, tt := range tests { + r := newTestRaft(id, 5, 1, newTestMemoryStorage(withPeers(tt.peers...))) + assert.Equal(t, tt.wp, r.promotable(), "#%d", i) + } +} + +func TestRaftNodes(t *testing.T) { + tests := []struct { + ids []uint64 + wids []uint64 + }{ + { + []uint64{1, 2, 3}, + []uint64{1, 2, 3}, + }, + { + []uint64{3, 2, 1}, + []uint64{1, 2, 3}, + }, + } + for i, tt := range tests { + r := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(tt.ids...))) + assert.Equal(t, tt.wids, r.trk.VoterNodes(), "#%d", i) + } +} + +func TestCampaignWhileLeader(t *testing.T) { + testCampaignWhileLeader(t, false) +} + +func TestPreCampaignWhileLeader(t *testing.T) { + testCampaignWhileLeader(t, true) +} + +func testCampaignWhileLeader(t *testing.T, preVote bool) { + cfg := newTestConfig(1, 5, 1, newTestMemoryStorage(withPeers(1))) + cfg.PreVote = preVote + r := newRaft(cfg) + assert.Equal(t, StateFollower, r.state) + // We don't call campaign() directly because it comes after the check + // for our current state. + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + r.advanceMessagesAfterAppend() + assert.Equal(t, StateLeader, r.state) + + term := r.Term + r.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + r.advanceMessagesAfterAppend() + assert.Equal(t, StateLeader, r.state) + assert.Equal(t, term, r.Term) +} + +// TestCommitAfterRemoveNode verifies that pending commands can become +// committed when a config change reduces the quorum requirements. +func TestCommitAfterRemoveNode(t *testing.T) { + // Create a cluster with two nodes. + s := newTestMemoryStorage(withPeers(1, 2)) + r := newTestRaft(1, 5, 1, s) + r.becomeCandidate() + r.becomeLeader() + + // Begin to remove the second node. + cc := &pb.ConfChange{ + Type: pb.ConfChangeRemoveNode.Enum(), + NodeId: new(uint64(2)), + } + ccData, err := proto.Marshal(cc) + require.NoError(t, err) + r.Step(&pb.Message{ + Type: pb.MsgProp.Enum(), + Entries: []*pb.Entry{{Type: pb.EntryConfChange.Enum(), Data: ccData}}, + }) + + // Stabilize the log and make sure nothing is committed yet. + require.Empty(t, nextEnts(r, s)) + ccIndex := r.raftLog.lastIndex() + + // While the config change is pending, make another proposal. + r.Step(&pb.Message{ + Type: pb.MsgProp.Enum(), + Entries: []*pb.Entry{{Type: pb.EntryNormal.Enum(), Data: []byte("hello")}}, + }) + + // Node 2 acknowledges the config change, committing it. + r.Step(&pb.Message{ + Type: pb.MsgAppResp.Enum(), + From: new(uint64(2)), + Index: new(ccIndex), + }) + ents := nextEnts(r, s) + require.Len(t, ents, 2) + require.Equal(t, pb.EntryNormal, ents[0].GetType()) + require.Nil(t, ents[0].GetData()) + require.Equal(t, pb.EntryConfChange, ents[1].GetType()) + + // Apply the config change. This reduces quorum requirements so the + // pending command can now commit. + r.applyConfChange(cc.AsV2()) + ents = nextEnts(r, s) + require.Len(t, ents, 1) + require.Equal(t, pb.EntryNormal, ents[0].GetType()) + require.Equal(t, []byte("hello"), ents[0].GetData()) +} + +// TestLeaderTransferToUpToDateNode verifies transferring should succeed +// if the transferee has the most up-to-date log entries when transfer starts. +func TestLeaderTransferToUpToDateNode(t *testing.T) { + nt := newNetwork(nil, nil, nil) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + lead := nt.peers[1].(*raft) + + require.Equal(t, uint64(1), lead.lead) + + // Transfer leadership to 2. + nt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + + checkLeaderTransferState(t, lead, StateFollower, 2) + + // After some log replication, transfer leadership back to 1. + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}}) + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(2)), Type: pb.MsgTransferLeader.Enum()}) + + checkLeaderTransferState(t, lead, StateLeader, 1) +} + +// TestLeaderTransferToUpToDateNodeFromFollower verifies transferring should succeed +// if the transferee has the most up-to-date log entries when transfer starts. +// Not like TestLeaderTransferToUpToDateNode, where the leader transfer message +// is sent to the leader, in this test case every leader transfer message is sent +// to the follower. +func TestLeaderTransferToUpToDateNodeFromFollower(t *testing.T) { + nt := newNetwork(nil, nil, nil) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + lead := nt.peers[1].(*raft) + + require.Equal(t, uint64(1), lead.lead) + + // Transfer leadership to 2. + nt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgTransferLeader.Enum()}) + + checkLeaderTransferState(t, lead, StateFollower, 2) + + // After some log replication, transfer leadership back to 1. + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}}) + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + + checkLeaderTransferState(t, lead, StateLeader, 1) +} + +// TestLeaderTransferWithCheckQuorum ensures transferring leader still works +// even the current leader is still under its leader lease +func TestLeaderTransferWithCheckQuorum(t *testing.T) { + nt := newNetwork(nil, nil, nil) + for i := 1; i < 4; i++ { + r := nt.peers[uint64(i)].(*raft) + r.checkQuorum = true + setRandomizedElectionTimeout(r, r.electionTimeout+i) + } + + // Letting peer 2 electionElapsed reach to timeout so that it can vote for peer 1 + f := nt.peers[2].(*raft) + for i := 0; i < f.electionTimeout; i++ { + f.tick() + } + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + lead := nt.peers[1].(*raft) + + require.Equal(t, StateLeader, lead.state) + + // Transfer leadership to 2. + nt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + + checkLeaderTransferState(t, lead, StateFollower, 2) + + // After some log replication, transfer leadership back to 1. + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}}) + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(2)), Type: pb.MsgTransferLeader.Enum()}) + + checkLeaderTransferState(t, lead, StateLeader, 1) +} + +func TestLeaderTransferToSlowFollower(t *testing.T) { + defaultLogger.EnableDebug() + nt := newNetwork(nil, nil, nil) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + nt.isolate(3) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}}) + + nt.recover() + lead := nt.peers[1].(*raft) + require.Equal(t, uint64(1), lead.trk.Progress[3].Match) + + // Transfer leadership to 3 when node 3 is lack of log. + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + + checkLeaderTransferState(t, lead, StateFollower, 3) +} + +func TestLeaderTransferAfterSnapshot(t *testing.T) { + nt := newNetwork(nil, nil, nil) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + nt.isolate(3) + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}}) + lead := nt.peers[1].(*raft) + nextEnts(lead, nt.storage[1]) + nt.storage[1].CreateSnapshot(lead.raftLog.applied, &pb.ConfState{Voters: lead.trk.VoterNodes()}, nil) + nt.storage[1].Compact(lead.raftLog.applied) + + nt.recover() + require.Equal(t, uint64(1), lead.trk.Progress[3].Match) + + filtered := &pb.Message{} + // Snapshot needs to be applied before sending MsgAppResp + nt.msgHook = func(m *pb.Message) bool { + if m.GetType() != pb.MsgAppResp || m.GetFrom() != 3 || m.GetReject() { + return true + } + filtered = m + return false + } + // Transfer leadership to 3 when node 3 is lack of snapshot. + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + require.Equal(t, StateLeader, lead.state) + require.NotEqual(t, &pb.Message{}, filtered) + + // Apply snapshot and resume progress + follower := nt.peers[3].(*raft) + snap := follower.raftLog.nextUnstableSnapshot() + nt.storage[3].ApplySnapshot(snap) + follower.appliedSnap(snap) + nt.msgHook = nil + nt.send(filtered) + + checkLeaderTransferState(t, lead, StateFollower, 3) +} + +func TestLeaderTransferToSelf(t *testing.T) { + nt := newNetwork(nil, nil, nil) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + lead := nt.peers[1].(*raft) + + // Transfer leadership to self, there will be noop. + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + checkLeaderTransferState(t, lead, StateLeader, 1) +} + +func TestLeaderTransferToNonExistingNode(t *testing.T) { + nt := newNetwork(nil, nil, nil) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + lead := nt.peers[1].(*raft) + // Transfer leadership to non-existing node, there will be noop. + nt.send(&pb.Message{From: new(uint64(4)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + checkLeaderTransferState(t, lead, StateLeader, 1) +} + +func TestLeaderTransferTimeout(t *testing.T) { + nt := newNetwork(nil, nil, nil) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + nt.isolate(3) + + lead := nt.peers[1].(*raft) + + // Transfer leadership to isolated node, wait for timeout. + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + require.Equal(t, uint64(3), lead.leadTransferee) + + for i := 0; i < lead.heartbeatTimeout; i++ { + lead.tick() + } + require.Equal(t, uint64(3), lead.leadTransferee) + + for i := 0; i < lead.electionTimeout-lead.heartbeatTimeout; i++ { + lead.tick() + } + + checkLeaderTransferState(t, lead, StateLeader, 1) +} + +func TestLeaderTransferIgnoreProposal(t *testing.T) { + s := newTestMemoryStorage(withPeers(1, 2, 3)) + r := newTestRaft(1, 10, 1, s) + nt := newNetwork(r, nil, nil) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + nt.isolate(3) + + lead := nt.peers[1].(*raft) + + nextEnts(r, s) // handle empty entry + + // Transfer leadership to isolated node to let transfer pending, then send proposal. + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + require.Equal(t, uint64(3), lead.leadTransferee) + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}}) + err := lead.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{}}}) + require.Equal(t, ErrProposalDropped, err) + + require.Equal(t, uint64(1), lead.trk.Progress[1].Match) +} + +func TestLeaderTransferReceiveHigherTermVote(t *testing.T) { + nt := newNetwork(nil, nil, nil) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + nt.isolate(3) + + lead := nt.peers[1].(*raft) + + // Transfer leadership to isolated node to let transfer pending. + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + require.Equal(t, uint64(3), lead.leadTransferee) + + nt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgHup.Enum(), Index: new(uint64(1)), Term: new(uint64(2))}) + + checkLeaderTransferState(t, lead, StateFollower, 2) +} + +func TestLeaderTransferRemoveNode(t *testing.T) { + nt := newNetwork(nil, nil, nil) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + nt.ignore(pb.MsgTimeoutNow) + + lead := nt.peers[1].(*raft) + + // The leadTransferee is removed when leadship transferring. + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + require.Equal(t, uint64(3), lead.leadTransferee) + + lead.applyConfChange((&pb.ConfChange{NodeId: new(uint64(3)), Type: pb.ConfChangeRemoveNode.Enum()}).AsV2()) + + checkLeaderTransferState(t, lead, StateLeader, 1) +} + +func TestLeaderTransferDemoteNode(t *testing.T) { + nt := newNetwork(nil, nil, nil) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + nt.ignore(pb.MsgTimeoutNow) + + lead := nt.peers[1].(*raft) + + // The leadTransferee is demoted when leadship transferring. + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + require.Equal(t, uint64(3), lead.leadTransferee) + + lead.applyConfChange(&pb.ConfChangeV2{ + Changes: []*pb.ConfChangeSingle{ + { + Type: pb.ConfChangeRemoveNode.Enum(), + NodeId: new(uint64(3)), + }, + { + Type: pb.ConfChangeAddLearnerNode.Enum(), + NodeId: new(uint64(3)), + }, + }, + }) + + // Make the Raft group commit the LeaveJoint entry. + lead.applyConfChange(&pb.ConfChangeV2{}) + checkLeaderTransferState(t, lead, StateLeader, 1) +} + +// TestLeaderTransferBack verifies leadership can transfer back to self when last transfer is pending. +func TestLeaderTransferBack(t *testing.T) { + nt := newNetwork(nil, nil, nil) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + nt.isolate(3) + + lead := nt.peers[1].(*raft) + + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + require.Equal(t, uint64(3), lead.leadTransferee) + + // Transfer leadership back to self. + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + + checkLeaderTransferState(t, lead, StateLeader, 1) +} + +// TestLeaderTransferSecondTransferToAnotherNode verifies leader can transfer to another node +// when last transfer is pending. +func TestLeaderTransferSecondTransferToAnotherNode(t *testing.T) { + nt := newNetwork(nil, nil, nil) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + nt.isolate(3) + + lead := nt.peers[1].(*raft) + + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + require.Equal(t, uint64(3), lead.leadTransferee) + + // Transfer leadership to another node. + nt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + + checkLeaderTransferState(t, lead, StateFollower, 2) +} + +// TestLeaderTransferSecondTransferToSameNode verifies second transfer leader request +// to the same node should not extend the timeout while the first one is pending. +func TestLeaderTransferSecondTransferToSameNode(t *testing.T) { + nt := newNetwork(nil, nil, nil) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + nt.isolate(3) + + lead := nt.peers[1].(*raft) + + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + require.Equal(t, uint64(3), lead.leadTransferee) + + for i := 0; i < lead.heartbeatTimeout; i++ { + lead.tick() + } + // Second transfer leadership request to the same node. + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + + for i := 0; i < lead.electionTimeout-lead.heartbeatTimeout; i++ { + lead.tick() + } + + checkLeaderTransferState(t, lead, StateLeader, 1) +} + +func checkLeaderTransferState(t *testing.T, r *raft, state StateType, lead uint64) { + require.Equal(t, state, r.state) + require.Equal(t, lead, r.lead) + require.Equal(t, None, r.leadTransferee) +} + +// TestTransferNonMember verifies that when a MsgTimeoutNow arrives at +// a node that has been removed from the group, nothing happens. +// (previously, if the node also got votes, it would panic as it +// transitioned to StateLeader) +func TestTransferNonMember(t *testing.T) { + r := newTestRaft(1, 5, 1, newTestMemoryStorage(withPeers(2, 3, 4))) + r.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgTimeoutNow.Enum()}) + + r.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgVoteResp.Enum()}) + r.Step(&pb.Message{From: new(uint64(3)), To: new(uint64(1)), Type: pb.MsgVoteResp.Enum()}) + require.Equal(t, StateFollower, r.state) +} + +// TestNodeWithSmallerTermCanCompleteElection tests the scenario where a node +// that has been partitioned away (and fallen behind) rejoins the cluster at +// about the same time the leader node gets partitioned away. +// Previously the cluster would come to a standstill when run with PreVote +// enabled. +func TestNodeWithSmallerTermCanCompleteElection(t *testing.T) { + n1 := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + n2 := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + n3 := newTestRaft(3, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + + n1.becomeFollower(1, None) + n2.becomeFollower(1, None) + n3.becomeFollower(1, None) + + n1.preVote = true + n2.preVote = true + n3.preVote = true + + // cause a network partition to isolate node 3 + nt := newNetwork(n1, n2, n3) + nt.cut(1, 3) + nt.cut(2, 3) + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + sm := nt.peers[1].(*raft) + assert.Equal(t, StateLeader, sm.state) + + sm = nt.peers[2].(*raft) + assert.Equal(t, StateFollower, sm.state) + + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + sm = nt.peers[3].(*raft) + assert.Equal(t, StatePreCandidate, sm.state) + + nt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgHup.Enum()}) + + // check whether the term values are expected + sm = nt.peers[1].(*raft) + assert.Equal(t, uint64(3), sm.Term) + sm = nt.peers[2].(*raft) + assert.Equal(t, uint64(3), sm.Term) + sm = nt.peers[3].(*raft) + assert.Equal(t, uint64(1), sm.Term) + + // check state + sm = nt.peers[1].(*raft) + assert.Equal(t, StateFollower, sm.state) + sm = nt.peers[2].(*raft) + assert.Equal(t, StateLeader, sm.state) + sm = nt.peers[3].(*raft) + assert.Equal(t, StatePreCandidate, sm.state) + + sm.logger.Infof("going to bring back peer 3 and kill peer 2") + // recover the network then immediately isolate b which is currently + // the leader, this is to emulate the crash of b. + nt.recover() + nt.cut(2, 1) + nt.cut(2, 3) + + // call for election + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + // do we have a leader? + sma := nt.peers[1].(*raft) + smb := nt.peers[3].(*raft) + assert.True(t, sma.state == StateLeader || smb.state == StateLeader) +} + +// TestPreVoteWithSplitVote verifies that after split vote, cluster can complete +// election in next round. +func TestPreVoteWithSplitVote(t *testing.T) { + n1 := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + n2 := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + n3 := newTestRaft(3, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + + n1.becomeFollower(1, None) + n2.becomeFollower(1, None) + n3.becomeFollower(1, None) + + n1.preVote = true + n2.preVote = true + n3.preVote = true + + nt := newNetwork(n1, n2, n3) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + // simulate leader down. followers start split vote. + nt.isolate(1) + nt.send([]*pb.Message{ + {From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgHup.Enum()}, + {From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}, + }...) + + // check whether the term values are expected + sm := nt.peers[2].(*raft) + assert.Equal(t, uint64(3), sm.Term) + sm = nt.peers[3].(*raft) + assert.Equal(t, uint64(3), sm.Term) + + // check state + sm = nt.peers[2].(*raft) + assert.Equal(t, StateCandidate, sm.state) + sm = nt.peers[3].(*raft) + assert.Equal(t, StateCandidate, sm.state) + + // node 2 election timeout first + nt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgHup.Enum()}) + + // check whether the term values are expected + sm = nt.peers[2].(*raft) + assert.Equal(t, uint64(4), sm.Term) + sm = nt.peers[3].(*raft) + assert.Equal(t, uint64(4), sm.Term) + + // check state + sm = nt.peers[2].(*raft) + assert.Equal(t, StateLeader, sm.state) + sm = nt.peers[3].(*raft) + assert.Equal(t, StateFollower, sm.state) +} + +// TestPreVoteWithCheckQuorum ensures that after a node become pre-candidate, +// it will checkQuorum correctly. +func TestPreVoteWithCheckQuorum(t *testing.T) { + n1 := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + n2 := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + n3 := newTestRaft(3, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + + n1.becomeFollower(1, None) + n2.becomeFollower(1, None) + n3.becomeFollower(1, None) + + n1.preVote = true + n2.preVote = true + n3.preVote = true + + n1.checkQuorum = true + n2.checkQuorum = true + n3.checkQuorum = true + + nt := newNetwork(n1, n2, n3) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + // isolate node 1. node 2 and node 3 have leader info + nt.isolate(1) + + // check state + sm := nt.peers[1].(*raft) + require.Equal(t, StateLeader, sm.state) + + sm = nt.peers[2].(*raft) + require.Equal(t, StateFollower, sm.state) + + sm = nt.peers[3].(*raft) + require.Equal(t, StateFollower, sm.state) + + // node 2 will ignore node 3's PreVote + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + nt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgHup.Enum()}) + + // Do we have a leader? + assert.True(t, n2.state == StateLeader || n3.state == StateFollower) +} + +// TestLearnerCampaign verifies that a learner won't campaign even if it receives +// a MsgHup or MsgTimeoutNow. +func TestLearnerCampaign(t *testing.T) { + n1 := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1))) + n1.applyConfChange((&pb.ConfChange{NodeId: new(uint64(2)), Type: pb.ConfChangeAddLearnerNode.Enum()}).AsV2()) + n2 := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1))) + n2.applyConfChange((&pb.ConfChange{NodeId: new(uint64(2)), Type: pb.ConfChangeAddLearnerNode.Enum()}).AsV2()) + nt := newNetwork(n1, n2) + nt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgHup.Enum()}) + + require.True(t, n2.isLearner) + + require.Equal(t, StateFollower, n2.state) + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + require.True(t, n1.state == StateLeader && n1.lead == 1) + + // NB: TransferLeader already checks that the recipient is not a learner, but + // the check could have happened by the time the recipient becomes a learner, + // in which case it will receive MsgTimeoutNow as in this test case and we + // verify that it's ignored. + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(2)), Type: pb.MsgTimeoutNow.Enum()}) + require.Equal(t, StateFollower, n2.state) +} + +// simulate rolling update a cluster for Pre-Vote. cluster has 3 nodes [n1, n2, n3]. +// n1 is leader with term 2 +// n2 is follower with term 2 +// n3 is partitioned, with term 4 and less log, state is candidate +func newPreVoteMigrationCluster(t *testing.T) *network { + n1 := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + n2 := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + n3 := newTestRaft(3, 10, 1, newTestMemoryStorage(withPeers(1, 2, 3))) + + n1.becomeFollower(1, None) + n2.becomeFollower(1, None) + n3.becomeFollower(1, None) + + n1.preVote = true + n2.preVote = true + // We intentionally do not enable PreVote for n3, this is done so in order + // to simulate a rolling restart process where it's possible to have a mixed + // version cluster with replicas with PreVote enabled, and replicas without. + + nt := newNetwork(n1, n2, n3) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + + // Cause a network partition to isolate n3. + nt.isolate(3) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgProp.Enum(), Entries: []*pb.Entry{{Data: []byte("some data")}}}) + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + + // check state + require.Equal(t, StateLeader, n1.state) + require.Equal(t, StateFollower, n2.state) + require.Equal(t, StateCandidate, n3.state) + + // check term + require.Equal(t, uint64(2), n1.Term) + require.Equal(t, uint64(2), n2.Term) + require.Equal(t, uint64(4), n3.Term) + + // Enable prevote on n3, then recover the network + n3.preVote = true + nt.recover() + + return nt +} + +func TestPreVoteMigrationCanCompleteElection(t *testing.T) { + nt := newPreVoteMigrationCluster(t) + + // n1 is leader with term 2 + // n2 is follower with term 2 + // n3 is pre-candidate with term 4, and less log + n2 := nt.peers[2].(*raft) + n3 := nt.peers[3].(*raft) + + // simulate leader down + nt.isolate(1) + + // Call for elections from both n2 and n3. + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + nt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgHup.Enum()}) + + // check state + assert.Equal(t, StateFollower, n2.state) + assert.Equal(t, StatePreCandidate, n3.state) + + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + nt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(2)), Type: pb.MsgHup.Enum()}) + + // Do we have a leader? + assert.True(t, n2.state == StateLeader || n3.state == StateFollower) +} + +func TestPreVoteMigrationWithFreeStuckPreCandidate(t *testing.T) { + nt := newPreVoteMigrationCluster(t) + + // n1 is leader with term 2 + // n2 is follower with term 2 + // n3 is pre-candidate with term 4, and less log + n1 := nt.peers[1].(*raft) + n2 := nt.peers[2].(*raft) + n3 := nt.peers[3].(*raft) + + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + + assert.Equal(t, StateLeader, n1.state) + assert.Equal(t, StateFollower, n2.state) + assert.Equal(t, StatePreCandidate, n3.state) + + // Pre-Vote again for safety + nt.send(&pb.Message{From: new(uint64(3)), To: new(uint64(3)), Type: pb.MsgHup.Enum()}) + + assert.Equal(t, StateLeader, n1.state) + assert.Equal(t, StateFollower, n2.state) + assert.Equal(t, StatePreCandidate, n3.state) + + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(3)), Type: pb.MsgHeartbeat.Enum(), Term: new(n1.Term)}) + + // Disrupt the leader so that the stuck peer is freed + assert.Equal(t, StateFollower, n1.state) + assert.Equal(t, n1.Term, n3.Term) + +} + +func testConfChangeCheckBeforeCampaign(t *testing.T, v2 bool) { + nt := newNetwork(nil, nil, nil) + n1 := nt.peers[1].(*raft) + n2 := nt.peers[2].(*raft) + nt.send(&pb.Message{From: new(uint64(1)), To: new(uint64(1)), Type: pb.MsgHup.Enum()}) + assert.Equal(t, StateLeader, n1.state) + + // Begin to remove the third node. + cc := &pb.ConfChange{ + Type: pb.ConfChangeRemoveNode.Enum(), + NodeId: new(uint64(2)), + } + var ccData []byte + var err error + var ty pb.EntryType + if v2 { + ccv2 := cc.AsV2() + ccData, err = proto.Marshal(ccv2) + ty = pb.EntryConfChangeV2 + } else { + ccData, err = proto.Marshal(cc) + ty = pb.EntryConfChange + } + require.NoError(t, err) + nt.send(&pb.Message{ + From: new(uint64(1)), + To: new(uint64(1)), + Type: pb.MsgProp.Enum(), + Entries: []*pb.Entry{{Type: ty.Enum(), Data: ccData}}, + }) + + // Trigger campaign in node 2 + for i := 0; i < n2.randomizedElectionTimeout; i++ { + n2.tick() + } + // It's still follower because committed conf change is not applied. + assert.Equal(t, StateFollower, n2.state) + + // Transfer leadership to peer 2. + nt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + assert.Equal(t, StateLeader, n1.state) + // It's still follower because committed conf change is not applied. + assert.Equal(t, StateFollower, n2.state) + + // Abort transfer leader + for i := 0; i < n1.electionTimeout; i++ { + n1.tick() + } + + // Advance apply + nextEnts(n2, nt.storage[2]) + + // Transfer leadership to peer 2 again. + nt.send(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgTransferLeader.Enum()}) + assert.Equal(t, StateFollower, n1.state) + assert.Equal(t, StateLeader, n2.state) + + nextEnts(n1, nt.storage[1]) + // Trigger campaign in node 2 + for i := 0; i < n1.randomizedElectionTimeout; i++ { + n1.tick() + } + assert.Equal(t, StateCandidate, n1.state) +} + +// TestConfChangeCheckBeforeCampaign tests if unapplied ConfChange is checked before campaign. +func TestConfChangeCheckBeforeCampaign(t *testing.T) { + testConfChangeCheckBeforeCampaign(t, false) +} + +// TestConfChangeV2CheckBeforeCampaign tests if unapplied ConfChangeV2 is checked before campaign. +func TestConfChangeV2CheckBeforeCampaign(t *testing.T) { + testConfChangeCheckBeforeCampaign(t, true) +} + +func TestFastLogRejection(t *testing.T) { + tests := []struct { + leaderLog []*pb.Entry // Logs on the leader + followerLog []*pb.Entry // Logs on the follower + followerCompact uint64 // Index at which the follower log is compacted. + rejectHintTerm uint64 // Expected term included in rejected MsgAppResp. + rejectHintIndex uint64 // Expected index included in rejected MsgAppResp. + nextAppendTerm uint64 // Expected term when leader appends after rejected. + nextAppendIndex uint64 // Expected index when leader appends after rejected. + }{ + // This case tests that leader can find the conflict index quickly. + // Firstly leader appends (type=MsgApp,index=7,logTerm=4, entries=...); + // After rejected leader appends (type=MsgApp,index=3,logTerm=2). + { + leaderLog: index(1).terms(1, 2, 2, 4, 4, 4, 4), + followerLog: index(1).terms(1, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3), + rejectHintTerm: 3, + rejectHintIndex: 7, + nextAppendTerm: 2, + nextAppendIndex: 3, + }, + // This case tests that leader can find the conflict index quickly. + // Firstly leader appends (type=MsgApp,index=8,logTerm=5, entries=...); + // After rejected leader appends (type=MsgApp,index=4,logTerm=3). + { + leaderLog: index(1).terms(1, 2, 2, 3, 4, 4, 4, 5), + followerLog: index(1).terms(1, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3), + rejectHintTerm: 3, + rejectHintIndex: 8, + nextAppendTerm: 3, + nextAppendIndex: 4, + }, + // This case tests that follower can find the conflict index quickly. + // Firstly leader appends (type=MsgApp,index=4,logTerm=1, entries=...); + // After rejected leader appends (type=MsgApp,index=1,logTerm=1). + { + leaderLog: index(1).terms(1, 1, 1, 1), + followerLog: index(1).terms(1, 2, 2, 4), + rejectHintTerm: 1, + rejectHintIndex: 1, + nextAppendTerm: 1, + nextAppendIndex: 1, + }, + // This case is similar to the previous case. However, this time, the + // leader has a longer uncommitted log tail than the follower. + // Firstly leader appends (type=MsgApp,index=6,logTerm=1, entries=...); + // After rejected leader appends (type=MsgApp,index=1,logTerm=1). + { + leaderLog: index(1).terms(1, 1, 1, 1, 1, 1), + followerLog: index(1).terms(1, 2, 2, 4), + rejectHintTerm: 1, + rejectHintIndex: 1, + nextAppendTerm: 1, + nextAppendIndex: 1, + }, + // This case is similar to the previous case. However, this time, the + // follower has a longer uncommitted log tail than the leader. + // Firstly leader appends (type=MsgApp,index=4,logTerm=1, entries=...); + // After rejected leader appends (type=MsgApp,index=1,logTerm=1). + { + leaderLog: index(1).terms(1, 1, 1, 1), + followerLog: index(1).terms(1, 2, 2, 4, 4, 4), + rejectHintTerm: 1, + rejectHintIndex: 1, + nextAppendTerm: 1, + nextAppendIndex: 1, + }, + // An normal case that there are no log conflicts. + // Firstly leader appends (type=MsgApp,index=5,logTerm=5, entries=...); + // After rejected leader appends (type=MsgApp,index=4,logTerm=4). + { + leaderLog: index(1).terms(1, 1, 1, 4, 5), + followerLog: index(1).terms(1, 1, 1, 4), + rejectHintTerm: 4, + rejectHintIndex: 4, + nextAppendTerm: 4, + nextAppendIndex: 4, + }, + // Test case from example comment in stepLeader (on leader). + { + leaderLog: index(1).terms(2, 5, 5, 5, 5, 5, 5, 5, 5), + followerLog: index(1).terms(2, 4, 4, 4, 4, 4), + rejectHintTerm: 4, + rejectHintIndex: 6, + nextAppendTerm: 2, + nextAppendIndex: 1, + }, + // Test case from example comment in handleAppendEntries (on follower). + { + leaderLog: index(1).terms(2, 2, 2, 2, 2), + followerLog: index(1).terms(2, 4, 4, 4, 4, 4, 4, 4), + rejectHintTerm: 2, + rejectHintIndex: 1, + nextAppendTerm: 2, + nextAppendIndex: 1, + }, + // A case when a stale MsgApp from leader arrives after the corresponding + // log index got compacted. + // A stale (type=MsgApp,index=3,logTerm=3,entries=[(term=3,index=4)]) is + // delivered to a follower who has already compacted beyond log index 3. The + // MsgAppResp rejection will return same index=3, with logTerm=0. The leader + // will rollback by one entry, and send MsgApp with index=2,logTerm=1. + { + leaderLog: index(1).terms(1, 1, 3), + followerLog: index(1).terms(1, 1, 3, 3, 3), + followerCompact: 5, // entries <= index 5 are compacted + rejectHintTerm: 0, + rejectHintIndex: 3, + nextAppendTerm: 1, + nextAppendIndex: 2, + }, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + s1 := NewMemoryStorage() + s1.snapshot.Metadata.ConfState = &pb.ConfState{Voters: []uint64{1, 2, 3}} + s1.Append(test.leaderLog) + last := test.leaderLog[len(test.leaderLog)-1] + s1.SetHardState(&pb.HardState{ + Term: new(last.GetTerm() - 1), + Commit: new(last.GetIndex()), + }) + n1 := newTestRaft(1, 10, 1, s1) + n1.becomeCandidate() // bumps Term to last.GetTerm() + n1.becomeLeader() + + s2 := NewMemoryStorage() + s2.snapshot.Metadata.ConfState = &pb.ConfState{Voters: []uint64{1, 2, 3}} + s2.Append(test.followerLog) + s2.SetHardState(&pb.HardState{ + Term: new(last.GetTerm()), + Vote: new(uint64(1)), + Commit: new(uint64(0)), + }) + n2 := newTestRaft(2, 10, 1, s2) + if test.followerCompact != 0 { + s2.Compact(test.followerCompact) + // NB: the state of n2 after this compaction isn't realistic because the + // commit index is still at 0. We do this to exercise a "doesn't happen" + // edge case behaviour, in case it still does happen in some other way. + } + + require.NoError(t, n2.Step(&pb.Message{From: new(uint64(1)), To: new(uint64(2)), Type: pb.MsgHeartbeat.Enum()})) + msgs := n2.readMessages() + require.Len(t, msgs, 1, "can't read 1 message from peer 2") + require.Equal(t, pb.MsgHeartbeatResp, msgs[0].GetType()) + + require.NoError(t, n1.Step(msgs[0])) + msgs = n1.readMessages() + require.Len(t, msgs, 1, "can't read 1 message from peer 1") + require.Equal(t, pb.MsgApp, msgs[0].GetType()) + + require.NoError(t, n2.Step(msgs[0]), "peer 2 step append fail") + msgs = n2.readMessages() + require.Len(t, msgs, 1, "can't read 1 message from peer 2") + require.Equal(t, pb.MsgAppResp, msgs[0].GetType()) + require.True(t, msgs[0].GetReject(), "expected rejected append response from peer 2") + require.Equal(t, test.rejectHintTerm, msgs[0].GetLogTerm(), "hint log term mismatch") + require.Equal(t, test.rejectHintIndex, msgs[0].GetRejectHint(), "hint log index mismatch") + + require.NoError(t, n1.Step(msgs[0]), "peer 1 step append fail") + msgs = n1.readMessages() + require.Equal(t, test.nextAppendTerm, msgs[0].GetLogTerm()) + require.Equal(t, test.nextAppendIndex, msgs[0].GetIndex()) + }) + } +} + +func entsWithConfig(configFunc func(*Config), terms ...uint64) *raft { + storage := NewMemoryStorage() + for i, term := range terms { + storage.Append([]*pb.Entry{{Index: new(uint64(i + 1)), Term: new(term)}}) + } + cfg := newTestConfig(1, 5, 1, storage) + if configFunc != nil { + configFunc(cfg) + } + sm := newRaft(cfg) + sm.reset(terms[len(terms)-1]) + return sm +} + +// votedWithConfig creates a raft state machine with Vote and Term set +// to the given value but no log entries (indicating that it voted in +// the given term but has not received any logs). +func votedWithConfig(configFunc func(*Config), vote, term uint64) *raft { + storage := NewMemoryStorage() + storage.SetHardState(&pb.HardState{Vote: new(vote), Term: new(term)}) + cfg := newTestConfig(1, 5, 1, storage) + if configFunc != nil { + configFunc(cfg) + } + sm := newRaft(cfg) + sm.reset(term) + return sm +} + +func TestLogReplicationWithReorderedMessage(t *testing.T) { + r1 := newTestRaft(1, 10, 1, newTestMemoryStorage(withPeers(1, 2))) + r1.becomeCandidate() + r1.becomeLeader() + r1.readMessages() + r1.trk.Progress[2].BecomeReplicate() + + r2 := newTestRaft(2, 10, 1, newTestMemoryStorage(withPeers(1, 2))) + + // r1 sends 2 MsgApp messages to r2. + mustAppendEntry(r1, &pb.Entry{Data: []byte("somedata")}) + r1.sendAppend(2) + req1 := expectOneMessage(t, r1) + mustAppendEntry(r1, &pb.Entry{Data: []byte("somedata")}) + r1.sendAppend(2) + req2 := expectOneMessage(t, r1) + + // r2 receives the second MsgApp first due to reordering. + r2.Step(req2) + resp2 := expectOneMessage(t, r2) + // r2 rejects req2 + require.True(t, resp2.GetReject()) + require.Zero(t, resp2.GetRejectHint()) + require.Equal(t, uint64(2), resp2.GetIndex()) + + // r2 handles the first MsgApp and responses to r1. + // And r1 updates match index accordingly. + r2.Step(req1) + m := expectOneMessage(t, r2) + require.False(t, m.GetReject()) + require.Equal(t, uint64(2), m.GetIndex()) + r1.Step(m) + _ = expectOneMessage(t, r1) + require.Equal(t, uint64(2), r1.trk.Progress[2].Match) + + // r1 observes a transient network issue to r2, hence transits to probe state. + r1.Step(&pb.Message{From: new(uint64(2)), To: new(uint64(1)), Type: pb.MsgUnreachable.Enum()}) + require.Equal(t, tracker.StateProbe, r1.trk.Progress[2].State) + + // now r1 receives the delayed resp2. + r1.Step(resp2) + m = expectOneMessage(t, r1) + // r1 shall re-send MsgApp from match index even if resp2's reject hint is less than matching index. + require.Equal(t, r1.trk.Progress[2].Match, m.GetIndex()) +} + +func expectOneMessage(t *testing.T, r *raft) *pb.Message { + msgs := r.readMessages() + require.Len(t, msgs, 1, "expect one message") + return msgs[0] +} + +type network struct { + t *testing.T // optional + + peers map[uint64]stateMachine + storage map[uint64]*MemoryStorage + dropm map[connem]float64 + ignorem map[pb.MessageType]bool + + // msgHook is called for each message sent. It may inspect the + // message and return true to send it or false to drop it. + msgHook func(*pb.Message) bool +} + +// newNetwork initializes a network from peers. +// A nil node will be replaced with a new *stateMachine. +// A *stateMachine will get its k, id. +// When using stateMachine, the address list is always [1, n]. +func newNetwork(peers ...stateMachine) *network { + return newNetworkWithConfig(nil, peers...) +} + +// newNetworkWithConfig is like newNetwork but calls the given func to +// modify the configuration of any state machines it creates. +func newNetworkWithConfig(configFunc func(*Config), peers ...stateMachine) *network { + size := len(peers) + peerAddrs := idsBySize(size) + + npeers := make(map[uint64]stateMachine, size) + nstorage := make(map[uint64]*MemoryStorage, size) + + for j, p := range peers { + id := peerAddrs[j] + switch v := p.(type) { + case nil: + nstorage[id] = newTestMemoryStorage(withPeers(peerAddrs...)) + cfg := newTestConfig(id, 10, 1, nstorage[id]) + if configFunc != nil { + configFunc(cfg) + } + sm := newRaft(cfg) + npeers[id] = sm + case *raft: + // TODO(tbg): this is all pretty confused. Clean this up. + learners := make(map[uint64]bool, len(v.trk.Learners)) + for i := range v.trk.Learners { + learners[i] = true + } + v.id = id + v.trk = tracker.MakeProgressTracker(v.trk.MaxInflight, v.trk.MaxInflightBytes) + if len(learners) > 0 { + v.trk.Learners = map[uint64]struct{}{} + } + for i := 0; i < size; i++ { + pr := &tracker.Progress{} + if _, ok := learners[peerAddrs[i]]; ok { + pr.IsLearner = true + v.trk.Learners[peerAddrs[i]] = struct{}{} + } else { + v.trk.Voters[0][peerAddrs[i]] = struct{}{} + } + v.trk.Progress[peerAddrs[i]] = pr + } + v.reset(v.Term) + npeers[id] = v + case *blackHole: + npeers[id] = v + default: + panic(fmt.Sprintf("unexpected state machine type: %T", p)) + } + } + return &network{ + peers: npeers, + storage: nstorage, + dropm: make(map[connem]float64), + ignorem: make(map[pb.MessageType]bool), + } +} + +func preVoteConfig(c *Config) { + c.PreVote = true +} + +func (nw *network) send(msgs ...*pb.Message) { + for len(msgs) > 0 { + m := msgs[0] + p := nw.peers[m.GetTo()] + if nw.t != nil { + nw.t.Log(DescribeMessage(m, nil)) + } + _ = p.Step(m) + p.advanceMessagesAfterAppend() + msgs = append(msgs[1:], nw.filter(p.readMessages())...) + } +} + +func (nw *network) drop(from, to uint64, perc float64) { + nw.dropm[connem{from, to}] = perc +} + +func (nw *network) cut(one, other uint64) { + nw.drop(one, other, 2.0) // always drop + nw.drop(other, one, 2.0) // always drop +} + +func (nw *network) isolate(id uint64) { + for i := 0; i < len(nw.peers); i++ { + nid := uint64(i) + 1 + if nid != id { + nw.drop(id, nid, 1.0) // always drop + nw.drop(nid, id, 1.0) // always drop + } + } +} + +func (nw *network) ignore(t pb.MessageType) { + nw.ignorem[t] = true +} + +func (nw *network) recover() { + nw.dropm = make(map[connem]float64) + nw.ignorem = make(map[pb.MessageType]bool) +} + +func (nw *network) filter(msgs []*pb.Message) []*pb.Message { + var mm []*pb.Message + for _, m := range msgs { + if nw.ignorem[m.GetType()] { + continue + } + switch m.GetType() { + case pb.MsgHup: + // hups never go over the network, so don't drop them but panic + panic("unexpected msgHup") + default: + perc := nw.dropm[connem{m.GetFrom(), m.GetTo()}] + if n := rand.Float64(); n < perc { + continue + } + } + if nw.msgHook != nil { + if !nw.msgHook(m) { + continue + } + } + mm = append(mm, m) + } + return mm +} + +type connem struct { + from, to uint64 +} + +type blackHole struct{} + +func (blackHole) Step(*pb.Message) error { return nil } +func (blackHole) readMessages() []*pb.Message { return nil } +func (blackHole) advanceMessagesAfterAppend() {} + +var nopStepper = &blackHole{} + +func idsBySize(size int) []uint64 { + ids := make([]uint64, size) + for i := 0; i < size; i++ { + ids[i] = 1 + uint64(i) + } + return ids +} + +// setRandomizedElectionTimeout set up the value by caller instead of choosing +// by system, in some test scenario we need to fill in some expected value to +// ensure the certainty +func setRandomizedElectionTimeout(r *raft, v int) { + r.randomizedElectionTimeout = v +} + +// SetRandomizedElectionTimeout is like setRandomizedElectionTimeout, but +// exported for use by tests that are not in the raft package, using RawNode. +func SetRandomizedElectionTimeout(r *RawNode, v int) { + setRandomizedElectionTimeout(r.raft, v) +} + +func newTestConfig(id uint64, election, heartbeat int, storage Storage) *Config { + return &Config{ + ID: id, + ElectionTick: election, + HeartbeatTick: heartbeat, + Storage: storage, + MaxSizePerMsg: noLimit, + MaxInflightMsgs: 256, + } +} + +type testMemoryStorageOptions func(*MemoryStorage) + +func withPeers(peers ...uint64) testMemoryStorageOptions { + return func(ms *MemoryStorage) { + ms.snapshot.Metadata.ConfState.Voters = peers + } +} + +func withLearners(learners ...uint64) testMemoryStorageOptions { + return func(ms *MemoryStorage) { + ms.snapshot.Metadata.ConfState.Learners = learners + } +} + +func newTestMemoryStorage(opts ...testMemoryStorageOptions) *MemoryStorage { + ms := NewMemoryStorage() + for _, o := range opts { + o(ms) + } + return ms +} + +func newTestRaft(id uint64, election, heartbeat int, storage Storage) *raft { + return newRaft(newTestConfig(id, election, heartbeat, storage)) +} + +func newTestLearnerRaft(id uint64, election, heartbeat int, storage Storage) *raft { + cfg := newTestConfig(id, election, heartbeat, storage) + return newRaft(cfg) +} + +// newTestRawNode sets up a RawNode with the given peers. The configuration will +// not be reflected in the Storage. +func newTestRawNode(id uint64, election, heartbeat int, storage Storage) *RawNode { + cfg := newTestConfig(id, election, heartbeat, storage) + rn, err := NewRawNode(cfg) + if err != nil { + panic(err) + } + return rn +} diff --git a/third_party/raft/raftpb/alias.go b/third_party/raft/raftpb/alias.go new file mode 100644 index 0000000000..57511cd394 --- /dev/null +++ b/third_party/raft/raftpb/alias.go @@ -0,0 +1,55 @@ +// Copyright 2026 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raftpb + +const ( + EntryNormal = EntryType_EntryNormal + EntryConfChange = EntryType_EntryConfChange + EntryConfChangeV2 EntryType = EntryType_EntryConfChangeV2 + + MsgHup = MessageType_MsgHup + MsgBeat = MessageType_MsgBeat + MsgProp = MessageType_MsgProp + MsgApp = MessageType_MsgApp + MsgAppResp = MessageType_MsgAppResp + MsgVote = MessageType_MsgVote + MsgVoteResp = MessageType_MsgVoteResp + MsgSnap = MessageType_MsgSnap + MsgHeartbeat = MessageType_MsgHeartbeat + MsgHeartbeatResp = MessageType_MsgHeartbeatResp + MsgUnreachable = MessageType_MsgUnreachable + MsgSnapStatus = MessageType_MsgSnapStatus + MsgCheckQuorum = MessageType_MsgCheckQuorum + MsgTransferLeader = MessageType_MsgTransferLeader + MsgTimeoutNow = MessageType_MsgTimeoutNow + MsgReadIndex = MessageType_MsgReadIndex + MsgReadIndexResp = MessageType_MsgReadIndexResp + MsgPreVote = MessageType_MsgPreVote + MsgPreVoteResp = MessageType_MsgPreVoteResp + MsgStorageAppend = MessageType_MsgStorageAppend + MsgStorageAppendResp = MessageType_MsgStorageAppendResp + MsgStorageApply = MessageType_MsgStorageApply + MsgStorageApplyResp = MessageType_MsgStorageApplyResp + MsgForgetLeader = MessageType_MsgForgetLeader + + ConfChangeTransitionAuto = ConfChangeTransition_ConfChangeTransitionAuto + ConfChangeTransitionJointImplicit = ConfChangeTransition_ConfChangeTransitionJointImplicit + ConfChangeTransitionJointExplicit = ConfChangeTransition_ConfChangeTransitionJointExplicit + + ConfChangeAddNode = ConfChangeType_ConfChangeAddNode + ConfChangeRemoveNode = ConfChangeType_ConfChangeRemoveNode + ConfChangeUpdateNode = ConfChangeType_ConfChangeUpdateNode + ConfChangeAddLearnerNode = ConfChangeType_ConfChangeAddLearnerNode +) diff --git a/third_party/raft/raftpb/confchange.go b/third_party/raft/raftpb/confchange.go new file mode 100644 index 0000000000..2b245df4fe --- /dev/null +++ b/third_party/raft/raftpb/confchange.go @@ -0,0 +1,177 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raftpb + +import ( + "fmt" + "strconv" + "strings" + + "google.golang.org/protobuf/proto" +) + +// ConfChangeI abstracts over ConfChangeV2 and (legacy) ConfChange to allow +// treating them in a unified manner. +type ConfChangeI interface { + // AsV2 should always return a non-nil object + AsV2() *ConfChangeV2 + // AsV1 should always return a non-nil object + AsV1() (*ConfChange, bool) +} + +// MarshalConfChange calls Marshal on the underlying ConfChange or ConfChangeV2 +// and returns the result along with the corresponding EntryType. +func MarshalConfChange(c ConfChangeI) (EntryType, []byte, error) { + var typ EntryType + var ccdata []byte + var err error + if c == nil { + // A nil data unmarshals into an empty ConfChangeV2 and has the benefit + // that appendEntry can never refuse it based on its size (which + // registers as zero). + typ = EntryConfChangeV2 + ccdata = nil + } else if ccv1, ok := c.AsV1(); ok { + typ = EntryConfChange + ccdata, err = proto.Marshal(ccv1) + } else { + ccv2 := c.AsV2() + typ = EntryConfChangeV2 + ccdata, err = proto.Marshal(ccv2) + } + return typ, ccdata, err +} + +// AsV2 returns a V2 configuration change carrying out the same operation. +func (c *ConfChange) AsV2() *ConfChangeV2 { + return &ConfChangeV2{ + Changes: []*ConfChangeSingle{{ + Type: c.GetType().Enum(), + NodeId: new(c.GetNodeId()), + }}, + Context: c.Context, + } +} + +// AsV1 returns the ConfChange and true. +func (c *ConfChange) AsV1() (*ConfChange, bool) { + return c, true +} + +// AsV2 is the identity. +func (c *ConfChangeV2) AsV2() *ConfChangeV2 { return c } + +// AsV1 returns nil and false. +func (c *ConfChangeV2) AsV1() (*ConfChange, bool) { return nil, false } + +// EnterJoint returns two bools. The second bool is true if and only if this +// config change will use Joint Consensus, which is the case if it contains more +// than one change or if the use of Joint Consensus was requested explicitly. +// The first bool can only be true if second one is, and indicates whether the +// Joint State will be left automatically. +func (c *ConfChangeV2) EnterJoint() (autoLeave bool, ok bool) { + // NB: in theory, more config changes could qualify for the "simple" + // protocol but it depends on the config on top of which the changes apply. + // For example, adding two learners is not OK if both nodes are part of the + // base config (i.e. two voters are turned into learners in the process of + // applying the conf change). In practice, these distinctions should not + // matter, so we keep it simple and use Joint Consensus liberally. + if c.GetTransition() != ConfChangeTransitionAuto || len(c.Changes) > 1 { + // Use Joint Consensus. + var autoLeave bool + switch c.GetTransition() { + case ConfChangeTransitionAuto: + autoLeave = true + case ConfChangeTransitionJointImplicit: + autoLeave = true + case ConfChangeTransitionJointExplicit: + default: + panic(fmt.Sprintf("unknown transition: %+v", c)) + } + return autoLeave, true + } + return false, false +} + +// LeaveJoint is true if the configuration change leaves a joint configuration. +// This is the case if the ConfChangeV2 is zero, with the possible exception of +// the Context field. +func (c *ConfChangeV2) LeaveJoint() bool { + return c.GetTransition() == ConfChangeTransition_ConfChangeTransitionAuto && + len(c.GetChanges()) == 0 +} + +// ConfChangesFromString parses a Space-delimited sequence of operations into a +// slice of ConfChangeSingle. The supported operations are: +// - vn: make n a voter, +// - ln: make n a learner, +// - rn: remove n, and +// - un: update n. +func ConfChangesFromString(s string) ([]*ConfChangeSingle, error) { + var ccs []*ConfChangeSingle + toks := strings.Split(strings.TrimSpace(s), " ") + if toks[0] == "" { + toks = nil + } + for _, tok := range toks { + if len(tok) < 2 { + return nil, fmt.Errorf("unknown token %s", tok) + } + cc := &ConfChangeSingle{} + switch tok[0] { + case 'v': + cc.Type = ConfChangeAddNode.Enum() + case 'l': + cc.Type = ConfChangeAddLearnerNode.Enum() + case 'r': + cc.Type = ConfChangeRemoveNode.Enum() + case 'u': + cc.Type = ConfChangeUpdateNode.Enum() + default: + return nil, fmt.Errorf("unknown input: %s", tok) + } + id, err := strconv.ParseUint(tok[1:], 10, 64) + if err != nil { + return nil, err + } + cc.NodeId = new(id) + ccs = append(ccs, cc) + } + return ccs, nil +} + +// ConfChangesToString is the inverse to ConfChangesFromString. +func ConfChangesToString(ccs []*ConfChangeSingle) string { + var buf strings.Builder + for i, cc := range ccs { + if i > 0 { + buf.WriteByte(' ') + } + switch cc.GetType() { + case ConfChangeAddNode: + buf.WriteByte('v') + case ConfChangeAddLearnerNode: + buf.WriteByte('l') + case ConfChangeRemoveNode: + buf.WriteByte('r') + case ConfChangeUpdateNode: + buf.WriteByte('u') + default: + buf.WriteString("unknown") + } + fmt.Fprintf(&buf, "%d", cc.GetNodeId()) + } + return buf.String() +} diff --git a/third_party/raft/raftpb/confchange_test.go b/third_party/raft/raftpb/confchange_test.go new file mode 100644 index 0000000000..110506bfda --- /dev/null +++ b/third_party/raft/raftpb/confchange_test.go @@ -0,0 +1,130 @@ +// Copyright 2026 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raftpb + +import ( + "reflect" + "testing" +) + +func TestLeaveJoint(t *testing.T) { + // Verify that ConfChangeV2 has exactly the expected fields. + // This ensures that if new fields are added to ConfChangeV2, this test will fail. + ccv2Type := reflect.TypeOf((*ConfChangeV2)(nil)).Elem() + // 3 user-visible fields (Transition, Changes, Context) + 3 internal protobuf + // fields (state, unknownFields, sizeCache) added by protoc-gen-go v2. + expectedFieldCount := 6 + if ccv2Type.NumField() != expectedFieldCount { + t.Fatalf("ConfChangeV2 has %d fields, expected %d. If new fields were added, please update this test.", + ccv2Type.NumField(), expectedFieldCount) + } + + tests := []struct { + name string + cc *ConfChangeV2 + expected bool + }{ + { + name: "empty conf change (all zero values)", + cc: &ConfChangeV2{ + Transition: nil, + Changes: nil, + Context: nil, + }, + expected: true, + }, + { + name: "empty conf change with auto transition explicitly set", + cc: &ConfChangeV2{ + Transition: ConfChangeTransition_ConfChangeTransitionAuto.Enum(), + Changes: nil, + Context: nil, + }, + expected: true, + }, + { + name: "empty changes with context (context should be ignored)", + cc: &ConfChangeV2{ + Transition: ConfChangeTransition_ConfChangeTransitionAuto.Enum(), + Changes: nil, + Context: []byte("some context"), + }, + expected: true, + }, + + { + name: "empty changes slice with auto transition", + cc: &ConfChangeV2{ + Transition: ConfChangeTransition_ConfChangeTransitionAuto.Enum(), + Changes: []*ConfChangeSingle{}, + Context: nil, + }, + expected: true, + }, + { + name: "non-empty changes", + cc: &ConfChangeV2{ + Transition: ConfChangeTransition_ConfChangeTransitionAuto.Enum(), + Changes: []*ConfChangeSingle{ + { + Type: ConfChangeAddNode.Enum(), + NodeId: new(uint64), + }, + }, + Context: nil, + }, + expected: false, + }, + { + name: "joint implicit transition with empty changes", + cc: &ConfChangeV2{ + Transition: ConfChangeTransition_ConfChangeTransitionJointImplicit.Enum(), + Changes: nil, + Context: nil, + }, + expected: false, + }, + { + name: "joint explicit transition with empty changes", + cc: &ConfChangeV2{ + Transition: ConfChangeTransition_ConfChangeTransitionJointExplicit.Enum(), + Changes: nil, + Context: nil, + }, + expected: false, + }, + { + name: "auto transition with multiple changes", + cc: &ConfChangeV2{ + Transition: ConfChangeTransition_ConfChangeTransitionAuto.Enum(), + Changes: []*ConfChangeSingle{ + {Type: ConfChangeAddNode.Enum(), NodeId: new(uint64)}, + {Type: ConfChangeRemoveNode.Enum(), NodeId: new(uint64)}, + }, + Context: nil, + }, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.cc.LeaveJoint() + if got != tt.expected { + t.Errorf("LeaveJoint() = %v, want %v", got, tt.expected) + } + }) + } +} diff --git a/third_party/raft/raftpb/confstate.go b/third_party/raft/raftpb/confstate.go new file mode 100644 index 0000000000..462b2d9306 --- /dev/null +++ b/third_party/raft/raftpb/confstate.go @@ -0,0 +1,52 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raftpb + +import ( + "fmt" + "slices" + + "google.golang.org/protobuf/proto" +) + +// Equivalent returns a nil error if the inputs describe the same configuration. +// On mismatch, returns a descriptive error showing the differences. +func (cs *ConfState) Equivalent(cs2 *ConfState) error { + if cs == nil || cs2 == nil { + return fmt.Errorf("cannot compare ConfState: nil input (left=%v, right=%v)", cs == nil, cs2 == nil) + } + + cs1v := proto.Clone(cs).(*ConfState) + cs2v := proto.Clone(cs2).(*ConfState) + + s := func(sl *[]uint64) { + slices.Sort(*sl) + } + for _, c := range []*ConfState{cs1v, cs2v} { + s(&c.Voters) + s(&c.Learners) + s(&c.VotersOutgoing) + s(&c.LearnersNext) + + // Treat nil AutoLeave as false. + autoLeave := c.GetAutoLeave() + c.AutoLeave = &autoLeave + } + + if !proto.Equal(cs1v, cs2v) { + return fmt.Errorf("ConfStates not equivalent after sorting:\n%+#v\n%+#v\nInputs were:\n%+#v\n%+#v", cs1v, cs2v, cs, cs2) + } + return nil +} diff --git a/third_party/raft/raftpb/confstate_test.go b/third_party/raft/raftpb/confstate_test.go new file mode 100644 index 0000000000..13f1e026f4 --- /dev/null +++ b/third_party/raft/raftpb/confstate_test.go @@ -0,0 +1,59 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raftpb + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestConfState_Equivalent(t *testing.T) { + type testCase struct { + cs, cs2 *ConfState + ok bool + } + + testCases := []testCase{ + // Reordered voters and learners. + {&ConfState{ + Voters: []uint64{1, 2, 3}, + Learners: []uint64{5, 4, 6}, + VotersOutgoing: []uint64{9, 8, 7}, + LearnersNext: []uint64{10, 20, 15}, + }, &ConfState{ + Voters: []uint64{1, 2, 3}, + Learners: []uint64{4, 5, 6}, + VotersOutgoing: []uint64{7, 9, 8}, + LearnersNext: []uint64{20, 10, 15}, + }, true}, + // Not sensitive to nil vs empty slice. + {&ConfState{Voters: []uint64{}}, &ConfState{Voters: []uint64(nil)}, true}, + // Non-equivalent voters. + {&ConfState{Voters: []uint64{1, 2, 3, 4}}, &ConfState{Voters: []uint64{2, 1, 3}}, false}, + {&ConfState{Voters: []uint64{1, 4, 3}}, &ConfState{Voters: []uint64{2, 1, 3}}, false}, + // Non-equivalent learners. + {&ConfState{Voters: []uint64{1, 2, 3, 4}}, &ConfState{Voters: []uint64{2, 1, 3}}, false}, + // Sensitive to AutoLeave flag. + {&ConfState{AutoLeave: new(true)}, &ConfState{}, false}, + } + + for _, tc := range testCases { + t.Run("", func(t *testing.T) { + cs, cs2 := EnsureConfState(tc.cs), EnsureConfState(tc.cs2) + require.Equal(t, tc.ok, cs.Equivalent(cs2) == nil) + }) + } +} diff --git a/third_party/raft/raftpb/raft.pb.go b/third_party/raft/raftpb/raft.pb.go new file mode 100644 index 0000000000..cb15132c0f --- /dev/null +++ b/third_party/raft/raftpb/raft.pb.go @@ -0,0 +1,1233 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.20.3 +// source: raft.proto + +package raftpb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type EntryType int32 + +const ( + EntryType_EntryNormal EntryType = 0 + EntryType_EntryConfChange EntryType = 1 // corresponds to pb.ConfChange + EntryType_EntryConfChangeV2 EntryType = 2 // corresponds to pb.ConfChangeV2 +) + +// Enum value maps for EntryType. +var ( + EntryType_name = map[int32]string{ + 0: "EntryNormal", + 1: "EntryConfChange", + 2: "EntryConfChangeV2", + } + EntryType_value = map[string]int32{ + "EntryNormal": 0, + "EntryConfChange": 1, + "EntryConfChangeV2": 2, + } +) + +func (x EntryType) Enum() *EntryType { + p := new(EntryType) + *p = x + return p +} + +func (x EntryType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (EntryType) Descriptor() protoreflect.EnumDescriptor { + return file_raft_proto_enumTypes[0].Descriptor() +} + +func (EntryType) Type() protoreflect.EnumType { + return &file_raft_proto_enumTypes[0] +} + +func (x EntryType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *EntryType) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = EntryType(num) + return nil +} + +// Deprecated: Use EntryType.Descriptor instead. +func (EntryType) EnumDescriptor() ([]byte, []int) { + return file_raft_proto_rawDescGZIP(), []int{0} +} + +// For description of different message types, see: +// https://pkg.go.dev/go.etcd.io/raft/v3#hdr-MessageType +type MessageType int32 + +const ( + MessageType_MsgHup MessageType = 0 + MessageType_MsgBeat MessageType = 1 + MessageType_MsgProp MessageType = 2 + MessageType_MsgApp MessageType = 3 + MessageType_MsgAppResp MessageType = 4 + MessageType_MsgVote MessageType = 5 + MessageType_MsgVoteResp MessageType = 6 + MessageType_MsgSnap MessageType = 7 + MessageType_MsgHeartbeat MessageType = 8 + MessageType_MsgHeartbeatResp MessageType = 9 + MessageType_MsgUnreachable MessageType = 10 + MessageType_MsgSnapStatus MessageType = 11 + MessageType_MsgCheckQuorum MessageType = 12 + MessageType_MsgTransferLeader MessageType = 13 + MessageType_MsgTimeoutNow MessageType = 14 + MessageType_MsgReadIndex MessageType = 15 + MessageType_MsgReadIndexResp MessageType = 16 + MessageType_MsgPreVote MessageType = 17 + MessageType_MsgPreVoteResp MessageType = 18 + MessageType_MsgStorageAppend MessageType = 19 + MessageType_MsgStorageAppendResp MessageType = 20 + MessageType_MsgStorageApply MessageType = 21 + MessageType_MsgStorageApplyResp MessageType = 22 + MessageType_MsgForgetLeader MessageType = 23 +) + +// Enum value maps for MessageType. +var ( + MessageType_name = map[int32]string{ + 0: "MsgHup", + 1: "MsgBeat", + 2: "MsgProp", + 3: "MsgApp", + 4: "MsgAppResp", + 5: "MsgVote", + 6: "MsgVoteResp", + 7: "MsgSnap", + 8: "MsgHeartbeat", + 9: "MsgHeartbeatResp", + 10: "MsgUnreachable", + 11: "MsgSnapStatus", + 12: "MsgCheckQuorum", + 13: "MsgTransferLeader", + 14: "MsgTimeoutNow", + 15: "MsgReadIndex", + 16: "MsgReadIndexResp", + 17: "MsgPreVote", + 18: "MsgPreVoteResp", + 19: "MsgStorageAppend", + 20: "MsgStorageAppendResp", + 21: "MsgStorageApply", + 22: "MsgStorageApplyResp", + 23: "MsgForgetLeader", + } + MessageType_value = map[string]int32{ + "MsgHup": 0, + "MsgBeat": 1, + "MsgProp": 2, + "MsgApp": 3, + "MsgAppResp": 4, + "MsgVote": 5, + "MsgVoteResp": 6, + "MsgSnap": 7, + "MsgHeartbeat": 8, + "MsgHeartbeatResp": 9, + "MsgUnreachable": 10, + "MsgSnapStatus": 11, + "MsgCheckQuorum": 12, + "MsgTransferLeader": 13, + "MsgTimeoutNow": 14, + "MsgReadIndex": 15, + "MsgReadIndexResp": 16, + "MsgPreVote": 17, + "MsgPreVoteResp": 18, + "MsgStorageAppend": 19, + "MsgStorageAppendResp": 20, + "MsgStorageApply": 21, + "MsgStorageApplyResp": 22, + "MsgForgetLeader": 23, + } +) + +func (x MessageType) Enum() *MessageType { + p := new(MessageType) + *p = x + return p +} + +func (x MessageType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MessageType) Descriptor() protoreflect.EnumDescriptor { + return file_raft_proto_enumTypes[1].Descriptor() +} + +func (MessageType) Type() protoreflect.EnumType { + return &file_raft_proto_enumTypes[1] +} + +func (x MessageType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *MessageType) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = MessageType(num) + return nil +} + +// Deprecated: Use MessageType.Descriptor instead. +func (MessageType) EnumDescriptor() ([]byte, []int) { + return file_raft_proto_rawDescGZIP(), []int{1} +} + +// ConfChangeTransition specifies the behavior of a configuration change with +// respect to joint consensus. +type ConfChangeTransition int32 + +const ( + // Automatically use the simple protocol if possible, otherwise fall back + // to ConfChangeJointImplicit. Most applications will want to use this. + ConfChangeTransition_ConfChangeTransitionAuto ConfChangeTransition = 0 + // Use joint consensus unconditionally, and transition out of them + // automatically (by proposing a zero configuration change). + // + // This option is suitable for applications that want to minimize the time + // spent in the joint configuration and do not store the joint configuration + // in the state machine (outside of InitialState). + ConfChangeTransition_ConfChangeTransitionJointImplicit ConfChangeTransition = 1 + // Use joint consensus and remain in the joint configuration until the + // application proposes a no-op configuration change. This is suitable for + // applications that want to explicitly control the transitions, for example + // to use a custom payload (via the Context field). + ConfChangeTransition_ConfChangeTransitionJointExplicit ConfChangeTransition = 2 +) + +// Enum value maps for ConfChangeTransition. +var ( + ConfChangeTransition_name = map[int32]string{ + 0: "ConfChangeTransitionAuto", + 1: "ConfChangeTransitionJointImplicit", + 2: "ConfChangeTransitionJointExplicit", + } + ConfChangeTransition_value = map[string]int32{ + "ConfChangeTransitionAuto": 0, + "ConfChangeTransitionJointImplicit": 1, + "ConfChangeTransitionJointExplicit": 2, + } +) + +func (x ConfChangeTransition) Enum() *ConfChangeTransition { + p := new(ConfChangeTransition) + *p = x + return p +} + +func (x ConfChangeTransition) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ConfChangeTransition) Descriptor() protoreflect.EnumDescriptor { + return file_raft_proto_enumTypes[2].Descriptor() +} + +func (ConfChangeTransition) Type() protoreflect.EnumType { + return &file_raft_proto_enumTypes[2] +} + +func (x ConfChangeTransition) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *ConfChangeTransition) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = ConfChangeTransition(num) + return nil +} + +// Deprecated: Use ConfChangeTransition.Descriptor instead. +func (ConfChangeTransition) EnumDescriptor() ([]byte, []int) { + return file_raft_proto_rawDescGZIP(), []int{2} +} + +type ConfChangeType int32 + +const ( + ConfChangeType_ConfChangeAddNode ConfChangeType = 0 + ConfChangeType_ConfChangeRemoveNode ConfChangeType = 1 + ConfChangeType_ConfChangeUpdateNode ConfChangeType = 2 + ConfChangeType_ConfChangeAddLearnerNode ConfChangeType = 3 +) + +// Enum value maps for ConfChangeType. +var ( + ConfChangeType_name = map[int32]string{ + 0: "ConfChangeAddNode", + 1: "ConfChangeRemoveNode", + 2: "ConfChangeUpdateNode", + 3: "ConfChangeAddLearnerNode", + } + ConfChangeType_value = map[string]int32{ + "ConfChangeAddNode": 0, + "ConfChangeRemoveNode": 1, + "ConfChangeUpdateNode": 2, + "ConfChangeAddLearnerNode": 3, + } +) + +func (x ConfChangeType) Enum() *ConfChangeType { + p := new(ConfChangeType) + *p = x + return p +} + +func (x ConfChangeType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ConfChangeType) Descriptor() protoreflect.EnumDescriptor { + return file_raft_proto_enumTypes[3].Descriptor() +} + +func (ConfChangeType) Type() protoreflect.EnumType { + return &file_raft_proto_enumTypes[3] +} + +func (x ConfChangeType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *ConfChangeType) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = ConfChangeType(num) + return nil +} + +// Deprecated: Use ConfChangeType.Descriptor instead. +func (ConfChangeType) EnumDescriptor() ([]byte, []int) { + return file_raft_proto_rawDescGZIP(), []int{3} +} + +type Entry struct { + state protoimpl.MessageState `protogen:"open.v1"` + Term *uint64 `protobuf:"varint,2,opt,name=Term" json:"Term,omitempty"` + Index *uint64 `protobuf:"varint,3,opt,name=Index" json:"Index,omitempty"` + Type *EntryType `protobuf:"varint,1,opt,name=Type,enum=raftpb.EntryType" json:"Type,omitempty"` + Data []byte `protobuf:"bytes,4,opt,name=Data" json:"Data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Entry) Reset() { + *x = Entry{} + mi := &file_raft_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Entry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Entry) ProtoMessage() {} + +func (x *Entry) ProtoReflect() protoreflect.Message { + mi := &file_raft_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Entry.ProtoReflect.Descriptor instead. +func (*Entry) Descriptor() ([]byte, []int) { + return file_raft_proto_rawDescGZIP(), []int{0} +} + +func (x *Entry) GetTerm() uint64 { + if x != nil && x.Term != nil { + return *x.Term + } + return 0 +} + +func (x *Entry) GetIndex() uint64 { + if x != nil && x.Index != nil { + return *x.Index + } + return 0 +} + +func (x *Entry) GetType() EntryType { + if x != nil && x.Type != nil { + return *x.Type + } + return EntryType_EntryNormal +} + +func (x *Entry) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +type SnapshotMetadata struct { + state protoimpl.MessageState `protogen:"open.v1"` + ConfState *ConfState `protobuf:"bytes,1,opt,name=conf_state,json=confState" json:"conf_state,omitempty"` + Index *uint64 `protobuf:"varint,2,opt,name=index" json:"index,omitempty"` + Term *uint64 `protobuf:"varint,3,opt,name=term" json:"term,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SnapshotMetadata) Reset() { + *x = SnapshotMetadata{} + mi := &file_raft_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SnapshotMetadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SnapshotMetadata) ProtoMessage() {} + +func (x *SnapshotMetadata) ProtoReflect() protoreflect.Message { + mi := &file_raft_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SnapshotMetadata.ProtoReflect.Descriptor instead. +func (*SnapshotMetadata) Descriptor() ([]byte, []int) { + return file_raft_proto_rawDescGZIP(), []int{1} +} + +func (x *SnapshotMetadata) GetConfState() *ConfState { + if x != nil { + return x.ConfState + } + return nil +} + +func (x *SnapshotMetadata) GetIndex() uint64 { + if x != nil && x.Index != nil { + return *x.Index + } + return 0 +} + +func (x *SnapshotMetadata) GetTerm() uint64 { + if x != nil && x.Term != nil { + return *x.Term + } + return 0 +} + +type Snapshot struct { + state protoimpl.MessageState `protogen:"open.v1"` + Data []byte `protobuf:"bytes,1,opt,name=data" json:"data,omitempty"` + Metadata *SnapshotMetadata `protobuf:"bytes,2,opt,name=metadata" json:"metadata,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Snapshot) Reset() { + *x = Snapshot{} + mi := &file_raft_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Snapshot) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Snapshot) ProtoMessage() {} + +func (x *Snapshot) ProtoReflect() protoreflect.Message { + mi := &file_raft_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Snapshot.ProtoReflect.Descriptor instead. +func (*Snapshot) Descriptor() ([]byte, []int) { + return file_raft_proto_rawDescGZIP(), []int{2} +} + +func (x *Snapshot) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *Snapshot) GetMetadata() *SnapshotMetadata { + if x != nil { + return x.Metadata + } + return nil +} + +type Message struct { + state protoimpl.MessageState `protogen:"open.v1"` + Type *MessageType `protobuf:"varint,1,opt,name=type,enum=raftpb.MessageType" json:"type,omitempty"` + To *uint64 `protobuf:"varint,2,opt,name=to" json:"to,omitempty"` + From *uint64 `protobuf:"varint,3,opt,name=from" json:"from,omitempty"` + Term *uint64 `protobuf:"varint,4,opt,name=term" json:"term,omitempty"` + // logTerm is generally used for appending Raft logs to followers. For example, + // (type=MsgApp,index=100,logTerm=5) means the leader appends entries starting + // at index=101, and the term of the entry at index 100 is 5. + // (type=MsgAppResp,reject=true,index=100,logTerm=5) means follower rejects some + // entries from its leader as it already has an entry with term 5 at index 100. + // (type=MsgStorageAppendResp,index=100,logTerm=5) means the local node wrote + // entries up to index=100 in stable storage, and the term of the entry at index + // 100 was 5. This doesn't always mean that the corresponding MsgStorageAppend + // message was the one that carried these entries, just that those entries were + // stable at the time of processing the corresponding MsgStorageAppend. + LogTerm *uint64 `protobuf:"varint,5,opt,name=logTerm" json:"logTerm,omitempty"` + Index *uint64 `protobuf:"varint,6,opt,name=index" json:"index,omitempty"` + Entries []*Entry `protobuf:"bytes,7,rep,name=entries" json:"entries,omitempty"` + Commit *uint64 `protobuf:"varint,8,opt,name=commit" json:"commit,omitempty"` + // (type=MsgStorageAppend,vote=5,term=10) means the local node is voting for + // peer 5 in term 10. For MsgStorageAppends, the term, vote, and commit fields + // will either all be set (to facilitate the construction of a HardState) if + // any of the fields have changed or will all be unset if none of the fields + // have changed. + Vote *uint64 `protobuf:"varint,13,opt,name=vote" json:"vote,omitempty"` + // snapshot is non-nil and non-empty for MsgSnap messages and nil for all other + // message types. However, peer nodes running older binary versions may send a + // non-nil, empty value for the snapshot field of non-MsgSnap messages. Code + // should be prepared to handle such messages. + Snapshot *Snapshot `protobuf:"bytes,9,opt,name=snapshot" json:"snapshot,omitempty"` + Reject *bool `protobuf:"varint,10,opt,name=reject" json:"reject,omitempty"` + RejectHint *uint64 `protobuf:"varint,11,opt,name=rejectHint" json:"rejectHint,omitempty"` + Context []byte `protobuf:"bytes,12,opt,name=context" json:"context,omitempty"` + // responses are populated by a raft node to instruct storage threads on how + // to respond and who to respond to when the work associated with a message + // is complete. Populated for MsgStorageAppend and MsgStorageApply messages. + Responses []*Message `protobuf:"bytes,14,rep,name=responses" json:"responses,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Message) Reset() { + *x = Message{} + mi := &file_raft_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Message) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Message) ProtoMessage() {} + +func (x *Message) ProtoReflect() protoreflect.Message { + mi := &file_raft_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Message.ProtoReflect.Descriptor instead. +func (*Message) Descriptor() ([]byte, []int) { + return file_raft_proto_rawDescGZIP(), []int{3} +} + +func (x *Message) GetType() MessageType { + if x != nil && x.Type != nil { + return *x.Type + } + return MessageType_MsgHup +} + +func (x *Message) GetTo() uint64 { + if x != nil && x.To != nil { + return *x.To + } + return 0 +} + +func (x *Message) GetFrom() uint64 { + if x != nil && x.From != nil { + return *x.From + } + return 0 +} + +func (x *Message) GetTerm() uint64 { + if x != nil && x.Term != nil { + return *x.Term + } + return 0 +} + +func (x *Message) GetLogTerm() uint64 { + if x != nil && x.LogTerm != nil { + return *x.LogTerm + } + return 0 +} + +func (x *Message) GetIndex() uint64 { + if x != nil && x.Index != nil { + return *x.Index + } + return 0 +} + +func (x *Message) GetEntries() []*Entry { + if x != nil { + return x.Entries + } + return nil +} + +func (x *Message) GetCommit() uint64 { + if x != nil && x.Commit != nil { + return *x.Commit + } + return 0 +} + +func (x *Message) GetVote() uint64 { + if x != nil && x.Vote != nil { + return *x.Vote + } + return 0 +} + +func (x *Message) GetSnapshot() *Snapshot { + if x != nil { + return x.Snapshot + } + return nil +} + +func (x *Message) GetReject() bool { + if x != nil && x.Reject != nil { + return *x.Reject + } + return false +} + +func (x *Message) GetRejectHint() uint64 { + if x != nil && x.RejectHint != nil { + return *x.RejectHint + } + return 0 +} + +func (x *Message) GetContext() []byte { + if x != nil { + return x.Context + } + return nil +} + +func (x *Message) GetResponses() []*Message { + if x != nil { + return x.Responses + } + return nil +} + +type HardState struct { + state protoimpl.MessageState `protogen:"open.v1"` + Term *uint64 `protobuf:"varint,1,opt,name=term" json:"term,omitempty"` + Vote *uint64 `protobuf:"varint,2,opt,name=vote" json:"vote,omitempty"` + Commit *uint64 `protobuf:"varint,3,opt,name=commit" json:"commit,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HardState) Reset() { + *x = HardState{} + mi := &file_raft_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HardState) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HardState) ProtoMessage() {} + +func (x *HardState) ProtoReflect() protoreflect.Message { + mi := &file_raft_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HardState.ProtoReflect.Descriptor instead. +func (*HardState) Descriptor() ([]byte, []int) { + return file_raft_proto_rawDescGZIP(), []int{4} +} + +func (x *HardState) GetTerm() uint64 { + if x != nil && x.Term != nil { + return *x.Term + } + return 0 +} + +func (x *HardState) GetVote() uint64 { + if x != nil && x.Vote != nil { + return *x.Vote + } + return 0 +} + +func (x *HardState) GetCommit() uint64 { + if x != nil && x.Commit != nil { + return *x.Commit + } + return 0 +} + +type ConfState struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The voters in the incoming config. (If the configuration is not joint, + // then the outgoing config is empty). + Voters []uint64 `protobuf:"varint,1,rep,name=voters" json:"voters,omitempty"` + // The learners in the incoming config. + Learners []uint64 `protobuf:"varint,2,rep,name=learners" json:"learners,omitempty"` + // The voters in the outgoing config. + VotersOutgoing []uint64 `protobuf:"varint,3,rep,name=voters_outgoing,json=votersOutgoing" json:"voters_outgoing,omitempty"` + // The nodes that will become learners when the outgoing config is removed. + // These nodes are necessarily currently in nodes_joint (or they would have + // been added to the incoming config right away). + LearnersNext []uint64 `protobuf:"varint,4,rep,name=learners_next,json=learnersNext" json:"learners_next,omitempty"` + // If set, the config is joint and Raft will automatically transition into + // the final config (i.e. remove the outgoing config) when this is safe. + AutoLeave *bool `protobuf:"varint,5,opt,name=auto_leave,json=autoLeave" json:"auto_leave,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConfState) Reset() { + *x = ConfState{} + mi := &file_raft_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConfState) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConfState) ProtoMessage() {} + +func (x *ConfState) ProtoReflect() protoreflect.Message { + mi := &file_raft_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConfState.ProtoReflect.Descriptor instead. +func (*ConfState) Descriptor() ([]byte, []int) { + return file_raft_proto_rawDescGZIP(), []int{5} +} + +func (x *ConfState) GetVoters() []uint64 { + if x != nil { + return x.Voters + } + return nil +} + +func (x *ConfState) GetLearners() []uint64 { + if x != nil { + return x.Learners + } + return nil +} + +func (x *ConfState) GetVotersOutgoing() []uint64 { + if x != nil { + return x.VotersOutgoing + } + return nil +} + +func (x *ConfState) GetLearnersNext() []uint64 { + if x != nil { + return x.LearnersNext + } + return nil +} + +func (x *ConfState) GetAutoLeave() bool { + if x != nil && x.AutoLeave != nil { + return *x.AutoLeave + } + return false +} + +type ConfChange struct { + state protoimpl.MessageState `protogen:"open.v1"` + Type *ConfChangeType `protobuf:"varint,2,opt,name=type,enum=raftpb.ConfChangeType" json:"type,omitempty"` + NodeId *uint64 `protobuf:"varint,3,opt,name=node_id,json=nodeId" json:"node_id,omitempty"` + Context []byte `protobuf:"bytes,4,opt,name=context" json:"context,omitempty"` + // NB: this is used only by etcd to thread through a unique identifier. + // Ideally it should really use the Context instead. No counterpart to + // this field exists in ConfChangeV2. + Id *uint64 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConfChange) Reset() { + *x = ConfChange{} + mi := &file_raft_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConfChange) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConfChange) ProtoMessage() {} + +func (x *ConfChange) ProtoReflect() protoreflect.Message { + mi := &file_raft_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConfChange.ProtoReflect.Descriptor instead. +func (*ConfChange) Descriptor() ([]byte, []int) { + return file_raft_proto_rawDescGZIP(), []int{6} +} + +func (x *ConfChange) GetType() ConfChangeType { + if x != nil && x.Type != nil { + return *x.Type + } + return ConfChangeType_ConfChangeAddNode +} + +func (x *ConfChange) GetNodeId() uint64 { + if x != nil && x.NodeId != nil { + return *x.NodeId + } + return 0 +} + +func (x *ConfChange) GetContext() []byte { + if x != nil { + return x.Context + } + return nil +} + +func (x *ConfChange) GetId() uint64 { + if x != nil && x.Id != nil { + return *x.Id + } + return 0 +} + +// ConfChangeSingle is an individual configuration change operation. Multiple +// such operations can be carried out atomically via a ConfChangeV2. +type ConfChangeSingle struct { + state protoimpl.MessageState `protogen:"open.v1"` + Type *ConfChangeType `protobuf:"varint,1,opt,name=type,enum=raftpb.ConfChangeType" json:"type,omitempty"` + NodeId *uint64 `protobuf:"varint,2,opt,name=node_id,json=nodeId" json:"node_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConfChangeSingle) Reset() { + *x = ConfChangeSingle{} + mi := &file_raft_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConfChangeSingle) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConfChangeSingle) ProtoMessage() {} + +func (x *ConfChangeSingle) ProtoReflect() protoreflect.Message { + mi := &file_raft_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConfChangeSingle.ProtoReflect.Descriptor instead. +func (*ConfChangeSingle) Descriptor() ([]byte, []int) { + return file_raft_proto_rawDescGZIP(), []int{7} +} + +func (x *ConfChangeSingle) GetType() ConfChangeType { + if x != nil && x.Type != nil { + return *x.Type + } + return ConfChangeType_ConfChangeAddNode +} + +func (x *ConfChangeSingle) GetNodeId() uint64 { + if x != nil && x.NodeId != nil { + return *x.NodeId + } + return 0 +} + +// ConfChangeV2 messages initiate configuration changes. They support both the +// simple "one at a time" membership change protocol and full Joint Consensus +// allowing for arbitrary changes in membership. +// +// The supplied context is treated as an opaque payload and can be used to +// attach an action on the state machine to the application of the config change +// proposal. Note that contrary to Joint Consensus as outlined in the Raft +// paper[1], configuration changes become active when they are *applied* to the +// state machine (not when they are appended to the log). +// +// The simple protocol can be used whenever only a single change is made. +// +// Non-simple changes require the use of Joint Consensus, for which two +// configuration changes are run. The first configuration change specifies the +// desired changes and transitions the Raft group into the joint configuration, +// in which quorum requires a majority of both the pre-changes and post-changes +// configuration. Joint Consensus avoids entering fragile intermediate +// configurations that could compromise survivability. For example, without the +// use of Joint Consensus and running across three availability zones with a +// replication factor of three, it is not possible to replace a voter without +// entering an intermediate configuration that does not survive the outage of +// one availability zone. +// +// The provided ConfChangeTransition specifies how (and whether) Joint Consensus +// is used, and assigns the task of leaving the joint configuration either to +// Raft or the application. Leaving the joint configuration is accomplished by +// proposing a ConfChangeV2 with only and optionally the Context field +// populated. +// +// For details on Raft membership changes, see: +// +// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf +type ConfChangeV2 struct { + state protoimpl.MessageState `protogen:"open.v1"` + Transition *ConfChangeTransition `protobuf:"varint,1,opt,name=transition,enum=raftpb.ConfChangeTransition" json:"transition,omitempty"` + Changes []*ConfChangeSingle `protobuf:"bytes,2,rep,name=changes" json:"changes,omitempty"` + Context []byte `protobuf:"bytes,3,opt,name=context" json:"context,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConfChangeV2) Reset() { + *x = ConfChangeV2{} + mi := &file_raft_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConfChangeV2) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConfChangeV2) ProtoMessage() {} + +func (x *ConfChangeV2) ProtoReflect() protoreflect.Message { + mi := &file_raft_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConfChangeV2.ProtoReflect.Descriptor instead. +func (*ConfChangeV2) Descriptor() ([]byte, []int) { + return file_raft_proto_rawDescGZIP(), []int{8} +} + +func (x *ConfChangeV2) GetTransition() ConfChangeTransition { + if x != nil && x.Transition != nil { + return *x.Transition + } + return ConfChangeTransition_ConfChangeTransitionAuto +} + +func (x *ConfChangeV2) GetChanges() []*ConfChangeSingle { + if x != nil { + return x.Changes + } + return nil +} + +func (x *ConfChangeV2) GetContext() []byte { + if x != nil { + return x.Context + } + return nil +} + +var File_raft_proto protoreflect.FileDescriptor + +const file_raft_proto_rawDesc = "" + + "\n" + + "\n" + + "raft.proto\x12\x06raftpb\"l\n" + + "\x05Entry\x12\x12\n" + + "\x04Term\x18\x02 \x01(\x04R\x04Term\x12\x14\n" + + "\x05Index\x18\x03 \x01(\x04R\x05Index\x12%\n" + + "\x04Type\x18\x01 \x01(\x0e2\x11.raftpb.EntryTypeR\x04Type\x12\x12\n" + + "\x04Data\x18\x04 \x01(\fR\x04Data\"n\n" + + "\x10SnapshotMetadata\x120\n" + + "\n" + + "conf_state\x18\x01 \x01(\v2\x11.raftpb.ConfStateR\tconfState\x12\x14\n" + + "\x05index\x18\x02 \x01(\x04R\x05index\x12\x12\n" + + "\x04term\x18\x03 \x01(\x04R\x04term\"T\n" + + "\bSnapshot\x12\x12\n" + + "\x04data\x18\x01 \x01(\fR\x04data\x124\n" + + "\bmetadata\x18\x02 \x01(\v2\x18.raftpb.SnapshotMetadataR\bmetadata\"\x9e\x03\n" + + "\aMessage\x12'\n" + + "\x04type\x18\x01 \x01(\x0e2\x13.raftpb.MessageTypeR\x04type\x12\x0e\n" + + "\x02to\x18\x02 \x01(\x04R\x02to\x12\x12\n" + + "\x04from\x18\x03 \x01(\x04R\x04from\x12\x12\n" + + "\x04term\x18\x04 \x01(\x04R\x04term\x12\x18\n" + + "\alogTerm\x18\x05 \x01(\x04R\alogTerm\x12\x14\n" + + "\x05index\x18\x06 \x01(\x04R\x05index\x12'\n" + + "\aentries\x18\a \x03(\v2\r.raftpb.EntryR\aentries\x12\x16\n" + + "\x06commit\x18\b \x01(\x04R\x06commit\x12\x12\n" + + "\x04vote\x18\r \x01(\x04R\x04vote\x12,\n" + + "\bsnapshot\x18\t \x01(\v2\x10.raftpb.SnapshotR\bsnapshot\x12\x16\n" + + "\x06reject\x18\n" + + " \x01(\bR\x06reject\x12\x1e\n" + + "\n" + + "rejectHint\x18\v \x01(\x04R\n" + + "rejectHint\x12\x18\n" + + "\acontext\x18\f \x01(\fR\acontext\x12-\n" + + "\tresponses\x18\x0e \x03(\v2\x0f.raftpb.MessageR\tresponses\"K\n" + + "\tHardState\x12\x12\n" + + "\x04term\x18\x01 \x01(\x04R\x04term\x12\x12\n" + + "\x04vote\x18\x02 \x01(\x04R\x04vote\x12\x16\n" + + "\x06commit\x18\x03 \x01(\x04R\x06commit\"\xac\x01\n" + + "\tConfState\x12\x16\n" + + "\x06voters\x18\x01 \x03(\x04R\x06voters\x12\x1a\n" + + "\blearners\x18\x02 \x03(\x04R\blearners\x12'\n" + + "\x0fvoters_outgoing\x18\x03 \x03(\x04R\x0evotersOutgoing\x12#\n" + + "\rlearners_next\x18\x04 \x03(\x04R\flearnersNext\x12\x1d\n" + + "\n" + + "auto_leave\x18\x05 \x01(\bR\tautoLeave\"{\n" + + "\n" + + "ConfChange\x12*\n" + + "\x04type\x18\x02 \x01(\x0e2\x16.raftpb.ConfChangeTypeR\x04type\x12\x17\n" + + "\anode_id\x18\x03 \x01(\x04R\x06nodeId\x12\x18\n" + + "\acontext\x18\x04 \x01(\fR\acontext\x12\x0e\n" + + "\x02id\x18\x01 \x01(\x04R\x02id\"W\n" + + "\x10ConfChangeSingle\x12*\n" + + "\x04type\x18\x01 \x01(\x0e2\x16.raftpb.ConfChangeTypeR\x04type\x12\x17\n" + + "\anode_id\x18\x02 \x01(\x04R\x06nodeId\"\x9a\x01\n" + + "\fConfChangeV2\x12<\n" + + "\n" + + "transition\x18\x01 \x01(\x0e2\x1c.raftpb.ConfChangeTransitionR\n" + + "transition\x122\n" + + "\achanges\x18\x02 \x03(\v2\x18.raftpb.ConfChangeSingleR\achanges\x12\x18\n" + + "\acontext\x18\x03 \x01(\fR\acontext*H\n" + + "\tEntryType\x12\x0f\n" + + "\vEntryNormal\x10\x00\x12\x13\n" + + "\x0fEntryConfChange\x10\x01\x12\x15\n" + + "\x11EntryConfChangeV2\x10\x02*\xc6\x03\n" + + "\vMessageType\x12\n" + + "\n" + + "\x06MsgHup\x10\x00\x12\v\n" + + "\aMsgBeat\x10\x01\x12\v\n" + + "\aMsgProp\x10\x02\x12\n" + + "\n" + + "\x06MsgApp\x10\x03\x12\x0e\n" + + "\n" + + "MsgAppResp\x10\x04\x12\v\n" + + "\aMsgVote\x10\x05\x12\x0f\n" + + "\vMsgVoteResp\x10\x06\x12\v\n" + + "\aMsgSnap\x10\a\x12\x10\n" + + "\fMsgHeartbeat\x10\b\x12\x14\n" + + "\x10MsgHeartbeatResp\x10\t\x12\x12\n" + + "\x0eMsgUnreachable\x10\n" + + "\x12\x11\n" + + "\rMsgSnapStatus\x10\v\x12\x12\n" + + "\x0eMsgCheckQuorum\x10\f\x12\x15\n" + + "\x11MsgTransferLeader\x10\r\x12\x11\n" + + "\rMsgTimeoutNow\x10\x0e\x12\x10\n" + + "\fMsgReadIndex\x10\x0f\x12\x14\n" + + "\x10MsgReadIndexResp\x10\x10\x12\x0e\n" + + "\n" + + "MsgPreVote\x10\x11\x12\x12\n" + + "\x0eMsgPreVoteResp\x10\x12\x12\x14\n" + + "\x10MsgStorageAppend\x10\x13\x12\x18\n" + + "\x14MsgStorageAppendResp\x10\x14\x12\x13\n" + + "\x0fMsgStorageApply\x10\x15\x12\x17\n" + + "\x13MsgStorageApplyResp\x10\x16\x12\x13\n" + + "\x0fMsgForgetLeader\x10\x17*\x82\x01\n" + + "\x14ConfChangeTransition\x12\x1c\n" + + "\x18ConfChangeTransitionAuto\x10\x00\x12%\n" + + "!ConfChangeTransitionJointImplicit\x10\x01\x12%\n" + + "!ConfChangeTransitionJointExplicit\x10\x02*y\n" + + "\x0eConfChangeType\x12\x15\n" + + "\x11ConfChangeAddNode\x10\x00\x12\x18\n" + + "\x14ConfChangeRemoveNode\x10\x01\x12\x18\n" + + "\x14ConfChangeUpdateNode\x10\x02\x12\x1c\n" + + "\x18ConfChangeAddLearnerNode\x10\x03B\x1bZ\x19go.etcd.io/raft/v3/raftpb" + +var ( + file_raft_proto_rawDescOnce sync.Once + file_raft_proto_rawDescData []byte +) + +func file_raft_proto_rawDescGZIP() []byte { + file_raft_proto_rawDescOnce.Do(func() { + file_raft_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_raft_proto_rawDesc), len(file_raft_proto_rawDesc))) + }) + return file_raft_proto_rawDescData +} + +var file_raft_proto_enumTypes = make([]protoimpl.EnumInfo, 4) +var file_raft_proto_msgTypes = make([]protoimpl.MessageInfo, 9) +var file_raft_proto_goTypes = []any{ + (EntryType)(0), // 0: raftpb.EntryType + (MessageType)(0), // 1: raftpb.MessageType + (ConfChangeTransition)(0), // 2: raftpb.ConfChangeTransition + (ConfChangeType)(0), // 3: raftpb.ConfChangeType + (*Entry)(nil), // 4: raftpb.Entry + (*SnapshotMetadata)(nil), // 5: raftpb.SnapshotMetadata + (*Snapshot)(nil), // 6: raftpb.Snapshot + (*Message)(nil), // 7: raftpb.Message + (*HardState)(nil), // 8: raftpb.HardState + (*ConfState)(nil), // 9: raftpb.ConfState + (*ConfChange)(nil), // 10: raftpb.ConfChange + (*ConfChangeSingle)(nil), // 11: raftpb.ConfChangeSingle + (*ConfChangeV2)(nil), // 12: raftpb.ConfChangeV2 +} +var file_raft_proto_depIdxs = []int32{ + 0, // 0: raftpb.Entry.Type:type_name -> raftpb.EntryType + 9, // 1: raftpb.SnapshotMetadata.conf_state:type_name -> raftpb.ConfState + 5, // 2: raftpb.Snapshot.metadata:type_name -> raftpb.SnapshotMetadata + 1, // 3: raftpb.Message.type:type_name -> raftpb.MessageType + 4, // 4: raftpb.Message.entries:type_name -> raftpb.Entry + 6, // 5: raftpb.Message.snapshot:type_name -> raftpb.Snapshot + 7, // 6: raftpb.Message.responses:type_name -> raftpb.Message + 3, // 7: raftpb.ConfChange.type:type_name -> raftpb.ConfChangeType + 3, // 8: raftpb.ConfChangeSingle.type:type_name -> raftpb.ConfChangeType + 2, // 9: raftpb.ConfChangeV2.transition:type_name -> raftpb.ConfChangeTransition + 11, // 10: raftpb.ConfChangeV2.changes:type_name -> raftpb.ConfChangeSingle + 11, // [11:11] is the sub-list for method output_type + 11, // [11:11] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name +} + +func init() { file_raft_proto_init() } +func file_raft_proto_init() { + if File_raft_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_raft_proto_rawDesc), len(file_raft_proto_rawDesc)), + NumEnums: 4, + NumMessages: 9, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_raft_proto_goTypes, + DependencyIndexes: file_raft_proto_depIdxs, + EnumInfos: file_raft_proto_enumTypes, + MessageInfos: file_raft_proto_msgTypes, + }.Build() + File_raft_proto = out.File + file_raft_proto_goTypes = nil + file_raft_proto_depIdxs = nil +} diff --git a/third_party/raft/raftpb/raft.proto b/third_party/raft/raftpb/raft.proto new file mode 100644 index 0000000000..31eead0458 --- /dev/null +++ b/third_party/raft/raftpb/raft.proto @@ -0,0 +1,205 @@ +syntax = "proto2"; +package raftpb; + +option go_package="go.etcd.io/raft/v3/raftpb"; + +enum EntryType { + EntryNormal = 0; + EntryConfChange = 1; // corresponds to pb.ConfChange + EntryConfChangeV2 = 2; // corresponds to pb.ConfChangeV2 +} + +message Entry { + optional uint64 Term = 2; + optional uint64 Index = 3; + optional EntryType Type = 1; + optional bytes Data = 4; +} + +message SnapshotMetadata { + optional ConfState conf_state = 1; + optional uint64 index = 2; + optional uint64 term = 3; +} + +message Snapshot { + optional bytes data = 1; + optional SnapshotMetadata metadata = 2; +} + +// For description of different message types, see: +// https://pkg.go.dev/go.etcd.io/raft/v3#hdr-MessageType +enum MessageType { + MsgHup = 0; + MsgBeat = 1; + MsgProp = 2; + MsgApp = 3; + MsgAppResp = 4; + MsgVote = 5; + MsgVoteResp = 6; + MsgSnap = 7; + MsgHeartbeat = 8; + MsgHeartbeatResp = 9; + MsgUnreachable = 10; + MsgSnapStatus = 11; + MsgCheckQuorum = 12; + MsgTransferLeader = 13; + MsgTimeoutNow = 14; + MsgReadIndex = 15; + MsgReadIndexResp = 16; + MsgPreVote = 17; + MsgPreVoteResp = 18; + MsgStorageAppend = 19; + MsgStorageAppendResp = 20; + MsgStorageApply = 21; + MsgStorageApplyResp = 22; + MsgForgetLeader = 23; + // NOTE: when adding new message types, remember to update the isLocalMsg and + // isResponseMsg arrays in raft/util.go and update the corresponding tests in + // raft/util_test.go. +} + +message Message { + optional MessageType type = 1; + optional uint64 to = 2; + optional uint64 from = 3; + optional uint64 term = 4; + // logTerm is generally used for appending Raft logs to followers. For example, + // (type=MsgApp,index=100,logTerm=5) means the leader appends entries starting + // at index=101, and the term of the entry at index 100 is 5. + // (type=MsgAppResp,reject=true,index=100,logTerm=5) means follower rejects some + // entries from its leader as it already has an entry with term 5 at index 100. + // (type=MsgStorageAppendResp,index=100,logTerm=5) means the local node wrote + // entries up to index=100 in stable storage, and the term of the entry at index + // 100 was 5. This doesn't always mean that the corresponding MsgStorageAppend + // message was the one that carried these entries, just that those entries were + // stable at the time of processing the corresponding MsgStorageAppend. + optional uint64 logTerm = 5; + optional uint64 index = 6; + repeated Entry entries = 7; + optional uint64 commit = 8; + // (type=MsgStorageAppend,vote=5,term=10) means the local node is voting for + // peer 5 in term 10. For MsgStorageAppends, the term, vote, and commit fields + // will either all be set (to facilitate the construction of a HardState) if + // any of the fields have changed or will all be unset if none of the fields + // have changed. + optional uint64 vote = 13; + // snapshot is non-nil and non-empty for MsgSnap messages and nil for all other + // message types. However, peer nodes running older binary versions may send a + // non-nil, empty value for the snapshot field of non-MsgSnap messages. Code + // should be prepared to handle such messages. + optional Snapshot snapshot = 9; + optional bool reject = 10; + optional uint64 rejectHint = 11; + optional bytes context = 12; + // responses are populated by a raft node to instruct storage threads on how + // to respond and who to respond to when the work associated with a message + // is complete. Populated for MsgStorageAppend and MsgStorageApply messages. + repeated Message responses = 14; +} + +message HardState { + optional uint64 term = 1; + optional uint64 vote = 2; + optional uint64 commit = 3; +} + +// ConfChangeTransition specifies the behavior of a configuration change with +// respect to joint consensus. +enum ConfChangeTransition { + // Automatically use the simple protocol if possible, otherwise fall back + // to ConfChangeJointImplicit. Most applications will want to use this. + ConfChangeTransitionAuto = 0; + // Use joint consensus unconditionally, and transition out of them + // automatically (by proposing a zero configuration change). + // + // This option is suitable for applications that want to minimize the time + // spent in the joint configuration and do not store the joint configuration + // in the state machine (outside of InitialState). + ConfChangeTransitionJointImplicit = 1; + // Use joint consensus and remain in the joint configuration until the + // application proposes a no-op configuration change. This is suitable for + // applications that want to explicitly control the transitions, for example + // to use a custom payload (via the Context field). + ConfChangeTransitionJointExplicit = 2; +} + +message ConfState { + // The voters in the incoming config. (If the configuration is not joint, + // then the outgoing config is empty). + repeated uint64 voters = 1; + // The learners in the incoming config. + repeated uint64 learners = 2; + // The voters in the outgoing config. + repeated uint64 voters_outgoing = 3; + // The nodes that will become learners when the outgoing config is removed. + // These nodes are necessarily currently in nodes_joint (or they would have + // been added to the incoming config right away). + repeated uint64 learners_next = 4; + // If set, the config is joint and Raft will automatically transition into + // the final config (i.e. remove the outgoing config) when this is safe. + optional bool auto_leave = 5; +} + +enum ConfChangeType { + ConfChangeAddNode = 0; + ConfChangeRemoveNode = 1; + ConfChangeUpdateNode = 2; + ConfChangeAddLearnerNode = 3; +} + +message ConfChange { + optional ConfChangeType type = 2; + optional uint64 node_id = 3; + optional bytes context = 4; + + // NB: this is used only by etcd to thread through a unique identifier. + // Ideally it should really use the Context instead. No counterpart to + // this field exists in ConfChangeV2. + optional uint64 id = 1; +} + +// ConfChangeSingle is an individual configuration change operation. Multiple +// such operations can be carried out atomically via a ConfChangeV2. +message ConfChangeSingle { + optional ConfChangeType type = 1; + optional uint64 node_id = 2; +} + +// ConfChangeV2 messages initiate configuration changes. They support both the +// simple "one at a time" membership change protocol and full Joint Consensus +// allowing for arbitrary changes in membership. +// +// The supplied context is treated as an opaque payload and can be used to +// attach an action on the state machine to the application of the config change +// proposal. Note that contrary to Joint Consensus as outlined in the Raft +// paper[1], configuration changes become active when they are *applied* to the +// state machine (not when they are appended to the log). +// +// The simple protocol can be used whenever only a single change is made. +// +// Non-simple changes require the use of Joint Consensus, for which two +// configuration changes are run. The first configuration change specifies the +// desired changes and transitions the Raft group into the joint configuration, +// in which quorum requires a majority of both the pre-changes and post-changes +// configuration. Joint Consensus avoids entering fragile intermediate +// configurations that could compromise survivability. For example, without the +// use of Joint Consensus and running across three availability zones with a +// replication factor of three, it is not possible to replace a voter without +// entering an intermediate configuration that does not survive the outage of +// one availability zone. +// +// The provided ConfChangeTransition specifies how (and whether) Joint Consensus +// is used, and assigns the task of leaving the joint configuration either to +// Raft or the application. Leaving the joint configuration is accomplished by +// proposing a ConfChangeV2 with only and optionally the Context field +// populated. +// +// For details on Raft membership changes, see: +// +// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf +message ConfChangeV2 { + optional ConfChangeTransition transition = 1; + repeated ConfChangeSingle changes = 2; + optional bytes context = 3; +} diff --git a/third_party/raft/raftpb/raft_test.go b/third_party/raft/raftpb/raft_test.go new file mode 100644 index 0000000000..06ff338745 --- /dev/null +++ b/third_party/raft/raftpb/raft_test.go @@ -0,0 +1,60 @@ +// Copyright 2021 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raftpb + +import ( + "math/bits" + "testing" + "unsafe" + + "github.com/stretchr/testify/assert" +) + +func TestProtoMemorySizes(t *testing.T) { + if64Bit := func(yes, no uintptr) uintptr { + if bits.UintSize == 64 { + return yes + } + return no + } + + // TODO: use proto.Size to get Entry's size, or remove this test case + // var e Entry + // assert.Equal(t, if64Bit(48, 32), unsafe.Sizeof(e), "Entry size check") + + var sm SnapshotMetadata + assert.Equal(t, if64Bit(64, 32), unsafe.Sizeof(sm), "SnapshotMetadata size check") + + var s Snapshot + assert.Equal(t, if64Bit(72, 36), unsafe.Sizeof(s), "Snapshot size check") + + var m Message + assert.Equal(t, if64Bit(200, 100), unsafe.Sizeof(m), "Message size check") + + var hs HardState + assert.Equal(t, if64Bit(64, 32), unsafe.Sizeof(hs), "HardState size check") + + var cs ConfState + assert.Equal(t, if64Bit(144, 72), unsafe.Sizeof(cs), "ConfState size check") + + var cc ConfChange + assert.Equal(t, if64Bit(88, 44), unsafe.Sizeof(cc), "ConfChange size check") + + var ccs ConfChangeSingle + assert.Equal(t, if64Bit(56, 28), unsafe.Sizeof(ccs), "ConfChangeSingle size check") + + var ccv2 ConfChangeV2 + assert.Equal(t, if64Bit(96, 48), unsafe.Sizeof(ccv2), "ConfChangeV2 size check") +} diff --git a/third_party/raft/raftpb/util.go b/third_party/raft/raftpb/util.go new file mode 100644 index 0000000000..6b1abcebb8 --- /dev/null +++ b/third_party/raft/raftpb/util.go @@ -0,0 +1,67 @@ +// Copyright 2026 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raftpb + +// EnsureConfState ensures that cs and all of its pointer fields are non-nil. +// If cs is nil, a new ConfState is allocated. Any nil pointer field is set to +// point to its zero value. Returns the resulting cs. +func EnsureConfState(cs *ConfState) *ConfState { + if cs == nil { + cs = new(ConfState) + } + if cs.AutoLeave == nil { + cs.AutoLeave = new(false) + } + return cs +} + +// EnsureSnapshotMetadata ensures that m and all of its pointer fields are +// non-nil. If m is nil, a new SnapshotMetadata is allocated. Any nil pointer +// field is set to point to its zero value. Returns the resulting m. +func EnsureSnapshotMetadata(m *SnapshotMetadata) *SnapshotMetadata { + if m == nil { + m = new(SnapshotMetadata) + } + newConfState := EnsureConfState(m.ConfState) + if m.ConfState == nil { + // ConfState in a SnapshotMetadata is guaranteed to be non-nil, so in most cases + // we don't need to update m.ConfState, which also avoids potential races. + m.ConfState = newConfState + } + + if m.Index == nil { + m.Index = new(uint64) + } + if m.Term == nil { + m.Term = new(uint64) + } + return m +} + +// EnsureSnapshot ensures that s and all of its pointer fields are non-nil. +// If s is nil, a new Snapshot is allocated. Any nil pointer field is set to +// point to its zero value. Returns the resulting s. +func EnsureSnapshot(s *Snapshot) *Snapshot { + if s == nil { + s = new(Snapshot) + } + newMedata := EnsureSnapshotMetadata(s.Metadata) + if s.Metadata == nil { + // Metadata in a snapshot is guaranteed to be non-nil, so in most cases + // we don't need to update s.Metadata, which also avoids potential races. + s.Metadata = newMedata + } + return s +} diff --git a/third_party/raft/rafttest/doc.go b/third_party/raft/rafttest/doc.go new file mode 100644 index 0000000000..bba9a1a386 --- /dev/null +++ b/third_party/raft/rafttest/doc.go @@ -0,0 +1,16 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package rafttest provides functional tests for etcd's raft implementation. +package rafttest diff --git a/third_party/raft/rafttest/interaction_env.go b/third_party/raft/rafttest/interaction_env.go new file mode 100644 index 0000000000..28c0695472 --- /dev/null +++ b/third_party/raft/rafttest/interaction_env.go @@ -0,0 +1,107 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "bufio" + "fmt" + "math" + "strings" + + "go.etcd.io/raft/v3" + pb "go.etcd.io/raft/v3/raftpb" +) + +// InteractionOpts groups the options for an InteractionEnv. +type InteractionOpts struct { + OnConfig func(*raft.Config) + + // SetRandomizedElectionTimeout is used to plumb this function down from the + // raft test package. + SetRandomizedElectionTimeout func(node *raft.RawNode, timeout int) +} + +// Node is a member of a raft group tested via an InteractionEnv. +type Node struct { + *raft.RawNode + Storage + + Config *raft.Config + AppendWork []*pb.Message // []MsgStorageAppend + ApplyWork []*pb.Message // []MsgStorageApply + History []*pb.Snapshot +} + +// InteractionEnv facilitates testing of complex interactions between the +// members of a raft group. +type InteractionEnv struct { + Options *InteractionOpts + Nodes []Node + Messages []*pb.Message // in-flight messages + + Output *RedirectLogger +} + +// NewInteractionEnv initializes an InteractionEnv. opts may be nil. +func NewInteractionEnv(opts *InteractionOpts) *InteractionEnv { + if opts == nil { + opts = &InteractionOpts{} + } + return &InteractionEnv{ + Options: opts, + Output: &RedirectLogger{ + Builder: &strings.Builder{}, + }, + } +} + +func (env *InteractionEnv) withIndent(f func()) { + orig := env.Output.Builder + env.Output.Builder = &strings.Builder{} + f() + + scanner := bufio.NewScanner(strings.NewReader(env.Output.Builder.String())) + for scanner.Scan() { + orig.WriteString(" " + scanner.Text() + "\n") + } + env.Output.Builder = orig +} + +// Storage is the interface used by InteractionEnv. It is comprised of raft's +// Storage interface plus access to operations that maintain the log and drive +// the Ready handling loop. +type Storage interface { + raft.Storage + SetHardState(state *pb.HardState) error + ApplySnapshot(*pb.Snapshot) error + Compact(newFirstIndex uint64) error + Append([]*pb.Entry) error +} + +// raftConfigStub sets up a raft.Config stub with reasonable testing defaults. +// In particular, no limits are set. It is not a complete config: ID and Storage +// must be set for each node using the stub as a template. +func raftConfigStub() raft.Config { + return raft.Config{ + ElectionTick: 3, + HeartbeatTick: 1, + MaxSizePerMsg: math.MaxUint64, + MaxInflightMsgs: math.MaxInt32, + } +} + +func defaultEntryFormatter(b []byte) string { + return fmt.Sprintf("%q", b) +} diff --git a/third_party/raft/rafttest/interaction_env_handler.go b/third_party/raft/rafttest/interaction_env_handler.go new file mode 100644 index 0000000000..2ffec2b400 --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler.go @@ -0,0 +1,241 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "fmt" + "strconv" + "testing" + + "github.com/cockroachdb/datadriven" +) + +// Handle is the entrypoint for data-driven interaction testing. Commands and +// parameters are parsed from the supplied TestData. Errors during data parsing +// are reported via the supplied *testing.T; errors from the raft nodes and the +// storage engine are reported to the output buffer. +func (env *InteractionEnv) Handle(t *testing.T, d datadriven.TestData) string { + env.Output.Reset() + var err error + switch d.Cmd { + case "_breakpoint": + // This is a helper case to attach a debugger to when a problem needs + // to be investigated in a longer test file. In such a case, add the + // following stanza immediately before the interesting behavior starts: + // + // _breakpoint: + // ---- + // ok + // + // and set a breakpoint on the `case` above. + case "add-nodes": + // Example: + // + // add-nodes voters=(1 2 3) learners=(4 5) index=2 content=foo async-storage-writes=true + err = env.handleAddNodes(t, d) + case "campaign": + // Example: + // + // campaign + err = env.handleCampaign(t, d) + case "compact": + // Example: + // + // compact + err = env.handleCompact(t, d) + case "deliver-msgs": + // Deliver the messages for a given recipient. + // + // Example: + // + // deliver-msgs type=MsgApp drop=(2,3) + err = env.handleDeliverMsgs(t, d) + case "process-ready": + // Example: + // + // process-ready 3 + err = env.handleProcessReady(t, d) + case "process-append-thread": + // Example: + // + // process-append-thread 3 + err = env.handleProcessAppendThread(t, d) + case "process-apply-thread": + // Example: + // + // process-apply-thread 3 + err = env.handleProcessApplyThread(t, d) + case "log-level": + // Set the log level. NONE disables all output, including from the test + // harness (except errors). + // + // Example: + // + // log-level WARN + err = env.handleLogLevel(d) + case "raft-log": + // Print the Raft log. + // + // Example: + // + // raft-log 3 + err = env.handleRaftLog(t, d) + case "raft-state": + // Print Raft state of all nodes (whether the node is leading, + // following, etc.). The information for node n is based on + // n's view. + err = env.handleRaftState() + case "set-randomized-election-timeout": + // Set the randomized election timeout for the given node. Will be reset + // again when the node changes state. + // + // Example: + // + // set-randomized-election-timeout 1 timeout=5 + err = env.handleSetRandomizedElectionTimeout(t, d) + case "stabilize": + // Deliver messages to and run process-ready on the set of IDs until + // no more work is to be done. If no ids are given, all nodes are used. + // + // Example: + // + // stabilize 1 4 + err = env.handleStabilize(t, d) + case "status": + // Print Raft status. + // + // Example: + // + // status 5 + err = env.handleStatus(t, d) + case "tick-election": + // Tick an election timeout interval for the given node (but beware the + // randomized timeout). + // + // Example: + // + // tick-election 3 + err = env.handleTickElection(t, d) + case "tick-heartbeat": + // Tick a heartbeat interval. + // + // Example: + // + // tick-heartbeat 3 + err = env.handleTickHeartbeat(t, d) + case "transfer-leadership": + // Transfer the Raft leader. + // + // Example: + // + // transfer-leadership from=1 to=4 + err = env.handleTransferLeadership(t, d) + case "forget-leader": + // Forgets the current leader of the given node. + // + // Example: + // + // forget-leader 1 + err = env.handleForgetLeader(t, d) + case "send-snapshot": + // Sends a snapshot to a node. Takes the source and destination node. + // The message will be queued, but not delivered automatically. + // + // Example: send-snapshot 1 3 + env.handleSendSnapshot(t, d) + case "propose": + // Propose an entry. + // + // Example: + // + // propose 1 foo + err = env.handlePropose(t, d) + case "propose-conf-change": + // Propose a configuration change, or transition out of a previously + // proposed joint configuration change that requested explicit + // transitions. When adding nodes, this command can be used to + // logically add nodes to the configuration, but add-nodes is needed + // to "create" the nodes. + // + // propose-conf-change node_id [v1=] [transition=] + // command string + // See ConfChangesFromString for command string format. + // Arguments are: + // node_id - the node proposing the configuration change. + // v1 - make one change at a time, false by default. + // transition - "auto" (the default), "explicit" or "implicit". + // Example: + // + // propose-conf-change 1 transition=explicit + // v1 v3 l4 r5 + // + // Example: + // + // propose-conf-change 2 v1=true + // v5 + err = env.handleProposeConfChange(t, d) + case "report-unreachable": + // Calls <1st>.ReportUnreachable(<2nd>). + // + // Example: + // report-unreachable 1 2 + err = env.handleReportUnreachable(t, d) + default: + err = fmt.Errorf("unknown command") + } + // NB: the highest log level suppresses all output, including that of the + // handlers. This comes in useful during setup which can be chatty. + // However, errors are always logged. + if err != nil { + if env.Output.Quiet() { + return err.Error() + } + env.Output.WriteString(err.Error()) + } + if env.Output.Len() == 0 { + return "ok" + } + return env.Output.String() +} + +func firstAsInt(t *testing.T, d datadriven.TestData) int { + t.Helper() + n, err := strconv.Atoi(d.CmdArgs[0].Key) + if err != nil { + t.Fatal(err) + } + return n +} + +func firstAsNodeIdx(t *testing.T, d datadriven.TestData) int { + t.Helper() + n := firstAsInt(t, d) + return n - 1 +} + +func nodeIdxs(t *testing.T, d datadriven.TestData) []int { + var ints []int + for i := 0; i < len(d.CmdArgs); i++ { + if len(d.CmdArgs[i].Vals) != 0 { + continue + } + n, err := strconv.Atoi(d.CmdArgs[i].Key) + if err != nil { + t.Fatal(err) + } + ints = append(ints, n-1) + } + return ints +} diff --git a/third_party/raft/rafttest/interaction_env_handler_add_nodes.go b/third_party/raft/rafttest/interaction_env_handler_add_nodes.go new file mode 100644 index 0000000000..70fb2ed5ed --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_add_nodes.go @@ -0,0 +1,167 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "errors" + "fmt" + "testing" + + "github.com/cockroachdb/datadriven" + "google.golang.org/protobuf/proto" + + "go.etcd.io/raft/v3" + pb "go.etcd.io/raft/v3/raftpb" +) + +func (env *InteractionEnv) handleAddNodes(t *testing.T, d datadriven.TestData) error { + n := firstAsInt(t, d) + snap := pb.EnsureSnapshot(nil) + cfg := raftConfigStub() + for _, arg := range d.CmdArgs[1:] { + for i := range arg.Vals { + switch arg.Key { + case "voters": + var id uint64 + arg.Scan(t, i, &id) + snap.Metadata.ConfState.Voters = append(snap.Metadata.ConfState.Voters, id) + case "learners": + var id uint64 + arg.Scan(t, i, &id) + snap.Metadata.ConfState.Learners = append(snap.Metadata.ConfState.Learners, id) + case "inflight": + arg.Scan(t, i, &cfg.MaxInflightMsgs) + case "index": + var idx uint64 + arg.Scan(t, i, &idx) + snap.Metadata.Index = new(idx) + cfg.Applied = snap.GetMetadata().GetIndex() + case "content": + arg.Scan(t, i, &snap.Data) + case "async-storage-writes": + arg.Scan(t, i, &cfg.AsyncStorageWrites) + case "prevote": + arg.Scan(t, i, &cfg.PreVote) + case "checkquorum": + arg.Scan(t, i, &cfg.CheckQuorum) + case "max-committed-size-per-ready": + arg.Scan(t, i, &cfg.MaxCommittedSizePerReady) + case "disable-conf-change-validation": + arg.Scan(t, i, &cfg.DisableConfChangeValidation) + case "read-only": + switch arg.Vals[i] { + case "safe": + cfg.ReadOnlyOption = raft.ReadOnlySafe + case "lease-based": + cfg.ReadOnlyOption = raft.ReadOnlyLeaseBased + default: + return fmt.Errorf("invalid read-only option %q", arg.Vals[i]) + } + case "step-down-on-removal": + arg.Scan(t, i, &cfg.StepDownOnRemoval) + } + } + } + return env.AddNodes(n, cfg, snap) +} + +type snapOverrideStorage struct { + Storage + snapshotOverride func() (*pb.Snapshot, error) +} + +func (s snapOverrideStorage) Snapshot() (*pb.Snapshot, error) { + if s.snapshotOverride != nil { + return s.snapshotOverride() + } + return s.Storage.Snapshot() +} + +var _ raft.Storage = snapOverrideStorage{} + +// AddNodes adds n new nodes initialized from the given snapshot (which may be +// empty), and using the cfg as template. They will be assigned consecutive IDs. +func (env *InteractionEnv) AddNodes(n int, cfg raft.Config, snap *pb.Snapshot) error { + emptySnapshot := pb.EnsureSnapshot(nil) + snap = pb.EnsureSnapshot(snap) + bootstrap := !proto.Equal(snap, emptySnapshot) + for i := 0; i < n; i++ { + id := uint64(1 + len(env.Nodes)) + s := snapOverrideStorage{ + Storage: raft.NewMemoryStorage(), + // When you ask for a snapshot, you get the most recent snapshot. + // + // TODO(tbg): this is sort of clunky, but MemoryStorage itself will + // give you some fixed snapshot and also the snapshot changes + // whenever you compact the logs and vice versa, so it's all a bit + // awkward to use. + snapshotOverride: func() (*pb.Snapshot, error) { + snaps := env.Nodes[int(id-1)].History + return snaps[len(snaps)-1], nil + }, + } + if bootstrap { + // NB: we could make this work with 1, but MemoryStorage just + // doesn't play well with that and it's not a loss of generality. + if snap.GetMetadata().GetIndex() <= 1 { + return errors.New("index must be specified as > 1 due to bootstrap") + } + snap.Metadata.Term = new(uint64(1)) + if err := s.ApplySnapshot(snap); err != nil { + return err + } + fi, err := s.FirstIndex() + if err != nil { + return err + } + // At the time of writing and for *MemoryStorage, applying a + // snapshot also truncates appropriately, but this would change with + // other storage engines potentially. + if exp := snap.GetMetadata().GetIndex() + 1; fi != exp { + return fmt.Errorf("failed to establish first index %d; got %d", exp, fi) + } + } + cfg := cfg // fork the config stub + cfg.ID, cfg.Storage = id, s + if env.Options.OnConfig != nil { + env.Options.OnConfig(&cfg) + if cfg.ID != id { + // This could be supported but then we need to do more work + // translating back and forth -- not worth it. + return errors.New("OnConfig must not change the ID") + } + } + if cfg.Logger != nil { + return errors.New("OnConfig must not set Logger") + } + cfg.Logger = env.Output + + rn, err := raft.NewRawNode(&cfg) + if err != nil { + return err + } + + node := Node{ + RawNode: rn, + // TODO(tbg): allow a more general Storage, as long as it also allows + // us to apply snapshots, append entries, and update the HardState. + Storage: s, + Config: &cfg, + History: []*pb.Snapshot{snap}, + } + env.Nodes = append(env.Nodes, node) + } + return nil +} diff --git a/third_party/raft/rafttest/interaction_env_handler_campaign.go b/third_party/raft/rafttest/interaction_env_handler_campaign.go new file mode 100644 index 0000000000..8b6ccbc1f4 --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_campaign.go @@ -0,0 +1,31 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "testing" + + "github.com/cockroachdb/datadriven" +) + +func (env *InteractionEnv) handleCampaign(t *testing.T, d datadriven.TestData) error { + idx := firstAsNodeIdx(t, d) + return env.Campaign(idx) +} + +// Campaign the node at the given index. +func (env *InteractionEnv) Campaign(idx int) error { + return env.Nodes[idx].Campaign() +} diff --git a/third_party/raft/rafttest/interaction_env_handler_compact.go b/third_party/raft/rafttest/interaction_env_handler_compact.go new file mode 100644 index 0000000000..25fa1d22c9 --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_compact.go @@ -0,0 +1,40 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "strconv" + "testing" + + "github.com/cockroachdb/datadriven" +) + +func (env *InteractionEnv) handleCompact(t *testing.T, d datadriven.TestData) error { + idx := firstAsNodeIdx(t, d) + newFirstIndex, err := strconv.ParseUint(d.CmdArgs[1].Key, 10, 64) + if err != nil { + return err + } + return env.Compact(idx, newFirstIndex) +} + +// Compact truncates the log on the node at index idx so that the supplied new +// first index results. +func (env *InteractionEnv) Compact(idx int, newFirstIndex uint64) error { + if err := env.Nodes[idx].Compact(newFirstIndex); err != nil { + return err + } + return env.RaftLog(idx) +} diff --git a/third_party/raft/rafttest/interaction_env_handler_deliver_msgs.go b/third_party/raft/rafttest/interaction_env_handler_deliver_msgs.go new file mode 100644 index 0000000000..f06dc2500b --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_deliver_msgs.go @@ -0,0 +1,104 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "fmt" + "strconv" + "testing" + + "github.com/cockroachdb/datadriven" + + "go.etcd.io/raft/v3" + "go.etcd.io/raft/v3/raftpb" +) + +func (env *InteractionEnv) handleDeliverMsgs(t *testing.T, d datadriven.TestData) error { + var typ raftpb.MessageType = -1 + var rs []Recipient + for _, arg := range d.CmdArgs { + if len(arg.Vals) == 0 { + id, err := strconv.ParseUint(arg.Key, 10, 64) + if err != nil { + t.Fatal(err) + } + rs = append(rs, Recipient{ID: id}) + } + for i := range arg.Vals { + switch arg.Key { + case "drop": + var id uint64 + arg.Scan(t, i, &id) + var found bool + for _, r := range rs { + if r.ID == id { + found = true + } + } + if found { + t.Fatalf("can't both deliver and drop msgs to %d", id) + } + rs = append(rs, Recipient{ID: id, Drop: true}) + case "type": + var s string + arg.Scan(t, i, &s) + v, ok := raftpb.MessageType_value[s] + if !ok { + t.Fatalf("unknown message type %s", s) + } + typ = raftpb.MessageType(v) + } + } + } + + if n := env.DeliverMsgs(typ, rs...); n == 0 { + env.Output.WriteString("no messages\n") + } + return nil +} + +type Recipient struct { + ID uint64 + Drop bool +} + +// DeliverMsgs goes through env.Messages and, depending on the Drop flag, +// delivers or drops messages to the specified Recipients. Only messages of type +// typ are delivered (-1 for all types). Returns the number of messages handled +// (i.e. delivered or dropped). A handled message is removed from env.Messages. +func (env *InteractionEnv) DeliverMsgs(typ raftpb.MessageType, rs ...Recipient) int { + var n int + for _, r := range rs { + var msgs []*raftpb.Message + msgs, env.Messages = splitMsgs(env.Messages, r.ID, typ, r.Drop) + n += len(msgs) + for _, msg := range msgs { + if r.Drop { + fmt.Fprint(env.Output, "dropped: ") + } + fmt.Fprintln(env.Output, raft.DescribeMessage(msg, defaultEntryFormatter)) + if r.Drop { + // NB: it's allowed to drop messages to nodes that haven't been instantiated yet, + // we haven't used msg.To yet. + continue + } + toIdx := int(msg.GetTo() - 1) + if err := env.Nodes[toIdx].Step(msg); err != nil { + fmt.Fprintln(env.Output, err) + } + } + } + return n +} diff --git a/third_party/raft/rafttest/interaction_env_handler_forget_leader.go b/third_party/raft/rafttest/interaction_env_handler_forget_leader.go new file mode 100644 index 0000000000..b239db75e1 --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_forget_leader.go @@ -0,0 +1,32 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "testing" + + "github.com/cockroachdb/datadriven" +) + +func (env *InteractionEnv) handleForgetLeader(t *testing.T, d datadriven.TestData) error { + idx := firstAsNodeIdx(t, d) + env.ForgetLeader(idx) + return nil +} + +// ForgetLeader makes the follower at the given index forget its leader. +func (env *InteractionEnv) ForgetLeader(idx int) { + env.Nodes[idx].ForgetLeader() +} diff --git a/third_party/raft/rafttest/interaction_env_handler_log_level.go b/third_party/raft/rafttest/interaction_env_handler_log_level.go new file mode 100644 index 0000000000..c517a4d77d --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_log_level.go @@ -0,0 +1,36 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "fmt" + "strings" + + "github.com/cockroachdb/datadriven" +) + +func (env *InteractionEnv) handleLogLevel(d datadriven.TestData) error { + return env.LogLevel(d.CmdArgs[0].Key) +} + +func (env *InteractionEnv) LogLevel(name string) error { + for i, s := range lvlNames { + if strings.EqualFold(s, name) { + env.Output.Lvl = i + return nil + } + } + return fmt.Errorf("log levels must be either of %v", lvlNames) +} diff --git a/third_party/raft/rafttest/interaction_env_handler_process_append_thread.go b/third_party/raft/rafttest/interaction_env_handler_process_append_thread.go new file mode 100644 index 0000000000..4588e4fecb --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_process_append_thread.go @@ -0,0 +1,100 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "errors" + "fmt" + "testing" + + "github.com/cockroachdb/datadriven" + "google.golang.org/protobuf/proto" + + "go.etcd.io/raft/v3" + "go.etcd.io/raft/v3/raftpb" +) + +func (env *InteractionEnv) handleProcessAppendThread(t *testing.T, d datadriven.TestData) error { + idxs := nodeIdxs(t, d) + for _, idx := range idxs { + var err error + if len(idxs) > 1 { + fmt.Fprintf(env.Output, "> %d processing append thread\n", idx+1) + env.withIndent(func() { err = env.ProcessAppendThread(idx) }) + } else { + err = env.ProcessAppendThread(idx) + } + if err != nil { + return err + } + } + return nil +} + +// ProcessAppendThread runs processes a single message on the "append" thread of +// the node with the given index. +func (env *InteractionEnv) ProcessAppendThread(idx int) error { + n := &env.Nodes[idx] + if len(n.AppendWork) == 0 { + env.Output.WriteString("no append work to perform") + return nil + } + m := n.AppendWork[0] + n.AppendWork = n.AppendWork[1:] + + resps := m.GetResponses() + m.Responses = nil + env.Output.WriteString("Processing:\n") + env.Output.WriteString(raft.DescribeMessage(m, defaultEntryFormatter) + "\n") + st := &raftpb.HardState{ + Term: new(m.GetTerm()), + Vote: new(m.GetVote()), + Commit: new(m.GetCommit()), + } + snap := m.GetSnapshot() + var cloned *raftpb.Snapshot + if snap != nil { + cloned = proto.Clone(snap).(*raftpb.Snapshot) + } + cloned = raftpb.EnsureSnapshot(cloned) + if err := processAppend(n, st, m.GetEntries(), cloned); err != nil { + return err + } + + env.Output.WriteString("Responses:\n") + for _, m := range resps { + env.Output.WriteString(raft.DescribeMessage(m, defaultEntryFormatter) + "\n") + } + env.Messages = append(env.Messages, resps...) + return nil +} + +func processAppend(n *Node, st *raftpb.HardState, ents []*raftpb.Entry, snap *raftpb.Snapshot) error { + // TODO(tbg): the order of operations here is not necessarily safe. See: + // https://github.com/etcd-io/etcd/pull/10861 + s := n.Storage + if !raft.IsEmptyHardState(st) { + if err := s.SetHardState(st); err != nil { + return err + } + } + if !raft.IsEmptySnap(snap) { + if len(ents) > 0 { + return errors.New("can't apply snapshot and entries at the same time") + } + return s.ApplySnapshot(snap) + } + return s.Append(ents) +} diff --git a/third_party/raft/rafttest/interaction_env_handler_process_apply_thread.go b/third_party/raft/rafttest/interaction_env_handler_process_apply_thread.go new file mode 100644 index 0000000000..d938594c09 --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_process_apply_thread.go @@ -0,0 +1,112 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "fmt" + "testing" + + "github.com/cockroachdb/datadriven" + "google.golang.org/protobuf/proto" + + "go.etcd.io/raft/v3" + "go.etcd.io/raft/v3/raftpb" +) + +func (env *InteractionEnv) handleProcessApplyThread(t *testing.T, d datadriven.TestData) error { + idxs := nodeIdxs(t, d) + for _, idx := range idxs { + var err error + if len(idxs) > 1 { + fmt.Fprintf(env.Output, "> %d processing apply thread\n", idx+1) + env.withIndent(func() { err = env.ProcessApplyThread(idx) }) + } else { + err = env.ProcessApplyThread(idx) + } + if err != nil { + return err + } + } + return nil +} + +// ProcessApplyThread runs processes a single message on the "apply" thread of +// the node with the given index. +func (env *InteractionEnv) ProcessApplyThread(idx int) error { + n := &env.Nodes[idx] + if len(n.ApplyWork) == 0 { + env.Output.WriteString("no apply work to perform") + return nil + } + m := n.ApplyWork[0] + n.ApplyWork = n.ApplyWork[1:] + + resps := m.GetResponses() + m.Responses = nil + env.Output.WriteString("Processing:\n") + env.Output.WriteString(raft.DescribeMessage(m, defaultEntryFormatter) + "\n") + if err := processApply(n, m.GetEntries()); err != nil { + return err + } + + env.Output.WriteString("Responses:\n") + for _, m := range resps { + env.Output.WriteString(raft.DescribeMessage(m, defaultEntryFormatter) + "\n") + } + env.Messages = append(env.Messages, resps...) + return nil +} + +func processApply(n *Node, ents []*raftpb.Entry) error { + for _, ent := range ents { + var update []byte + var cs *raftpb.ConfState + switch ent.GetType() { + case raftpb.EntryConfChange: + cc := &raftpb.ConfChange{} + if err := proto.Unmarshal(ent.GetData(), cc); err != nil { + return err + } + update = cc.Context + cs = n.RawNode.ApplyConfChange(cc) + case raftpb.EntryConfChangeV2: + cc := &raftpb.ConfChangeV2{} + if err := proto.Unmarshal(ent.GetData(), cc); err != nil { + return err + } + cs = n.RawNode.ApplyConfChange(cc) + update = cc.Context + default: + update = ent.GetData() + } + + // Record the new state by starting with the current state and applying + // the command. + lastSnap := n.History[len(n.History)-1] + snap := raftpb.EnsureSnapshot(nil) + snap.Data = append(snap.Data, lastSnap.Data...) + // NB: this hard-codes an "appender" state machine. + snap.Data = append(snap.Data, update...) + snap.Metadata.Index = new(ent.GetIndex()) + snap.Metadata.Term = new(ent.GetTerm()) + if cs == nil { + sl := n.History + cs = sl[len(sl)-1].GetMetadata().GetConfState() + } + snap.Metadata.ConfState = cs + n.History = append(n.History, snap) + } + return nil +} diff --git a/third_party/raft/rafttest/interaction_env_handler_process_ready.go b/third_party/raft/rafttest/interaction_env_handler_process_ready.go new file mode 100644 index 0000000000..6875b4ddc7 --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_process_ready.go @@ -0,0 +1,82 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "fmt" + "testing" + + "github.com/cockroachdb/datadriven" + + "go.etcd.io/raft/v3" + "go.etcd.io/raft/v3/raftpb" +) + +func (env *InteractionEnv) handleProcessReady(t *testing.T, d datadriven.TestData) error { + idxs := nodeIdxs(t, d) + for _, idx := range idxs { + var err error + if len(idxs) > 1 { + fmt.Fprintf(env.Output, "> %d handling Ready\n", idx+1) + env.withIndent(func() { err = env.ProcessReady(idx) }) + } else { + err = env.ProcessReady(idx) + } + if err != nil { + return err + } + } + return nil +} + +// ProcessReady runs Ready handling on the node with the given index. +func (env *InteractionEnv) ProcessReady(idx int) error { + // TODO(tbg): Allow simulating crashes here. + n := &env.Nodes[idx] + rd := n.Ready() + env.Output.WriteString(raft.DescribeReady(rd, defaultEntryFormatter)) + + if !n.Config.AsyncStorageWrites { + if err := processAppend(n, rd.HardState, rd.Entries, rd.Snapshot); err != nil { + return err + } + if err := processApply(n, rd.CommittedEntries); err != nil { + return err + } + } + + for _, m := range rd.Messages { + if raft.IsLocalMsgTarget(m.GetTo()) { + if !n.Config.AsyncStorageWrites { + panic("unexpected local msg target") + } + switch m.GetType() { + case raftpb.MsgStorageAppend: + n.AppendWork = append(n.AppendWork, m) + case raftpb.MsgStorageApply: + n.ApplyWork = append(n.ApplyWork, m) + default: + panic(fmt.Sprintf("unexpected message type %s", m.GetType())) + } + } else { + env.Messages = append(env.Messages, m) + } + } + + if !n.Config.AsyncStorageWrites { + n.Advance(rd) + } + return nil +} diff --git a/third_party/raft/rafttest/interaction_env_handler_propose.go b/third_party/raft/rafttest/interaction_env_handler_propose.go new file mode 100644 index 0000000000..7e8832340c --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_propose.go @@ -0,0 +1,34 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "testing" + + "github.com/cockroachdb/datadriven" +) + +func (env *InteractionEnv) handlePropose(t *testing.T, d datadriven.TestData) error { + idx := firstAsNodeIdx(t, d) + if len(d.CmdArgs) != 2 || len(d.CmdArgs[1].Vals) > 0 { + t.Fatalf("expected exactly one key with no vals: %+v", d.CmdArgs[1:]) + } + return env.Propose(idx, []byte(d.CmdArgs[1].Key)) +} + +// Propose a regular entry. +func (env *InteractionEnv) Propose(idx int, data []byte) error { + return env.Nodes[idx].Propose(data) +} diff --git a/third_party/raft/rafttest/interaction_env_handler_propose_conf_change.go b/third_party/raft/rafttest/interaction_env_handler_propose_conf_change.go new file mode 100644 index 0000000000..2761a16885 --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_propose_conf_change.go @@ -0,0 +1,83 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "fmt" + "strconv" + "testing" + + "github.com/cockroachdb/datadriven" + + "go.etcd.io/raft/v3/raftpb" +) + +func (env *InteractionEnv) handleProposeConfChange(t *testing.T, d datadriven.TestData) error { + idx := firstAsNodeIdx(t, d) + var v1 bool + transition := raftpb.ConfChangeTransitionAuto + for _, arg := range d.CmdArgs[1:] { + for _, val := range arg.Vals { + switch arg.Key { + case "v1": + var err error + v1, err = strconv.ParseBool(val) + if err != nil { + return err + } + case "transition": + switch val { + case "auto": + transition = raftpb.ConfChangeTransitionAuto + case "implicit": + transition = raftpb.ConfChangeTransitionJointImplicit + case "explicit": + transition = raftpb.ConfChangeTransitionJointExplicit + default: + return fmt.Errorf("unknown transition %s", val) + } + default: + return fmt.Errorf("unknown command %s", arg.Key) + } + } + } + + ccs, err := raftpb.ConfChangesFromString(d.Input) + if err != nil { + return err + } + + var c raftpb.ConfChangeI + if v1 { + if len(ccs) > 1 || transition != raftpb.ConfChangeTransitionAuto { + return fmt.Errorf("v1 conf change can only have one operation and no transition") + } + c = &raftpb.ConfChange{ + Type: ccs[0].GetType().Enum(), + NodeId: new(ccs[0].GetNodeId()), + } + } else { + c = &raftpb.ConfChangeV2{ + Transition: transition.Enum(), + Changes: ccs, + } + } + return env.ProposeConfChange(idx, c) +} + +// ProposeConfChange proposes a configuration change on the node with the given index. +func (env *InteractionEnv) ProposeConfChange(idx int, c raftpb.ConfChangeI) error { + return env.Nodes[idx].ProposeConfChange(c) +} diff --git a/third_party/raft/rafttest/interaction_env_handler_raft_log.go b/third_party/raft/rafttest/interaction_env_handler_raft_log.go new file mode 100644 index 0000000000..d4bc1b1376 --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_raft_log.go @@ -0,0 +1,55 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "fmt" + "math" + "testing" + + "github.com/cockroachdb/datadriven" + + "go.etcd.io/raft/v3" +) + +func (env *InteractionEnv) handleRaftLog(t *testing.T, d datadriven.TestData) error { + idx := firstAsNodeIdx(t, d) + return env.RaftLog(idx) +} + +// RaftLog pretty prints the raft log to the output buffer. +func (env *InteractionEnv) RaftLog(idx int) error { + s := env.Nodes[idx].Storage + fi, err := s.FirstIndex() + if err != nil { + return err + } + li, err := s.LastIndex() + if err != nil { + return err + } + if li < fi { + // TODO(tbg): this is what MemoryStorage returns, but unclear if it's + // the "correct" thing to do. + fmt.Fprintf(env.Output, "log is empty: first index=%d, last index=%d", fi, li) + return nil + } + ents, err := s.Entries(fi, li+1, math.MaxUint64) + if err != nil { + return err + } + env.Output.WriteString(raft.DescribeEntries(ents, defaultEntryFormatter)) + return err +} diff --git a/third_party/raft/rafttest/interaction_env_handler_raftstate.go b/third_party/raft/rafttest/interaction_env_handler_raftstate.go new file mode 100644 index 0000000000..8da52ec020 --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_raftstate.go @@ -0,0 +1,49 @@ +// Copyright 2021 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "fmt" + + "go.etcd.io/raft/v3" +) + +// isVoter checks whether node id is in the voter list within st. +func isVoter(id uint64, st raft.Status) bool { + idMap := st.Config.Voters.IDs() + for idx := range idMap { + if id == idx { + return true + } + } + return false +} + +// handleRaftState pretty-prints the raft state for all nodes to the output buffer. +// For each node, the information is based on its own configuration view. +func (env *InteractionEnv) handleRaftState() error { + for _, n := range env.Nodes { + st := n.Status() + var voterStatus string + if isVoter(st.ID, st) { + voterStatus = "(Voter)" + } else { + voterStatus = "(Non-Voter)" + } + fmt.Fprintf(env.Output, "%d: %s %s Term:%d Lead:%d\n", + st.ID, st.RaftState, voterStatus, st.GetTerm(), st.Lead) + } + return nil +} diff --git a/third_party/raft/rafttest/interaction_env_handler_report_unreachable.go b/third_party/raft/rafttest/interaction_env_handler_report_unreachable.go new file mode 100644 index 0000000000..dce46ba2b7 --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_report_unreachable.go @@ -0,0 +1,30 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package rafttest + +import ( + "errors" + "testing" + + "github.com/cockroachdb/datadriven" +) + +func (env *InteractionEnv) handleReportUnreachable(t *testing.T, d datadriven.TestData) error { + sl := nodeIdxs(t, d) + if len(sl) != 2 { + return errors.New("must specify exactly two node indexes: node on which to report, and reported node") + } + env.Nodes[sl[0]].ReportUnreachable(env.Nodes[sl[1]].Config.ID) + return nil +} diff --git a/third_party/raft/rafttest/interaction_env_handler_send_snapshot.go b/third_party/raft/rafttest/interaction_env_handler_send_snapshot.go new file mode 100644 index 0000000000..a69b89d56b --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_send_snapshot.go @@ -0,0 +1,51 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "testing" + + "github.com/cockroachdb/datadriven" + "github.com/stretchr/testify/require" + + "go.etcd.io/raft/v3" + "go.etcd.io/raft/v3/raftpb" +) + +func (env *InteractionEnv) handleSendSnapshot(t *testing.T, d datadriven.TestData) error { + idxs := nodeIdxs(t, d) + require.Len(t, idxs, 2) + return env.SendSnapshot(idxs[0], idxs[1]) +} + +// SendSnapshot sends a snapshot. +func (env *InteractionEnv) SendSnapshot(fromIdx, toIdx int) error { + snap, err := env.Nodes[fromIdx].Snapshot() + if err != nil { + return err + } + from, to := uint64(fromIdx+1), uint64(toIdx+1) + fromStatus := env.Nodes[fromIdx].BasicStatus() + msg := &raftpb.Message{ + Type: raftpb.MsgSnap.Enum(), + Term: new(fromStatus.GetTerm()), + From: new(from), + To: new(to), + Snapshot: snap, + } + env.Messages = append(env.Messages, msg) + _, _ = env.Output.WriteString(raft.DescribeMessage(msg, nil)) + return nil +} diff --git a/third_party/raft/rafttest/interaction_env_handler_set_randomized_election_timeout.go b/third_party/raft/rafttest/interaction_env_handler_set_randomized_election_timeout.go new file mode 100644 index 0000000000..36f0cd2d8d --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_set_randomized_election_timeout.go @@ -0,0 +1,34 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "testing" + + "github.com/cockroachdb/datadriven" + "github.com/stretchr/testify/require" +) + +func (env *InteractionEnv) handleSetRandomizedElectionTimeout( + t *testing.T, d datadriven.TestData, +) error { + idx := firstAsNodeIdx(t, d) + var timeout int + d.ScanArgs(t, "timeout", &timeout) + require.NotZero(t, timeout) + + env.Options.SetRandomizedElectionTimeout(env.Nodes[idx].RawNode, timeout) + return nil +} diff --git a/third_party/raft/rafttest/interaction_env_handler_stabilize.go b/third_party/raft/rafttest/interaction_env_handler_stabilize.go new file mode 100644 index 0000000000..6872378e86 --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_stabilize.go @@ -0,0 +1,132 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "fmt" + "testing" + + "github.com/cockroachdb/datadriven" + + "go.etcd.io/raft/v3" + "go.etcd.io/raft/v3/raftpb" +) + +func (env *InteractionEnv) handleStabilize(t *testing.T, d datadriven.TestData) error { + idxs := nodeIdxs(t, d) // skips key=value args + for _, arg := range d.CmdArgs { + for i := range arg.Vals { + switch arg.Key { + case "log-level": + defer func(old int) { + env.Output.Lvl = old + }(env.Output.Lvl) + var level string + arg.Scan(t, i, &level) + if err := env.LogLevel(level); err != nil { + return err + } + } + } + } + return env.Stabilize(idxs...) +} + +// Stabilize repeatedly runs Ready handling on and message delivery to the set +// of nodes specified via the idxs slice until reaching a fixed point. +func (env *InteractionEnv) Stabilize(idxs ...int) error { + var nodes []*Node + if len(idxs) != 0 { + for _, idx := range idxs { + nodes = append(nodes, &env.Nodes[idx]) + } + } else { + for i := range env.Nodes { + nodes = append(nodes, &env.Nodes[i]) + } + } + + for { + done := true + for _, rn := range nodes { + if rn.HasReady() { + idx := int(rn.Status().ID - 1) + fmt.Fprintf(env.Output, "> %d handling Ready\n", idx+1) + var err error + env.withIndent(func() { err = env.ProcessReady(idx) }) + if err != nil { + return err + } + done = false + } + } + for _, rn := range nodes { + id := rn.Status().ID + // NB: we grab the messages just to see whether to print the header. + // DeliverMsgs will do it again. + if msgs, _ := splitMsgs(env.Messages, id, -1 /* typ */, false /* drop */); len(msgs) > 0 { + fmt.Fprintf(env.Output, "> %d receiving messages\n", id) + env.withIndent(func() { env.DeliverMsgs(-1 /* typ */, Recipient{ID: id}) }) + done = false + } + } + for _, rn := range nodes { + idx := int(rn.Status().ID - 1) + if len(rn.AppendWork) > 0 { + fmt.Fprintf(env.Output, "> %d processing append thread\n", idx+1) + for len(rn.AppendWork) > 0 { + var err error + env.withIndent(func() { err = env.ProcessAppendThread(idx) }) + if err != nil { + return err + } + } + done = false + } + } + for _, rn := range nodes { + idx := int(rn.Status().ID - 1) + if len(rn.ApplyWork) > 0 { + fmt.Fprintf(env.Output, "> %d processing apply thread\n", idx+1) + for len(rn.ApplyWork) > 0 { + env.withIndent(func() { env.ProcessApplyThread(idx) }) + } + done = false + } + } + if done { + return nil + } + } +} + +// splitMsgs extracts messages for the given recipient of the given type (-1 for +// all types) from msgs, and returns them along with the remainder of msgs. +func splitMsgs(msgs []*raftpb.Message, to uint64, typ raftpb.MessageType, drop bool) (toMsgs []*raftpb.Message, rmdr []*raftpb.Message) { + // NB: this method does not reorder messages. + for _, msg := range msgs { + if msg.GetTo() == to && !(drop && isLocalMsg(msg)) && (typ < 0 || msg.GetType() == typ) { + toMsgs = append(toMsgs, msg) + } else { + rmdr = append(rmdr, msg) + } + } + return toMsgs, rmdr +} + +// Don't drop local messages, which require reliable delivery. +func isLocalMsg(msg *raftpb.Message) bool { + return msg.GetFrom() == msg.GetTo() || raft.IsLocalMsgTarget(msg.GetFrom()) || raft.IsLocalMsgTarget(msg.GetTo()) +} diff --git a/third_party/raft/rafttest/interaction_env_handler_status.go b/third_party/raft/rafttest/interaction_env_handler_status.go new file mode 100644 index 0000000000..40bb7d34af --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_status.go @@ -0,0 +1,43 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "fmt" + "testing" + + "github.com/cockroachdb/datadriven" + + "go.etcd.io/raft/v3/tracker" +) + +func (env *InteractionEnv) handleStatus(t *testing.T, d datadriven.TestData) error { + idx := firstAsNodeIdx(t, d) + return env.Status(idx) +} + +// Status pretty-prints the raft status for the node at the given index to the output +// buffer. +func (env *InteractionEnv) Status(idx int) error { + // TODO(tbg): actually print the full status. + st := env.Nodes[idx].Status() + m := tracker.ProgressMap{} + for id, pr := range st.Progress { + pr := pr // loop-local copy + m[id] = &pr + } + fmt.Fprint(env.Output, m) + return nil +} diff --git a/third_party/raft/rafttest/interaction_env_handler_tick.go b/third_party/raft/rafttest/interaction_env_handler_tick.go new file mode 100644 index 0000000000..10c9ba2130 --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_tick.go @@ -0,0 +1,39 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "testing" + + "github.com/cockroachdb/datadriven" +) + +func (env *InteractionEnv) handleTickElection(t *testing.T, d datadriven.TestData) error { + idx := firstAsNodeIdx(t, d) + return env.Tick(idx, env.Nodes[idx].Config.ElectionTick) +} + +func (env *InteractionEnv) handleTickHeartbeat(t *testing.T, d datadriven.TestData) error { + idx := firstAsNodeIdx(t, d) + return env.Tick(idx, env.Nodes[idx].Config.HeartbeatTick) +} + +// Tick the node at the given index the given number of times. +func (env *InteractionEnv) Tick(idx int, num int) error { + for i := 0; i < num; i++ { + env.Nodes[idx].Tick() + } + return nil +} diff --git a/third_party/raft/rafttest/interaction_env_handler_transfer_leadership.go b/third_party/raft/rafttest/interaction_env_handler_transfer_leadership.go new file mode 100644 index 0000000000..dc7f366d1f --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_handler_transfer_leadership.go @@ -0,0 +1,41 @@ +// Copyright 2021 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "testing" + + "github.com/cockroachdb/datadriven" +) + +func (env *InteractionEnv) handleTransferLeadership(t *testing.T, d datadriven.TestData) error { + var from, to uint64 + d.ScanArgs(t, "from", &from) + d.ScanArgs(t, "to", &to) + if from == 0 || from > uint64(len(env.Nodes)) { + t.Fatalf(`expected valid "from" argument`) + } + if to == 0 || to > uint64(len(env.Nodes)) { + t.Fatalf(`expected valid "to" argument`) + } + return env.transferLeadership(from, to) +} + +// Initiate leadership transfer. +func (env *InteractionEnv) transferLeadership(from, to uint64) error { + fromIdx := from - 1 + env.Nodes[fromIdx].TransferLeader(to) + return nil +} diff --git a/third_party/raft/rafttest/interaction_env_logger.go b/third_party/raft/rafttest/interaction_env_logger.go new file mode 100644 index 0000000000..0fcf94d659 --- /dev/null +++ b/third_party/raft/rafttest/interaction_env_logger.go @@ -0,0 +1,138 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "fmt" + "strings" + + "go.etcd.io/raft/v3" +) + +type logLevels [6]string + +var lvlNames logLevels = [...]string{"DEBUG", "INFO", "WARN", "ERROR", "FATAL", "NONE"} + +type RedirectLogger struct { + *strings.Builder + Lvl int // 0 = DEBUG, 1 = INFO, 2 = WARNING, 3 = ERROR, 4 = FATAL, 5 = NONE +} + +var _ raft.Logger = (*RedirectLogger)(nil) + +func (l *RedirectLogger) printf(lvl int, format string, args ...interface{}) { + if l.Lvl <= lvl { + fmt.Fprint(l, lvlNames[lvl], " ") + fmt.Fprintf(l, format, args...) + if n := len(format); n > 0 && format[n-1] != '\n' { + l.WriteByte('\n') + } + } +} +func (l *RedirectLogger) print(lvl int, args ...interface{}) { + if l.Lvl <= lvl { + fmt.Fprint(l, lvlNames[lvl], " ") + fmt.Fprintln(l, args...) + } +} + +func (l *RedirectLogger) Debug(v ...interface{}) { + l.print(0, v...) +} + +func (l *RedirectLogger) Debugf(format string, v ...interface{}) { + l.printf(0, format, v...) +} + +func (l *RedirectLogger) Info(v ...interface{}) { + l.print(1, v...) +} + +func (l *RedirectLogger) Infof(format string, v ...interface{}) { + l.printf(1, format, v...) +} + +func (l *RedirectLogger) Warning(v ...interface{}) { + l.print(2, v...) +} + +func (l *RedirectLogger) Warningf(format string, v ...interface{}) { + l.printf(2, format, v...) +} + +func (l *RedirectLogger) Error(v ...interface{}) { + l.print(3, v...) +} + +func (l *RedirectLogger) Errorf(format string, v ...interface{}) { + l.printf(3, format, v...) +} + +func (l *RedirectLogger) Fatal(v ...interface{}) { + l.print(4, v...) + panic(fmt.Sprint(v...)) +} + +func (l *RedirectLogger) Fatalf(format string, v ...interface{}) { + l.printf(4, format, v...) + panic(fmt.Sprintf(format, v...)) +} + +func (l *RedirectLogger) Panic(v ...interface{}) { + l.print(4, v...) + panic(fmt.Sprint(v...)) +} + +func (l *RedirectLogger) Panicf(format string, v ...interface{}) { + l.printf(4, format, v...) + // TODO(pavelkalinnikov): catch the panic gracefully in datadriven package. + // This would allow observing all the intermediate logging while debugging, + // and testing the cases when panic is expected. + panic(fmt.Sprintf(format, v...)) +} + +// Override StringBuilder write methods to silence them under NONE. + +func (l *RedirectLogger) Quiet() bool { + return l.Lvl == len(lvlNames)-1 +} + +func (l *RedirectLogger) Write(p []byte) (int, error) { + if l.Quiet() { + return 0, nil + } + return l.Builder.Write(p) +} + +func (l *RedirectLogger) WriteByte(c byte) error { + if l.Quiet() { + return nil + } + return l.Builder.WriteByte(c) +} + +func (l *RedirectLogger) WriteRune(r rune) (int, error) { + if l.Quiet() { + return 0, nil + } + return l.Builder.WriteRune(r) +} + +func (l *RedirectLogger) WriteString(s string) (int, error) { + if l.Quiet() { + return 0, nil + } + return l.Builder.WriteString(s) +} diff --git a/third_party/raft/rafttest/network.go b/third_party/raft/rafttest/network.go new file mode 100644 index 0000000000..1e4113eca0 --- /dev/null +++ b/third_party/raft/rafttest/network.go @@ -0,0 +1,167 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "math/rand" + "sync" + "time" + + "google.golang.org/protobuf/proto" + + "go.etcd.io/raft/v3/raftpb" +) + +// a network interface +type iface interface { + send(m *raftpb.Message) + recv() chan *raftpb.Message + disconnect() + connect() +} + +type raftNetwork struct { + rand *rand.Rand + mu sync.Mutex + disconnected map[uint64]bool + dropmap map[conn]float64 + delaymap map[conn]delay + recvQueues map[uint64]chan *raftpb.Message +} + +type conn struct { + from, to uint64 +} + +type delay struct { + d time.Duration + rate float64 +} + +func newRaftNetwork(nodes ...uint64) *raftNetwork { + pn := &raftNetwork{ + rand: rand.New(rand.NewSource(1)), + recvQueues: make(map[uint64]chan *raftpb.Message), + dropmap: make(map[conn]float64), + delaymap: make(map[conn]delay), + disconnected: make(map[uint64]bool), + } + + for _, n := range nodes { + pn.recvQueues[n] = make(chan *raftpb.Message, 1024) + } + return pn +} + +func (rn *raftNetwork) nodeNetwork(id uint64) iface { + return &nodeNetwork{id: id, raftNetwork: rn} +} + +func (rn *raftNetwork) send(m *raftpb.Message) { + rn.mu.Lock() + to := rn.recvQueues[m.GetTo()] + if rn.disconnected[m.GetTo()] { + to = nil + } + drop := rn.dropmap[conn{m.GetFrom(), m.GetTo()}] + dl := rn.delaymap[conn{m.GetFrom(), m.GetTo()}] + rn.mu.Unlock() + + if to == nil { + return + } + if drop != 0 && rn.rand.Float64() < drop { + return + } + // TODO: shall we dl without blocking the send call? + if dl.d != 0 && rn.rand.Float64() < dl.rate { + rd := rn.rand.Int63n(int64(dl.d)) + time.Sleep(time.Duration(rd)) + } + + // use marshal/unmarshal to copy message to avoid data race. + b, err := proto.Marshal(m) + if err != nil { + panic(err) + } + + cm := &raftpb.Message{} + err = proto.Unmarshal(b, cm) + if err != nil { + panic(err) + } + + select { + case to <- cm: + default: + // drop messages when the receiver queue is full. + } +} + +func (rn *raftNetwork) recvFrom(from uint64) chan *raftpb.Message { + rn.mu.Lock() + fromc := rn.recvQueues[from] + if rn.disconnected[from] { + fromc = nil + } + rn.mu.Unlock() + + return fromc +} + +func (rn *raftNetwork) drop(from, to uint64, rate float64) { + rn.mu.Lock() + defer rn.mu.Unlock() + rn.dropmap[conn{from, to}] = rate +} + +func (rn *raftNetwork) delay(from, to uint64, d time.Duration, rate float64) { + rn.mu.Lock() + defer rn.mu.Unlock() + rn.delaymap[conn{from, to}] = delay{d, rate} +} + +func (rn *raftNetwork) disconnect(id uint64) { + rn.mu.Lock() + defer rn.mu.Unlock() + rn.disconnected[id] = true +} + +func (rn *raftNetwork) connect(id uint64) { + rn.mu.Lock() + defer rn.mu.Unlock() + rn.disconnected[id] = false +} + +type nodeNetwork struct { + id uint64 + *raftNetwork +} + +func (nt *nodeNetwork) connect() { + nt.raftNetwork.connect(nt.id) +} + +func (nt *nodeNetwork) disconnect() { + nt.raftNetwork.disconnect(nt.id) +} + +func (nt *nodeNetwork) send(m *raftpb.Message) { + nt.raftNetwork.send(m) +} + +func (nt *nodeNetwork) recv() chan *raftpb.Message { + return nt.recvFrom(nt.id) +} diff --git a/third_party/raft/rafttest/network_test.go b/third_party/raft/rafttest/network_test.go new file mode 100644 index 0000000000..48985d57fc --- /dev/null +++ b/third_party/raft/rafttest/network_test.go @@ -0,0 +1,71 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "go.etcd.io/raft/v3/raftpb" +) + +func TestNetworkDrop(t *testing.T) { + // drop around 10% messages + sent := 1000 + droprate := 0.1 + nt := newRaftNetwork(1, 2) + nt.drop(1, 2, droprate) + for i := 0; i < sent; i++ { + nt.send(&raftpb.Message{From: new(uint64(1)), To: new(uint64(2))}) + } + + c := nt.recvFrom(2) + + received := 0 + done := false + for !done { + select { + case <-c: + received++ + default: + done = true + } + } + + drop := sent - received + assert.LessOrEqual(t, drop, int((droprate+0.1)*float64(sent))) + assert.GreaterOrEqual(t, drop, int((droprate-0.1)*float64(sent))) +} + +func TestNetworkDelay(t *testing.T) { + sent := 1000 + delay := time.Millisecond + delayrate := 0.1 + nt := newRaftNetwork(1, 2) + + nt.delay(1, 2, delay, delayrate) + var total time.Duration + for i := 0; i < sent; i++ { + s := time.Now() + nt.send(&raftpb.Message{From: new(uint64(1)), To: new(uint64(2))}) + total += time.Since(s) + } + + w := time.Duration(float64(sent)*delayrate/2) * delay + // there is some overhead in the send call since it generates random numbers. + assert.GreaterOrEqual(t, total, w) +} diff --git a/third_party/raft/rafttest/node.go b/third_party/raft/rafttest/node.go new file mode 100644 index 0000000000..8e570dfc48 --- /dev/null +++ b/third_party/raft/rafttest/node.go @@ -0,0 +1,158 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "context" + "log" + "math/rand" + "sync" + "time" + + "go.etcd.io/raft/v3" + "go.etcd.io/raft/v3/raftpb" +) + +type node struct { + raft.Node + id uint64 + iface iface + stopc chan struct{} + pausec chan bool + + // stable + storage *raft.MemoryStorage + + mu sync.Mutex // guards state + state *raftpb.HardState +} + +func startNode(id uint64, peers []raft.Peer, iface iface) *node { + st := raft.NewMemoryStorage() + c := &raft.Config{ + ID: id, + ElectionTick: 10, + HeartbeatTick: 1, + Storage: st, + MaxSizePerMsg: 1024 * 1024, + MaxInflightMsgs: 256, + MaxUncommittedEntriesSize: 1 << 30, + } + rn := raft.StartNode(c, peers) + n := &node{ + Node: rn, + id: id, + storage: st, + iface: iface, + pausec: make(chan bool), + } + n.start() + return n +} + +func (n *node) start() { + n.stopc = make(chan struct{}) + ticker := time.NewTicker(5 * time.Millisecond).C + + go func() { + for { + select { + case <-ticker: + n.Tick() + case rd := <-n.Ready(): + if !raft.IsEmptyHardState(rd.HardState) { + n.mu.Lock() + n.state = rd.HardState + n.mu.Unlock() + n.storage.SetHardState(n.state) + } + n.storage.Append(rd.Entries) + time.Sleep(time.Millisecond) + + // simulate async send, more like real world... + for _, m := range rd.Messages { + mlocal := m + go func() { + time.Sleep(time.Duration(rand.Int63n(10)) * time.Millisecond) + n.iface.send(mlocal) + }() + } + n.Advance() + case m := <-n.iface.recv(): + go n.Step(context.TODO(), m) + case <-n.stopc: + n.Stop() + log.Printf("raft.%d: stop", n.id) + n.Node = nil + close(n.stopc) + return + case p := <-n.pausec: + recvms := make([]*raftpb.Message, 0) + for p { + select { + case m := <-n.iface.recv(): + recvms = append(recvms, m) + case p = <-n.pausec: + } + } + // step all pending messages + for _, m := range recvms { + n.Step(context.TODO(), m) + } + } + } + }() +} + +// stop stops the node. stop a stopped node might panic. +// All in memory state of node is discarded. +// All stable MUST be unchanged. +func (n *node) stop() { + n.iface.disconnect() + n.stopc <- struct{}{} + // wait for the shutdown + <-n.stopc +} + +// restart restarts the node. restart a started node +// blocks and might affect the future stop operation. +func (n *node) restart() { + // wait for the shutdown + <-n.stopc + c := &raft.Config{ + ID: n.id, + ElectionTick: 10, + HeartbeatTick: 1, + Storage: n.storage, + MaxSizePerMsg: 1024 * 1024, + MaxInflightMsgs: 256, + MaxUncommittedEntriesSize: 1 << 30, + } + n.Node = raft.RestartNode(c) + n.start() + n.iface.connect() +} + +// pause pauses the node. +// The paused node buffers the received messages and replies +// all of them when it resumes. +func (n *node) pause() { + n.pausec <- true +} + +// resume resumes the paused node. +func (n *node) resume() { + n.pausec <- false +} diff --git a/third_party/raft/rafttest/node_bench_test.go b/third_party/raft/rafttest/node_bench_test.go new file mode 100644 index 0000000000..5b20048c3d --- /dev/null +++ b/third_party/raft/rafttest/node_bench_test.go @@ -0,0 +1,52 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "testing" + "time" + + "go.etcd.io/raft/v3" +) + +func BenchmarkProposal3Nodes(b *testing.B) { + peers := []raft.Peer{{ID: 1, Context: nil}, {ID: 2, Context: nil}, {ID: 3, Context: nil}} + nt := newRaftNetwork(1, 2, 3) + + nodes := make([]*node, 0) + + for i := 1; i <= 3; i++ { + n := startNode(uint64(i), peers, nt.nodeNetwork(uint64(i))) + nodes = append(nodes, n) + } + // get ready and warm up + time.Sleep(50 * time.Millisecond) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + nodes[0].Propose(b.Context(), []byte("somedata")) + } + + for _, n := range nodes { + if n.state.GetCommit() != uint64(b.N+4) { + continue + } + } + b.StopTimer() + + for _, n := range nodes { + n.stop() + } +} diff --git a/third_party/raft/rafttest/node_test.go b/third_party/raft/rafttest/node_test.go new file mode 100644 index 0000000000..6cb8f06e6b --- /dev/null +++ b/third_party/raft/rafttest/node_test.go @@ -0,0 +1,171 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "go.etcd.io/raft/v3" +) + +func TestBasicProgress(t *testing.T) { + peers := []raft.Peer{{ID: 1, Context: nil}, {ID: 2, Context: nil}, {ID: 3, Context: nil}, {ID: 4, Context: nil}, {ID: 5, Context: nil}} + nt := newRaftNetwork(1, 2, 3, 4, 5) + + nodes := make([]*node, 0) + + for i := 1; i <= 5; i++ { + n := startNode(uint64(i), peers, nt.nodeNetwork(uint64(i))) + nodes = append(nodes, n) + } + + waitLeader(nodes) + + for i := 0; i < 100; i++ { + nodes[0].Propose(t.Context(), []byte("somedata")) + } + + assert.True(t, waitCommitConverge(nodes, 100)) + + for _, n := range nodes { + n.stop() + } +} + +func TestRestart(t *testing.T) { + peers := []raft.Peer{{ID: 1, Context: nil}, {ID: 2, Context: nil}, {ID: 3, Context: nil}, {ID: 4, Context: nil}, {ID: 5, Context: nil}} + nt := newRaftNetwork(1, 2, 3, 4, 5) + + nodes := make([]*node, 0) + + for i := 1; i <= 5; i++ { + n := startNode(uint64(i), peers, nt.nodeNetwork(uint64(i))) + nodes = append(nodes, n) + } + + l := waitLeader(nodes) + k1, k2 := (l+1)%5, (l+2)%5 + + for i := 0; i < 30; i++ { + nodes[l].Propose(t.Context(), []byte("somedata")) + } + nodes[k1].stop() + for i := 0; i < 30; i++ { + nodes[(l+3)%5].Propose(t.Context(), []byte("somedata")) + } + nodes[k2].stop() + for i := 0; i < 30; i++ { + nodes[(l+4)%5].Propose(t.Context(), []byte("somedata")) + } + nodes[k2].restart() + for i := 0; i < 30; i++ { + nodes[l].Propose(t.Context(), []byte("somedata")) + } + nodes[k1].restart() + + assert.True(t, waitCommitConverge(nodes, 120)) + + for _, n := range nodes { + n.stop() + } +} + +func TestPause(t *testing.T) { + peers := []raft.Peer{{ID: 1, Context: nil}, {ID: 2, Context: nil}, {ID: 3, Context: nil}, {ID: 4, Context: nil}, {ID: 5, Context: nil}} + nt := newRaftNetwork(1, 2, 3, 4, 5) + + nodes := make([]*node, 0) + + for i := 1; i <= 5; i++ { + n := startNode(uint64(i), peers, nt.nodeNetwork(uint64(i))) + nodes = append(nodes, n) + } + + waitLeader(nodes) + + for i := 0; i < 30; i++ { + nodes[0].Propose(t.Context(), []byte("somedata")) + } + nodes[1].pause() + for i := 0; i < 30; i++ { + nodes[0].Propose(t.Context(), []byte("somedata")) + } + nodes[2].pause() + for i := 0; i < 30; i++ { + nodes[0].Propose(t.Context(), []byte("somedata")) + } + nodes[2].resume() + for i := 0; i < 30; i++ { + nodes[0].Propose(t.Context(), []byte("somedata")) + } + nodes[1].resume() + + assert.True(t, waitCommitConverge(nodes, 120)) + + for _, n := range nodes { + n.stop() + } +} + +func waitLeader(ns []*node) int { + var l map[uint64]struct{} + var lindex = -1 + + for { + l = make(map[uint64]struct{}) + + for i, n := range ns { + lead := n.Status().SoftState.Lead + if lead != 0 { + l[lead] = struct{}{} + if n.id == lead { + lindex = i + } + } + } + + if len(l) == 1 && lindex != -1 { + return lindex + } + } +} + +func waitCommitConverge(ns []*node, target uint64) bool { + var c map[uint64]struct{} + + for i := 0; i < 50; i++ { + c = make(map[uint64]struct{}) + var good int + + for _, n := range ns { + hs := n.Node.Status().HardState + commit := hs.GetCommit() + c[commit] = struct{}{} + if commit > target { + good++ + } + } + + if len(c) == 1 && good == len(ns) { + return true + } + time.Sleep(100 * time.Millisecond) + } + + return false +} diff --git a/third_party/raft/rawnode.go b/third_party/raft/rawnode.go new file mode 100644 index 0000000000..dddb798f61 --- /dev/null +++ b/third_party/raft/rawnode.go @@ -0,0 +1,557 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "errors" + + pb "go.etcd.io/raft/v3/raftpb" + "go.etcd.io/raft/v3/tracker" +) + +// ErrStepLocalMsg is returned when try to step a local raft message +var ErrStepLocalMsg = errors.New("raft: cannot step raft local message") + +// ErrStepPeerNotFound is returned when try to step a response message +// but there is no peer found in raft.trk for that node. +var ErrStepPeerNotFound = errors.New("raft: cannot step as peer not found") + +// RawNode is a thread-unsafe Node. +// The methods of this struct correspond to the methods of Node and are described +// more fully there. +type RawNode struct { + raft *raft + asyncStorageWrites bool + + // Mutable fields. + prevSoftSt *SoftState + prevHardSt *pb.HardState + stepsOnAdvance []*pb.Message +} + +// NewRawNode instantiates a RawNode from the given configuration. +// +// See Bootstrap() for bootstrapping an initial state; this replaces the former +// 'peers' argument to this method (with identical behavior). However, It is +// recommended that instead of calling Bootstrap, applications bootstrap their +// state manually by setting up a Storage that has a first index > 1 and which +// stores the desired ConfState as its InitialState. +func NewRawNode(config *Config) (*RawNode, error) { + r := newRaft(config) + rn := &RawNode{ + raft: r, + } + rn.asyncStorageWrites = config.AsyncStorageWrites + ss := r.softState() + rn.prevSoftSt = &ss + rn.prevHardSt = r.hardState() + return rn, nil +} + +// Tick advances the internal logical clock by a single tick. +func (rn *RawNode) Tick() { + rn.raft.tick() +} + +// TickQuiesced advances the internal logical clock by a single tick without +// performing any other state machine processing. It allows the caller to avoid +// periodic heartbeats and elections when all of the peers in a Raft group are +// known to be at the same state. Expected usage is to periodically invoke Tick +// or TickQuiesced depending on whether the group is "active" or "quiesced". +// +// WARNING: Be very careful about using this method as it subverts the Raft +// state machine. You should probably be using Tick instead. +// +// DEPRECATED: This method will be removed in a future release. +func (rn *RawNode) TickQuiesced() { + rn.raft.electionElapsed++ +} + +// Campaign causes this RawNode to transition to candidate state. +func (rn *RawNode) Campaign() error { + return rn.raft.Step(&pb.Message{ + Type: pb.MsgHup.Enum(), + }) +} + +// Propose proposes data be appended to the raft log. +func (rn *RawNode) Propose(data []byte) error { + return rn.raft.Step(&pb.Message{ + Type: pb.MsgProp.Enum(), + From: new(rn.raft.id), + Entries: []*pb.Entry{{Data: data}}, + }) +} + +// ProposeConfChange proposes a config change. See (Node).ProposeConfChange for +// details. +func (rn *RawNode) ProposeConfChange(cc pb.ConfChangeI) error { + m, err := confChangeToMsg(cc) + if err != nil { + return err + } + return rn.raft.Step(m) +} + +// ApplyConfChange applies a config change to the local node. The app must call +// this when it applies a configuration change, except when it decides to reject +// the configuration change, in which case no call must take place. +func (rn *RawNode) ApplyConfChange(cc pb.ConfChangeI) *pb.ConfState { + return rn.raft.applyConfChange(cc.AsV2()) +} + +// Step advances the state machine using the given message. +func (rn *RawNode) Step(m *pb.Message) error { + // Ignore unexpected local messages receiving over network. + if IsLocalMsg(m.GetType()) && !IsLocalMsgTarget(m.GetFrom()) { + return ErrStepLocalMsg + } + if IsResponseMsg(m.GetType()) && !IsLocalMsgTarget(m.GetFrom()) && rn.raft.trk.Progress[m.GetFrom()] == nil { + return ErrStepPeerNotFound + } + return rn.raft.Step(m) +} + +// Ready returns the outstanding work that the application needs to handle. This +// includes appending and applying entries or a snapshot, updating the HardState, +// and sending messages. The returned Ready() *must* be handled and subsequently +// passed back via Advance(). +func (rn *RawNode) Ready() Ready { + rd := rn.readyWithoutAccept() + rn.acceptReady(rd) + return rd +} + +// readyWithoutAccept returns a Ready. This is a read-only operation, i.e. there +// is no obligation that the Ready must be handled. +func (rn *RawNode) readyWithoutAccept() Ready { + r := rn.raft + + rd := Ready{ + Entries: r.raftLog.nextUnstableEnts(), + CommittedEntries: r.raftLog.nextCommittedEnts(rn.applyUnstableEntries()), + Messages: r.msgs, + } + if softSt := r.softState(); !softSt.equal(rn.prevSoftSt) { + // Allocate only when SoftState changes. + escapingSoftSt := softSt + rd.SoftState = &escapingSoftSt + } + if hardSt := r.hardState(); !isHardStateEqual(hardSt, rn.prevHardSt) { + rd.HardState = hardSt + } + if r.raftLog.hasNextUnstableSnapshot() { + rd.Snapshot = r.raftLog.nextUnstableSnapshot() + } + if len(r.readStates) != 0 { + rd.ReadStates = r.readStates + } + rd.MustSync = MustSync(r.hardState(), rn.prevHardSt, len(rd.Entries)) + + if rn.asyncStorageWrites { + // If async storage writes are enabled, enqueue messages to + // local storage threads, where applicable. + if needStorageAppendMsg(r, rd) { + m := newStorageAppendMsg(r, rd) + rd.Messages = append(rd.Messages, m) + } + if needStorageApplyMsg(rd) { + m := newStorageApplyMsg(r, rd) + rd.Messages = append(rd.Messages, m) + } + } else { + // If async storage writes are disabled, immediately enqueue + // msgsAfterAppend to be sent out. The Ready struct contract + // mandates that Messages cannot be sent until after Entries + // are written to stable storage. + for _, m := range r.msgsAfterAppend { + if m.GetTo() != r.id { + rd.Messages = append(rd.Messages, m) + } + } + } + + return rd +} + +// MustSync returns true if the hard state and count of Raft entries indicate +// that a synchronous write to persistent storage is required. +func MustSync(st, prevst *pb.HardState, entsnum int) bool { + // Persistent state on all servers: + // (Updated on stable storage before responding to RPCs) + // currentTerm + // votedFor + // log entries[] + return entsnum != 0 || st.GetVote() != prevst.GetVote() || st.GetTerm() != prevst.GetTerm() +} + +func needStorageAppendMsg(r *raft, rd Ready) bool { + // Return true if log entries, hard state, or a snapshot need to be written + // to stable storage. Also return true if any messages are contingent on all + // prior MsgStorageAppend being processed. + return len(rd.Entries) > 0 || + !IsEmptyHardState(rd.HardState) || + !IsEmptySnap(rd.Snapshot) || + len(r.msgsAfterAppend) > 0 +} + +func needStorageAppendRespMsg(r *raft, rd Ready) bool { + // Return true if raft needs to hear about stabilized entries or an applied + // snapshot. See the comment in newStorageAppendRespMsg, which explains why + // we check hasNextOrInProgressUnstableEnts instead of len(rd.Entries) > 0. + return r.raftLog.hasNextOrInProgressUnstableEnts() || + !IsEmptySnap(rd.Snapshot) +} + +// newStorageAppendMsg creates the message that should be sent to the local +// append thread to instruct it to append log entries, write an updated hard +// state, and apply a snapshot. The message also carries a set of responses +// that should be delivered after the rest of the message is processed. Used +// with AsyncStorageWrites. +func newStorageAppendMsg(r *raft, rd Ready) *pb.Message { + m := &pb.Message{ + Type: pb.MsgStorageAppend.Enum(), + To: new(LocalAppendThread), + From: new(r.id), + Entries: rd.Entries, + } + if !IsEmptyHardState(rd.HardState) { + // If the Ready includes a HardState update, assign each of its fields + // to the corresponding fields in the Message. This allows clients to + // reconstruct the HardState and save it to stable storage. + // + // If the Ready does not include a HardState update, make sure to not + // assign a value to any of the fields so that a HardState reconstructed + // from them will be empty (return true from raft.IsEmptyHardState). + m.Term = new(rd.GetTerm()) + m.Vote = new(rd.GetVote()) + m.Commit = new(rd.GetCommit()) + } + if !IsEmptySnap(rd.Snapshot) { + m.Snapshot = rd.Snapshot + } + // Attach all messages in msgsAfterAppend as responses to be delivered after + // the message is processed, along with a self-directed MsgStorageAppendResp + // to acknowledge the entry stability. + // + // NB: it is important for performance that MsgStorageAppendResp message be + // handled after self-directed MsgAppResp messages on the leader (which will + // be contained in msgsAfterAppend). This ordering allows the MsgAppResp + // handling to use a fast-path in r.raftLog.term() before the newly appended + // entries are removed from the unstable log. + m.Responses = r.msgsAfterAppend + if needStorageAppendRespMsg(r, rd) { + resp := newStorageAppendRespMsg(r, rd) + m.Responses = append(m.GetResponses(), resp) + } + return m +} + +// newStorageAppendRespMsg creates the message that should be returned to node +// after the unstable log entries, hard state, and snapshot in the current Ready +// (along with those in all prior Ready structs) have been saved to stable +// storage. +func newStorageAppendRespMsg(r *raft, rd Ready) *pb.Message { + m := &pb.Message{ + Type: pb.MsgStorageAppendResp.Enum(), + To: new(r.id), + From: new(LocalAppendThread), + // Dropped after term change, see below. + Term: new(r.Term), + } + if r.raftLog.hasNextOrInProgressUnstableEnts() { + // If the raft log has unstable entries, attach the last index and term of the + // append to the response message. This (index, term) tuple will be handed back + // and consulted when the stability of those log entries is signaled to the + // unstable. If the (index, term) match the unstable log by the time the + // response is received (unstable.stableTo), the unstable log can be truncated. + // + // However, with just this logic, there would be an ABA problem[^1] that could + // lead to the unstable log and the stable log getting out of sync temporarily + // and leading to an inconsistent view. Consider the following example with 5 + // nodes, A B C D E: + // + // 1. A is the leader. + // 2. A proposes some log entries but only B receives these entries. + // 3. B gets the Ready and the entries are appended asynchronously. + // 4. A crashes and C becomes leader after getting a vote from D and E. + // 5. C proposes some log entries and B receives these entries, overwriting the + // previous unstable log entries that are in the process of being appended. + // The entries have a larger term than the previous entries but the same + // indexes. It begins appending these new entries asynchronously. + // 6. C crashes and A restarts and becomes leader again after getting the vote + // from D and E. + // 7. B receives the entries from A which are the same as the ones from step 2, + // overwriting the previous unstable log entries that are in the process of + // being appended from step 5. The entries have the original terms and + // indexes from step 2. Recall that log entries retain their original term + // numbers when a leader replicates entries from previous terms. It begins + // appending these new entries asynchronously. + // 8. The asynchronous log appends from the first Ready complete and stableTo + // is called. + // 9. However, the log entries from the second Ready are still in the + // asynchronous append pipeline and will overwrite (in stable storage) the + // entries from the first Ready at some future point. We can't truncate the + // unstable log yet or a future read from Storage might see the entries from + // step 5 before they have been replaced by the entries from step 7. + // Instead, we must wait until we are sure that the entries are stable and + // that no in-progress appends might overwrite them before removing entries + // from the unstable log. + // + // To prevent these kinds of problems, we also attach the current term to the + // MsgStorageAppendResp (above). If the term has changed by the time the + // MsgStorageAppendResp if returned, the response is ignored and the unstable + // log is not truncated. The unstable log is only truncated when the term has + // remained unchanged from the time that the MsgStorageAppend was sent to the + // time that the MsgStorageAppendResp is received, indicating that no-one else + // is in the process of truncating the stable log. + // + // However, this replaces a correctness problem with a liveness problem. If we + // only attempted to truncate the unstable log when appending new entries but + // also occasionally dropped these responses, then quiescence of new log entries + // could lead to the unstable log never being truncated. + // + // To combat this, we attempt to truncate the log on all MsgStorageAppendResp + // messages where the unstable log is not empty, not just those associated with + // entry appends. This includes MsgStorageAppendResp messages associated with an + // updated HardState, which occur after a term change. + // + // In other words, we set Index and LogTerm in a block that looks like: + // + // if r.raftLog.hasNextOrInProgressUnstableEnts() { ... } + // + // not like: + // + // if len(rd.Entries) > 0 { ... } + // + // To do so, we attach r.raftLog.lastIndex() and r.raftLog.lastTerm(), not the + // (index, term) of the last entry in rd.Entries. If rd.Entries is not empty, + // these will be the same. However, if rd.Entries is empty, we still want to + // attest that this (index, term) is correct at the current term, in case the + // MsgStorageAppend that contained the last entry in the unstable slice carried + // an earlier term and was dropped. + // + // A MsgStorageAppend with a new term is emitted on each term change. This is + // the same condition that causes MsgStorageAppendResp messages with earlier + // terms to be ignored. As a result, we are guaranteed that, assuming a bounded + // number of term changes, there will eventually be a MsgStorageAppendResp + // message that is not ignored. This means that entries in the unstable log + // which have been appended to stable storage will eventually be truncated and + // dropped from memory. + // + // [^1]: https://en.wikipedia.org/wiki/ABA_problem + last := r.raftLog.lastEntryID() + m.Index = new(last.index) + m.LogTerm = new(last.term) + } + if !IsEmptySnap(rd.Snapshot) { + m.Snapshot = rd.Snapshot + } + return m +} + +func needStorageApplyMsg(rd Ready) bool { return len(rd.CommittedEntries) > 0 } +func needStorageApplyRespMsg(rd Ready) bool { return needStorageApplyMsg(rd) } + +// newStorageApplyMsg creates the message that should be sent to the local +// apply thread to instruct it to apply committed log entries. The message +// also carries a response that should be delivered after the rest of the +// message is processed. Used with AsyncStorageWrites. +func newStorageApplyMsg(r *raft, rd Ready) *pb.Message { + ents := rd.CommittedEntries + return &pb.Message{ + Type: pb.MsgStorageApply.Enum(), + To: new(LocalApplyThread), + From: new(r.id), + Term: new(uint64(0)), // committed entries don't apply under a specific term + Entries: ents, + Responses: []*pb.Message{newStorageApplyRespMsg(r, ents)}, + } +} + +// newStorageApplyRespMsg creates the message that should be returned to node +// after the committed entries in the current Ready (along with those in all +// prior Ready structs) have been applied to the local state machine. +func newStorageApplyRespMsg(r *raft, ents []*pb.Entry) *pb.Message { + return &pb.Message{ + Type: pb.MsgStorageApplyResp.Enum(), + To: new(r.id), + From: new(LocalApplyThread), + Term: new(uint64(0)), // committed entries don't apply under a specific term + Entries: ents, + } +} + +// acceptReady is called when the consumer of the RawNode has decided to go +// ahead and handle a Ready. Nothing must alter the state of the RawNode between +// this call and the prior call to Ready(). +func (rn *RawNode) acceptReady(rd Ready) { + if rd.SoftState != nil { + rn.prevSoftSt = rd.SoftState + } + if !IsEmptyHardState(rd.HardState) { + rn.prevHardSt = rd.HardState + } + if len(rd.ReadStates) != 0 { + rn.raft.readStates = nil + } + if !rn.asyncStorageWrites { + if len(rn.stepsOnAdvance) != 0 { + rn.raft.logger.Panicf("two accepted Ready structs without call to Advance") + } + for _, m := range rn.raft.msgsAfterAppend { + if m.GetTo() == rn.raft.id { + rn.stepsOnAdvance = append(rn.stepsOnAdvance, m) + } + } + if needStorageAppendRespMsg(rn.raft, rd) { + m := newStorageAppendRespMsg(rn.raft, rd) + rn.stepsOnAdvance = append(rn.stepsOnAdvance, m) + } + if needStorageApplyRespMsg(rd) { + m := newStorageApplyRespMsg(rn.raft, rd.CommittedEntries) + rn.stepsOnAdvance = append(rn.stepsOnAdvance, m) + } + } + rn.raft.msgs = nil + rn.raft.msgsAfterAppend = nil + rn.raft.raftLog.acceptUnstable() + if len(rd.CommittedEntries) > 0 { + ents := rd.CommittedEntries + index := ents[len(ents)-1].GetIndex() + rn.raft.raftLog.acceptApplying(index, entsSize(ents), rn.applyUnstableEntries()) + } + + traceReady(rn.raft) +} + +// applyUnstableEntries returns whether entries are allowed to be applied once +// they are known to be committed but before they have been written locally to +// stable storage. +func (rn *RawNode) applyUnstableEntries() bool { + return !rn.asyncStorageWrites +} + +// HasReady called when RawNode user need to check if any Ready pending. +func (rn *RawNode) HasReady() bool { + // TODO(nvanbenschoten): order these cases in terms of cost and frequency. + r := rn.raft + if softSt := r.softState(); !softSt.equal(rn.prevSoftSt) { + return true + } + if hardSt := r.hardState(); !IsEmptyHardState(hardSt) && !isHardStateEqual(hardSt, rn.prevHardSt) { + return true + } + if r.raftLog.hasNextUnstableSnapshot() { + return true + } + if len(r.msgs) > 0 || len(r.msgsAfterAppend) > 0 { + return true + } + if r.raftLog.hasNextUnstableEnts() || r.raftLog.hasNextCommittedEnts(rn.applyUnstableEntries()) { + return true + } + if len(r.readStates) != 0 { + return true + } + return false +} + +// Advance notifies the RawNode that the application has applied and saved progress in the +// last Ready results. +// +// NOTE: Advance must not be called when using AsyncStorageWrites. Response messages from +// the local append and apply threads take its place. +func (rn *RawNode) Advance(_ Ready) { + // The actions performed by this function are encoded into stepsOnAdvance in + // acceptReady. In earlier versions of this library, they were computed from + // the provided Ready struct. Retain the unused parameter for compatibility. + if rn.asyncStorageWrites { + rn.raft.logger.Panicf("Advance must not be called when using AsyncStorageWrites") + } + for i, m := range rn.stepsOnAdvance { + _ = rn.raft.Step(m) + rn.stepsOnAdvance[i] = nil + } + rn.stepsOnAdvance = rn.stepsOnAdvance[:0] +} + +// Status returns the current status of the given group. This allocates, see +// BasicStatus and WithProgress for allocation-friendlier choices. +func (rn *RawNode) Status() Status { + status := getStatus(rn.raft) + return status +} + +// BasicStatus returns a BasicStatus. Notably this does not contain the +// Progress map; see WithProgress for an allocation-free way to inspect it. +func (rn *RawNode) BasicStatus() BasicStatus { + return getBasicStatus(rn.raft) +} + +// ProgressType indicates the type of replica a Progress corresponds to. +type ProgressType byte + +const ( + // ProgressTypePeer accompanies a Progress for a regular peer replica. + ProgressTypePeer ProgressType = iota + // ProgressTypeLearner accompanies a Progress for a learner replica. + ProgressTypeLearner +) + +// WithProgress is a helper to introspect the Progress for this node and its +// peers. +func (rn *RawNode) WithProgress(visitor func(id uint64, typ ProgressType, pr tracker.Progress)) { + rn.raft.trk.Visit(func(id uint64, pr *tracker.Progress) { + typ := ProgressTypePeer + if pr.IsLearner { + typ = ProgressTypeLearner + } + p := *pr + p.Inflights = nil + visitor(id, typ, p) + }) +} + +// ReportUnreachable reports the given node is not reachable for the last send. +func (rn *RawNode) ReportUnreachable(id uint64) { + _ = rn.raft.Step(&pb.Message{Type: pb.MsgUnreachable.Enum(), From: new(id)}) +} + +// ReportSnapshot reports the status of the sent snapshot. +func (rn *RawNode) ReportSnapshot(id uint64, status SnapshotStatus) { + rej := status == SnapshotFailure + + _ = rn.raft.Step(&pb.Message{Type: pb.MsgSnapStatus.Enum(), From: new(id), Reject: new(rej)}) +} + +// TransferLeader tries to transfer leadership to the given transferee. +func (rn *RawNode) TransferLeader(transferee uint64) { + _ = rn.raft.Step(&pb.Message{Type: pb.MsgTransferLeader.Enum(), From: new(transferee)}) +} + +// ForgetLeader forgets a follower's current leader, changing it to None. +// See (Node).ForgetLeader for details. +func (rn *RawNode) ForgetLeader() error { + return rn.raft.Step(&pb.Message{Type: pb.MsgForgetLeader.Enum()}) +} + +// ReadIndex requests a read state. The read state will be set in ready. +// Read State has a read index. Once the application advances further than the read +// index, any linearizable read requests issued before the read request can be +// processed safely. The read state will have the same rctx attached. +func (rn *RawNode) ReadIndex(rctx []byte) { + _ = rn.raft.Step(&pb.Message{Type: pb.MsgReadIndex.Enum(), Entries: []*pb.Entry{{Data: rctx}}}) +} diff --git a/third_party/raft/rawnode_test.go b/third_party/raft/rawnode_test.go new file mode 100644 index 0000000000..24b92a36f5 --- /dev/null +++ b/third_party/raft/rawnode_test.go @@ -0,0 +1,1068 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "context" + "fmt" + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "go.etcd.io/raft/v3/quorum" + pb "go.etcd.io/raft/v3/raftpb" + "go.etcd.io/raft/v3/tracker" +) + +// rawNodeAdapter is essentially a lint that makes sure that RawNode implements +// "most of" Node. The exceptions (some of which are easy to fix) are listed +// below. +type rawNodeAdapter struct { + *RawNode +} + +var _ Node = (*rawNodeAdapter)(nil) + +// TransferLeadership is to test when node specifies lead, which is pointless, can just be filled in. +func (a *rawNodeAdapter) TransferLeadership(_ context.Context, _, transferee uint64) { + a.RawNode.TransferLeader(transferee) +} + +// ForgetLeader takes a context, RawNode doesn't need it. +func (a *rawNodeAdapter) ForgetLeader(context.Context) error { return a.RawNode.ForgetLeader() } + +// Stop when node has a goroutine, RawNode doesn't need this. +func (a *rawNodeAdapter) Stop() {} + +// Status retirns RawNode's status as *Status. +func (a *rawNodeAdapter) Status() Status { return a.RawNode.Status() } + +// Advance is when RawNode takes a Ready. It doesn't really have to do that I think? It can hold on +// to it internally. But maybe that approach is frail. +func (a *rawNodeAdapter) Advance() { a.RawNode.Advance(Ready{}) } + +// Ready when RawNode returns a Ready, not a chan of one. +func (a *rawNodeAdapter) Ready() <-chan Ready { return nil } + +// Node takes more contexts. Easy enough to fix. + +func (a *rawNodeAdapter) Campaign(context.Context) error { return a.RawNode.Campaign() } +func (a *rawNodeAdapter) ReadIndex(_ context.Context, rctx []byte) error { + a.RawNode.ReadIndex(rctx) + // RawNode swallowed the error in ReadIndex, it probably should not do that. + return nil +} +func (a *rawNodeAdapter) Step(_ context.Context, m *pb.Message) error { return a.RawNode.Step(m) } +func (a *rawNodeAdapter) Propose(_ context.Context, data []byte) error { + return a.RawNode.Propose(data) +} +func (a *rawNodeAdapter) ProposeConfChange(_ context.Context, cc pb.ConfChangeI) error { + return a.RawNode.ProposeConfChange(cc) +} + +// TestRawNodeStep ensures that RawNode.Step ignore local message. +func TestRawNodeStep(t *testing.T) { + for i, msgn := range pb.MessageType_name { + t.Run(msgn, func(t *testing.T) { + s := NewMemoryStorage() + s.SetHardState(&pb.HardState{Term: new(uint64(1)), Commit: new(uint64(1))}) + s.Append([]*pb.Entry{{Term: new(uint64(1)), Index: new(uint64(1))}}) + require.NoError(t, s.ApplySnapshot(&pb.Snapshot{Metadata: &pb.SnapshotMetadata{ + ConfState: &pb.ConfState{ + Voters: []uint64{1}, + }, + Index: new(uint64(1)), + Term: new(uint64(1)), + }}), "#%d", i) + // Append an empty entry to make sure the non-local messages (like + // vote requests) are ignored and don't trigger assertions. + rawNode, err := NewRawNode(newTestConfig(1, 10, 1, s)) + require.NoError(t, err, "#%d", i) + msgt := pb.MessageType(i) + err = rawNode.Step(&pb.Message{Type: msgt.Enum()}) + // LocalMsg should be ignored. + if IsLocalMsg(msgt) { + assert.Equal(t, ErrStepLocalMsg, err, "#%d", i) + } + }) + } +} + +// TestNodeStepUnblock from node_test.go has no equivalent in rawNode because there is +// no goroutine in RawNode. + +// TestRawNodeProposeAndConfChange tests the configuration change mechanism. Each +// test case sends a configuration change which is either simple or joint, verifies +// that it applies and that the resulting ConfState matches expectations, and for +// joint configurations makes sure that they are exited successfully. +func TestRawNodeProposeAndConfChange(t *testing.T) { + testCases := []struct { + cc pb.ConfChangeI + exp *pb.ConfState + exp2 *pb.ConfState + }{ + // V1 config change. + { + &pb.ConfChange{Type: pb.ConfChangeAddNode.Enum(), NodeId: new(uint64(2))}, + &pb.ConfState{Voters: []uint64{1, 2}}, + nil, + }, + // Proposing the same as a V2 change works just the same, without entering + // a joint config. + { + &pb.ConfChangeV2{Changes: []*pb.ConfChangeSingle{ + {Type: pb.ConfChangeAddNode.Enum(), NodeId: new(uint64(2))}, + }, + }, + &pb.ConfState{Voters: []uint64{1, 2}}, + nil, + }, + // Ditto if we add it as a learner instead. + { + &pb.ConfChangeV2{Changes: []*pb.ConfChangeSingle{ + {Type: pb.ConfChangeAddLearnerNode.Enum(), NodeId: new(uint64(2))}, + }, + }, + &pb.ConfState{Voters: []uint64{1}, Learners: []uint64{2}}, + nil, + }, + // We can ask explicitly for joint consensus if we want it. + { + &pb.ConfChangeV2{Changes: []*pb.ConfChangeSingle{ + {Type: pb.ConfChangeAddLearnerNode.Enum(), NodeId: new(uint64(2))}, + }, + Transition: pb.ConfChangeTransitionJointExplicit.Enum(), + }, + &pb.ConfState{Voters: []uint64{1}, VotersOutgoing: []uint64{1}, Learners: []uint64{2}}, + &pb.ConfState{Voters: []uint64{1}, Learners: []uint64{2}}, + }, + // Ditto, but with implicit transition (the harness checks this). + { + &pb.ConfChangeV2{Changes: []*pb.ConfChangeSingle{ + {Type: pb.ConfChangeAddLearnerNode.Enum(), NodeId: new(uint64(2))}, + }, + Transition: pb.ConfChangeTransitionJointImplicit.Enum(), + }, + &pb.ConfState{ + Voters: []uint64{1}, VotersOutgoing: []uint64{1}, Learners: []uint64{2}, + AutoLeave: new(true), + }, + &pb.ConfState{Voters: []uint64{1}, Learners: []uint64{2}}, + }, + // Add a new node and demote n1. This exercises the interesting case in + // which we really need joint config changes and also need LearnersNext. + { + &pb.ConfChangeV2{Changes: []*pb.ConfChangeSingle{ + {NodeId: new(uint64(2)), Type: pb.ConfChangeAddNode.Enum()}, + {NodeId: new(uint64(1)), Type: pb.ConfChangeAddLearnerNode.Enum()}, + {NodeId: new(uint64(3)), Type: pb.ConfChangeAddLearnerNode.Enum()}, + }, + }, + &pb.ConfState{ + Voters: []uint64{2}, + VotersOutgoing: []uint64{1}, + Learners: []uint64{3}, + LearnersNext: []uint64{1}, + AutoLeave: new(true), + }, + &pb.ConfState{Voters: []uint64{2}, Learners: []uint64{1, 3}}, + }, + // Ditto explicit. + { + &pb.ConfChangeV2{Changes: []*pb.ConfChangeSingle{ + {NodeId: new(uint64(2)), Type: pb.ConfChangeAddNode.Enum()}, + {NodeId: new(uint64(1)), Type: pb.ConfChangeAddLearnerNode.Enum()}, + {NodeId: new(uint64(3)), Type: pb.ConfChangeAddLearnerNode.Enum()}, + }, + Transition: pb.ConfChangeTransitionJointExplicit.Enum(), + }, + &pb.ConfState{ + Voters: []uint64{2}, + VotersOutgoing: []uint64{1}, + Learners: []uint64{3}, + LearnersNext: []uint64{1}, + }, + &pb.ConfState{Voters: []uint64{2}, Learners: []uint64{1, 3}}, + }, + // Ditto implicit. + { + &pb.ConfChangeV2{ + Changes: []*pb.ConfChangeSingle{ + {NodeId: new(uint64(2)), Type: pb.ConfChangeAddNode.Enum()}, + {NodeId: new(uint64(1)), Type: pb.ConfChangeAddLearnerNode.Enum()}, + {NodeId: new(uint64(3)), Type: pb.ConfChangeAddLearnerNode.Enum()}, + }, + Transition: pb.ConfChangeTransitionJointImplicit.Enum(), + }, + &pb.ConfState{ + Voters: []uint64{2}, + VotersOutgoing: []uint64{1}, + Learners: []uint64{3}, + LearnersNext: []uint64{1}, + AutoLeave: new(true), + }, + &pb.ConfState{Voters: []uint64{2}, Learners: []uint64{1, 3}}, + }, + } + + for _, tc := range testCases { + t.Run("", func(t *testing.T) { + pb.EnsureConfState(tc.exp) + pb.EnsureConfState(tc.exp2) + s := newTestMemoryStorage(withPeers(1)) + rawNode, err := NewRawNode(newTestConfig(1, 10, 1, s)) + require.NoError(t, err) + + rawNode.Campaign() + proposed := false + var ( + lastIndex uint64 + ccdata []byte + ) + // Propose the ConfChange, wait until it applies, save the resulting + // ConfState. + var cs *pb.ConfState + for cs == nil { + rd := rawNode.Ready() + s.Append(rd.Entries) + for _, ent := range rd.CommittedEntries { + var cc pb.ConfChangeI + if ent.GetType() == pb.EntryConfChange { + ccc := &pb.ConfChange{} + require.NoError(t, proto.Unmarshal(ent.GetData(), ccc)) + cc = ccc + } else if ent.GetType() == pb.EntryConfChangeV2 { + ccc := &pb.ConfChangeV2{} + require.NoError(t, proto.Unmarshal(ent.GetData(), ccc)) + cc = ccc + } + if cc != nil { + cs = rawNode.ApplyConfChange(cc) + } + } + rawNode.Advance(rd) + // Once we are the leader, propose a command and a ConfChange. + if !proposed && rd.SoftState.Lead == rawNode.raft.id { + require.NoError(t, rawNode.Propose([]byte("somedata"))) + if ccv1, ok := tc.cc.AsV1(); ok { + ccdata, err = proto.Marshal(ccv1) + require.NoError(t, err) + rawNode.ProposeConfChange(ccv1) + } else { + ccv2 := tc.cc.AsV2() + ccdata, err = proto.Marshal(ccv2) + require.NoError(t, err) + rawNode.ProposeConfChange(ccv2) + } + proposed = true + } + } + + // Check that the last index is exactly the conf change we put in, + // down to the bits. Note that this comes from the Storage, which + // will not reflect any unstable entries that we'll only be presented + // with in the next Ready. + lastIndex, err = s.LastIndex() + require.NoError(t, err) + + entries, err := s.Entries(lastIndex-1, lastIndex+1, noLimit) + require.NoError(t, err) + require.Len(t, entries, 2) + assert.Equal(t, []byte("somedata"), entries[0].GetData()) + + typ := pb.EntryConfChange + if _, ok := tc.cc.AsV1(); !ok { + typ = pb.EntryConfChangeV2 + } + require.Equal(t, typ, entries[1].GetType()) + assert.Equal(t, ccdata, entries[1].GetData()) + + require.Equal(t, tc.exp, cs) + + var maybePlusOne uint64 + if autoLeave, ok := tc.cc.AsV2().EnterJoint(); ok && autoLeave { + // If this is an auto-leaving joint conf change, it will have + // appended the entry that auto-leaves, so add one to the last + // index that forms the basis of our expectations on + // pendingConfIndex. (Recall that lastIndex was taken from stable + // storage, but this auto-leaving entry isn't on stable storage + // yet). + maybePlusOne = 1 + } + require.Equal(t, lastIndex+maybePlusOne, rawNode.raft.pendingConfIndex) + + // Move the RawNode along. If the ConfChange was simple, nothing else + // should happen. Otherwise, we're in a joint state, which is either + // left automatically or not. If not, we add the proposal that leaves + // it manually. + rd := rawNode.Ready() + var context []byte + if !tc.exp.GetAutoLeave() { + require.Empty(t, rd.Entries) + rawNode.Advance(rd) + if tc.exp2 == nil { + return + } + context = []byte("manual") + t.Log("leaving joint state manually") + require.NoError(t, rawNode.ProposeConfChange(&pb.ConfChangeV2{Context: context})) + rd = rawNode.Ready() + } + + // Check that the right ConfChange comes out. + require.Len(t, rd.Entries, 1) + require.Equal(t, pb.EntryConfChangeV2, rd.Entries[0].GetType()) + cc := &pb.ConfChangeV2{} + require.NoError(t, proto.Unmarshal(rd.Entries[0].GetData(), cc)) + require.True(t, proto.Equal(&pb.ConfChangeV2{Context: context}, cc)) + + // Lie and pretend the ConfChange applied. It won't do so because now + // we require the joint quorum and we're only running one node. + cs = rawNode.ApplyConfChange(cc) + require.Equal(t, tc.exp2, cs) + + rawNode.Advance(rd) + }) + } +} + +// TestRawNodeJointAutoLeave tests the configuration change auto leave even leader +// lost leadership. +func TestRawNodeJointAutoLeave(t *testing.T) { + testCc := &pb.ConfChangeV2{Changes: []*pb.ConfChangeSingle{ + {Type: pb.ConfChangeAddLearnerNode.Enum(), NodeId: new(uint64(2))}, + }, + Transition: pb.ConfChangeTransitionJointImplicit.Enum(), + } + expCs := &pb.ConfState{ + Voters: []uint64{1}, VotersOutgoing: []uint64{1}, Learners: []uint64{2}, + AutoLeave: new(true), + } + exp2Cs := &pb.ConfState{Voters: []uint64{1}, Learners: []uint64{2}, AutoLeave: new(false)} + + s := newTestMemoryStorage(withPeers(1)) + rawNode, err := NewRawNode(newTestConfig(1, 10, 1, s)) + require.NoError(t, err) + + rawNode.Campaign() + proposed := false + var ( + lastIndex uint64 + ccdata []byte + ) + // Propose the ConfChange, wait until it applies, save the resulting + // ConfState. + var cs *pb.ConfState + for cs == nil { + rd := rawNode.Ready() + s.Append(rd.Entries) + for _, ent := range rd.CommittedEntries { + var cc pb.ConfChangeI + if ent.GetType() == pb.EntryConfChangeV2 { + ccc := &pb.ConfChangeV2{} + require.NoError(t, proto.Unmarshal(ent.GetData(), ccc)) + cc = ccc + } + if cc != nil { + // Force it step down. + rawNode.Step(&pb.Message{Type: pb.MsgHeartbeatResp.Enum(), From: new(uint64(1)), Term: new(rawNode.raft.Term + 1)}) + cs = rawNode.ApplyConfChange(cc) + } + } + rawNode.Advance(rd) + // Once we are the leader, propose a command and a ConfChange. + if !proposed && rd.SoftState.Lead == rawNode.raft.id { + require.NoError(t, rawNode.Propose([]byte("somedata"))) + ccdata, err = proto.Marshal(testCc) + require.NoError(t, err) + rawNode.ProposeConfChange(testCc) + proposed = true + } + } + + // Check that the last index is exactly the conf change we put in, + // down to the bits. Note that this comes from the Storage, which + // will not reflect any unstable entries that we'll only be presented + // with in the next Ready. + lastIndex, err = s.LastIndex() + require.NoError(t, err) + + entries, err := s.Entries(lastIndex-1, lastIndex+1, noLimit) + require.NoError(t, err) + require.Len(t, entries, 2) + assert.Equal(t, []byte("somedata"), entries[0].GetData()) + require.Equal(t, pb.EntryConfChangeV2, entries[1].GetType()) + assert.Equal(t, ccdata, entries[1].GetData()) + + require.Equal(t, expCs, cs) + + require.Zero(t, rawNode.raft.pendingConfIndex) + + // Move the RawNode along. It should not leave joint because it's follower. + rd := rawNode.readyWithoutAccept() + // Check that the right ConfChange comes out. + require.Empty(t, rd.Entries) + + // Make it leader again. It should leave joint automatically after moving apply index. + rawNode.Campaign() + rd = rawNode.Ready() + t.Log(DescribeReady(rd, nil)) + s.Append(rd.Entries) + rawNode.Advance(rd) + rd = rawNode.Ready() + t.Log(DescribeReady(rd, nil)) + s.Append(rd.Entries) + rawNode.Advance(rd) + rd = rawNode.Ready() + t.Log(DescribeReady(rd, nil)) + s.Append(rd.Entries) + rawNode.Advance(rd) + rd = rawNode.Ready() + t.Log(DescribeReady(rd, nil)) + s.Append(rd.Entries) + // Check that the right ConfChange comes out. + require.Len(t, rd.Entries, 1) + require.Equal(t, pb.EntryConfChangeV2, rd.Entries[0].GetType()) + cc := &pb.ConfChangeV2{} + require.NoError(t, proto.Unmarshal(rd.Entries[0].GetData(), cc)) + require.True(t, proto.Equal(&pb.ConfChangeV2{}, cc)) + // Lie and pretend the ConfChange applied. It won't do so because now + // we require the joint quorum and we're only running one node. + cs = rawNode.ApplyConfChange(cc) + require.Equal(t, exp2Cs, cs) +} + +// TestRawNodeProposeAddDuplicateNode ensures that two proposes to add the same node should +// not affect the later propose to add new node. +func TestRawNodeProposeAddDuplicateNode(t *testing.T) { + s := newTestMemoryStorage(withPeers(1)) + rawNode, err := NewRawNode(newTestConfig(1, 10, 1, s)) + require.NoError(t, err) + + rd := rawNode.Ready() + s.Append(rd.Entries) + rawNode.Advance(rd) + + rawNode.Campaign() + for { + rd = rawNode.Ready() + s.Append(rd.Entries) + if rd.SoftState.Lead == rawNode.raft.id { + rawNode.Advance(rd) + break + } + rawNode.Advance(rd) + } + + proposeConfChangeAndApply := func(cc *pb.ConfChange) { + rawNode.ProposeConfChange(cc) + rd = rawNode.Ready() + s.Append(rd.Entries) + for _, entry := range rd.CommittedEntries { + if entry.GetType() == pb.EntryConfChange { + cc := &pb.ConfChange{} + proto.Unmarshal(entry.GetData(), cc) + rawNode.ApplyConfChange(cc) + } + } + rawNode.Advance(rd) + } + + cc1 := &pb.ConfChange{Type: pb.ConfChangeAddNode.Enum(), NodeId: new(uint64(1))} + ccdata1, err := proto.Marshal(cc1) + require.NoError(t, err) + proposeConfChangeAndApply(cc1) + + // try to add the same node again + proposeConfChangeAndApply(cc1) + + // the new node join should be ok + cc2 := &pb.ConfChange{Type: pb.ConfChangeAddNode.Enum(), NodeId: new(uint64(2))} + ccdata2, err := proto.Marshal(cc2) + require.NoError(t, err) + proposeConfChangeAndApply(cc2) + + lastIndex, err := s.LastIndex() + require.NoError(t, err) + + // the last three entries should be: ConfChange cc1, cc1, cc2 + entries, err := s.Entries(lastIndex-2, lastIndex+1, noLimit) + require.NoError(t, err) + require.Len(t, entries, 3) + assert.Equal(t, ccdata1, entries[0].GetData()) + assert.Equal(t, ccdata2, entries[2].GetData()) +} + +// TestRawNodeReadIndex ensures that Rawnode.ReadIndex sends the MsgReadIndex message +// to the underlying raft. It also ensures that ReadState can be read out. +func TestRawNodeReadIndex(t *testing.T) { + var msgs []*pb.Message + appendStep := func(_ *raft, m *pb.Message) error { + msgs = append(msgs, m) + return nil + } + wrs := []ReadState{{Index: uint64(1), RequestCtx: []byte("somedata")}} + + s := newTestMemoryStorage(withPeers(1)) + c := newTestConfig(1, 10, 1, s) + rawNode, err := NewRawNode(c) + require.NoError(t, err) + + rawNode.raft.readStates = wrs + // ensure the ReadStates can be read out + assert.True(t, rawNode.HasReady()) + rd := rawNode.Ready() + assert.Equal(t, wrs, rd.ReadStates) + s.Append(rd.Entries) + rawNode.Advance(rd) + // ensure raft.readStates is reset after advance + assert.Nil(t, rawNode.raft.readStates) + + wrequestCtx := []byte("somedata2") + rawNode.Campaign() + for { + rd = rawNode.Ready() + s.Append(rd.Entries) + + if rd.SoftState.Lead == rawNode.raft.id { + rawNode.Advance(rd) + + // Once we are the leader, issue a ReadIndex request + rawNode.raft.step = appendStep + rawNode.ReadIndex(wrequestCtx) + break + } + rawNode.Advance(rd) + } + // ensure that MsgReadIndex message is sent to the underlying raft + require.Len(t, msgs, 1) + assert.Equal(t, pb.MsgReadIndex, msgs[0].GetType()) + assert.Equal(t, wrequestCtx, msgs[0].GetEntries()[0].GetData()) +} + +// TestBlockProposal from node_test.go has no equivalent in rawNode because there is +// no leader check in RawNode. + +// TestNodeTick from node_test.go has no equivalent in rawNode because +// it reaches into the raft object which is not exposed. + +// TestNodeStop from node_test.go has no equivalent in rawNode because there is +// no goroutine in RawNode. + +// TestRawNodeStart ensures that a node can be started correctly. Note that RawNode +// requires the application to bootstrap the state, i.e. it does not accept peers +// and will not create faux configuration change entries. +func TestRawNodeStart(t *testing.T) { + entries := []*pb.Entry{ + {Term: new(uint64(1)), Index: new(uint64(2)), Data: nil}, // empty entry + {Term: new(uint64(1)), Index: new(uint64(3)), Data: []byte("foo")}, // non-empty entry + } + want := Ready{ + SoftState: &SoftState{Lead: 1, RaftState: StateLeader}, + HardState: &pb.HardState{Term: new(uint64(1)), Commit: new(uint64(3)), Vote: new(uint64(1))}, + Entries: nil, // emitted & checked in intermediate Ready cycle + CommittedEntries: entries, + MustSync: false, // since we're only applying, not appending + } + + storage := NewMemoryStorage() + storage.ents[0].Index = new(uint64(1)) + + // TODO(tbg): this is a first prototype of what bootstrapping could look + // like (without the annoying faux ConfChanges). We want to persist a + // ConfState at some index and make sure that this index can't be reached + // from log position 1, so that followers are forced to pick up the + // ConfState in order to move away from log position 1 (unless they got + // bootstrapped in the same way already). Failing to do so would mean that + // followers diverge from the bootstrapped nodes and don't learn about the + // initial config. + // + // NB: this is exactly what CockroachDB does. The Raft log really begins at + // index 10, so empty followers (at index 1) always need a snapshot first. + type appenderStorage interface { + Storage + ApplySnapshot(*pb.Snapshot) error + } + bootstrap := func(storage appenderStorage, cs *pb.ConfState) error { + require.NotEmpty(t, cs.Voters, "no voters specified") + fi, err := storage.FirstIndex() + require.NoError(t, err) + require.GreaterOrEqual(t, fi, uint64(2), "FirstIndex >= 2 is prerequisite for bootstrap") + + _, err = storage.Entries(fi, fi, math.MaxUint64) + // TODO(tbg): match exact error + require.Error(t, err, "should not have been able to load first index") + + li, err := storage.LastIndex() + require.NoError(t, err) + + _, err = storage.Entries(li, li, math.MaxUint64) + require.Error(t, err, "should not have been able to load last index") + + hs, ics, err := storage.InitialState() + require.NoError(t, err) + require.True(t, IsEmptyHardState(hs)) + require.Empty(t, ics.Voters) + + snap := &pb.Snapshot{Metadata: &pb.SnapshotMetadata{ + Index: new(uint64(1)), + Term: new(uint64(0)), + ConfState: cs, + }} + return storage.ApplySnapshot(snap) + } + + require.NoError(t, bootstrap(storage, &pb.ConfState{Voters: []uint64{1}})) + + rawNode, err := NewRawNode(newTestConfig(1, 10, 1, storage)) + require.NoError(t, err) + require.False(t, rawNode.HasReady()) + + rawNode.Campaign() + rd := rawNode.Ready() + storage.Append(rd.Entries) + rawNode.Advance(rd) + rawNode.Propose([]byte("foo")) + require.True(t, rawNode.HasReady()) + + rd = rawNode.Ready() + requireEqualEntries(t, entries, rd.Entries) + storage.Append(rd.Entries) + rawNode.Advance(rd) + + require.True(t, rawNode.HasReady()) + rd = rawNode.Ready() + require.Empty(t, rd.Entries) + require.False(t, rd.MustSync) + rawNode.Advance(rd) + + rd.SoftState, want.SoftState = nil, nil + + requireEqualReady(t, want, rd) + assert.False(t, rawNode.HasReady()) +} + +func TestRawNodeRestart(t *testing.T) { + entries := []*pb.Entry{ + {Term: new(uint64(1)), Index: new(uint64(1))}, + {Term: new(uint64(1)), Index: new(uint64(2)), Data: []byte("foo")}, + } + st := &pb.HardState{Term: new(uint64(1)), Commit: new(uint64(1))} + + want := Ready{ + HardState: nil, + // commit up to commit index in st + CommittedEntries: entries[:st.GetCommit()], + MustSync: false, + } + + storage := newTestMemoryStorage(withPeers(1)) + storage.SetHardState(st) + storage.Append(entries) + rawNode, err := NewRawNode(newTestConfig(1, 10, 1, storage)) + require.NoError(t, err) + rd := rawNode.Ready() + assert.Equal(t, want, rd) + rawNode.Advance(rd) + assert.False(t, rawNode.HasReady()) +} + +func TestRawNodeRestartFromSnapshot(t *testing.T) { + snap := &pb.Snapshot{ + Metadata: &pb.SnapshotMetadata{ + ConfState: &pb.ConfState{Voters: []uint64{1, 2}}, + Index: new(uint64(2)), + Term: new(uint64(1)), + }, + } + entries := []*pb.Entry{ + {Term: new(uint64(1)), Index: new(uint64(3)), Data: []byte("foo")}, + } + st := &pb.HardState{Term: new(uint64(1)), Commit: new(uint64(3))} + + want := Ready{ + HardState: nil, + // commit up to commit index in st + CommittedEntries: entries, + MustSync: false, + } + + s := NewMemoryStorage() + s.SetHardState(st) + s.ApplySnapshot(snap) + s.Append(entries) + rawNode, err := NewRawNode(newTestConfig(1, 10, 1, s)) + require.NoError(t, err) + rd := rawNode.Ready() + if assert.Equal(t, want, rd) { + rawNode.Advance(rd) + } + assert.False(t, rawNode.HasReady()) +} + +// TestNodeAdvance from node_test.go has no equivalent in rawNode because there is +// no dependency check between Ready() and Advance() +func TestRawNodeStatus(t *testing.T) { + s := newTestMemoryStorage(withPeers(1)) + rn, err := NewRawNode(newTestConfig(1, 10, 1, s)) + require.NoError(t, err) + require.Nil(t, rn.Status().Progress) + require.NoError(t, rn.Campaign()) + + rd := rn.Ready() + s.Append(rd.Entries) + rn.Advance(rd) + status := rn.Status() + require.Equal(t, uint64(1), status.Lead) + require.Equal(t, StateLeader, status.RaftState) + require.Equal(t, *rn.raft.trk.Progress[1], status.Progress[1]) + + expCfg := tracker.Config{Voters: quorum.JointConfig{ + quorum.MajorityConfig{1: {}}, + nil, + }} + require.Equal(t, expCfg, status.Config) +} + +// TestRawNodeCommitPaginationAfterRestart is the RawNode version of +// TestNodeCommitPaginationAfterRestart. The anomaly here was even worse as the +// Raft group would forget to apply entries: +// +// - node learns that index 11 is committed +// - nextCommittedEnts returns index 1..10 in CommittedEntries (but index 10 +// already exceeds maxBytes), which isn't noticed internally by Raft +// - Commit index gets bumped to 10 +// - the node persists the HardState, but crashes before applying the entries +// - upon restart, the storage returns the same entries, but `slice` takes a +// different code path and removes the last entry. +// - Raft does not emit a HardState, but when the app calls Advance(), it bumps +// its internal applied index cursor to 10 (when it should be 9) +// - the next Ready asks the app to apply index 11 (omitting index 10), losing a +// write. +func TestRawNodeCommitPaginationAfterRestart(t *testing.T) { + s := &ignoreSizeHintMemStorage{ + MemoryStorage: newTestMemoryStorage(withPeers(1)), + } + persistedHardState := &pb.HardState{ + Term: new(uint64(1)), + Vote: new(uint64(1)), + Commit: new(uint64(10)), + } + + s.hardState = persistedHardState + s.ents = make([]*pb.Entry, 10) + var size uint64 + for i := range s.ents { + ent := &pb.Entry{Term: new(uint64(1)), Index: new(uint64(i + 1)), Type: pb.EntryNormal.Enum(), Data: []byte("a")} + + s.ents[i] = ent + size += uint64(proto.Size(ent)) + } + + cfg := newTestConfig(1, 10, 1, s) + // Set a MaxSizePerMsg that would suggest to Raft that the last committed entry should + // not be included in the initial rd.CommittedEntries. However, our storage will ignore + // this and *will* return it (which is how the Commit index ended up being 10 initially). + cfg.MaxSizePerMsg = size - uint64(proto.Size(s.ents[len(s.ents)-1])) - 1 + + s.ents = append(s.ents, &pb.Entry{Term: new(uint64(1)), Index: new(uint64(11)), Type: pb.EntryNormal.Enum(), Data: []byte("boom")}) + + rawNode, err := NewRawNode(cfg) + require.NoError(t, err) + + for highestApplied := uint64(0); highestApplied != 11; { + rd := rawNode.Ready() + n := len(rd.CommittedEntries) + require.NotZero(t, n, "stopped applying entries at index %d", highestApplied) + next := rd.CommittedEntries[0].GetIndex() + require.False(t, highestApplied != 0 && highestApplied+1 != next, + "attempting to apply index %d after index %d, leaving a gap", next, highestApplied) + + highestApplied = rd.CommittedEntries[n-1].GetIndex() + rawNode.Advance(rd) + rawNode.Step(&pb.Message{ + Type: pb.MsgHeartbeat.Enum(), + To: new(uint64(1)), + From: new(uint64(2)), // illegal, but we get away with it + Term: new(uint64(1)), + Commit: new(uint64(11)), + }) + } +} + +// TestRawNodeBoundedLogGrowthWithPartition tests a scenario where a leader is +// partitioned from a quorum of nodes. It verifies that the leader's log is +// protected from unbounded growth even as new entries continue to be proposed. +// This protection is provided by the MaxUncommittedEntriesSize configuration. +func TestRawNodeBoundedLogGrowthWithPartition(t *testing.T) { + const maxEntries = 16 + data := []byte("testdata") + testEntry := &pb.Entry{Data: data} + maxEntrySize := maxEntries * payloadSize(testEntry) + t.Log("maxEntrySize", maxEntrySize) + + s := newTestMemoryStorage(withPeers(1)) + cfg := newTestConfig(1, 10, 1, s) + cfg.MaxUncommittedEntriesSize = uint64(maxEntrySize) + rawNode, err := NewRawNode(cfg) + require.NoError(t, err) + + // Become the leader and apply empty entry. + rawNode.Campaign() + for { + rd := rawNode.Ready() + s.Append(rd.Entries) + rawNode.Advance(rd) + if len(rd.CommittedEntries) > 0 { + break + } + } + + // Simulate a network partition while we make our proposals by never + // committing anything. These proposals should not cause the leader's + // log to grow indefinitely. + for i := 0; i < 1024; i++ { + rawNode.Propose(data) + } + + // Check the size of leader's uncommitted log tail. It should not exceed the + // MaxUncommittedEntriesSize limit. + checkUncommitted := func(exp entryPayloadSize) { + t.Helper() + require.Equal(t, exp, rawNode.raft.uncommittedSize) + } + checkUncommitted(maxEntrySize) + + // Recover from the partition. The uncommitted tail of the Raft log should + // disappear as entries are committed. + rd := rawNode.Ready() + require.Len(t, rd.Entries, maxEntries) + s.Append(rd.Entries) + rawNode.Advance(rd) + + // Entries are appended, but not applied. + checkUncommitted(maxEntrySize) + + rd = rawNode.Ready() + require.Empty(t, rd.Entries) + require.Len(t, rd.CommittedEntries, maxEntries) + rawNode.Advance(rd) + + checkUncommitted(0) +} + +func BenchmarkStatus(b *testing.B) { + setup := func(members int) *RawNode { + peers := make([]uint64, members) + for i := range peers { + peers[i] = uint64(i + 1) + } + cfg := newTestConfig(1, 3, 1, newTestMemoryStorage(withPeers(peers...))) + cfg.Logger = discardLogger + r := newRaft(cfg) + r.becomeFollower(1, 1) + r.becomeCandidate() + r.becomeLeader() + return &RawNode{raft: r} + } + + for _, members := range []int{1, 3, 5, 100} { + b.Run(fmt.Sprintf("members=%d", members), func(b *testing.B) { + rn := setup(members) + + b.Run("Status", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _ = rn.Status() + } + }) + + b.Run("Status-example", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + s := rn.Status() + var n uint64 + for _, pr := range s.Progress { + n += pr.Match + } + _ = n + } + }) + + b.Run("BasicStatus", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _ = rn.BasicStatus() + } + }) + + b.Run("WithProgress", func(b *testing.B) { + b.ReportAllocs() + visit := func(uint64, ProgressType, tracker.Progress) {} + + for i := 0; i < b.N; i++ { + rn.WithProgress(visit) + } + }) + b.Run("WithProgress-example", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + var n uint64 + visit := func(_ uint64, _ ProgressType, pr tracker.Progress) { + n += pr.Match + } + rn.WithProgress(visit) + _ = n + } + }) + }) + } +} + +func TestRawNodeConsumeReady(t *testing.T) { + // Check that readyWithoutAccept() does not call acceptReady (which resets + // the messages) but Ready() does. + s := newTestMemoryStorage(withPeers(1)) + rn := newTestRawNode(1, 3, 1, s) + m1 := &pb.Message{Context: []byte("foo")} + m2 := &pb.Message{Context: []byte("bar")} + + // Inject first message, make sure it's visible via readyWithoutAccept. + rn.raft.msgs = append(rn.raft.msgs, m1) + rd := rn.readyWithoutAccept() + require.Len(t, rd.Messages, 1) + require.Equal(t, m1, rd.Messages[0]) + require.Len(t, rn.raft.msgs, 1) + require.Equal(t, m1, rn.raft.msgs[0]) + + // Now call Ready() which should move the message into the Ready (as opposed + // to leaving it in both places). + rd = rn.Ready() + require.Empty(t, rn.raft.msgs) + require.Len(t, rd.Messages, 1) + require.Equal(t, m1, rd.Messages[0]) + + // Add a message to raft to make sure that Advance() doesn't drop it. + rn.raft.msgs = append(rn.raft.msgs, m2) + rn.Advance(rd) + require.Len(t, rn.raft.msgs, 1) + require.Equal(t, m2, rn.raft.msgs[0]) +} + +func BenchmarkRawNode(b *testing.B) { + cases := []struct { + name string + peers []uint64 + }{ + { + name: "single-voter", + peers: []uint64{1}, + }, + { + name: "two-voters", + peers: []uint64{1, 2}, + }, + // You can easily add more cases here. + } + + for _, tc := range cases { + b.Run(tc.name, func(b *testing.B) { + benchmarkRawNodeImpl(b, tc.peers...) + }) + } +} + +func benchmarkRawNodeImpl(b *testing.B, peers ...uint64) { + + const debug = false + + s := newTestMemoryStorage(withPeers(peers...)) + cfg := newTestConfig(1, 10, 1, s) + if !debug { + cfg.Logger = discardLogger // avoid distorting benchmark output + } + rn, err := NewRawNode(cfg) + if err != nil { + b.Fatal(err) + } + + run := make(chan struct{}, 1) + defer close(run) + + var numReady uint64 + stabilize := func() (applied uint64) { + for rn.HasReady() { + numReady++ + rd := rn.Ready() + if debug { + b.Log(DescribeReady(rd, nil)) + } + if n := len(rd.CommittedEntries); n > 0 { + applied = rd.CommittedEntries[n-1].GetIndex() + } + s.Append(rd.Entries) + for _, m := range rd.Messages { + if m.GetType() == pb.MsgVote { + resp := &pb.Message{To: m.From, From: m.To, Term: m.Term, Type: pb.MsgVoteResp.Enum()} + if debug { + b.Log(DescribeMessage(resp, nil)) + } + rn.Step(resp) + } + if m.GetType() == pb.MsgApp { + idx := m.GetIndex() + if n := len(m.GetEntries()); n > 0 { + idx = m.GetEntries()[n-1].GetIndex() + } + resp := &pb.Message{To: m.From, From: m.To, Type: pb.MsgAppResp.Enum(), Term: m.Term, Index: new(idx)} + if debug { + b.Log(DescribeMessage(resp, nil)) + } + rn.Step(resp) + } + } + rn.Advance(rd) + } + return applied + } + + rn.Campaign() + stabilize() + + if debug { + // nolint:staticcheck + b.N = 1 + } + + var applied uint64 + for i := 0; i < b.N; i++ { + if err := rn.Propose([]byte("foo")); err != nil { + b.Fatal(err) + } + applied = stabilize() + } + if applied < uint64(b.N) { + b.Fatalf("did not apply everything: %d < %d", applied, b.N) + } + b.ReportAllocs() + b.ReportMetric(float64(s.callStats.firstIndex)/float64(b.N), "firstIndex/op") + b.ReportMetric(float64(s.callStats.lastIndex)/float64(b.N), "lastIndex/op") + b.ReportMetric(float64(s.callStats.term)/float64(b.N), "term/op") + b.ReportMetric(float64(numReady)/float64(b.N), "ready/op") + b.Logf("storage access stats: %+v", s.callStats) +} diff --git a/third_party/raft/read_only.go b/third_party/raft/read_only.go new file mode 100644 index 0000000000..23fc9d32db --- /dev/null +++ b/third_party/raft/read_only.go @@ -0,0 +1,101 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "encoding/binary" + + "go.etcd.io/raft/v3/quorum" + pb "go.etcd.io/raft/v3/raftpb" +) + +// ReadState provides state for read only query. +// It's caller's responsibility to call ReadIndex first before getting +// this state from ready, it's also caller's duty to differentiate if this +// state is what it requests through RequestCtx, eg. given a unique id as +// RequestCtx +type ReadState struct { + Index uint64 + RequestCtx []byte +} + +type readIndexRequest struct { + req *pb.Message + index uint64 +} + +type readOnly struct { + option ReadOnlyOption + acks map[uint64]uint64 + + unconfirmedReads []*readIndexRequest + // Number of readIndexRequests that were confirmed in the past by this + // readOnly, which were removed from the beginning of `unconfirmedReads`. + confirmedReads uint64 +} + +func newReadOnly(option ReadOnlyOption) *readOnly { + return &readOnly{ + option: option, + acks: make(map[uint64]uint64), + } +} + +// addRequest adds a read only request into the `readOnly`. +// `commitIndex` is the commit index of the raft state machine when it received +// the read only request. +// `req` is the original read only request message from the local or remote node. +func (ro *readOnly) addRequest(commitIndex uint64, req *pb.Message) { + ro.unconfirmedReads = append(ro.unconfirmedReads, &readIndexRequest{req: req, index: commitIndex}) +} + +// recvAck notifies the `readOnly` of an acknowledgment of a heartbeat response. +func (ro *readOnly) recvAck(from uint64, ctx []byte) { + if len(ctx) != 0 { + ro.acks[from] = max(ro.acks[from], binary.LittleEndian.Uint64(ctx)) + } +} + +// AckedIndex allows for using `CommittedIndex` in `maybeAdvance`. +func (ro *readOnly) AckedIndex(voterID uint64) (quorum.Index, bool) { + idx, found := ro.acks[voterID] + return quorum.Index(idx), found +} + +// maybeAdvance uses the existing acknowledgements and current raft +// configuration to confirm and return as many unconfirmed reads as possible. +func (ro *readOnly) maybeAdvance(c quorum.JointConfig) []*readIndexRequest { + // Use `CommittedIndex` to figure out how many reads are now confirmed. + newConfirmedReads := uint64(c.CommittedIndex(ro)) + if newConfirmedReads <= ro.confirmedReads { + return nil + } + readStates := ro.unconfirmedReads[:newConfirmedReads-ro.confirmedReads] + ro.unconfirmedReads = ro.unconfirmedReads[newConfirmedReads-ro.confirmedReads:] + ro.confirmedReads = newConfirmedReads + return readStates +} + +// heartbeatCtx returns the `Context` that should be sent in order to confirm +// all currently unconfirmed reads. +func (ro *readOnly) heartbeatCtx() []byte { + if len(ro.unconfirmedReads) == 0 { + return nil + } + unconfirmedReadPosition := ro.confirmedReads + uint64(len(ro.unconfirmedReads)) + encLastIndex := make([]byte, 8) + binary.LittleEndian.PutUint64(encLastIndex, unconfirmedReadPosition) + return encLastIndex +} diff --git a/third_party/raft/scripts/fix.sh b/third_party/raft/scripts/fix.sh new file mode 100644 index 0000000000..ae53551b85 --- /dev/null +++ b/third_party/raft/scripts/fix.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -eo pipefail + +source ./scripts/test_lib.sh + +function mod_tidy_fix { + rm ./go.sum + go mod tidy || return 2 +} + +function go_fmt_fix { + go fmt -n . || return 2 +} + +log_callout -e "\\nFixing raft code for you...\\n" + +mod_tidy_fix || exit 2 +go_fmt_fix || exit 2 + +log_success -e "\\nSUCCESS: raft code is fixed :)" + diff --git a/third_party/raft/scripts/genproto.sh b/third_party/raft/scripts/genproto.sh new file mode 100644 index 0000000000..9f6fad7b43 --- /dev/null +++ b/third_party/raft/scripts/genproto.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Generate all etcd protobuf bindings. +# Run from repository root directory named etcd. +# +set -e +shopt -s globstar + +if ! [[ "$0" =~ scripts/genproto.sh ]]; then + echo "must be run from repository root" + exit 255 +fi + +source ./scripts/test_lib.sh + +if [[ $(protoc --version | cut -f2 -d' ') != "3.20.3" ]]; then + echo "could not find protoc 3.20.3, is it installed + in PATH?" + exit 255 +fi + +GOGEN_BIN=$(tool_get_bin google.golang.org/protobuf/cmd/protoc-gen-go) + +echo +echo "Resolved binary and packages versions:" +echo " - protoc-gen-go: ${GOGEN_BIN}" + +# directories containing protos to be built +DIRS="./raftpb" + +log_callout -e "\\nRunning protoc-gen-go proto generation..." + +for dir in ${DIRS}; do + pushd "${dir}" + protoc --go_out=. -I=".:${RAFT_ROOT_DIR}/..:${RAFT_ROOT_DIR}" \ + --plugin=protoc-gen-go="${GOGEN_BIN}" \ + --go_opt=paths=source_relative ./**/*.proto + + rm -f ./**/*.bak + gofmt -s -w ./**/*.pb.go + run_go_tool "golang.org/x/tools/cmd/goimports" -w ./**/*.pb.go + popd +done + +log_success -e "\\n./genproto SUCCESS" diff --git a/third_party/raft/scripts/test.sh b/third_party/raft/scripts/test.sh new file mode 100644 index 0000000000..b8ea371b31 --- /dev/null +++ b/third_party/raft/scripts/test.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash + +set -eo pipefail + +source ./scripts/test_lib.sh + +# set default GOARCH if not set +if [ -z "$GOARCH" ]; then + GOARCH=$(go env GOARCH); +fi + +# determine whether target supports race detection +if [ -z "${RACE}" ] ; then + if [ "$GOARCH" == "amd64" ]; then + RACE="--race" + else + RACE="--race=false" + fi +else + RACE="--race=${RACE:-true}" +fi + +# This options make sense for cases where SUT (System Under Test) is compiled by test. +COMMON_TEST_FLAGS=("${RACE}") +if [[ -n "${CPU}" ]]; then + COMMON_TEST_FLAGS+=("--cpu=${CPU}") +fi + +######### Code formatting checkers ############################################# + +# generic_checker [cmd...] +# executes given command in the current module, and clearly fails if it +# failed or returned output. +function generic_checker { + local cmd=("$@") + if ! output=$("${cmd[@]}"); then + echo "${output}" + log_error -e "FAIL: '${cmd[*]}' checking failed (!=0 return code)" + return 255 + fi + if [ -n "${output}" ]; then + echo "${output}" + log_error -e "FAIL: '${cmd[*]}' checking failed (printed output)" + return 255 + fi +} + +function go_fmt_for_package { + # We utilize 'go fmt' to find all files suitable for formatting, + # but reuse full power gofmt to perform just RO check. + go fmt -n . | sed 's| -w | -d |g' | sh +} + +function gofmt_pass { + generic_checker go_fmt_for_package +} + +function genproto_pass { + "${RAFT_ROOT_DIR}/scripts/verify_genproto.sh" +} + +######## VARIOUS CHECKERS ###################################################### + +function dump_deps_of_module() { + local module + if ! module=$(go list -m); then + return 255 + fi + go list -f "{{if not .Indirect}}{{if .Version}}{{.Path}},{{.Version}},${module}{{end}}{{end}}" -m all +} + +# Checks whether dependencies are consistent across modules +function dep_pass { + local all_dependencies + all_dependencies=$(dump_deps_of_module | sort) || return 2 + + local duplicates + duplicates=$(echo "${all_dependencies}" | cut -d ',' -f 1,2 | sort | uniq | cut -d ',' -f 1 | sort | uniq -d) || return 2 + + for dup in ${duplicates}; do + log_error "FAIL: inconsistent versions for depencency: ${dup}" + echo "${all_dependencies}" | grep "${dup}" | sed "s|\\([^,]*\\),\\([^,]*\\),\\([^,]*\\)| - \\1@\\2 from: \\3|g" + done + if [[ -n "${duplicates}" ]]; then + log_error "FAIL: inconsistent dependencies" + return 2 + else + log_success "SUCCESS: dependencies are consistent across modules" + fi +} + +function mod_tidy_for_module { + # Watch for upstream solution: https://github.com/golang/go/issues/27005 + local tmpModDir + tmpModDir=$(mktemp -d -t 'tmpModDir.XXXXXX') + cp "./go.mod" "${tmpModDir}" || return 2 + + # Guarantees keeping go.sum minimal + # If this is causing too much problems, we should + # stop controlling go.sum at all. + rm go.sum + go mod tidy || return 2 + + set +e + local tmpFileGoModInSync + diff -C 5 "${tmpModDir}/go.mod" "./go.mod" + tmpFileGoModInSync="$?" + + # Bring back initial state + mv "${tmpModDir}/go.mod" "./go.mod" + + if [ "${tmpFileGoModInSync}" -ne 0 ]; then + log_error "${PWD}/go.mod is not in sync with 'go mod tidy'" + return 255 + fi +} + +function mod_tidy_pass { + mod_tidy_for_module +} + +################# REGULAR TESTS ################################################ + +# run_unit_tests [pkgs] runs unit tests for a current module and givesn set of [pkgs] +function run_unit_tests { + shift 1 + # shellcheck disable=SC2086 + GOLANG_TEST_SHORT=true go test ./... -short -timeout="${TIMEOUT:-3m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" +} + +function unit_pass { + run_unit_tests "$@" +} + +########### MAIN ############################################################### + +function run_pass { + local pass="${1}" + shift 1 + log_callout -e "\\n'${pass}' started at $(date)" + if "${pass}_pass" "$@" ; then + log_success "'${pass}' completed at $(date)" + else + log_error "FAIL: '${pass}' failed at $(date)" + exit 255 + fi +} + +log_callout "Starting at: $(date)" +for pass in $PASSES; do + run_pass "${pass}" "${@}" +done + +log_success "SUCCESS" + diff --git a/third_party/raft/scripts/test_lib.sh b/third_party/raft/scripts/test_lib.sh new file mode 100644 index 0000000000..2472f1de9e --- /dev/null +++ b/third_party/raft/scripts/test_lib.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +ROOT_MODULE="go.etcd.io/raft" + +function set_root_dir { + RAFT_ROOT_DIR=$(go list -f '{{.Dir}}' "${ROOT_MODULE}/v3") +} + +set_root_dir + +#### Convenient IO methods ##### + +COLOR_RED='\033[0;31m' +COLOR_ORANGE='\033[0;33m' +COLOR_GREEN='\033[0;32m' +COLOR_LIGHTCYAN='\033[0;36m' +COLOR_BLUE='\033[0;94m' +COLOR_MAGENTA='\033[95m' +COLOR_BOLD='\033[1m' +COLOR_NONE='\033[0m' # No Color + + +function log_error { + >&2 echo -n -e "${COLOR_BOLD}${COLOR_RED}" + >&2 echo "$@" + >&2 echo -n -e "${COLOR_NONE}" +} + +function log_warning { + >&2 echo -n -e "${COLOR_ORANGE}" + >&2 echo "$@" + >&2 echo -n -e "${COLOR_NONE}" +} + +function log_callout { + >&2 echo -n -e "${COLOR_LIGHTCYAN}" + >&2 echo "$@" + >&2 echo -n -e "${COLOR_NONE}" +} + +function log_cmd { + >&2 echo -n -e "${COLOR_BLUE}" + >&2 echo "$@" + >&2 echo -n -e "${COLOR_NONE}" +} + +function log_success { + >&2 echo -n -e "${COLOR_GREEN}" + >&2 echo "$@" + >&2 echo -n -e "${COLOR_NONE}" +} + +function log_info { + >&2 echo -n -e "${COLOR_NONE}" + >&2 echo "$@" + >&2 echo -n -e "${COLOR_NONE}" +} + +# run_for_module [module] [cmd] +# executes given command in the given module for given pkgs. +# module_name - "." (in future: tests, client, server) +# cmd - cmd to be executed - that takes package as last argument +function run_for_module { + local module=${1:-"."} + shift 1 + ( + cd "${RAFT_ROOT_DIR}/${module}" && "$@" + ) +} + +# tool_pkg_dir [pkg] - returns absolute path to a directory that stores given pkg. +# The pkg versions must be defined in ./tools/mod directory. +function tool_pkg_dir { + run_for_module ./tools/mod go list -f '{{.Dir}}' "${1}" +} + +# tool_get_bin [tool] - returns absolute path to a tool binary (or returns error) +function tool_get_bin { + local tool="$1" + local pkg_part="$1" + if [[ "$tool" == *"@"* ]]; then + pkg_part=$(echo "${tool}" | cut -d'@' -f1) + # shellcheck disable=SC2086 + go install ${GOBINARGS:-} "${tool}" || return 2 + else + # shellcheck disable=SC2086 + run_for_module ./tools/mod go install ${GOBINARGS:-} "${tool}" || return 2 + fi + + # remove the version suffix, such as removing "/v3" from "go.etcd.io/etcd/v3". + local cmd_base_name + cmd_base_name=$(basename "${pkg_part}") + if [[ ${cmd_base_name} =~ ^v[0-9]*$ ]]; then + pkg_part=$(dirname "${pkg_part}") + fi + + run_for_module ./tools/mod go list -f '{{.Target}}' "${pkg_part}" +} + +# tool_get_bin [tool] +function run_go_tool { + local cmdbin + if ! cmdbin=$(GOARCH="" tool_get_bin "${1}"); then + log_warning "Failed to install tool '${1}'" + return 2 + fi + shift 1 + GOARCH="" "${cmdbin}" "$@" || return 2 +} diff --git a/third_party/raft/scripts/verify_genproto.sh b/third_party/raft/scripts/verify_genproto.sh new file mode 100644 index 0000000000..301af36be7 --- /dev/null +++ b/third_party/raft/scripts/verify_genproto.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# This scripts is automatically run by CI to prevent pull requests missing running genproto.sh +# after changing *.proto file. + +set -o errexit +set -o nounset +set -o pipefail + +tmpWorkDir=$(mktemp -d -t 'twd.XXXXXX') +mkdir "$tmpWorkDir/raft" +tmpWorkDir="$tmpWorkDir/raft" +cp -r . "$tmpWorkDir" +pushd "$tmpWorkDir" +git add -A +git commit -m init || true # maybe fail because nothing to commit +./scripts/genproto.sh +diff=$(git diff --numstat | awk '{print $3}') +popd +if [ -z "$diff" ]; then + echo "PASSED genproto-verification!" + exit 0 +fi +echo "Failed genproto-verification!" >&2 +printf "* Found changed files:\n%s\n" "$diff" >&2 +echo "* Please rerun genproto.sh after changing *.proto file" >&2 +echo "* Run ./scripts/genproto.sh" >&2 +exit 1 diff --git a/third_party/raft/state_trace.go b/third_party/raft/state_trace.go new file mode 100644 index 0000000000..caccd705b8 --- /dev/null +++ b/third_party/raft/state_trace.go @@ -0,0 +1,339 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build with_tla + +package raft + +import ( + "strconv" + "time" + + "go.etcd.io/raft/v3/raftpb" + "go.etcd.io/raft/v3/tracker" +) + +const StateTraceDeployed = true + +type stateMachineEventType int + +const ( + rsmInitState stateMachineEventType = iota + rsmBecomeCandidate + rsmBecomeFollower + rsmBecomeLeader + rsmCommit + rsmReplicate + rsmChangeConf + rsmApplyConfChange + rsmReady + rsmSendAppendEntriesRequest + rsmReceiveAppendEntriesRequest + rsmSendAppendEntriesResponse + rsmReceiveAppendEntriesResponse + rsmSendRequestVoteRequest + rsmReceiveRequestVoteRequest + rsmSendRequestVoteResponse + rsmReceiveRequestVoteResponse + rsmSendSnapshot + rsmReceiveSnapshot +) + +func (e stateMachineEventType) String() string { + return []string{ + "InitState", + "BecomeCandidate", + "BecomeFollower", + "BecomeLeader", + "Commit", + "Replicate", + "ChangeConf", + "ApplyConfChange", + "Ready", + "SendAppendEntriesRequest", + "ReceiveAppendEntriesRequest", + "SendAppendEntriesResponse", + "ReceiveAppendEntriesResponse", + "SendRequestVoteRequest", + "ReceiveRequestVoteRequest", + "SendRequestVoteResponse", + "ReceiveRequestVoteResponse", + "SendSnapshot", + "ReceiveSnapshot", + }[e] +} + +const ( + ConfChangeAddNewServer string = "AddNewServer" + ConfChangeRemoveServer string = "RemoveServer" + ConfChangeAddLearner string = "AddLearner" +) + +type TracingEvent struct { + Name string `json:"name"` + NodeID string `json:"nid"` + State TracingState `json:"state"` + Role string `json:"role"` + LogSize uint64 `json:"log"` + Conf [2][]string `json:"conf"` + Message *TracingMessage `json:"msg,omitempty"` + ConfChange *TracingConfChange `json:"cc,omitempty"` + Properties map[string]any `json:"prop,omitempty"` +} + +type TracingState struct { + Term uint64 `json:"term"` + Vote string `json:"vote"` + Commit uint64 `json:"commit"` +} + +type TracingMessage struct { + Type string `json:"type"` + Term uint64 `json:"term"` + From string `json:"from"` + To string `json:"to"` + EntryLength int `json:"entries"` + LogTerm uint64 `json:"logTerm"` + Index uint64 `json:"index"` + Commit uint64 `json:"commit"` + Vote string `json:"vote"` + Reject bool `json:"reject"` + RejectHint uint64 `json:"rejectHint"` +} + +type SingleConfChange struct { + NodeID string `json:"nid"` + Action string `json:"action"` +} + +type TracingConfChange struct { + Changes []SingleConfChange `json:"changes,omitempty"` + NewConf []string `json:"newconf,omitempty"` +} + +func makeTracingState(r *raft) TracingState { + hs := r.hardState() + return TracingState{ + Term: hs.GetTerm(), + Vote: strconv.FormatUint(hs.GetVote(), 10), + Commit: hs.GetCommit(), + } +} + +func makeTracingMessage(m *raftpb.Message) *TracingMessage { + if m == nil { + return nil + } + + logTerm := m.GetLogTerm() + entries := len(m.GetEntries()) + index := m.GetIndex() + if m.GetType() == raftpb.MsgSnap { + index = 0 + logTerm = 0 + entries = int(m.GetSnapshot().GetMetadata().GetIndex()) + } + return &TracingMessage{ + Type: m.GetType().String(), + Term: m.GetTerm(), + From: strconv.FormatUint(m.GetFrom(), 10), + To: strconv.FormatUint(m.GetTo(), 10), + EntryLength: entries, + LogTerm: logTerm, + Index: index, + Commit: m.GetCommit(), + Vote: strconv.FormatUint(m.GetVote(), 10), + Reject: m.GetReject(), + RejectHint: m.GetRejectHint(), + } +} + +type TraceLogger interface { + TraceEvent(*TracingEvent) +} + +func traceEvent(evt stateMachineEventType, r *raft, m *raftpb.Message, prop map[string]any) { + if r.traceLogger == nil { + return + } + + r.traceLogger.TraceEvent(&TracingEvent{ + Name: evt.String(), + NodeID: strconv.FormatUint(r.id, 10), + State: makeTracingState(r), + LogSize: r.raftLog.lastIndex(), + Conf: [2][]string{formatConf(r.trk.Voters[0].Slice()), formatConf(r.trk.Voters[1].Slice())}, + Role: r.state.String(), + Message: makeTracingMessage(m), + Properties: prop, + }) +} + +func traceNodeEvent(evt stateMachineEventType, r *raft) { + traceEvent(evt, r, nil, nil) +} + +func formatConf(s []uint64) []string { + if s == nil { + return []string{} + } + + r := make([]string, len(s)) + for i, v := range s { + r[i] = strconv.FormatUint(v, 10) + } + return r +} + +// Use following helper functions to trace specific state and/or +// transition at corresponding code lines +func traceInitState(r *raft) { + if r.traceLogger == nil { + return + } + + traceNodeEvent(rsmInitState, r) +} + +func traceReady(r *raft) { + traceNodeEvent(rsmReady, r) +} + +func traceCommit(r *raft) { + traceNodeEvent(rsmCommit, r) +} + +func traceReplicate(r *raft, es ...*raftpb.Entry) { + for i := range es { + if es[i].GetType() == raftpb.EntryNormal { + traceNodeEvent(rsmReplicate, r) + } + } +} + +func traceBecomeFollower(r *raft) { + traceNodeEvent(rsmBecomeFollower, r) +} + +func traceBecomeCandidate(r *raft) { + traceNodeEvent(rsmBecomeCandidate, r) +} + +func traceBecomeLeader(r *raft) { + traceNodeEvent(rsmBecomeLeader, r) +} + +func traceChangeConfEvent(cci raftpb.ConfChangeI, r *raft) { + cc2 := cci.AsV2() + cc := &TracingConfChange{ + Changes: []SingleConfChange{}, + NewConf: []string{}, + } + for _, c := range cc2.Changes { + switch c.GetType() { + case raftpb.ConfChangeAddNode: + cc.Changes = append(cc.Changes, SingleConfChange{ + NodeID: strconv.FormatUint(c.GetNodeId(), 10), + Action: ConfChangeAddNewServer, + }) + case raftpb.ConfChangeRemoveNode: + cc.Changes = append(cc.Changes, SingleConfChange{ + NodeID: strconv.FormatUint(c.GetNodeId(), 10), + Action: ConfChangeRemoveServer, + }) + case raftpb.ConfChangeAddLearnerNode: + cc.Changes = append(cc.Changes, SingleConfChange{ + NodeID: strconv.FormatUint(c.GetNodeId(), 10), + Action: ConfChangeAddLearner, + }) + } + } + + if len(cc.Changes) == 0 { + return + } + + p := map[string]any{} + p["cc"] = cc + traceEvent(rsmChangeConf, r, nil, p) +} + +func traceConfChangeEvent(cfg tracker.Config, r *raft) { + if r.traceLogger == nil { + return + } + + cc := &TracingConfChange{ + Changes: []SingleConfChange{}, + NewConf: formatConf(cfg.Voters[0].Slice()), + } + + p := map[string]any{} + p["cc"] = cc + traceEvent(rsmApplyConfChange, r, nil, p) +} + +func traceSendMessage(r *raft, m *raftpb.Message) { + if r.traceLogger == nil { + return + } + + prop := map[string]any{} + + var evt stateMachineEventType + switch m.GetType() { + case raftpb.MsgApp: + evt = rsmSendAppendEntriesRequest + if p, exist := r.trk.Progress[m.GetFrom()]; exist { + prop["match"] = p.Match + prop["next"] = p.Next + } + + case raftpb.MsgHeartbeat, raftpb.MsgSnap: + evt = rsmSendAppendEntriesRequest + case raftpb.MsgAppResp, raftpb.MsgHeartbeatResp: + evt = rsmSendAppendEntriesResponse + case raftpb.MsgVote: + evt = rsmSendRequestVoteRequest + case raftpb.MsgVoteResp: + evt = rsmSendRequestVoteResponse + default: + return + } + + traceEvent(evt, r, m, prop) +} + +func traceReceiveMessage(r *raft, m *raftpb.Message) { + if r.traceLogger == nil { + return + } + + var evt stateMachineEventType + switch m.GetType() { + case raftpb.MsgApp, raftpb.MsgHeartbeat, raftpb.MsgSnap: + evt = rsmReceiveAppendEntriesRequest + case raftpb.MsgAppResp, raftpb.MsgHeartbeatResp: + evt = rsmReceiveAppendEntriesResponse + case raftpb.MsgVote: + evt = rsmReceiveRequestVoteRequest + case raftpb.MsgVoteResp: + evt = rsmReceiveRequestVoteResponse + default: + return + } + + time.Sleep(time.Millisecond) // sleep 1ms to reduce time shift impact accross node + traceEvent(evt, r, m, nil) +} diff --git a/third_party/raft/state_trace_nop.go b/third_party/raft/state_trace_nop.go new file mode 100644 index 0000000000..f954c05620 --- /dev/null +++ b/third_party/raft/state_trace_nop.go @@ -0,0 +1,50 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !with_tla + +package raft + +import ( + "go.etcd.io/raft/v3/raftpb" + "go.etcd.io/raft/v3/tracker" +) + +const StateTraceDeployed = false + +type TraceLogger interface{} + +type TracingEvent struct{} + +func traceInitState(*raft) {} + +func traceReady(*raft) {} + +func traceCommit(*raft) {} + +func traceReplicate(*raft, ...*raftpb.Entry) {} + +func traceBecomeFollower(*raft) {} + +func traceBecomeCandidate(*raft) {} + +func traceBecomeLeader(*raft) {} + +func traceChangeConfEvent(raftpb.ConfChangeI, *raft) {} + +func traceConfChangeEvent(tracker.Config, *raft) {} + +func traceSendMessage(*raft, *raftpb.Message) {} + +func traceReceiveMessage(*raft, *raftpb.Message) {} diff --git a/third_party/raft/status.go b/third_party/raft/status.go new file mode 100644 index 0000000000..9bc019a71f --- /dev/null +++ b/third_party/raft/status.go @@ -0,0 +1,105 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "fmt" + + pb "go.etcd.io/raft/v3/raftpb" + "go.etcd.io/raft/v3/tracker" +) + +// Status contains information about this Raft peer and its view of the system. +// The Progress is only populated on the leader. +type Status struct { + BasicStatus + Config tracker.Config + Progress map[uint64]tracker.Progress +} + +// BasicStatus contains basic information about the Raft peer. It does not allocate. +type BasicStatus struct { + ID uint64 + + *pb.HardState + SoftState + + Applied uint64 + + LeadTransferee uint64 +} + +func getProgressCopy(r *raft) map[uint64]tracker.Progress { + m := make(map[uint64]tracker.Progress) + r.trk.Visit(func(id uint64, pr *tracker.Progress) { + p := *pr + p.Inflights = pr.Inflights.Clone() + pr = nil + + m[id] = p + }) + return m +} + +func getBasicStatus(r *raft) BasicStatus { + s := BasicStatus{ + ID: r.id, + LeadTransferee: r.leadTransferee, + } + s.HardState = r.hardState() + s.SoftState = r.softState() + s.Applied = r.raftLog.applied + return s +} + +// getStatus gets a copy of the current raft status. +func getStatus(r *raft) Status { + var s Status + s.BasicStatus = getBasicStatus(r) + if s.RaftState == StateLeader { + s.Progress = getProgressCopy(r) + } + s.Config = r.trk.Config.Clone() + return s +} + +// MarshalJSON translates the raft status into JSON. +// TODO: try to simplify this by introducing ID type into raft +func (s Status) MarshalJSON() ([]byte, error) { + j := fmt.Sprintf(`{"id":"%x","term":%d,"vote":"%x","commit":%d,"lead":"%x","raftState":%q,"applied":%d,"progress":{`, + s.ID, s.GetTerm(), s.GetVote(), s.GetCommit(), s.Lead, s.RaftState, s.Applied) + + if len(s.Progress) == 0 { + j += "}," + } else { + for k, v := range s.Progress { + subj := fmt.Sprintf(`"%x":{"match":%d,"next":%d,"state":%q},`, k, v.Match, v.Next, v.State) + j += subj + } + // remove the trailing "," + j = j[:len(j)-1] + "}," + } + + j += fmt.Sprintf(`"leadtransferee":"%x"}`, s.LeadTransferee) + return []byte(j), nil +} + +func (s Status) String() string { + b, err := s.MarshalJSON() + if err != nil { + getLogger().Panicf("unexpected error: %v", err) + } + return string(b) +} diff --git a/third_party/raft/storage.go b/third_party/raft/storage.go new file mode 100644 index 0000000000..8027283833 --- /dev/null +++ b/third_party/raft/storage.go @@ -0,0 +1,326 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "errors" + "sync" + + "google.golang.org/protobuf/proto" + + pb "go.etcd.io/raft/v3/raftpb" +) + +// ErrCompacted is returned by Storage.Entries/Compact when a requested +// index is unavailable because it predates the last snapshot. +var ErrCompacted = errors.New("requested index is unavailable due to compaction") + +// ErrSnapOutOfDate is returned by Storage.CreateSnapshot when a requested +// index is older than the existing snapshot. +var ErrSnapOutOfDate = errors.New("requested index is older than the existing snapshot") + +// ErrUnavailable is returned by Storage interface when the requested log entries +// are unavailable. +var ErrUnavailable = errors.New("requested entry at index is unavailable") + +// ErrSnapshotTemporarilyUnavailable is returned by the Storage interface when the required +// snapshot is temporarily unavailable. +var ErrSnapshotTemporarilyUnavailable = errors.New("snapshot is temporarily unavailable") + +// Storage is an interface that may be implemented by the application +// to retrieve log entries from storage. +// +// If any Storage method returns an error, the raft instance will +// become inoperable and refuse to participate in elections; the +// application is responsible for cleanup and recovery in this case. +type Storage interface { + // TODO(tbg): split this into two interfaces, LogStorage and StateStorage. + + // InitialState returns the saved HardState and ConfState information. + // The returned HardState may be nil to indicate that no HardState has been + // persisted yet; IsEmptyHardState treats nil as empty. + // The returned ConfState must not be nil; if no ConfState has been + // persisted yet, return an empty ConfState instead. + InitialState() (*pb.HardState, *pb.ConfState, error) + + // Entries returns a slice of consecutive log entries in the range [lo, hi), + // starting from lo. The maxSize limits the total size of the log entries + // returned, but Entries returns at least one entry if any. + // + // The caller of Entries owns the returned slice, and may append to it. The + // individual entries in the slice must not be mutated, neither by the Storage + // implementation nor the caller. Note that raft may forward these entries + // back to the application via Ready struct, so the corresponding handler must + // not mutate entries either (see comments in Ready struct). + // + // Since the caller may append to the returned slice, Storage implementation + // must protect its state from corruption that such appends may cause. For + // example, common ways to do so are: + // - allocate the slice before returning it (safest option), + // - return a slice protected by Go full slice expression, which causes + // copying on appends (see MemoryStorage). + // + // Returns ErrCompacted if entry lo has been compacted, or ErrUnavailable if + // encountered an unavailable entry in [lo, hi). + Entries(lo, hi, maxSize uint64) ([]*pb.Entry, error) + + // Term returns the term of entry i, which must be in the range + // [FirstIndex()-1, LastIndex()]. The term of the entry before + // FirstIndex is retained for matching purposes even though the + // rest of that entry may not be available. + Term(i uint64) (uint64, error) + // LastIndex returns the index of the last entry in the log. + LastIndex() (uint64, error) + // FirstIndex returns the index of the first log entry that is + // possibly available via Entries (older entries have been incorporated + // into the latest Snapshot; if storage only contains the dummy entry the + // first log entry is not available). + FirstIndex() (uint64, error) + // Snapshot returns the most recent snapshot. + // If snapshot is temporarily unavailable, it should return ErrSnapshotTemporarilyUnavailable, + // so raft state machine could know that Storage needs some time to prepare + // snapshot and call Snapshot later. + Snapshot() (*pb.Snapshot, error) +} + +type inMemStorageCallStats struct { + initialState, firstIndex, lastIndex, entries, term, snapshot int +} + +// MemoryStorage implements the Storage interface backed by an +// in-memory array. +type MemoryStorage struct { + // Protects access to all fields. Most methods of MemoryStorage are + // run on the raft goroutine, but Append() is run on an application + // goroutine. + sync.Mutex + + hardState *pb.HardState + snapshot *pb.Snapshot + // ents[i] has raft log position i+ms.snapshot.GetMetadata().GetIndex() + ents []*pb.Entry + + callStats inMemStorageCallStats +} + +// NewMemoryStorage creates an empty MemoryStorage. +func NewMemoryStorage() *MemoryStorage { + ms := &MemoryStorage{ + // When starting from scratch populate the list with a dummy entry at term zero. + ents: []*pb.Entry{{}}, + } + ms.snapshot = pb.EnsureSnapshot(ms.snapshot) + return ms +} + +// InitialState implements the Storage interface. +func (ms *MemoryStorage) InitialState() (*pb.HardState, *pb.ConfState, error) { + ms.callStats.initialState++ + cs := ms.snapshot.GetMetadata().GetConfState() + cs = pb.EnsureConfState(cs) + return ms.hardState, cs, nil +} + +// SetHardState saves the current HardState. +func (ms *MemoryStorage) SetHardState(st *pb.HardState) error { + ms.Lock() + defer ms.Unlock() + ms.hardState = st + return nil +} + +// Entries implements the Storage interface. +func (ms *MemoryStorage) Entries(lo, hi, maxSize uint64) ([]*pb.Entry, error) { + ms.Lock() + defer ms.Unlock() + ms.callStats.entries++ + offset := ms.ents[0].GetIndex() + if lo <= offset { + return nil, ErrCompacted + } + if hi > ms.lastIndex()+1 { + getLogger().Panicf("entries' hi(%d) is out of bound lastindex(%d)", hi, ms.lastIndex()) + } + // only contains dummy entries. + if len(ms.ents) == 1 { + return nil, ErrUnavailable + } + + ents := limitSize(ms.ents[lo-offset:hi-offset], entryEncodingSize(maxSize)) + // NB: use the full slice expression to limit what the caller can do with the + // returned slice. For example, an append will reallocate and copy this slice + // instead of corrupting the neighbouring ms.ents. + return ents[:len(ents):len(ents)], nil +} + +// Term implements the Storage interface. +func (ms *MemoryStorage) Term(i uint64) (uint64, error) { + ms.Lock() + defer ms.Unlock() + ms.callStats.term++ + offset := ms.ents[0].GetIndex() + if i < offset { + return 0, ErrCompacted + } + if int(i-offset) >= len(ms.ents) { + return 0, ErrUnavailable + } + return ms.ents[i-offset].GetTerm(), nil +} + +// LastIndex implements the Storage interface. +func (ms *MemoryStorage) LastIndex() (uint64, error) { + ms.Lock() + defer ms.Unlock() + ms.callStats.lastIndex++ + return ms.lastIndex(), nil +} + +func (ms *MemoryStorage) lastIndex() uint64 { + return ms.ents[0].GetIndex() + uint64(len(ms.ents)) - 1 +} + +// FirstIndex implements the Storage interface. +func (ms *MemoryStorage) FirstIndex() (uint64, error) { + ms.Lock() + defer ms.Unlock() + ms.callStats.firstIndex++ + return ms.firstIndex(), nil +} + +func (ms *MemoryStorage) firstIndex() uint64 { + return ms.ents[0].GetIndex() + 1 +} + +// Snapshot implements the Storage interface. +func (ms *MemoryStorage) Snapshot() (*pb.Snapshot, error) { + ms.Lock() + defer ms.Unlock() + ms.callStats.snapshot++ + ms.snapshot = pb.EnsureSnapshot(ms.snapshot) + return proto.Clone(ms.snapshot).(*pb.Snapshot), nil +} + +// ApplySnapshot overwrites the contents of this Storage object with +// those of the given snapshot. +func (ms *MemoryStorage) ApplySnapshot(snap *pb.Snapshot) error { + ms.Lock() + defer ms.Unlock() + + snap = pb.EnsureSnapshot(snap) + + //handle check for old snapshot being applied + msIndex := ms.snapshot.GetMetadata().GetIndex() + snapIndex := snap.GetMetadata().GetIndex() + // During bootstrap, applications (e.g., etcd) may initialize only the + // ConfState in the snapshot. In this case, both the snapshot index and + // term are 0. + if msIndex != 0 && msIndex >= snapIndex { + return ErrSnapOutOfDate + } + + ms.snapshot = proto.Clone(snap).(*pb.Snapshot) + ms.ents = []*pb.Entry{{Term: new(snap.GetMetadata().GetTerm()), Index: new(snap.GetMetadata().GetIndex())}} + return nil +} + +// CreateSnapshot makes a snapshot which can be retrieved with Snapshot() and +// can be used to reconstruct the state at that point. +// If any configuration changes have been made since the last compaction, +// the result of the last ApplyConfChange must be passed in. +func (ms *MemoryStorage) CreateSnapshot(i uint64, cs *pb.ConfState, data []byte) (*pb.Snapshot, error) { + ms.Lock() + defer ms.Unlock() + if i <= ms.snapshot.GetMetadata().GetIndex() { + return nil, ErrSnapOutOfDate + } + + offset := ms.ents[0].GetIndex() + if i > ms.lastIndex() { + getLogger().Panicf("snapshot %d is out of bound lastindex(%d)", i, ms.lastIndex()) + } + + ms.snapshot = pb.EnsureSnapshot(ms.snapshot) + ms.snapshot.Metadata.Index = new(i) + ms.snapshot.Metadata.Term = new(ms.ents[i-offset].GetTerm()) + if cs != nil { + ms.snapshot.Metadata.ConfState = proto.Clone(cs).(*pb.ConfState) + } + ms.snapshot.Data = data + return proto.Clone(ms.snapshot).(*pb.Snapshot), nil +} + +// Compact discards all log entries prior to compactIndex. +// It is the application's responsibility to not attempt to compact an index +// greater than raftLog.applied. +func (ms *MemoryStorage) Compact(compactIndex uint64) error { + ms.Lock() + defer ms.Unlock() + offset := ms.ents[0].GetIndex() + if compactIndex <= offset { + return ErrCompacted + } + if compactIndex > ms.lastIndex() { + getLogger().Panicf("compact %d is out of bound lastindex(%d)", compactIndex, ms.lastIndex()) + } + + i := compactIndex - offset + // NB: allocate a new slice instead of reusing the old ms.ents. Entries in + // ms.ents are immutable, and can be referenced from outside MemoryStorage + // through slices returned by ms.Entries(). + ents := make([]*pb.Entry, 1, uint64(len(ms.ents))-i) + ents[0] = &pb.Entry{Index: new(ms.ents[i].GetIndex()), Term: new(ms.ents[i].GetTerm())} + ents = append(ents, ms.ents[i+1:]...) + ms.ents = ents + return nil +} + +// Append the new entries to storage. +// TODO (xiangli): ensure the entries are continuous and +// entries[0].Index > ms.entries[0].Index +func (ms *MemoryStorage) Append(entries []*pb.Entry) error { + if len(entries) == 0 { + return nil + } + + ms.Lock() + defer ms.Unlock() + + first := ms.firstIndex() + last := entries[0].GetIndex() + uint64(len(entries)) - 1 + + // shortcut if there is no new entry. + if last < first { + return nil + } + // truncate compacted entries + if first > entries[0].GetIndex() { + entries = entries[first-entries[0].GetIndex():] + } + + offset := entries[0].GetIndex() - ms.ents[0].GetIndex() + switch { + case uint64(len(ms.ents)) > offset: + // NB: full slice expression protects ms.ents at index >= offset from + // rewrites, as they may still be referenced from outside MemoryStorage. + ms.ents = append(ms.ents[:offset:offset], entries...) + case uint64(len(ms.ents)) == offset: + ms.ents = append(ms.ents, entries...) + default: + getLogger().Panicf("missing log entry [last: %d, append at: %d]", + ms.lastIndex(), entries[0].GetIndex()) + } + return nil +} diff --git a/third_party/raft/storage_test.go b/third_party/raft/storage_test.go new file mode 100644 index 0000000000..6bef3ff10f --- /dev/null +++ b/third_party/raft/storage_test.go @@ -0,0 +1,277 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "math" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + pb "go.etcd.io/raft/v3/raftpb" +) + +func TestStorageTerm(t *testing.T) { + ents := index(3).terms(3, 4, 5) + tests := []struct { + i uint64 + + werr error + wterm uint64 + wpanic bool + }{ + {2, ErrCompacted, 0, false}, + {3, nil, 3, false}, + {4, nil, 4, false}, + {5, nil, 5, false}, + {6, ErrUnavailable, 0, false}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + s := &MemoryStorage{ents: ents} + + if tt.wpanic { + require.Panics(t, func() { + _, _ = s.Term(tt.i) + }) + } + term, err := s.Term(tt.i) + require.Equal(t, tt.werr, err) + require.Equal(t, tt.wterm, term) + }) + } +} + +func TestStorageEntries(t *testing.T) { + ents := index(3).terms(3, 4, 5, 6) + tests := []struct { + lo, hi, maxsize uint64 + + werr error + wentries []*pb.Entry + }{ + {2, 6, math.MaxUint64, ErrCompacted, nil}, + {3, 4, math.MaxUint64, ErrCompacted, nil}, + {4, 5, math.MaxUint64, nil, index(4).terms(4)}, + {4, 6, math.MaxUint64, nil, index(4).terms(4, 5)}, + {4, 7, math.MaxUint64, nil, index(4).terms(4, 5, 6)}, + // even if maxsize is zero, the first entry should be returned + {4, 7, 0, nil, index(4).terms(4)}, + // limit to 2 + {4, 7, uint64(proto.Size(ents[1]) + proto.Size(ents[2])), nil, index(4).terms(4, 5)}, + // limit to 2 + {4, 7, uint64(proto.Size(ents[1]) + proto.Size(ents[2]) + proto.Size(ents[3])/2), nil, index(4).terms(4, 5)}, + {4, 7, uint64(proto.Size(ents[1]) + proto.Size(ents[2]) + proto.Size(ents[3]) - 1), nil, index(4).terms(4, 5)}, + // all + {4, 7, uint64(proto.Size(ents[1]) + proto.Size(ents[2]) + proto.Size(ents[3])), nil, index(4).terms(4, 5, 6)}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + s := &MemoryStorage{ents: ents} + entries, err := s.Entries(tt.lo, tt.hi, tt.maxsize) + require.Equal(t, tt.werr, err) + requireEqualEntries(t, tt.wentries, entries) + }) + } +} + +func TestStorageLastIndex(t *testing.T) { + ents := index(3).terms(3, 4, 5) + s := &MemoryStorage{ents: ents} + + last, err := s.LastIndex() + require.NoError(t, err) + require.Equal(t, uint64(5), last) + + require.NoError(t, s.Append(index(6).terms(5))) + last, err = s.LastIndex() + require.NoError(t, err) + require.Equal(t, uint64(6), last) +} + +func TestStorageFirstIndex(t *testing.T) { + ents := index(3).terms(3, 4, 5) + s := &MemoryStorage{ents: ents} + + first, err := s.FirstIndex() + require.NoError(t, err) + require.Equal(t, uint64(4), first) + + require.NoError(t, s.Compact(4)) + first, err = s.FirstIndex() + require.NoError(t, err) + require.Equal(t, uint64(5), first) +} + +func TestStorageCompact(t *testing.T) { + ents := index(3).terms(3, 4, 5) + tests := []struct { + i uint64 + + werr error + windex uint64 + wterm uint64 + wlen int + }{ + {2, ErrCompacted, 3, 3, 3}, + {3, ErrCompacted, 3, 3, 3}, + {4, nil, 4, 4, 2}, + {5, nil, 5, 5, 1}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + s := &MemoryStorage{ents: ents} + require.Equal(t, tt.werr, s.Compact(tt.i)) + require.Equal(t, tt.windex, s.ents[0].GetIndex()) + require.Equal(t, tt.wterm, s.ents[0].GetTerm()) + require.Equal(t, tt.wlen, len(s.ents)) + }) + } +} + +func TestStorageCreateSnapshot(t *testing.T) { + ents := index(3).terms(3, 4, 5) + cs := &pb.ConfState{Voters: []uint64{1, 2, 3}} + data := []byte("data") + + tests := []struct { + i uint64 + + werr error + wsnap *pb.Snapshot + }{ + {4, nil, &pb.Snapshot{Data: data, Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(4)), ConfState: &pb.ConfState{Voters: []uint64{1, 2, 3}}}}}, + {5, nil, &pb.Snapshot{Data: data, Metadata: &pb.SnapshotMetadata{Index: new(uint64(5)), Term: new(uint64(5)), ConfState: &pb.ConfState{Voters: []uint64{1, 2, 3}}}}}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + s := &MemoryStorage{ents: ents} + snap, err := s.CreateSnapshot(tt.i, cs, data) + require.Equal(t, tt.werr, err) + require.True(t, proto.Equal(tt.wsnap, snap)) + }) + } +} + +func TestStorageAppend(t *testing.T) { + ents := index(3).terms(3, 4, 5) + tests := []struct { + entries []*pb.Entry + + werr error + wentries []*pb.Entry + }{ + { + index(1).terms(1, 2), + nil, + index(3).terms(3, 4, 5), + }, + { + index(3).terms(3, 4, 5), + nil, + index(3).terms(3, 4, 5), + }, + { + index(3).terms(3, 6, 6), + nil, + index(3).terms(3, 6, 6), + }, + { + index(3).terms(3, 4, 5, 5), + nil, + index(3).terms(3, 4, 5, 5), + }, + // Truncate incoming entries, truncate the existing entries and append. + { + index(2).terms(3, 3, 5), + nil, + index(3).terms(3, 5), + }, + // Truncate the existing entries and append. + { + index(4).terms(5), + nil, + index(3).terms(3, 5), + }, + // Direct append. + { + index(6).terms(5), + nil, + index(3).terms(3, 4, 5, 5), + }, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + s := &MemoryStorage{ents: ents} + require.Equal(t, tt.werr, s.Append(tt.entries)) + require.Equal(t, tt.wentries, s.ents) + }) + } +} + +func TestStorageApplySnapshot(t *testing.T) { + cs := &pb.ConfState{Voters: []uint64{1, 2, 3}} + data := []byte("data") + + testCases := []struct { + name string + snapshots []*pb.Snapshot + expectedError error + }{ + { + name: "normal case", + snapshots: []*pb.Snapshot{ + {Data: data, Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(4)), ConfState: cs}}, + }, + expectedError: nil, + }, + { + name: "snapshot out of date", + snapshots: []*pb.Snapshot{ + {Data: data, Metadata: &pb.SnapshotMetadata{Index: new(uint64(4)), Term: new(uint64(4)), ConfState: cs}}, + {Data: data, Metadata: &pb.SnapshotMetadata{Index: new(uint64(3)), Term: new(uint64(3)), ConfState: cs}}, + }, + expectedError: ErrSnapOutOfDate, + }, + { + name: "bootstrap with confState", + snapshots: []*pb.Snapshot{ + {Data: data, Metadata: &pb.SnapshotMetadata{ConfState: cs}}, + }, + expectedError: nil, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + s := NewMemoryStorage() + var err error + for _, sn := range tt.snapshots { + err = s.ApplySnapshot(sn) + } + if tt.expectedError != nil { + require.Equal(t, tt.expectedError, err) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/third_party/raft/testdata/async_storage_writes.txt b/third_party/raft/testdata/async_storage_writes.txt new file mode 100644 index 0000000000..02e59e5a80 --- /dev/null +++ b/third_party/raft/testdata/async_storage_writes.txt @@ -0,0 +1,869 @@ +# Build a pipeline of in-progress entries to append. Let the pipeline grow to a +# depth of 2 before we start allowing work on the append thread to complete. + +add-nodes 3 voters=(1,2,3) index=10 async-storage-writes=true +---- +INFO 1 switched to configuration voters=(1 2 3) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1,2,3], term: 0, commit: 10, applied: 10, lastindex: 10, lastterm: 1] +INFO 2 switched to configuration voters=(1 2 3) +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [1,2,3], term: 0, commit: 10, applied: 10, lastindex: 10, lastterm: 1] +INFO 3 switched to configuration voters=(1 2 3) +INFO 3 became follower at term 0 +INFO newRaft 3 [peers: [1,2,3], term: 0, commit: 10, applied: 10, lastindex: 10, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 +INFO 1 [logterm: 1, index: 10] sent MsgVote request to 2 at term 1 +INFO 1 [logterm: 1, index: 10] sent MsgVote request to 3 at term 1 + +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Lead:0 State:StateCandidate + HardState Term:1 Vote:1 Commit:10 + Messages: + 1->2 MsgVote Term:1 Log:1/10 + 1->3 MsgVote Term:1 Log:1/10 + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Responses:[ + 1->1 MsgVoteResp Term:1 Log:0/0 + ] +> 2 receiving messages + 1->2 MsgVote Term:1 Log:1/10 + INFO 2 [term: 0] received a MsgVote message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + INFO 2 [logterm: 1, index: 10, vote: 0] cast MsgVote for 1 [logterm: 1, index: 10] at term 1 +> 3 receiving messages + 1->3 MsgVote Term:1 Log:1/10 + INFO 3 [term: 0] received a MsgVote message with higher term from 1 [term: 1] + INFO 3 became follower at term 1 + INFO 3 [logterm: 1, index: 10, vote: 0] cast MsgVote for 1 [logterm: 1, index: 10] at term 1 +> 1 processing append thread + Processing: + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 + Responses: + 1->1 MsgVoteResp Term:1 Log:0/0 +> 2 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:10 + Messages: + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Responses:[ + 2->1 MsgVoteResp Term:1 Log:0/0 + ] +> 3 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:10 + Messages: + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Responses:[ + 3->1 MsgVoteResp Term:1 Log:0/0 + ] +> 1 receiving messages + 1->1 MsgVoteResp Term:1 Log:0/0 + INFO 1 received MsgVoteResp from 1 at term 1 + INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections +> 2 processing append thread + Processing: + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 + Responses: + 2->1 MsgVoteResp Term:1 Log:0/0 +> 3 processing append thread + Processing: + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 + Responses: + 3->1 MsgVoteResp Term:1 Log:0/0 +> 1 receiving messages + 2->1 MsgVoteResp Term:1 Log:0/0 + INFO 1 received MsgVoteResp from 2 at term 1 + INFO 1 has received 2 MsgVoteResp votes and 0 vote rejections + INFO 1 became leader at term 1 + 3->1 MsgVoteResp Term:1 Log:0/0 +> 1 handling Ready + Ready MustSync=true: + Lead:1 State:StateLeader + Entries: + 1/11 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] + 1->3 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] + 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ + 1->1 MsgAppResp Term:1 Log:0/11 + AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11 + ] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] +> 1 processing append thread + Processing: + 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] + Responses: + 1->1 MsgAppResp Term:1 Log:0/11 + AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11 +> 2 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + Entries: + 1/11 EntryNormal "" + Messages: + 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ + 2->1 MsgAppResp Term:1 Log:0/11 + AppendThread->2 MsgStorageAppendResp Term:1 Log:1/11 + ] +> 3 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + Entries: + 1/11 EntryNormal "" + Messages: + 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ + 3->1 MsgAppResp Term:1 Log:0/11 + AppendThread->3 MsgStorageAppendResp Term:1 Log:1/11 + ] +> 1 receiving messages + 1->1 MsgAppResp Term:1 Log:0/11 + AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11 +> 2 processing append thread + Processing: + 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] + Responses: + 2->1 MsgAppResp Term:1 Log:0/11 + AppendThread->2 MsgStorageAppendResp Term:1 Log:1/11 +> 3 processing append thread + Processing: + 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] + Responses: + 3->1 MsgAppResp Term:1 Log:0/11 + AppendThread->3 MsgStorageAppendResp Term:1 Log:1/11 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/11 + 3->1 MsgAppResp Term:1 Log:0/11 +> 2 receiving messages + AppendThread->2 MsgStorageAppendResp Term:1 Log:1/11 +> 3 receiving messages + AppendThread->3 MsgStorageAppendResp Term:1 Log:1/11 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:11 + CommittedEntries: + 1/11 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/11 Commit:11 + 1->3 MsgApp Term:1 Log:1/11 Commit:11 + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 + 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ + ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] + ] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/11 Commit:11 +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/11 Commit:11 +> 1 processing append thread + Processing: + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 + Responses: +> 1 processing apply thread + Processing: + 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] + Responses: + ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:11 + CommittedEntries: + 1/11 EntryNormal "" + Messages: + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses:[ + 2->1 MsgAppResp Term:1 Log:0/11 + ] + 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ + ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] + ] +> 3 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:11 + CommittedEntries: + 1/11 EntryNormal "" + Messages: + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses:[ + 3->1 MsgAppResp Term:1 Log:0/11 + ] + 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ + ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] + ] +> 1 receiving messages + ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] +> 2 processing append thread + Processing: + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 + Responses: + 2->1 MsgAppResp Term:1 Log:0/11 +> 3 processing append thread + Processing: + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 + Responses: + 3->1 MsgAppResp Term:1 Log:0/11 +> 2 processing apply thread + Processing: + 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] + Responses: + ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] +> 3 processing apply thread + Processing: + 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] + Responses: + ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/11 + 3->1 MsgAppResp Term:1 Log:0/11 +> 2 receiving messages + ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] +> 3 receiving messages + ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] + +propose 1 prop_1 +---- +ok + +process-ready 1 2 3 +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/12 EntryNormal "prop_1" + Messages: + 1->2 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1"] + 1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1"] + 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ + 1->1 MsgAppResp Term:1 Log:0/12 + AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12 + ] +> 2 handling Ready + +> 3 handling Ready + + +deliver-msgs 1 2 3 +---- +1->2 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1"] +1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1"] + +process-ready 1 2 3 +---- +> 1 handling Ready + +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/12 EntryNormal "prop_1" + Messages: + 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ + 2->1 MsgAppResp Term:1 Log:0/12 + AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12 + ] +> 3 handling Ready + Ready MustSync=true: + Entries: + 1/12 EntryNormal "prop_1" + Messages: + 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ + 3->1 MsgAppResp Term:1 Log:0/12 + AppendThread->3 MsgStorageAppendResp Term:1 Log:1/12 + ] + +propose 1 prop_2 +---- +ok + +process-ready 1 2 3 +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/13 EntryNormal "prop_2" + Messages: + 1->2 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "prop_2"] + 1->3 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "prop_2"] + 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ + 1->1 MsgAppResp Term:1 Log:0/13 + AppendThread->1 MsgStorageAppendResp Term:1 Log:1/13 + ] +> 2 handling Ready + +> 3 handling Ready + + +deliver-msgs 1 2 3 +---- +1->2 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "prop_2"] +1->3 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "prop_2"] + +process-ready 1 2 3 +---- +> 1 handling Ready + +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/13 EntryNormal "prop_2" + Messages: + 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ + 2->1 MsgAppResp Term:1 Log:0/13 + AppendThread->2 MsgStorageAppendResp Term:1 Log:1/13 + ] +> 3 handling Ready + Ready MustSync=true: + Entries: + 1/13 EntryNormal "prop_2" + Messages: + 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ + 3->1 MsgAppResp Term:1 Log:0/13 + AppendThread->3 MsgStorageAppendResp Term:1 Log:1/13 + ] + +process-append-thread 1 2 3 +---- +> 1 processing append thread + Processing: + 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] + Responses: + 1->1 MsgAppResp Term:1 Log:0/12 + AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12 +> 2 processing append thread + Processing: + 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] + Responses: + 2->1 MsgAppResp Term:1 Log:0/12 + AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12 +> 3 processing append thread + Processing: + 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] + Responses: + 3->1 MsgAppResp Term:1 Log:0/12 + AppendThread->3 MsgStorageAppendResp Term:1 Log:1/12 + +deliver-msgs 1 2 3 +---- +1->1 MsgAppResp Term:1 Log:0/12 +AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12 +2->1 MsgAppResp Term:1 Log:0/12 +3->1 MsgAppResp Term:1 Log:0/12 +AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12 +AppendThread->3 MsgStorageAppendResp Term:1 Log:1/12 + +propose 1 prop_3 +---- +ok + +process-ready 1 2 3 +---- +> 1 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:12 + Entries: + 1/14 EntryNormal "prop_3" + CommittedEntries: + 1/12 EntryNormal "prop_1" + Messages: + 1->2 MsgApp Term:1 Log:1/13 Commit:12 + 1->3 MsgApp Term:1 Log:1/13 Commit:12 + 1->2 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"] + 1->3 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"] + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[ + 1->1 MsgAppResp Term:1 Log:0/14 + AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14 + ] + 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ + ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] + ] +> 2 handling Ready + +> 3 handling Ready + + +deliver-msgs 1 2 3 +---- +1->2 MsgApp Term:1 Log:1/13 Commit:12 +1->2 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"] +1->3 MsgApp Term:1 Log:1/13 Commit:12 +1->3 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"] + +process-ready 1 2 3 +---- +> 1 handling Ready + +> 2 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:12 + Entries: + 1/14 EntryNormal "prop_3" + CommittedEntries: + 1/12 EntryNormal "prop_1" + Messages: + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[ + 2->1 MsgAppResp Term:1 Log:0/13 + 2->1 MsgAppResp Term:1 Log:0/14 + AppendThread->2 MsgStorageAppendResp Term:1 Log:1/14 + ] + 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ + ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] + ] +> 3 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:12 + Entries: + 1/14 EntryNormal "prop_3" + CommittedEntries: + 1/12 EntryNormal "prop_1" + Messages: + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[ + 3->1 MsgAppResp Term:1 Log:0/13 + 3->1 MsgAppResp Term:1 Log:0/14 + AppendThread->3 MsgStorageAppendResp Term:1 Log:1/14 + ] + 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ + ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] + ] + +process-append-thread 1 2 3 +---- +> 1 processing append thread + Processing: + 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] + Responses: + 1->1 MsgAppResp Term:1 Log:0/13 + AppendThread->1 MsgStorageAppendResp Term:1 Log:1/13 +> 2 processing append thread + Processing: + 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] + Responses: + 2->1 MsgAppResp Term:1 Log:0/13 + AppendThread->2 MsgStorageAppendResp Term:1 Log:1/13 +> 3 processing append thread + Processing: + 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] + Responses: + 3->1 MsgAppResp Term:1 Log:0/13 + AppendThread->3 MsgStorageAppendResp Term:1 Log:1/13 + +deliver-msgs 1 2 3 +---- +1->1 MsgAppResp Term:1 Log:0/13 +AppendThread->1 MsgStorageAppendResp Term:1 Log:1/13 +2->1 MsgAppResp Term:1 Log:0/13 +3->1 MsgAppResp Term:1 Log:0/13 +AppendThread->2 MsgStorageAppendResp Term:1 Log:1/13 +AppendThread->3 MsgStorageAppendResp Term:1 Log:1/13 + +propose 1 prop_4 +---- +ok + +process-ready 1 2 3 +---- +> 1 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:13 + Entries: + 1/15 EntryNormal "prop_4" + CommittedEntries: + 1/13 EntryNormal "prop_2" + Messages: + 1->2 MsgApp Term:1 Log:1/14 Commit:13 + 1->3 MsgApp Term:1 Log:1/14 Commit:13 + 1->2 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"] + 1->3 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"] + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[ + 1->1 MsgAppResp Term:1 Log:0/15 + AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15 + ] + 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ + ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] + ] +> 2 handling Ready + +> 3 handling Ready + + +deliver-msgs 1 2 3 +---- +1->2 MsgApp Term:1 Log:1/14 Commit:13 +1->2 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"] +1->3 MsgApp Term:1 Log:1/14 Commit:13 +1->3 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"] + +process-ready 1 2 3 +---- +> 1 handling Ready + +> 2 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:13 + Entries: + 1/15 EntryNormal "prop_4" + CommittedEntries: + 1/13 EntryNormal "prop_2" + Messages: + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[ + 2->1 MsgAppResp Term:1 Log:0/14 + 2->1 MsgAppResp Term:1 Log:0/15 + AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 + ] + 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ + ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] + ] +> 3 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:13 + Entries: + 1/15 EntryNormal "prop_4" + CommittedEntries: + 1/13 EntryNormal "prop_2" + Messages: + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[ + 3->1 MsgAppResp Term:1 Log:0/14 + 3->1 MsgAppResp Term:1 Log:0/15 + AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15 + ] + 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ + ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] + ] + +process-append-thread 1 2 3 +---- +> 1 processing append thread + Processing: + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] + Responses: + 1->1 MsgAppResp Term:1 Log:0/14 + AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14 +> 2 processing append thread + Processing: + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] + Responses: + 2->1 MsgAppResp Term:1 Log:0/13 + 2->1 MsgAppResp Term:1 Log:0/14 + AppendThread->2 MsgStorageAppendResp Term:1 Log:1/14 +> 3 processing append thread + Processing: + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] + Responses: + 3->1 MsgAppResp Term:1 Log:0/13 + 3->1 MsgAppResp Term:1 Log:0/14 + AppendThread->3 MsgStorageAppendResp Term:1 Log:1/14 + +process-apply-thread 1 2 3 +---- +> 1 processing apply thread + Processing: + 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] + Responses: + ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] +> 2 processing apply thread + Processing: + 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] + Responses: + ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] +> 3 processing apply thread + Processing: + 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] + Responses: + ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] + +deliver-msgs 1 2 3 +---- +1->1 MsgAppResp Term:1 Log:0/14 +AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14 +2->1 MsgAppResp Term:1 Log:0/13 +2->1 MsgAppResp Term:1 Log:0/14 +3->1 MsgAppResp Term:1 Log:0/13 +3->1 MsgAppResp Term:1 Log:0/14 +ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] +AppendThread->2 MsgStorageAppendResp Term:1 Log:1/14 +ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] +AppendThread->3 MsgStorageAppendResp Term:1 Log:1/14 +ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] + +process-ready 1 2 3 +---- +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:14 + CommittedEntries: + 1/14 EntryNormal "prop_3" + Messages: + 1->2 MsgApp Term:1 Log:1/15 Commit:14 + 1->3 MsgApp Term:1 Log:1/15 Commit:14 + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[ + AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15 + ] + 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] Responses:[ + ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] + ] +> 2 handling Ready + +> 3 handling Ready + + +deliver-msgs 1 2 3 +---- +1->2 MsgApp Term:1 Log:1/15 Commit:14 +1->3 MsgApp Term:1 Log:1/15 Commit:14 + +process-ready 1 2 3 +---- +> 1 handling Ready + +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:14 + CommittedEntries: + 1/14 EntryNormal "prop_3" + Messages: + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[ + 2->1 MsgAppResp Term:1 Log:0/15 + AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 + ] + 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] Responses:[ + ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] + ] +> 3 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:14 + CommittedEntries: + 1/14 EntryNormal "prop_3" + Messages: + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[ + 3->1 MsgAppResp Term:1 Log:0/15 + AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15 + ] + 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] Responses:[ + ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] + ] + +process-append-thread 1 2 3 +---- +> 1 processing append thread + Processing: + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] + Responses: + 1->1 MsgAppResp Term:1 Log:0/15 + AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15 +> 2 processing append thread + Processing: + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] + Responses: + 2->1 MsgAppResp Term:1 Log:0/14 + 2->1 MsgAppResp Term:1 Log:0/15 + AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 +> 3 processing append thread + Processing: + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] + Responses: + 3->1 MsgAppResp Term:1 Log:0/14 + 3->1 MsgAppResp Term:1 Log:0/15 + AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15 + +process-apply-thread 1 2 3 +---- +> 1 processing apply thread + Processing: + 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] + Responses: + ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] +> 2 processing apply thread + Processing: + 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] + Responses: + ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] +> 3 processing apply thread + Processing: + 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] + Responses: + ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] + +deliver-msgs 1 2 3 +---- +1->1 MsgAppResp Term:1 Log:0/15 +AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15 +2->1 MsgAppResp Term:1 Log:0/14 +2->1 MsgAppResp Term:1 Log:0/15 +3->1 MsgAppResp Term:1 Log:0/14 +3->1 MsgAppResp Term:1 Log:0/15 +ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] +AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 +ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] +AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15 +ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] + +process-ready 1 2 3 +---- +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:15 + CommittedEntries: + 1/15 EntryNormal "prop_4" + Messages: + 1->2 MsgApp Term:1 Log:1/15 Commit:15 + 1->3 MsgApp Term:1 Log:1/15 Commit:15 + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 + 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] Responses:[ + ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] + ] +> 2 handling Ready + +> 3 handling Ready + + +deliver-msgs 1 2 3 +---- +1->2 MsgApp Term:1 Log:1/15 Commit:15 +1->3 MsgApp Term:1 Log:1/15 Commit:15 + +process-ready 1 2 3 +---- +> 1 handling Ready + +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:15 + CommittedEntries: + 1/15 EntryNormal "prop_4" + Messages: + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses:[ + 2->1 MsgAppResp Term:1 Log:0/15 + ] + 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] Responses:[ + ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] + ] +> 3 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:15 + CommittedEntries: + 1/15 EntryNormal "prop_4" + Messages: + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses:[ + 3->1 MsgAppResp Term:1 Log:0/15 + ] + 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] Responses:[ + ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] + ] + +process-append-thread 2 3 +---- +> 2 processing append thread + Processing: + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 + Responses: + 2->1 MsgAppResp Term:1 Log:0/15 + AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 +> 3 processing append thread + Processing: + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 + Responses: + 3->1 MsgAppResp Term:1 Log:0/15 + AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15 + +process-apply-thread 1 2 3 +---- +> 1 processing apply thread + Processing: + 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] + Responses: + ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] +> 2 processing apply thread + Processing: + 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] + Responses: + ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] +> 3 processing apply thread + Processing: + 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] + Responses: + ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] + +deliver-msgs 1 2 3 +---- +2->1 MsgAppResp Term:1 Log:0/15 +3->1 MsgAppResp Term:1 Log:0/15 +ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] +AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 +INFO entry at index 15 missing from unstable log; ignoring +ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] +AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15 +INFO entry at index 15 missing from unstable log; ignoring +ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] + +process-ready 1 2 3 +---- +> 1 handling Ready + +> 2 handling Ready + +> 3 handling Ready + + +process-append-thread 2 3 +---- +> 2 processing append thread + Processing: + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 + Responses: + 2->1 MsgAppResp Term:1 Log:0/15 +> 3 processing append thread + Processing: + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 + Responses: + 3->1 MsgAppResp Term:1 Log:0/15 + +process-apply-thread 1 2 3 +---- +> 1 processing apply thread + Processing: + 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] + Responses: + ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] +> 2 processing apply thread + Processing: + 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] + Responses: + ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] +> 3 processing apply thread + Processing: + 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] + Responses: + ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] + +deliver-msgs 1 2 3 +---- +2->1 MsgAppResp Term:1 Log:0/15 +3->1 MsgAppResp Term:1 Log:0/15 +ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] +ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] +ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] + +process-ready 1 2 3 +---- +> 1 handling Ready + +> 2 handling Ready + +> 3 handling Ready + + +stabilize +---- +> 1 processing append thread + Processing: + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 + Responses: + AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15 + Processing: + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 + Responses: +> 1 receiving messages + AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15 + INFO entry at index 15 missing from unstable log; ignoring diff --git a/third_party/raft/testdata/async_storage_writes_append_aba_race.txt b/third_party/raft/testdata/async_storage_writes_append_aba_race.txt new file mode 100644 index 0000000000..83964fe7d0 --- /dev/null +++ b/third_party/raft/testdata/async_storage_writes_append_aba_race.txt @@ -0,0 +1,517 @@ +# This test reproduces a scenario similar to the one described in +# newStorageAppendRespMsg, exercising a few interesting interactions +# between asynchronous storage writes, term changes, and log truncation. + +log-level none +---- +ok + +add-nodes 7 voters=(1,2,3,4,5,6,7) index=10 async-storage-writes=true +---- +ok + +# Step 1: node 2 is the leader. + +campaign 2 +---- +ok + +stabilize +---- +ok + +log-level info +---- +ok + +# Step 2: node 2 proposes some log entries but only node 1 receives these entries. + +propose 2 init_prop +---- +ok + +process-ready 2 +---- +Ready MustSync=true: +Entries: +1/12 EntryNormal "init_prop" +Messages: +2->1 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] +2->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] +2->4 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] +2->5 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] +2->6 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] +2->7 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] +2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses:[ + 2->2 MsgAppResp Term:1 Log:0/12 + AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12 +] + +deliver-msgs 1 drop=(3,4,5,6,7) +---- +2->1 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] +dropped: 2->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] +dropped: 2->4 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] +dropped: 2->5 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] +dropped: 2->6 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] +dropped: 2->7 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] + +# Step 3: node 1 gets the Ready and the entries are appended asynchronously. + +process-ready 1 +---- +Ready MustSync=true: +Entries: +1/12 EntryNormal "init_prop" +Messages: +1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses:[ + 1->2 MsgAppResp Term:1 Log:0/12 + AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12 +] + +# Step 4: node 3 becomes the leader after getting a vote from nodes 4, 5, and 6. + +campaign 3 +---- +INFO 3 is starting a new election at term 1 +INFO 3 became candidate at term 2 +INFO 3 [logterm: 1, index: 11] sent MsgVote request to 1 at term 2 +INFO 3 [logterm: 1, index: 11] sent MsgVote request to 2 at term 2 +INFO 3 [logterm: 1, index: 11] sent MsgVote request to 4 at term 2 +INFO 3 [logterm: 1, index: 11] sent MsgVote request to 5 at term 2 +INFO 3 [logterm: 1, index: 11] sent MsgVote request to 6 at term 2 +INFO 3 [logterm: 1, index: 11] sent MsgVote request to 7 at term 2 + +process-ready 3 +---- +Ready MustSync=true: +Lead:0 State:StateCandidate +HardState Term:2 Vote:3 Commit:11 +Messages: +3->1 MsgVote Term:2 Log:1/11 +3->2 MsgVote Term:2 Log:1/11 +3->4 MsgVote Term:2 Log:1/11 +3->5 MsgVote Term:2 Log:1/11 +3->6 MsgVote Term:2 Log:1/11 +3->7 MsgVote Term:2 Log:1/11 +3->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[ + 3->3 MsgVoteResp Term:2 Log:0/0 +] + +deliver-msgs 4 5 6 +---- +3->4 MsgVote Term:2 Log:1/11 +INFO 4 [term: 1] received a MsgVote message with higher term from 3 [term: 2] +INFO 4 became follower at term 2 +INFO 4 [logterm: 1, index: 11, vote: 0] cast MsgVote for 3 [logterm: 1, index: 11] at term 2 +3->5 MsgVote Term:2 Log:1/11 +INFO 5 [term: 1] received a MsgVote message with higher term from 3 [term: 2] +INFO 5 became follower at term 2 +INFO 5 [logterm: 1, index: 11, vote: 0] cast MsgVote for 3 [logterm: 1, index: 11] at term 2 +3->6 MsgVote Term:2 Log:1/11 +INFO 6 [term: 1] received a MsgVote message with higher term from 3 [term: 2] +INFO 6 became follower at term 2 +INFO 6 [logterm: 1, index: 11, vote: 0] cast MsgVote for 3 [logterm: 1, index: 11] at term 2 + +process-ready 4 5 6 +---- +> 4 handling Ready + Ready MustSync=true: + Lead:0 State:StateFollower + HardState Term:2 Vote:3 Commit:11 + Messages: + 4->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[ + 4->3 MsgVoteResp Term:2 Log:0/0 + ] +> 5 handling Ready + Ready MustSync=true: + Lead:0 State:StateFollower + HardState Term:2 Vote:3 Commit:11 + Messages: + 5->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[ + 5->3 MsgVoteResp Term:2 Log:0/0 + ] +> 6 handling Ready + Ready MustSync=true: + Lead:0 State:StateFollower + HardState Term:2 Vote:3 Commit:11 + Messages: + 6->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[ + 6->3 MsgVoteResp Term:2 Log:0/0 + ] + +process-append-thread 3 4 5 6 +---- +> 3 processing append thread + Processing: + 3->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 + Responses: + 3->3 MsgVoteResp Term:2 Log:0/0 +> 4 processing append thread + Processing: + 4->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 + Responses: + 4->3 MsgVoteResp Term:2 Log:0/0 +> 5 processing append thread + Processing: + 5->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 + Responses: + 5->3 MsgVoteResp Term:2 Log:0/0 +> 6 processing append thread + Processing: + 6->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 + Responses: + 6->3 MsgVoteResp Term:2 Log:0/0 + +deliver-msgs 3 +---- +3->3 MsgVoteResp Term:2 Log:0/0 +INFO 3 received MsgVoteResp from 3 at term 2 +INFO 3 has received 1 MsgVoteResp votes and 0 vote rejections +4->3 MsgVoteResp Term:2 Log:0/0 +INFO 3 received MsgVoteResp from 4 at term 2 +INFO 3 has received 2 MsgVoteResp votes and 0 vote rejections +5->3 MsgVoteResp Term:2 Log:0/0 +INFO 3 received MsgVoteResp from 5 at term 2 +INFO 3 has received 3 MsgVoteResp votes and 0 vote rejections +6->3 MsgVoteResp Term:2 Log:0/0 +INFO 3 received MsgVoteResp from 6 at term 2 +INFO 3 has received 4 MsgVoteResp votes and 0 vote rejections +INFO 3 became leader at term 2 + +# Step 5: node 3 proposes some log entries and node 1 receives these entries, +# overwriting the previous unstable log entries that are in the process of being +# appended. The entries have a larger term than the previous entries but the +# same indexes. It begins appending these new entries asynchronously. + +process-ready 3 +---- +Ready MustSync=true: +Lead:3 State:StateLeader +Entries: +2/12 EntryNormal "" +Messages: +3->1 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] +3->2 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] +3->4 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] +3->5 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] +3->6 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] +3->7 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] +3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[2/12 EntryNormal ""] Responses:[ + 3->3 MsgAppResp Term:2 Log:0/12 + AppendThread->3 MsgStorageAppendResp Term:2 Log:2/12 +] + +deliver-msgs 1 drop=(2,4,5,6,7) +---- +3->1 MsgVote Term:2 Log:1/11 +INFO 1 [term: 1] received a MsgVote message with higher term from 3 [term: 2] +INFO 1 became follower at term 2 +INFO 1 [logterm: 1, index: 12, vote: 0] rejected MsgVote from 3 [logterm: 1, index: 11] at term 2 +3->1 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] +INFO found conflict at index 12 [existing term: 1, conflicting term: 2] +INFO replace the unstable entries from index 12 +dropped: 3->2 MsgVote Term:2 Log:1/11 +dropped: 3->2 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] +dropped: 3->4 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] +dropped: 3->5 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] +dropped: 3->6 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] +dropped: 3->7 MsgVote Term:2 Log:1/11 +dropped: 3->7 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] + +process-ready 1 +---- +Ready MustSync=true: +Lead:3 State:StateFollower +HardState Term:2 Commit:11 +Entries: +2/12 EntryNormal "" +Messages: +1->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Entries:[2/12 EntryNormal ""] Responses:[ + 1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0) + 1->3 MsgAppResp Term:2 Log:0/12 + AppendThread->1 MsgStorageAppendResp Term:2 Log:2/12 +] + +# Step 6: node 3 crashes and node 4 becomes leader getting the vote from 5, 6, and 7. + +campaign 4 +---- +INFO 4 is starting a new election at term 2 +INFO 4 became candidate at term 3 +INFO 4 [logterm: 1, index: 11] sent MsgVote request to 1 at term 3 +INFO 4 [logterm: 1, index: 11] sent MsgVote request to 2 at term 3 +INFO 4 [logterm: 1, index: 11] sent MsgVote request to 3 at term 3 +INFO 4 [logterm: 1, index: 11] sent MsgVote request to 5 at term 3 +INFO 4 [logterm: 1, index: 11] sent MsgVote request to 6 at term 3 +INFO 4 [logterm: 1, index: 11] sent MsgVote request to 7 at term 3 + +process-ready 4 +---- +Ready MustSync=true: +Lead:0 State:StateCandidate +HardState Term:3 Vote:4 Commit:11 +Messages: +4->1 MsgVote Term:3 Log:1/11 +4->2 MsgVote Term:3 Log:1/11 +4->3 MsgVote Term:3 Log:1/11 +4->5 MsgVote Term:3 Log:1/11 +4->6 MsgVote Term:3 Log:1/11 +4->7 MsgVote Term:3 Log:1/11 +4->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[ + 4->4 MsgVoteResp Term:3 Log:0/0 +] + +deliver-msgs 5 6 7 +---- +4->5 MsgVote Term:3 Log:1/11 +INFO 5 [term: 2] received a MsgVote message with higher term from 4 [term: 3] +INFO 5 became follower at term 3 +INFO 5 [logterm: 1, index: 11, vote: 0] cast MsgVote for 4 [logterm: 1, index: 11] at term 3 +4->6 MsgVote Term:3 Log:1/11 +INFO 6 [term: 2] received a MsgVote message with higher term from 4 [term: 3] +INFO 6 became follower at term 3 +INFO 6 [logterm: 1, index: 11, vote: 0] cast MsgVote for 4 [logterm: 1, index: 11] at term 3 +4->7 MsgVote Term:3 Log:1/11 +INFO 7 [term: 1] received a MsgVote message with higher term from 4 [term: 3] +INFO 7 became follower at term 3 +INFO 7 [logterm: 1, index: 11, vote: 0] cast MsgVote for 4 [logterm: 1, index: 11] at term 3 + +process-ready 5 6 7 +---- +> 5 handling Ready + Ready MustSync=true: + HardState Term:3 Vote:4 Commit:11 + Messages: + 5->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[ + 5->4 MsgVoteResp Term:3 Log:0/0 + ] +> 6 handling Ready + Ready MustSync=true: + HardState Term:3 Vote:4 Commit:11 + Messages: + 6->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[ + 6->4 MsgVoteResp Term:3 Log:0/0 + ] +> 7 handling Ready + Ready MustSync=true: + Lead:0 State:StateFollower + HardState Term:3 Vote:4 Commit:11 + Messages: + 7->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[ + 7->4 MsgVoteResp Term:3 Log:0/0 + ] + +process-append-thread 4 5 6 7 +---- +> 4 processing append thread + Processing: + 4->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 + Responses: + 4->4 MsgVoteResp Term:3 Log:0/0 +> 5 processing append thread + Processing: + 5->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 + Responses: + 5->4 MsgVoteResp Term:3 Log:0/0 +> 6 processing append thread + Processing: + 6->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 + Responses: + 6->4 MsgVoteResp Term:3 Log:0/0 +> 7 processing append thread + Processing: + 7->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 + Responses: + 7->4 MsgVoteResp Term:3 Log:0/0 + +deliver-msgs 4 +---- +4->4 MsgVoteResp Term:3 Log:0/0 +INFO 4 received MsgVoteResp from 4 at term 3 +INFO 4 has received 1 MsgVoteResp votes and 0 vote rejections +5->4 MsgVoteResp Term:3 Log:0/0 +INFO 4 received MsgVoteResp from 5 at term 3 +INFO 4 has received 2 MsgVoteResp votes and 0 vote rejections +6->4 MsgVoteResp Term:3 Log:0/0 +INFO 4 received MsgVoteResp from 6 at term 3 +INFO 4 has received 3 MsgVoteResp votes and 0 vote rejections +7->4 MsgVoteResp Term:3 Log:0/0 +INFO 4 received MsgVoteResp from 7 at term 3 +INFO 4 has received 4 MsgVoteResp votes and 0 vote rejections +INFO 4 became leader at term 3 + +process-ready 4 +---- +Ready MustSync=true: +Lead:4 State:StateLeader +Entries: +3/12 EntryNormal "" +Messages: +4->1 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] +4->2 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] +4->3 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] +4->5 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] +4->6 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] +4->7 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] +4->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[ + 4->4 MsgAppResp Term:3 Log:0/12 + AppendThread->4 MsgStorageAppendResp Term:3 Log:3/12 +] + +# Step 7: before the new entries reach node 1, it hears of the term change +# through a heartbeat and persists the new term. Node 1 then receives these +# entries, overwriting the previous unstable log entries that are in the process +# of being appended. The entries have a larger term than the previous entries +# but the same indexes. It begins appending these new entries asynchronously. + +deliver-msgs drop=1 +---- +dropped: 4->1 MsgVote Term:3 Log:1/11 +dropped: 4->1 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] + +tick-heartbeat 4 +---- +ok + +process-ready 4 +---- +Ready MustSync=false: +Messages: +4->1 MsgHeartbeat Term:3 Log:0/0 +4->2 MsgHeartbeat Term:3 Log:0/0 +4->3 MsgHeartbeat Term:3 Log:0/0 +4->5 MsgHeartbeat Term:3 Log:0/0 +4->6 MsgHeartbeat Term:3 Log:0/0 +4->7 MsgHeartbeat Term:3 Log:0/0 + +deliver-msgs 1 +---- +4->1 MsgHeartbeat Term:3 Log:0/0 +INFO 1 [term: 2] received a MsgHeartbeat message with higher term from 4 [term: 3] +INFO 1 became follower at term 3 + +process-ready 1 +---- +Ready MustSync=true: +Lead:4 State:StateFollower +HardState Term:3 Commit:11 +Messages: +1->4 MsgHeartbeatResp Term:3 Log:0/0 +1->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Responses:[ + AppendThread->1 MsgStorageAppendResp Term:3 Log:2/12 +] + +deliver-msgs 4 +---- +1->4 MsgHeartbeatResp Term:3 Log:0/0 + +process-ready 4 +---- +Ready MustSync=false: +Messages: +4->1 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] + +deliver-msgs 1 +---- +4->1 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] +INFO found conflict at index 12 [existing term: 2, conflicting term: 3] +INFO replace the unstable entries from index 12 + +process-ready 1 +---- +Ready MustSync=true: +Entries: +3/12 EntryNormal "" +Messages: +1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[ + 1->4 MsgAppResp Term:3 Log:0/12 + AppendThread->1 MsgStorageAppendResp Term:3 Log:3/12 +] + +# Step 8: The asynchronous log appends from the first Ready complete and the +# MsgStorageAppendResp is returned to the raft node state machine. A decision +# is made about whether to truncate the unstable log. + +raft-log 1 +---- +1/11 EntryNormal "" + +process-append-thread 1 +---- +Processing: +1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] +Responses: +1->2 MsgAppResp Term:1 Log:0/12 +AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12 + +raft-log 1 +---- +1/11 EntryNormal "" +1/12 EntryNormal "init_prop" + +# Step 9: However, the log entries from the second Ready are still in the +# asynchronous append pipeline and will overwrite (in stable storage) the +# entries from the first Ready at some future point. We can't truncate the +# unstable log yet or a future read from Storage might see the entries from step +# 5 before they have been replaced by the entries from step 7. Instead, we must +# wait until we are sure that the entries are stable and that no in-progress +# appends might overwrite them before removing entries from the unstable log. + +deliver-msgs 1 +---- +AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12 +INFO 1 [term: 3] ignored entry appends from a MsgStorageAppendResp message with lower term [term: 1] + +process-append-thread 1 +---- +Processing: +1->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Entries:[2/12 EntryNormal ""] +Responses: +1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0) +1->3 MsgAppResp Term:2 Log:0/12 +AppendThread->1 MsgStorageAppendResp Term:2 Log:2/12 + +raft-log 1 +---- +1/11 EntryNormal "" +2/12 EntryNormal "" + +deliver-msgs 1 +---- +AppendThread->1 MsgStorageAppendResp Term:2 Log:2/12 +INFO 1 [term: 3] ignored entry appends from a MsgStorageAppendResp message with lower term [term: 2] + +process-append-thread 1 +---- +Processing: +1->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 +Responses: +AppendThread->1 MsgStorageAppendResp Term:3 Log:2/12 + +raft-log 1 +---- +1/11 EntryNormal "" +2/12 EntryNormal "" + +deliver-msgs 1 +---- +AppendThread->1 MsgStorageAppendResp Term:3 Log:2/12 +INFO entry at (index,term)=(12,2) mismatched with entry at (12,3) in unstable log; ignoring + +process-append-thread 1 +---- +Processing: +1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] +Responses: +1->4 MsgAppResp Term:3 Log:0/12 +AppendThread->1 MsgStorageAppendResp Term:3 Log:3/12 + +raft-log 1 +---- +1/11 EntryNormal "" +3/12 EntryNormal "" + +deliver-msgs 1 +---- +AppendThread->1 MsgStorageAppendResp Term:3 Log:3/12 diff --git a/third_party/raft/testdata/campaign.txt b/third_party/raft/testdata/campaign.txt new file mode 100644 index 0000000000..4eefab36d6 --- /dev/null +++ b/third_party/raft/testdata/campaign.txt @@ -0,0 +1,118 @@ +log-level info +---- +ok + +add-nodes 3 voters=(1,2,3) index=2 +---- +INFO 1 switched to configuration voters=(1 2 3) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] +INFO 2 switched to configuration voters=(1 2 3) +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] +INFO 3 switched to configuration voters=(1 2 3) +INFO 3 became follower at term 0 +INFO newRaft 3 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 +INFO 1 [logterm: 1, index: 2] sent MsgVote request to 2 at term 1 +INFO 1 [logterm: 1, index: 2] sent MsgVote request to 3 at term 1 + +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Lead:0 State:StateCandidate + HardState Term:1 Vote:1 Commit:2 + Messages: + 1->2 MsgVote Term:1 Log:1/2 + 1->3 MsgVote Term:1 Log:1/2 + INFO 1 received MsgVoteResp from 1 at term 1 + INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections +> 2 receiving messages + 1->2 MsgVote Term:1 Log:1/2 + INFO 2 [term: 0] received a MsgVote message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + INFO 2 [logterm: 1, index: 2, vote: 0] cast MsgVote for 1 [logterm: 1, index: 2] at term 1 +> 3 receiving messages + 1->3 MsgVote Term:1 Log:1/2 + INFO 3 [term: 0] received a MsgVote message with higher term from 1 [term: 1] + INFO 3 became follower at term 1 + INFO 3 [logterm: 1, index: 2, vote: 0] cast MsgVote for 1 [logterm: 1, index: 2] at term 1 +> 2 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 + Messages: + 2->1 MsgVoteResp Term:1 Log:0/0 +> 3 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 + Messages: + 3->1 MsgVoteResp Term:1 Log:0/0 +> 1 receiving messages + 2->1 MsgVoteResp Term:1 Log:0/0 + INFO 1 received MsgVoteResp from 2 at term 1 + INFO 1 has received 2 MsgVoteResp votes and 0 vote rejections + INFO 1 became leader at term 1 + 3->1 MsgVoteResp Term:1 Log:0/0 +> 1 handling Ready + Ready MustSync=true: + Lead:1 State:StateLeader + Entries: + 1/3 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] + 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] +> 2 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + Entries: + 1/3 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 +> 3 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + Entries: + 1/3 EntryNormal "" + Messages: + 3->1 MsgAppResp Term:1 Log:0/3 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/3 + 3->1 MsgAppResp Term:1 Log:0/3 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:3 + CommittedEntries: + 1/3 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:3 + 1->3 MsgApp Term:1 Log:1/3 Commit:3 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:3 +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/3 Commit:3 +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:3 + CommittedEntries: + 1/3 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 +> 3 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:3 + CommittedEntries: + 1/3 EntryNormal "" + Messages: + 3->1 MsgAppResp Term:1 Log:0/3 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/3 + 3->1 MsgAppResp Term:1 Log:0/3 diff --git a/third_party/raft/testdata/campaign_learner_must_vote.txt b/third_party/raft/testdata/campaign_learner_must_vote.txt new file mode 100644 index 0000000000..d05de374f1 --- /dev/null +++ b/third_party/raft/testdata/campaign_learner_must_vote.txt @@ -0,0 +1,159 @@ +# Regression test that verifies that learners can vote. This holds only in the +# sense that if a learner is asked to vote, a candidate believes that they are a +# voter based on its current config, which may be more recent than that of the +# learner. If learners which are actually voters but don't know it yet don't +# vote in that situation, the raft group may end up unavailable despite a quorum +# of voters (as of the latest config) being available. +# +# See: +# https://github.com/etcd-io/etcd/pull/10998 + +# Turn output off during boilerplate. +log-level none +---- +ok + +# Bootstrap three nodes. +add-nodes 3 voters=(1,2) learners=(3) index=2 +---- +ok + +# n1 gets to be leader. +campaign 1 +---- +ok + +stabilize +---- +ok + +# Propose a conf change on n1 that promotes n3 to voter. +propose-conf-change 1 +v3 +---- +ok + +# Commit and fully apply said conf change. n1 and n2 now consider n3 a voter. +stabilize 1 2 +---- +ok + +# Drop all inflight messages to 3. We don't want it to be caught up when it is +# asked to vote. +deliver-msgs drop=(3) +---- +ok + +# We now pretend that n1 is dead, and n2 is trying to become leader. + +log-level debug +---- +ok + +campaign 2 +---- +INFO 2 is starting a new election at term 1 +INFO 2 became candidate at term 2 +INFO 2 [logterm: 1, index: 4] sent MsgVote request to 1 at term 2 +INFO 2 [logterm: 1, index: 4] sent MsgVote request to 3 at term 2 + +# Send out the MsgVote requests. +process-ready 2 +---- +Ready MustSync=true: +Lead:0 State:StateCandidate +HardState Term:2 Vote:2 Commit:4 +Messages: +2->1 MsgVote Term:2 Log:1/4 +2->3 MsgVote Term:2 Log:1/4 +INFO 2 received MsgVoteResp from 2 at term 2 +INFO 2 has received 1 MsgVoteResp votes and 0 vote rejections + +# n2 is now campaigning while n1 is down (does not respond). The latest config +# has n3 as a voter, but n3 doesn't even have the corresponding conf change in +# its log. Still, it casts a vote for n2 which can in turn become leader and +# catches up n3. +stabilize 3 +---- +> 3 receiving messages + 2->3 MsgVote Term:2 Log:1/4 + INFO 3 [term: 1] received a MsgVote message with higher term from 2 [term: 2] + INFO 3 became follower at term 2 + INFO 3 [logterm: 1, index: 3, vote: 0] cast MsgVote for 2 [logterm: 1, index: 4] at term 2 +> 3 handling Ready + Ready MustSync=true: + Lead:0 State:StateFollower + HardState Term:2 Vote:2 Commit:3 + Messages: + 3->2 MsgVoteResp Term:2 Log:0/0 + +stabilize 2 3 +---- +> 2 receiving messages + 3->2 MsgVoteResp Term:2 Log:0/0 + INFO 2 received MsgVoteResp from 3 at term 2 + INFO 2 has received 2 MsgVoteResp votes and 0 vote rejections + INFO 2 became leader at term 2 +> 2 handling Ready + Ready MustSync=true: + Lead:2 State:StateLeader + Entries: + 2/5 EntryNormal "" + Messages: + 2->1 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""] + 2->3 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""] +> 3 receiving messages + 2->3 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""] + DEBUG 3 [logterm: 0, index: 4] rejected MsgApp [logterm: 1, index: 4] from 2 +> 3 handling Ready + Ready MustSync=false: + Lead:2 State:StateFollower + Messages: + 3->2 MsgAppResp Term:2 Log:1/4 Rejected (Hint: 3) +> 2 receiving messages + 3->2 MsgAppResp Term:2 Log:1/4 Rejected (Hint: 3) + DEBUG 2 received MsgAppResp(rejected, hint: (index 3, term 1)) from 3 for index 4 + DEBUG 2 decreased progress of 3 to [StateProbe match=0 next=4] +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->3 MsgApp Term:2 Log:1/3 Commit:4 Entries:[ + 1/4 EntryConfChangeV2 v3 + 2/5 EntryNormal "" + ] +> 3 receiving messages + 2->3 MsgApp Term:2 Log:1/3 Commit:4 Entries:[ + 1/4 EntryConfChangeV2 v3 + 2/5 EntryNormal "" + ] +> 3 handling Ready + Ready MustSync=true: + HardState Term:2 Vote:2 Commit:4 + Entries: + 1/4 EntryConfChangeV2 v3 + 2/5 EntryNormal "" + CommittedEntries: + 1/4 EntryConfChangeV2 v3 + Messages: + 3->2 MsgAppResp Term:2 Log:0/5 + INFO 3 switched to configuration voters=(1 2 3) +> 2 receiving messages + 3->2 MsgAppResp Term:2 Log:0/5 +> 2 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:2 Commit:5 + CommittedEntries: + 2/5 EntryNormal "" + Messages: + 2->3 MsgApp Term:2 Log:2/5 Commit:5 +> 3 receiving messages + 2->3 MsgApp Term:2 Log:2/5 Commit:5 +> 3 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:2 Commit:5 + CommittedEntries: + 2/5 EntryNormal "" + Messages: + 3->2 MsgAppResp Term:2 Log:0/5 +> 2 receiving messages + 3->2 MsgAppResp Term:2 Log:0/5 diff --git a/third_party/raft/testdata/checkquorum.txt b/third_party/raft/testdata/checkquorum.txt new file mode 100644 index 0000000000..b25c1e63da --- /dev/null +++ b/third_party/raft/testdata/checkquorum.txt @@ -0,0 +1,236 @@ +# Tests that CheckQuorum causes a leader to step down if it hasn't heard from a +# quorum of followers in the past election timeout interval. +# +# Also tests that votes are rejected when there is a current leader. In the Raft +# thesis this is part of PreVote, but etcd/raft enables this via CheckQuorum. + +log-level none +---- +ok + +add-nodes 3 voters=(1,2,3) index=10 checkquorum=true +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok + +log-level debug +---- +ok + +# Campaigning will fail when there is an active leader. +campaign 2 +---- +INFO 2 is starting a new election at term 1 +INFO 2 became candidate at term 2 +INFO 2 [logterm: 1, index: 11] sent MsgVote request to 1 at term 2 +INFO 2 [logterm: 1, index: 11] sent MsgVote request to 3 at term 2 + +stabilize +---- +> 2 handling Ready + Ready MustSync=true: + Lead:0 State:StateCandidate + HardState Term:2 Vote:2 Commit:11 + Messages: + 2->1 MsgVote Term:2 Log:1/11 + 2->3 MsgVote Term:2 Log:1/11 + INFO 2 received MsgVoteResp from 2 at term 2 + INFO 2 has received 1 MsgVoteResp votes and 0 vote rejections +> 1 receiving messages + 2->1 MsgVote Term:2 Log:1/11 + INFO 1 [logterm: 1, index: 11, vote: 1] ignored MsgVote from 2 [logterm: 1, index: 11] at term 1: lease is not expired (remaining ticks: 3) +> 3 receiving messages + 2->3 MsgVote Term:2 Log:1/11 + INFO 3 [logterm: 1, index: 11, vote: 1] ignored MsgVote from 2 [logterm: 1, index: 11] at term 1: lease is not expired (remaining ticks: 3) + +# Tick the leader without processing any messages from followers. We have to +# tick 2 election timeouts, since the followers were active in the current +# interval (see messages above). +tick-election 1 +---- +ok + +tick-election 1 +---- +WARN 1 stepped down to follower since quorum is not active +INFO 1 became follower at term 1 + +# We'll now send all of the heartbeats that were buffered during the ticks +# above. Conceptually, "the network was slow". +stabilize +---- +> 1 handling Ready + Ready MustSync=false: + Lead:0 State:StateFollower + Messages: + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 2 receiving messages + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 3 receiving messages + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgAppResp Term:2 Log:0/0 + 2->1 MsgAppResp Term:2 Log:0/0 + 2->1 MsgAppResp Term:2 Log:0/0 + 2->1 MsgAppResp Term:2 Log:0/0 + 2->1 MsgAppResp Term:2 Log:0/0 +> 3 handling Ready + Ready MustSync=false: + Messages: + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + 3->1 MsgHeartbeatResp Term:1 Log:0/0 +> 1 receiving messages + 2->1 MsgAppResp Term:2 Log:0/0 + INFO 1 [term: 1] received a MsgAppResp message with higher term from 2 [term: 2] + INFO 1 became follower at term 2 + 2->1 MsgAppResp Term:2 Log:0/0 + 2->1 MsgAppResp Term:2 Log:0/0 + 2->1 MsgAppResp Term:2 Log:0/0 + 2->1 MsgAppResp Term:2 Log:0/0 + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + INFO 1 [term: 2] ignored a MsgHeartbeatResp message with lower term from 3 [term: 1] + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + INFO 1 [term: 2] ignored a MsgHeartbeatResp message with lower term from 3 [term: 1] + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + INFO 1 [term: 2] ignored a MsgHeartbeatResp message with lower term from 3 [term: 1] + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + INFO 1 [term: 2] ignored a MsgHeartbeatResp message with lower term from 3 [term: 1] + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + INFO 1 [term: 2] ignored a MsgHeartbeatResp message with lower term from 3 [term: 1] +> 1 handling Ready + Ready MustSync=true: + HardState Term:2 Commit:11 + +# Other nodes can now successfully campaign. Note that we haven't ticked 3, so +# it won't grant votes. +campaign 2 +---- +INFO 2 is starting a new election at term 2 +INFO 2 became candidate at term 3 +INFO 2 [logterm: 1, index: 11] sent MsgVote request to 1 at term 3 +INFO 2 [logterm: 1, index: 11] sent MsgVote request to 3 at term 3 + +process-ready 2 +---- +Ready MustSync=true: +HardState Term:3 Vote:2 Commit:11 +Messages: +2->1 MsgVote Term:3 Log:1/11 +2->3 MsgVote Term:3 Log:1/11 +INFO 2 received MsgVoteResp from 2 at term 3 +INFO 2 has received 1 MsgVoteResp votes and 0 vote rejections + +deliver-msgs 1 +---- +2->1 MsgVote Term:3 Log:1/11 +INFO 1 [term: 2] received a MsgVote message with higher term from 2 [term: 3] +INFO 1 became follower at term 3 +INFO 1 [logterm: 1, index: 11, vote: 0] cast MsgVote for 2 [logterm: 1, index: 11] at term 3 + +deliver-msgs 3 +---- +2->3 MsgVote Term:3 Log:1/11 +INFO 3 [logterm: 1, index: 11, vote: 1] ignored MsgVote from 2 [logterm: 1, index: 11] at term 1: lease is not expired (remaining ticks: 3) + +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + HardState Term:3 Vote:2 Commit:11 + Messages: + 1->2 MsgVoteResp Term:3 Log:0/0 +> 2 receiving messages + 1->2 MsgVoteResp Term:3 Log:0/0 + INFO 2 received MsgVoteResp from 1 at term 3 + INFO 2 has received 2 MsgVoteResp votes and 0 vote rejections + INFO 2 became leader at term 3 +> 2 handling Ready + Ready MustSync=true: + Lead:2 State:StateLeader + Entries: + 3/12 EntryNormal "" + Messages: + 2->1 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] + 2->3 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] +> 1 receiving messages + 2->1 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] +> 3 receiving messages + 2->3 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] + INFO 3 [term: 1] received a MsgApp message with higher term from 2 [term: 3] + INFO 3 became follower at term 3 +> 1 handling Ready + Ready MustSync=true: + Lead:2 State:StateFollower + Entries: + 3/12 EntryNormal "" + Messages: + 1->2 MsgAppResp Term:3 Log:0/12 +> 3 handling Ready + Ready MustSync=true: + Lead:2 State:StateFollower + HardState Term:3 Commit:11 + Entries: + 3/12 EntryNormal "" + Messages: + 3->2 MsgAppResp Term:3 Log:0/12 +> 2 receiving messages + 1->2 MsgAppResp Term:3 Log:0/12 + 3->2 MsgAppResp Term:3 Log:0/12 +> 2 handling Ready + Ready MustSync=false: + HardState Term:3 Vote:2 Commit:12 + CommittedEntries: + 3/12 EntryNormal "" + Messages: + 2->1 MsgApp Term:3 Log:3/12 Commit:12 + 2->3 MsgApp Term:3 Log:3/12 Commit:12 +> 1 receiving messages + 2->1 MsgApp Term:3 Log:3/12 Commit:12 +> 3 receiving messages + 2->3 MsgApp Term:3 Log:3/12 Commit:12 +> 1 handling Ready + Ready MustSync=false: + HardState Term:3 Vote:2 Commit:12 + CommittedEntries: + 3/12 EntryNormal "" + Messages: + 1->2 MsgAppResp Term:3 Log:0/12 +> 3 handling Ready + Ready MustSync=false: + HardState Term:3 Commit:12 + CommittedEntries: + 3/12 EntryNormal "" + Messages: + 3->2 MsgAppResp Term:3 Log:0/12 +> 2 receiving messages + 1->2 MsgAppResp Term:3 Log:0/12 + 3->2 MsgAppResp Term:3 Log:0/12 diff --git a/third_party/raft/testdata/confchange_disable_validation.txt b/third_party/raft/testdata/confchange_disable_validation.txt new file mode 100644 index 0000000000..1a2bc4fdf8 --- /dev/null +++ b/third_party/raft/testdata/confchange_disable_validation.txt @@ -0,0 +1,76 @@ +# This test verifies the DisableConfChangeValidation setting. +# With it set, raft should allow configuration changes to enter the log even +# if they appear to be incompatible with the currently active configuration. +# +# The test sets up a single-voter group that applies entries one at a time. +# Then it proposes a bogus entry followed by a conf change. When the bogus entry +# has applied, a second (compatible, but the node doesn't know this yet) +# configuration change is proposed. That configuration change is accepted into +# the log since due to DisableConfChangeValidation=true. +add-nodes 1 voters=(1) index=2 max-committed-size-per-ready=1 disable-conf-change-validation=true +---- +INFO 1 switched to configuration voters=(1) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 + +stabilize log-level=none +---- +ok + +# Dummy entry. +propose 1 foo +---- +ok + +propose-conf-change 1 transition=explicit +l2 l3 +---- +ok + +# Entries both get appended. +process-ready 1 +---- +Ready MustSync=true: +Entries: +1/4 EntryNormal "foo" +1/5 EntryConfChangeV2 l2 l3 + +# Dummy entry comes up for application. +process-ready 1 +---- +Ready MustSync=false: +HardState Term:1 Vote:1 Commit:5 +CommittedEntries: +1/4 EntryNormal "foo" + +# Propose new config change. Note how it isn't rejected, +# which is due to DisableConfChangeValidation=true. +propose-conf-change 1 +---- +ok + +# Turn on autopilot: the first config change applies, the +# second one gets committed and also applies. +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/6 EntryConfChangeV2 + CommittedEntries: + 1/5 EntryConfChangeV2 l2 l3 + INFO 1 switched to configuration voters=(1)&&(1) learners=(2 3) +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:6 + CommittedEntries: + 1/6 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/5 Commit:5 Entries:[1/6 EntryConfChangeV2] + 1->3 MsgApp Term:1 Log:1/5 Commit:5 Entries:[1/6 EntryConfChangeV2] + INFO 1 switched to configuration voters=(1) learners=(2 3) diff --git a/third_party/raft/testdata/confchange_v1_add_single.txt b/third_party/raft/testdata/confchange_v1_add_single.txt new file mode 100644 index 0000000000..f13d30777d --- /dev/null +++ b/third_party/raft/testdata/confchange_v1_add_single.txt @@ -0,0 +1,96 @@ +# Run a V1 membership change that adds a single voter. + +# Bootstrap n1. +add-nodes 1 voters=(1) index=2 +---- +INFO 1 switched to configuration voters=(1) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 + +process-ready 1 +---- +Ready MustSync=true: +Lead:0 State:StateCandidate +HardState Term:1 Vote:1 Commit:2 +INFO 1 received MsgVoteResp from 1 at term 1 +INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections +INFO 1 became leader at term 1 + +# Add v2 (with an auto transition). +propose-conf-change 1 v1=true +v2 +---- +ok + +# Pull n2 out of thin air. +add-nodes 1 +---- +INFO 2 switched to configuration voters=() +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +# n1 commits the conf change using itself as commit quorum, immediately transitions into +# the final config, and catches up n2. Note that it's using an EntryConfChange, not an +# EntryConfChangeV2, so this is compatible with nodes that don't know about V2 conf changes. +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Lead:1 State:StateLeader + Entries: + 1/3 EntryNormal "" + 1/4 EntryConfChange v2 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:4 + CommittedEntries: + 1/3 EntryNormal "" + 1/4 EntryConfChange v2 + INFO 1 switched to configuration voters=(1 2) +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChange v2] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChange v2] + INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 +> 2 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Commit:0 + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) + DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgSnap Term:1 Log:0/0 + Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false +> 2 receiving messages + 1->2 MsgSnap Term:1 Log:0/0 + Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + INFO log [committed=0, applied=0, applying=0, unstable.offset=1, unstable.offsetInProgress=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] + INFO 2 switched to configuration voters=(1 2) + INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] + INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:4 + Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] diff --git a/third_party/raft/testdata/confchange_v1_remove_leader.txt b/third_party/raft/testdata/confchange_v1_remove_leader.txt new file mode 100644 index 0000000000..cc91508a45 --- /dev/null +++ b/third_party/raft/testdata/confchange_v1_remove_leader.txt @@ -0,0 +1,250 @@ +# We'll turn this back on after the boilerplate. +log-level none +---- +ok + +# Run a V1 membership change that removes the leader. +# Bootstrap n1, n2, n3. +add-nodes 3 voters=(1,2,3) index=2 +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok + +log-level debug +---- +ok + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 + +# Start removing n1. +propose-conf-change 1 v1=true +r1 +---- +ok + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 + +# Propose an extra entry which will be sent out together with the conf change. +propose 1 foo +---- +ok + +# Send out the corresponding appends. +process-ready 1 +---- +Ready MustSync=true: +Entries: +1/4 EntryConfChange r1 +1/5 EntryNormal "foo" +Messages: +1->2 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1] +1->3 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1] +1->2 MsgApp Term:1 Log:1/4 Commit:3 Entries:[1/5 EntryNormal "foo"] +1->3 MsgApp Term:1 Log:1/4 Commit:3 Entries:[1/5 EntryNormal "foo"] + +# Send response from n2 (which is enough to commit the entries so far next time +# n1 runs). +stabilize 2 +---- +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1] + 1->2 MsgApp Term:1 Log:1/4 Commit:3 Entries:[1/5 EntryNormal "foo"] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/4 EntryConfChange r1 + 1/5 EntryNormal "foo" + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/5 + +# Put another entry in n1's log. +propose 1 bar +---- +ok + +# n1 applies the conf change, so it has now removed itself. But it still has +# an uncommitted entry in the log. If the leader unconditionally counted itself +# as part of the commit quorum, we'd be in trouble. In the block below, we see +# it send out appends to the other nodes for the 'bar' entry. +stabilize 1 +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/6 EntryNormal "bar" + Messages: + 1->2 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"] + 1->3 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"] +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/5 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:5 + CommittedEntries: + 1/4 EntryConfChange r1 + 1/5 EntryNormal "foo" + Messages: + 1->2 MsgApp Term:1 Log:1/6 Commit:4 + 1->3 MsgApp Term:1 Log:1/6 Commit:4 + 1->2 MsgApp Term:1 Log:1/6 Commit:5 + 1->3 MsgApp Term:1 Log:1/6 Commit:5 + INFO 1 switched to configuration voters=(2 3) + +raft-state +---- +1: StateLeader (Non-Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 + +# n2 responds, n3 doesn't yet. Quorum for 'bar' should not be reached... +stabilize 2 +---- +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"] + 1->2 MsgApp Term:1 Log:1/6 Commit:4 + 1->2 MsgApp Term:1 Log:1/6 Commit:5 +> 2 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:5 + Entries: + 1/6 EntryNormal "bar" + CommittedEntries: + 1/4 EntryConfChange r1 + 1/5 EntryNormal "foo" + Messages: + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 + INFO 2 switched to configuration voters=(2 3) + +# ... which thankfully is what we see on the leader. +stabilize 1 +---- +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 + +# When n3 responds, quorum is reached and everything falls into place. +stabilize +---- +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1] + 1->3 MsgApp Term:1 Log:1/4 Commit:3 Entries:[1/5 EntryNormal "foo"] + 1->3 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"] + 1->3 MsgApp Term:1 Log:1/6 Commit:4 + 1->3 MsgApp Term:1 Log:1/6 Commit:5 +> 3 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:5 + Entries: + 1/4 EntryConfChange r1 + 1/5 EntryNormal "foo" + 1/6 EntryNormal "bar" + CommittedEntries: + 1/4 EntryConfChange r1 + 1/5 EntryNormal "foo" + Messages: + 3->1 MsgAppResp Term:1 Log:0/4 + 3->1 MsgAppResp Term:1 Log:0/5 + 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/6 + INFO 3 switched to configuration voters=(2 3) +> 1 receiving messages + 3->1 MsgAppResp Term:1 Log:0/4 + 3->1 MsgAppResp Term:1 Log:0/5 + 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/6 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:6 + CommittedEntries: + 1/6 EntryNormal "bar" + Messages: + 1->2 MsgApp Term:1 Log:1/6 Commit:6 + 1->3 MsgApp Term:1 Log:1/6 Commit:6 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/6 Commit:6 +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/6 Commit:6 +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:6 + CommittedEntries: + 1/6 EntryNormal "bar" + Messages: + 2->1 MsgAppResp Term:1 Log:0/6 +> 3 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:6 + CommittedEntries: + 1/6 EntryNormal "bar" + Messages: + 3->1 MsgAppResp Term:1 Log:0/6 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/6 + +# However not all is well. n1 is still leader but unconditionally drops all +# proposals on the floor, so we're effectively stuck if it still heartbeats +# its followers... +propose 1 baz +---- +raft proposal dropped + +tick-heartbeat 1 +---- +ok + +# ... which, uh oh, it does. +# TODO(tbg): change behavior so that a leader that is removed immediately steps +# down, and initiates an optimistic handover. +stabilize +---- +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:6 + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:6 +> 2 receiving messages + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:6 +> 3 receiving messages + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:6 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgHeartbeatResp Term:1 Log:0/0 +> 3 handling Ready + Ready MustSync=false: + Messages: + 3->1 MsgHeartbeatResp Term:1 Log:0/0 +> 1 receiving messages + 2->1 MsgHeartbeatResp Term:1 Log:0/0 + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + +# Just confirming the issue above - leader does not automatically step down. +# Expected behavior: a new leader is elected after an election timeout. +raft-state +---- +1: StateLeader (Non-Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 diff --git a/third_party/raft/testdata/confchange_v1_remove_leader_stepdown.txt b/third_party/raft/testdata/confchange_v1_remove_leader_stepdown.txt new file mode 100644 index 0000000000..fe39765093 --- /dev/null +++ b/third_party/raft/testdata/confchange_v1_remove_leader_stepdown.txt @@ -0,0 +1,190 @@ +# We'll turn this back on after the boilerplate. +log-level none +---- +ok + +# Run a V1 membership change that removes the leader, asking it +# to step down on removal. +# Bootstrap n1, n2, n3. +add-nodes 3 voters=(1,2,3) index=2 step-down-on-removal=true +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok + +log-level debug +---- +ok + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 + +# Start removing n1. +propose-conf-change 1 v1=true +r1 +---- +ok + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 + +# Propose an extra entry which will be sent out together with the conf change. +propose 1 foo +---- +ok + +# Send out the corresponding appends. +process-ready 1 +---- +Ready MustSync=true: +Entries: +1/4 EntryConfChange r1 +1/5 EntryNormal "foo" +Messages: +1->2 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1] +1->3 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1] +1->2 MsgApp Term:1 Log:1/4 Commit:3 Entries:[1/5 EntryNormal "foo"] +1->3 MsgApp Term:1 Log:1/4 Commit:3 Entries:[1/5 EntryNormal "foo"] + +# Send response from n2 (which is enough to commit the entries so far next time +# n1 runs). +stabilize 2 +---- +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1] + 1->2 MsgApp Term:1 Log:1/4 Commit:3 Entries:[1/5 EntryNormal "foo"] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/4 EntryConfChange r1 + 1/5 EntryNormal "foo" + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/5 + +# Put another entry in n1's log. +propose 1 bar +---- +ok + +# n1 applies the conf change, removing itself and stepping down. But it still +# has an uncommitted 'bar' entry in the log that it sends out appends for first. +stabilize 1 +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/6 EntryNormal "bar" + Messages: + 1->2 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"] + 1->3 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"] +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + 2->1 MsgAppResp Term:1 Log:0/5 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:5 + CommittedEntries: + 1/4 EntryConfChange r1 + 1/5 EntryNormal "foo" + Messages: + 1->2 MsgApp Term:1 Log:1/6 Commit:4 + 1->3 MsgApp Term:1 Log:1/6 Commit:4 + 1->2 MsgApp Term:1 Log:1/6 Commit:5 + 1->3 MsgApp Term:1 Log:1/6 Commit:5 + INFO 1 switched to configuration voters=(2 3) + INFO 1 became follower at term 1 +> 1 handling Ready + Ready MustSync=false: + Lead:0 State:StateFollower + +raft-state +---- +1: StateFollower (Non-Voter) Term:1 Lead:0 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 + +# n2 responds, n3 doesn't yet. Quorum for 'bar' should not be reached... +stabilize 2 +---- +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"] + 1->2 MsgApp Term:1 Log:1/6 Commit:4 + 1->2 MsgApp Term:1 Log:1/6 Commit:5 +> 2 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:5 + Entries: + 1/6 EntryNormal "bar" + CommittedEntries: + 1/4 EntryConfChange r1 + 1/5 EntryNormal "foo" + Messages: + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 + INFO 2 switched to configuration voters=(2 3) + +# ...because the old leader n1 ignores the append responses. +stabilize 1 +---- +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 + +# When n3 responds, quorum is reached and everything falls into place. +stabilize +---- +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1] + 1->3 MsgApp Term:1 Log:1/4 Commit:3 Entries:[1/5 EntryNormal "foo"] + 1->3 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"] + 1->3 MsgApp Term:1 Log:1/6 Commit:4 + 1->3 MsgApp Term:1 Log:1/6 Commit:5 +> 3 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:5 + Entries: + 1/4 EntryConfChange r1 + 1/5 EntryNormal "foo" + 1/6 EntryNormal "bar" + CommittedEntries: + 1/4 EntryConfChange r1 + 1/5 EntryNormal "foo" + Messages: + 3->1 MsgAppResp Term:1 Log:0/4 + 3->1 MsgAppResp Term:1 Log:0/5 + 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/6 + INFO 3 switched to configuration voters=(2 3) +> 1 receiving messages + 3->1 MsgAppResp Term:1 Log:0/4 + 3->1 MsgAppResp Term:1 Log:0/5 + 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/6 + +# n1 can no longer propose. +propose 1 baz +---- +INFO 1 no leader at term 1; dropping proposal +raft proposal dropped + +# Nor can it campaign to become leader. +campaign 1 +---- +WARN 1 is unpromotable and can not campaign diff --git a/third_party/raft/testdata/confchange_v2_add_double_auto.txt b/third_party/raft/testdata/confchange_v2_add_double_auto.txt new file mode 100644 index 0000000000..f290c980e4 --- /dev/null +++ b/third_party/raft/testdata/confchange_v2_add_double_auto.txt @@ -0,0 +1,404 @@ +# Run a V2 membership change that adds two voters at once and auto-leaves the +# joint configuration. (This is the same as specifying an explicit transition +# since more than one change is being made atomically). + +# Bootstrap n1. +add-nodes 1 voters=(1) index=2 +---- +INFO 1 switched to configuration voters=(1) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 + +process-ready 1 +---- +Ready MustSync=true: +Lead:0 State:StateCandidate +HardState Term:1 Vote:1 Commit:2 +INFO 1 received MsgVoteResp from 1 at term 1 +INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections +INFO 1 became leader at term 1 + +propose-conf-change 1 transition=auto +v2 v3 +---- +ok + +# Add two "empty" nodes to the cluster, n2 and n3. +add-nodes 2 +---- +INFO 2 switched to configuration voters=() +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] +INFO 3 switched to configuration voters=() +INFO 3 became follower at term 0 +INFO newRaft 3 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +# Process n1 once, so that it can append the entry. +process-ready 1 +---- +Ready MustSync=true: +Lead:1 State:StateLeader +Entries: +1/3 EntryNormal "" +1/4 EntryConfChangeV2 v2 v3 + +# Now n1 applies the conf change. We see that it starts transitioning out of that joint +# configuration (though we will only see that proposal in the next ready handling +# loop, when it is emitted). We also see that this is using joint consensus, which +# it has to since we're carrying out two additions at once. +process-ready 1 +---- +Ready MustSync=false: +HardState Term:1 Vote:1 Commit:4 +CommittedEntries: +1/3 EntryNormal "" +1/4 EntryConfChangeV2 v2 v3 +INFO 1 switched to configuration voters=(1 2 3)&&(1) autoleave +INFO initiating automatic transition out of joint configuration voters=(1 2 3)&&(1) autoleave + +# n1 immediately probes n2 and n3. +stabilize 1 +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2 v3] + 1->3 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2 v3] + +# First, play out the whole interaction between n1 and n2. We see n1's probe to +# n2 get rejected (since n2 needs a snapshot); the snapshot is delivered at which +# point n2 switches to the correct config, and n1 catches it up. This notably +# includes the empty conf change which gets committed and applied by both and +# which transitions them out of their joint configuration into the final one (1 2 3). +stabilize 1 2 +---- +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2 v3] + INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 +> 2 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Commit:0 + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) + DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgSnap Term:1 Log:0/0 + Snapshot: Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true +> 2 receiving messages + 1->2 MsgSnap Term:1 Log:0/0 + Snapshot: Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true + INFO log [committed=0, applied=0, applying=0, unstable.offset=1, unstable.offsetInProgress=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] + INFO 2 switched to configuration voters=(1 2 3)&&(1) autoleave + INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] + INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:4 + Snapshot Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/5 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/5 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:5 + CommittedEntries: + 1/5 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/5 Commit:5 + INFO 1 switched to configuration voters=(1 2 3) +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/5 Commit:5 +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:5 + CommittedEntries: + 1/5 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/5 + INFO 2 switched to configuration voters=(1 2 3) +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/5 + +# n3 immediately receives a snapshot in the final configuration. +stabilize 1 3 +---- +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2 v3] + INFO 3 [term: 0] received a MsgApp message with higher term from 1 [term: 1] + INFO 3 became follower at term 1 + DEBUG 3 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 +> 3 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Commit:0 + Messages: + 3->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) +> 1 receiving messages + 3->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) + DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 3 for index 3 + DEBUG 1 decreased progress of 3 to [StateProbe match=0 next=1] + DEBUG 1 [firstindex: 3, commit: 5] sent snapshot[index: 5, term: 1] to 3 [StateProbe match=0 next=1] + DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 next=6 paused pendingSnap=5] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->3 MsgSnap Term:1 Log:0/0 + Snapshot: Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false +> 3 receiving messages + 1->3 MsgSnap Term:1 Log:0/0 + Snapshot: Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + INFO log [committed=0, applied=0, applying=0, unstable.offset=1, unstable.offsetInProgress=1, len(unstable.Entries)=0] starts to restore snapshot [index: 5, term: 1] + INFO 3 switched to configuration voters=(1 2 3) + INFO 3 [commit: 5, lastindex: 5, lastterm: 1] restored snapshot [index: 5, term: 1] + INFO 3 [commit: 5] restored snapshot [index: 5, term: 1] +> 3 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:5 + Snapshot Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + Messages: + 3->1 MsgAppResp Term:1 Log:0/5 +> 1 receiving messages + 3->1 MsgAppResp Term:1 Log:0/5 + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=5 next=6 paused pendingSnap=5] + +# Nothing else happens. +stabilize +---- +ok + +# Now remove two nodes. What's new here is that the leader will actually have +# to go to a quorum to commit the transition into the joint config. + +propose-conf-change 1 +r2 r3 +---- +ok + +# n1 sends out MsgApps. +stabilize 1 +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/6 EntryConfChangeV2 r2 r3 + Messages: + 1->2 MsgApp Term:1 Log:1/5 Commit:5 Entries:[1/6 EntryConfChangeV2 r2 r3] + 1->3 MsgApp Term:1 Log:1/5 Commit:5 Entries:[1/6 EntryConfChangeV2 r2 r3] + +# n2, n3 ack them. +stabilize 2 3 +---- +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/5 Commit:5 Entries:[1/6 EntryConfChangeV2 r2 r3] +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/5 Commit:5 Entries:[1/6 EntryConfChangeV2 r2 r3] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/6 EntryConfChangeV2 r2 r3 + Messages: + 2->1 MsgAppResp Term:1 Log:0/6 +> 3 handling Ready + Ready MustSync=true: + Entries: + 1/6 EntryConfChangeV2 r2 r3 + Messages: + 3->1 MsgAppResp Term:1 Log:0/6 + +# n1 gets some more proposals. This is part of a regression test: There used to +# be a bug in which these proposals would prompt the leader to transition out of +# the same joint state multiple times, which would cause a panic. +propose 1 foo +---- +ok + +propose 1 bar +---- +ok + +# n1 switches to the joint config, then initiates a transition into the final +# config. +stabilize 1 +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/7 EntryNormal "foo" + 1/8 EntryNormal "bar" + Messages: + 1->2 MsgApp Term:1 Log:1/6 Commit:5 Entries:[1/7 EntryNormal "foo"] + 1->3 MsgApp Term:1 Log:1/6 Commit:5 Entries:[1/7 EntryNormal "foo"] + 1->2 MsgApp Term:1 Log:1/7 Commit:5 Entries:[1/8 EntryNormal "bar"] + 1->3 MsgApp Term:1 Log:1/7 Commit:5 Entries:[1/8 EntryNormal "bar"] +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/6 + 3->1 MsgAppResp Term:1 Log:0/6 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:6 + CommittedEntries: + 1/6 EntryConfChangeV2 r2 r3 + Messages: + 1->2 MsgApp Term:1 Log:1/8 Commit:6 + 1->3 MsgApp Term:1 Log:1/8 Commit:6 + INFO 1 switched to configuration voters=(1)&&(1 2 3) autoleave + INFO initiating automatic transition out of joint configuration voters=(1)&&(1 2 3) autoleave +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/9 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/8 Commit:6 Entries:[1/9 EntryConfChangeV2] + 1->3 MsgApp Term:1 Log:1/8 Commit:6 Entries:[1/9 EntryConfChangeV2] + +# n2 and n3 also switch to the joint config, and ack the transition out of it. +stabilize 2 3 +---- +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/6 Commit:5 Entries:[1/7 EntryNormal "foo"] + 1->2 MsgApp Term:1 Log:1/7 Commit:5 Entries:[1/8 EntryNormal "bar"] + 1->2 MsgApp Term:1 Log:1/8 Commit:6 + 1->2 MsgApp Term:1 Log:1/8 Commit:6 Entries:[1/9 EntryConfChangeV2] +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/6 Commit:5 Entries:[1/7 EntryNormal "foo"] + 1->3 MsgApp Term:1 Log:1/7 Commit:5 Entries:[1/8 EntryNormal "bar"] + 1->3 MsgApp Term:1 Log:1/8 Commit:6 + 1->3 MsgApp Term:1 Log:1/8 Commit:6 Entries:[1/9 EntryConfChangeV2] +> 2 handling Ready + Ready MustSync=true: + HardState Term:1 Commit:6 + Entries: + 1/7 EntryNormal "foo" + 1/8 EntryNormal "bar" + 1/9 EntryConfChangeV2 + CommittedEntries: + 1/6 EntryConfChangeV2 r2 r3 + Messages: + 2->1 MsgAppResp Term:1 Log:0/7 + 2->1 MsgAppResp Term:1 Log:0/8 + 2->1 MsgAppResp Term:1 Log:0/8 + 2->1 MsgAppResp Term:1 Log:0/9 + INFO 2 switched to configuration voters=(1)&&(1 2 3) autoleave +> 3 handling Ready + Ready MustSync=true: + HardState Term:1 Commit:6 + Entries: + 1/7 EntryNormal "foo" + 1/8 EntryNormal "bar" + 1/9 EntryConfChangeV2 + CommittedEntries: + 1/6 EntryConfChangeV2 r2 r3 + Messages: + 3->1 MsgAppResp Term:1 Log:0/7 + 3->1 MsgAppResp Term:1 Log:0/8 + 3->1 MsgAppResp Term:1 Log:0/8 + 3->1 MsgAppResp Term:1 Log:0/9 + INFO 3 switched to configuration voters=(1)&&(1 2 3) autoleave + +# n2 and n3 also leave the joint config and the dust settles. We see at the very +# end that n1 receives some messages from them that it refuses because it does +# not have them in its config any more. +stabilize +---- +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/7 + 2->1 MsgAppResp Term:1 Log:0/8 + 2->1 MsgAppResp Term:1 Log:0/8 + 2->1 MsgAppResp Term:1 Log:0/9 + 3->1 MsgAppResp Term:1 Log:0/7 + 3->1 MsgAppResp Term:1 Log:0/8 + 3->1 MsgAppResp Term:1 Log:0/8 + 3->1 MsgAppResp Term:1 Log:0/9 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:9 + CommittedEntries: + 1/7 EntryNormal "foo" + 1/8 EntryNormal "bar" + 1/9 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/9 Commit:7 + 1->3 MsgApp Term:1 Log:1/9 Commit:7 + 1->2 MsgApp Term:1 Log:1/9 Commit:8 + 1->3 MsgApp Term:1 Log:1/9 Commit:8 + 1->2 MsgApp Term:1 Log:1/9 Commit:9 + 1->3 MsgApp Term:1 Log:1/9 Commit:9 + INFO 1 switched to configuration voters=(1) +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/9 Commit:7 + 1->2 MsgApp Term:1 Log:1/9 Commit:8 + 1->2 MsgApp Term:1 Log:1/9 Commit:9 +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/9 Commit:7 + 1->3 MsgApp Term:1 Log:1/9 Commit:8 + 1->3 MsgApp Term:1 Log:1/9 Commit:9 +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:9 + CommittedEntries: + 1/7 EntryNormal "foo" + 1/8 EntryNormal "bar" + 1/9 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/9 + 2->1 MsgAppResp Term:1 Log:0/9 + 2->1 MsgAppResp Term:1 Log:0/9 + INFO 2 switched to configuration voters=(1) +> 3 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:9 + CommittedEntries: + 1/7 EntryNormal "foo" + 1/8 EntryNormal "bar" + 1/9 EntryConfChangeV2 + Messages: + 3->1 MsgAppResp Term:1 Log:0/9 + 3->1 MsgAppResp Term:1 Log:0/9 + 3->1 MsgAppResp Term:1 Log:0/9 + INFO 3 switched to configuration voters=(1) +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/9 + raft: cannot step as peer not found + 2->1 MsgAppResp Term:1 Log:0/9 + raft: cannot step as peer not found + 2->1 MsgAppResp Term:1 Log:0/9 + raft: cannot step as peer not found + 3->1 MsgAppResp Term:1 Log:0/9 + raft: cannot step as peer not found + 3->1 MsgAppResp Term:1 Log:0/9 + raft: cannot step as peer not found + 3->1 MsgAppResp Term:1 Log:0/9 + raft: cannot step as peer not found diff --git a/third_party/raft/testdata/confchange_v2_add_double_implicit.txt b/third_party/raft/testdata/confchange_v2_add_double_implicit.txt new file mode 100644 index 0000000000..2db3b7c14c --- /dev/null +++ b/third_party/raft/testdata/confchange_v2_add_double_implicit.txt @@ -0,0 +1,136 @@ +# Run a V2 membership change that adds a single voter but explicitly asks for the +# use of joint consensus (with auto-leaving). + +# TODO(tbg): also verify that if the leader changes while in the joint state, the +# new leader will auto-transition out of the joint state just the same. + +# Bootstrap n1. +add-nodes 1 voters=(1) index=2 +---- +INFO 1 switched to configuration voters=(1) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 + +process-ready 1 +---- +Ready MustSync=true: +Lead:0 State:StateCandidate +HardState Term:1 Vote:1 Commit:2 +INFO 1 received MsgVoteResp from 1 at term 1 +INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections +INFO 1 became leader at term 1 + +propose-conf-change 1 transition=implicit +v2 +---- +ok + +# Add n2. +add-nodes 1 +---- +INFO 2 switched to configuration voters=() +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +# n1 commits the conf change using itself as commit quorum, then starts catching up n2. +# When that's done, it starts auto-transitioning out. Note that the snapshots propagating +# the joint config have the AutoLeave flag set in their config. +stabilize 1 2 +---- +> 1 handling Ready + Ready MustSync=true: + Lead:1 State:StateLeader + Entries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:4 + CommittedEntries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 + INFO 1 switched to configuration voters=(1 2)&&(1) autoleave + INFO initiating automatic transition out of joint configuration voters=(1 2)&&(1) autoleave +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] + INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 +> 2 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Commit:0 + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) + DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgSnap Term:1 Log:0/0 + Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true +> 2 receiving messages + 1->2 MsgSnap Term:1 Log:0/0 + Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true + INFO log [committed=0, applied=0, applying=0, unstable.offset=1, unstable.offsetInProgress=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] + INFO 2 switched to configuration voters=(1 2)&&(1) autoleave + INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] + INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:4 + Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/5 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/5 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:5 + CommittedEntries: + 1/5 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/5 Commit:5 + INFO 1 switched to configuration voters=(1 2) +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/5 Commit:5 +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:5 + CommittedEntries: + 1/5 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/5 + INFO 2 switched to configuration voters=(1 2) +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/5 diff --git a/third_party/raft/testdata/confchange_v2_add_single_auto.txt b/third_party/raft/testdata/confchange_v2_add_single_auto.txt new file mode 100644 index 0000000000..3c2341590f --- /dev/null +++ b/third_party/raft/testdata/confchange_v2_add_single_auto.txt @@ -0,0 +1,97 @@ +# Run a V2 membership change that adds a single voter in auto mode, which means +# that joint consensus is not used but a direct transition into the new config +# takes place. + +# Bootstrap n1. +add-nodes 1 voters=(1) index=2 +---- +INFO 1 switched to configuration voters=(1) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 + +process-ready 1 +---- +Ready MustSync=true: +Lead:0 State:StateCandidate +HardState Term:1 Vote:1 Commit:2 +INFO 1 received MsgVoteResp from 1 at term 1 +INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections +INFO 1 became leader at term 1 + +# Add v2 (with an auto transition). +propose-conf-change 1 +v2 +---- +ok + +# Pull n2 out of thin air. +add-nodes 1 +---- +INFO 2 switched to configuration voters=() +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +# n1 commits the conf change using itself as commit quorum, immediately transitions into +# the final config, and catches up n2. +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Lead:1 State:StateLeader + Entries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:4 + CommittedEntries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 + INFO 1 switched to configuration voters=(1 2) +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] + INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 +> 2 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Commit:0 + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) + DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgSnap Term:1 Log:0/0 + Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false +> 2 receiving messages + 1->2 MsgSnap Term:1 Log:0/0 + Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + INFO log [committed=0, applied=0, applying=0, unstable.offset=1, unstable.offsetInProgress=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] + INFO 2 switched to configuration voters=(1 2) + INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] + INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:4 + Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] diff --git a/third_party/raft/testdata/confchange_v2_add_single_explicit.txt b/third_party/raft/testdata/confchange_v2_add_single_explicit.txt new file mode 100644 index 0000000000..012827690e --- /dev/null +++ b/third_party/raft/testdata/confchange_v2_add_single_explicit.txt @@ -0,0 +1,205 @@ +# Run a V2 membership change that adds a single voter but explicitly asks for the +# use of joint consensus, including wanting to transition out of the joint config +# manually. + +# Bootstrap n1. +add-nodes 1 voters=(1) index=2 +---- +INFO 1 switched to configuration voters=(1) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 + +process-ready 1 +---- +Ready MustSync=true: +Lead:0 State:StateCandidate +HardState Term:1 Vote:1 Commit:2 +INFO 1 received MsgVoteResp from 1 at term 1 +INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections +INFO 1 became leader at term 1 + +# Add v2 with an explicit transition. +propose-conf-change 1 transition=explicit +v2 +---- +ok + +# Pull n2 out of thin air. +add-nodes 1 +---- +INFO 2 switched to configuration voters=() +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +# n1 commits the conf change using itself as commit quorum, then starts catching up n2. +# Everyone remains in the joint config. Note that the snapshot below has AutoLeave unset. +stabilize 1 2 +---- +> 1 handling Ready + Ready MustSync=true: + Lead:1 State:StateLeader + Entries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:4 + CommittedEntries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 + INFO 1 switched to configuration voters=(1 2)&&(1) +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] + INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 +> 2 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Commit:0 + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) + DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgSnap Term:1 Log:0/0 + Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:false +> 2 receiving messages + 1->2 MsgSnap Term:1 Log:0/0 + Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:false + INFO log [committed=0, applied=0, applying=0, unstable.offset=1, unstable.offsetInProgress=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] + INFO 2 switched to configuration voters=(1 2)&&(1) + INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] + INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:4 + Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:false + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] + +# Check that we're not allowed to change membership again while in the joint state. +# This leads to an empty entry being proposed instead (index 5 in the stabilize block +# below). +propose-conf-change 1 +v3 v4 v5 +---- +INFO 1 ignoring conf change transition:ConfChangeTransitionAuto changes:{type:ConfChangeAddNode node_id:3} changes:{type:ConfChangeAddNode node_id:4} changes:{type:ConfChangeAddNode node_id:5} at config voters=(1 2)&&(1): must transition out of joint config first + +# Propose a transition out of the joint config. We'll see this at index 6 below. +propose-conf-change 1 +---- +ok + +# The group commits the command and everyone switches to the final config. +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryNormal "" + 1/6 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryNormal ""] + 1->2 MsgApp Term:1 Log:1/5 Commit:4 Entries:[1/6 EntryConfChangeV2] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryNormal ""] + 1->2 MsgApp Term:1 Log:1/5 Commit:4 Entries:[1/6 EntryConfChangeV2] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryNormal "" + 1/6 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/6 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/6 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:6 + CommittedEntries: + 1/5 EntryNormal "" + 1/6 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/6 Commit:5 + 1->2 MsgApp Term:1 Log:1/6 Commit:6 + INFO 1 switched to configuration voters=(1 2) +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/6 Commit:5 + 1->2 MsgApp Term:1 Log:1/6 Commit:6 +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:6 + CommittedEntries: + 1/5 EntryNormal "" + 1/6 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 + INFO 2 switched to configuration voters=(1 2) +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 + +# Check that trying to transition out again won't do anything. +propose-conf-change 1 +---- +INFO 1 ignoring conf change transition:ConfChangeTransitionAuto at config voters=(1 2): not in joint state; refusing empty conf change + +# Finishes work for the empty entry we just proposed. +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/7 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/6 Commit:6 Entries:[1/7 EntryNormal ""] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/6 Commit:6 Entries:[1/7 EntryNormal ""] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/7 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:1 Log:0/7 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/7 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:7 + CommittedEntries: + 1/7 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/7 Commit:7 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/7 Commit:7 +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:7 + CommittedEntries: + 1/7 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:1 Log:0/7 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/7 diff --git a/third_party/raft/testdata/confchange_v2_replace_leader.txt b/third_party/raft/testdata/confchange_v2_replace_leader.txt new file mode 100644 index 0000000000..628cc447e1 --- /dev/null +++ b/third_party/raft/testdata/confchange_v2_replace_leader.txt @@ -0,0 +1,419 @@ +# Run a V2 membership change that removes the leader and adds another voter as +# a single operation, using joint consensus and explicitly determining when to +# transition out of the joint config. Leadership is transferred to new joiner +# while in the joint config. After the reconfiguration completes, we verify +# that the removed leader cannot campaign to become leader. + +# We'll turn this back on after the boilerplate. +log-level none +---- +ok + +# Bootstrap n1, n2, n3. +add-nodes 3 voters=(1,2,3) index=2 +---- +ok + +# n1 campaigns to become leader. +campaign 1 +---- +ok + +stabilize +---- +ok + +log-level info +---- +ok + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 + +# create n4 +add-nodes 1 +---- +INFO 4 switched to configuration voters=() +INFO 4 became follower at term 0 +INFO newRaft 4 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +# Start reconfiguration to remove n1 and add n4. +propose-conf-change 1 v1=false transition=explicit +r1 v4 +---- +ok + +# Enter joint config. +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/4 EntryConfChangeV2 r1 v4 + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChangeV2 r1 v4] + 1->3 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChangeV2 r1 v4] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChangeV2 r1 v4] +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChangeV2 r1 v4] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/4 EntryConfChangeV2 r1 v4 + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 3 handling Ready + Ready MustSync=true: + Entries: + 1/4 EntryConfChangeV2 r1 v4 + Messages: + 3->1 MsgAppResp Term:1 Log:0/4 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + 3->1 MsgAppResp Term:1 Log:0/4 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:4 + CommittedEntries: + 1/4 EntryConfChangeV2 r1 v4 + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 + 1->3 MsgApp Term:1 Log:1/4 Commit:4 + INFO 1 switched to configuration voters=(2 3 4)&&(1 2 3) +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/4 Commit:4 +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->4 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 r1 v4] +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:4 + CommittedEntries: + 1/4 EntryConfChangeV2 r1 v4 + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 + INFO 2 switched to configuration voters=(2 3 4)&&(1 2 3) +> 3 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:4 + CommittedEntries: + 1/4 EntryConfChangeV2 r1 v4 + Messages: + 3->1 MsgAppResp Term:1 Log:0/4 + INFO 3 switched to configuration voters=(2 3 4)&&(1 2 3) +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/4 + 3->1 MsgAppResp Term:1 Log:0/4 +> 4 receiving messages + 1->4 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 r1 v4] + INFO 4 [term: 0] received a MsgApp message with higher term from 1 [term: 1] + INFO 4 became follower at term 1 +> 4 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Commit:0 + Messages: + 4->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) +> 1 receiving messages + 4->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->4 MsgSnap Term:1 Log:0/0 + Snapshot: Index:4 Term:1 ConfState:Voters:[2 3 4] VotersOutgoing:[1 2 3] Learners:[] LearnersNext:[] AutoLeave:false +> 4 receiving messages + 1->4 MsgSnap Term:1 Log:0/0 + Snapshot: Index:4 Term:1 ConfState:Voters:[2 3 4] VotersOutgoing:[1 2 3] Learners:[] LearnersNext:[] AutoLeave:false + INFO log [committed=0, applied=0, applying=0, unstable.offset=1, unstable.offsetInProgress=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] + INFO 4 switched to configuration voters=(2 3 4)&&(1 2 3) + INFO 4 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] + INFO 4 [commit: 4] restored snapshot [index: 4, term: 1] +> 4 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:4 + Snapshot Index:4 Term:1 ConfState:Voters:[2 3 4] VotersOutgoing:[1 2 3] Learners:[] LearnersNext:[] AutoLeave:false + Messages: + 4->1 MsgAppResp Term:1 Log:0/4 +> 1 receiving messages + 4->1 MsgAppResp Term:1 Log:0/4 + + +# Transfer leadership while in the joint config. +transfer-leadership from=1 to=4 +---- +INFO 1 [term 1] starts to transfer leadership to 4 +INFO 1 sends MsgTimeoutNow to 4 immediately as 4 already has up-to-date log + +# Leadership transfer wasn't processed yet. +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 +4: StateFollower (Voter) Term:1 Lead:1 + +# Leadership transfer is happening here. +stabilize +---- +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->4 MsgTimeoutNow Term:1 Log:0/0 +> 4 receiving messages + 1->4 MsgTimeoutNow Term:1 Log:0/0 + INFO 4 [term 1] received MsgTimeoutNow from 1 and starts an election to get leadership. + INFO 4 is starting a new election at term 1 + INFO 4 became candidate at term 2 + INFO 4 [logterm: 1, index: 4] sent MsgVote request to 1 at term 2 + INFO 4 [logterm: 1, index: 4] sent MsgVote request to 2 at term 2 + INFO 4 [logterm: 1, index: 4] sent MsgVote request to 3 at term 2 +> 4 handling Ready + Ready MustSync=true: + Lead:0 State:StateCandidate + HardState Term:2 Vote:4 Commit:4 + Messages: + 4->1 MsgVote Term:2 Log:1/4 + 4->2 MsgVote Term:2 Log:1/4 + 4->3 MsgVote Term:2 Log:1/4 + INFO 4 received MsgVoteResp from 4 at term 2 + INFO 4 has received 1 MsgVoteResp votes and 0 vote rejections +> 1 receiving messages + 4->1 MsgVote Term:2 Log:1/4 + INFO 1 [term: 1] received a MsgVote message with higher term from 4 [term: 2] + INFO 1 became follower at term 2 + INFO 1 [logterm: 1, index: 4, vote: 0] cast MsgVote for 4 [logterm: 1, index: 4] at term 2 +> 2 receiving messages + 4->2 MsgVote Term:2 Log:1/4 + INFO 2 [term: 1] received a MsgVote message with higher term from 4 [term: 2] + INFO 2 became follower at term 2 + INFO 2 [logterm: 1, index: 4, vote: 0] cast MsgVote for 4 [logterm: 1, index: 4] at term 2 +> 3 receiving messages + 4->3 MsgVote Term:2 Log:1/4 + INFO 3 [term: 1] received a MsgVote message with higher term from 4 [term: 2] + INFO 3 became follower at term 2 + INFO 3 [logterm: 1, index: 4, vote: 0] cast MsgVote for 4 [logterm: 1, index: 4] at term 2 +> 1 handling Ready + Ready MustSync=true: + Lead:0 State:StateFollower + HardState Term:2 Vote:4 Commit:4 + Messages: + 1->4 MsgVoteResp Term:2 Log:0/0 +> 2 handling Ready + Ready MustSync=true: + Lead:0 State:StateFollower + HardState Term:2 Vote:4 Commit:4 + Messages: + 2->4 MsgVoteResp Term:2 Log:0/0 +> 3 handling Ready + Ready MustSync=true: + Lead:0 State:StateFollower + HardState Term:2 Vote:4 Commit:4 + Messages: + 3->4 MsgVoteResp Term:2 Log:0/0 +> 4 receiving messages + 1->4 MsgVoteResp Term:2 Log:0/0 + INFO 4 received MsgVoteResp from 1 at term 2 + INFO 4 has received 2 MsgVoteResp votes and 0 vote rejections + 2->4 MsgVoteResp Term:2 Log:0/0 + INFO 4 received MsgVoteResp from 2 at term 2 + INFO 4 has received 3 MsgVoteResp votes and 0 vote rejections + INFO 4 became leader at term 2 + 3->4 MsgVoteResp Term:2 Log:0/0 +> 4 handling Ready + Ready MustSync=true: + Lead:4 State:StateLeader + Entries: + 2/5 EntryNormal "" + Messages: + 4->1 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""] + 4->2 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""] + 4->3 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""] +> 1 receiving messages + 4->1 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""] +> 2 receiving messages + 4->2 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""] +> 3 receiving messages + 4->3 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""] +> 1 handling Ready + Ready MustSync=true: + Lead:4 State:StateFollower + Entries: + 2/5 EntryNormal "" + Messages: + 1->4 MsgAppResp Term:2 Log:0/5 +> 2 handling Ready + Ready MustSync=true: + Lead:4 State:StateFollower + Entries: + 2/5 EntryNormal "" + Messages: + 2->4 MsgAppResp Term:2 Log:0/5 +> 3 handling Ready + Ready MustSync=true: + Lead:4 State:StateFollower + Entries: + 2/5 EntryNormal "" + Messages: + 3->4 MsgAppResp Term:2 Log:0/5 +> 4 receiving messages + 1->4 MsgAppResp Term:2 Log:0/5 + 2->4 MsgAppResp Term:2 Log:0/5 + 3->4 MsgAppResp Term:2 Log:0/5 +> 4 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:4 Commit:5 + CommittedEntries: + 2/5 EntryNormal "" + Messages: + 4->1 MsgApp Term:2 Log:2/5 Commit:5 + 4->2 MsgApp Term:2 Log:2/5 Commit:5 + 4->3 MsgApp Term:2 Log:2/5 Commit:5 +> 1 receiving messages + 4->1 MsgApp Term:2 Log:2/5 Commit:5 +> 2 receiving messages + 4->2 MsgApp Term:2 Log:2/5 Commit:5 +> 3 receiving messages + 4->3 MsgApp Term:2 Log:2/5 Commit:5 +> 1 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:4 Commit:5 + CommittedEntries: + 2/5 EntryNormal "" + Messages: + 1->4 MsgAppResp Term:2 Log:0/5 +> 2 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:4 Commit:5 + CommittedEntries: + 2/5 EntryNormal "" + Messages: + 2->4 MsgAppResp Term:2 Log:0/5 +> 3 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:4 Commit:5 + CommittedEntries: + 2/5 EntryNormal "" + Messages: + 3->4 MsgAppResp Term:2 Log:0/5 +> 4 receiving messages + 1->4 MsgAppResp Term:2 Log:0/5 + 2->4 MsgAppResp Term:2 Log:0/5 + 3->4 MsgAppResp Term:2 Log:0/5 + +# Leadership transfer succeeded. +raft-state +---- +1: StateFollower (Voter) Term:2 Lead:4 +2: StateFollower (Voter) Term:2 Lead:4 +3: StateFollower (Voter) Term:2 Lead:4 +4: StateLeader (Voter) Term:2 Lead:4 + +# n4 will propose a transition out of the joint config. +propose-conf-change 4 +---- +ok + +# The group commits the command and everyone switches to the final config. +stabilize +---- +> 4 handling Ready + Ready MustSync=true: + Entries: + 2/6 EntryConfChangeV2 + Messages: + 4->1 MsgApp Term:2 Log:2/5 Commit:5 Entries:[2/6 EntryConfChangeV2] + 4->2 MsgApp Term:2 Log:2/5 Commit:5 Entries:[2/6 EntryConfChangeV2] + 4->3 MsgApp Term:2 Log:2/5 Commit:5 Entries:[2/6 EntryConfChangeV2] +> 1 receiving messages + 4->1 MsgApp Term:2 Log:2/5 Commit:5 Entries:[2/6 EntryConfChangeV2] +> 2 receiving messages + 4->2 MsgApp Term:2 Log:2/5 Commit:5 Entries:[2/6 EntryConfChangeV2] +> 3 receiving messages + 4->3 MsgApp Term:2 Log:2/5 Commit:5 Entries:[2/6 EntryConfChangeV2] +> 1 handling Ready + Ready MustSync=true: + Entries: + 2/6 EntryConfChangeV2 + Messages: + 1->4 MsgAppResp Term:2 Log:0/6 +> 2 handling Ready + Ready MustSync=true: + Entries: + 2/6 EntryConfChangeV2 + Messages: + 2->4 MsgAppResp Term:2 Log:0/6 +> 3 handling Ready + Ready MustSync=true: + Entries: + 2/6 EntryConfChangeV2 + Messages: + 3->4 MsgAppResp Term:2 Log:0/6 +> 4 receiving messages + 1->4 MsgAppResp Term:2 Log:0/6 + 2->4 MsgAppResp Term:2 Log:0/6 + 3->4 MsgAppResp Term:2 Log:0/6 +> 4 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:4 Commit:6 + CommittedEntries: + 2/6 EntryConfChangeV2 + Messages: + 4->1 MsgApp Term:2 Log:2/6 Commit:6 + 4->2 MsgApp Term:2 Log:2/6 Commit:6 + 4->3 MsgApp Term:2 Log:2/6 Commit:6 + INFO 4 switched to configuration voters=(2 3 4) +> 1 receiving messages + 4->1 MsgApp Term:2 Log:2/6 Commit:6 +> 2 receiving messages + 4->2 MsgApp Term:2 Log:2/6 Commit:6 +> 3 receiving messages + 4->3 MsgApp Term:2 Log:2/6 Commit:6 +> 1 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:4 Commit:6 + CommittedEntries: + 2/6 EntryConfChangeV2 + Messages: + 1->4 MsgAppResp Term:2 Log:0/6 + INFO 1 switched to configuration voters=(2 3 4) +> 2 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:4 Commit:6 + CommittedEntries: + 2/6 EntryConfChangeV2 + Messages: + 2->4 MsgAppResp Term:2 Log:0/6 + INFO 2 switched to configuration voters=(2 3 4) +> 3 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:4 Commit:6 + CommittedEntries: + 2/6 EntryConfChangeV2 + Messages: + 3->4 MsgAppResp Term:2 Log:0/6 + INFO 3 switched to configuration voters=(2 3 4) +> 4 receiving messages + 1->4 MsgAppResp Term:2 Log:0/6 + raft: cannot step as peer not found + 2->4 MsgAppResp Term:2 Log:0/6 + 3->4 MsgAppResp Term:2 Log:0/6 + +# n1 is out of the configuration. +raft-state +---- +1: StateFollower (Non-Voter) Term:2 Lead:4 +2: StateFollower (Voter) Term:2 Lead:4 +3: StateFollower (Voter) Term:2 Lead:4 +4: StateLeader (Voter) Term:2 Lead:4 + +# Make sure n1 cannot campaign to become leader. +campaign 1 +---- +WARN 1 is unpromotable and can not campaign diff --git a/third_party/raft/testdata/confchange_v2_replace_leader_stepdown.txt b/third_party/raft/testdata/confchange_v2_replace_leader_stepdown.txt new file mode 100644 index 0000000000..62d01d23b4 --- /dev/null +++ b/third_party/raft/testdata/confchange_v2_replace_leader_stepdown.txt @@ -0,0 +1,192 @@ +# Run a V2 membership change that removes the leader and adds another voter as a +# single operation, using joint consensus and explicitly determining when to +# transition out of the joint config. Leadership is transferred by campaigning a +# designated voter in the new config once the old leader steps down. After the +# reconfiguration completes, we verify that the removed leader cannot campaign +# to become leader. + +# We'll turn this back on after the boilerplate. +log-level none +---- +ok + +# Bootstrap n1, n2, n3. +add-nodes 3 voters=(1,2,3) index=2 step-down-on-removal=true +---- +ok + +# n1 campaigns to become leader. +campaign 1 +---- +ok + +stabilize +---- +ok + +log-level info +---- +ok + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 + +# create n4 +add-nodes 1 +---- +INFO 4 switched to configuration voters=() +INFO 4 became follower at term 0 +INFO newRaft 4 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +# Start reconfiguration to remove n1 and add n4. +propose-conf-change 1 v1=false transition=explicit +r1 v4 +---- +ok + +# Enter joint config. +stabilize log-level=none +---- +ok + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 +4: StateFollower (Voter) Term:1 Lead:1 + +# n4 will propose a transition out of the joint config. +propose-conf-change 4 +---- +ok + +# The group commits the command and everyone switches to the final config. +# n1 steps down as leader. +stabilize +---- +> 4 handling Ready + Ready MustSync=false: + Messages: + 4->1 MsgProp Term:0 Log:0/0 Entries:[0/0 EntryConfChangeV2] +> 1 receiving messages + 4->1 MsgProp Term:0 Log:0/0 Entries:[0/0 EntryConfChangeV2] +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2] + 1->3 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2] + 1->4 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2] +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2] +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2] +> 4 receiving messages + 1->4 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/5 +> 3 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryConfChangeV2 + Messages: + 3->1 MsgAppResp Term:1 Log:0/5 +> 4 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryConfChangeV2 + Messages: + 4->1 MsgAppResp Term:1 Log:0/5 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/5 + 3->1 MsgAppResp Term:1 Log:0/5 + 4->1 MsgAppResp Term:1 Log:0/5 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:5 + CommittedEntries: + 1/5 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/5 Commit:5 + 1->3 MsgApp Term:1 Log:1/5 Commit:5 + 1->4 MsgApp Term:1 Log:1/5 Commit:5 + INFO 1 switched to configuration voters=(2 3 4) + INFO 1 became follower at term 1 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/5 Commit:5 +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/5 Commit:5 +> 4 receiving messages + 1->4 MsgApp Term:1 Log:1/5 Commit:5 +> 1 handling Ready + Ready MustSync=false: + Lead:0 State:StateFollower +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:5 + CommittedEntries: + 1/5 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/5 + INFO 2 switched to configuration voters=(2 3 4) +> 3 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:5 + CommittedEntries: + 1/5 EntryConfChangeV2 + Messages: + 3->1 MsgAppResp Term:1 Log:0/5 + INFO 3 switched to configuration voters=(2 3 4) +> 4 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:5 + CommittedEntries: + 1/5 EntryConfChangeV2 + Messages: + 4->1 MsgAppResp Term:1 Log:0/5 + INFO 4 switched to configuration voters=(2 3 4) +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/5 + 3->1 MsgAppResp Term:1 Log:0/5 + 4->1 MsgAppResp Term:1 Log:0/5 + +# n1 is out of the configuration. +raft-state +---- +1: StateFollower (Non-Voter) Term:1 Lead:0 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 +4: StateFollower (Voter) Term:1 Lead:1 + +# Make sure n1 cannot campaign to become leader. +campaign 1 +---- +WARN 1 is unpromotable and can not campaign + +# Campaign the dedicated voter n2 to become the new leader. +campaign 2 +---- +INFO 2 is starting a new election at term 1 +INFO 2 became candidate at term 2 +INFO 2 [logterm: 1, index: 5] sent MsgVote request to 3 at term 2 +INFO 2 [logterm: 1, index: 5] sent MsgVote request to 4 at term 2 + +stabilize log-level=none +---- +ok + +raft-state +---- +1: StateFollower (Non-Voter) Term:1 Lead:0 +2: StateLeader (Voter) Term:2 Lead:2 +3: StateFollower (Voter) Term:2 Lead:2 +4: StateFollower (Voter) Term:2 Lead:2 diff --git a/third_party/raft/testdata/forget_leader.txt b/third_party/raft/testdata/forget_leader.txt new file mode 100644 index 0000000000..a674b346c6 --- /dev/null +++ b/third_party/raft/testdata/forget_leader.txt @@ -0,0 +1,192 @@ +log-level none +---- +ok + +add-nodes 4 voters=(1,2,3) learners=(4) index=10 +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok + +log-level debug +---- +ok + +# ForgetLeader is a noop on the leader. +forget-leader 1 +---- +ok + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 +4: StateFollower (Non-Voter) Term:1 Lead:1 + +# ForgetLeader causes a follower to forget its leader, but remain in the current +# term. It's a noop if it's called again. +forget-leader 2 +---- +INFO 2 forgetting leader 1 at term 1 + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:0 +3: StateFollower (Voter) Term:1 Lead:1 +4: StateFollower (Non-Voter) Term:1 Lead:1 + +forget-leader 2 +---- +ok + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:0 +3: StateFollower (Voter) Term:1 Lead:1 +4: StateFollower (Non-Voter) Term:1 Lead:1 + +# ForgetLeader also works on learners. +forget-leader 4 +---- +INFO 4 forgetting leader 1 at term 1 + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:0 +3: StateFollower (Voter) Term:1 Lead:1 +4: StateFollower (Non-Voter) Term:1 Lead:0 + +# When receiving a heartbeat from the leader, they revert to followers. +tick-heartbeat 1 +---- +ok + +stabilize +---- +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->4 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 2 handling Ready + Ready MustSync=false: + Lead:0 State:StateFollower +> 4 handling Ready + Ready MustSync=false: + Lead:0 State:StateFollower +> 2 receiving messages + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 3 receiving messages + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 4 receiving messages + 1->4 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 2 handling Ready + Ready MustSync=false: + Lead:1 State:StateFollower + Messages: + 2->1 MsgHeartbeatResp Term:1 Log:0/0 +> 3 handling Ready + Ready MustSync=false: + Messages: + 3->1 MsgHeartbeatResp Term:1 Log:0/0 +> 4 handling Ready + Ready MustSync=false: + Lead:1 State:StateFollower + Messages: + 4->1 MsgHeartbeatResp Term:1 Log:0/0 +> 1 receiving messages + 2->1 MsgHeartbeatResp Term:1 Log:0/0 + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + 4->1 MsgHeartbeatResp Term:1 Log:0/0 + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 +4: StateFollower (Non-Voter) Term:1 Lead:1 + +# ForgetLeader is a noop on candidates. +campaign 3 +---- +INFO 3 is starting a new election at term 1 +INFO 3 became candidate at term 2 +INFO 3 [logterm: 1, index: 11] sent MsgVote request to 1 at term 2 +INFO 3 [logterm: 1, index: 11] sent MsgVote request to 2 at term 2 + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateCandidate (Voter) Term:2 Lead:0 +4: StateFollower (Non-Voter) Term:1 Lead:1 + +forget-leader 3 +---- +ok + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateCandidate (Voter) Term:2 Lead:0 +4: StateFollower (Non-Voter) Term:1 Lead:1 + +stabilize log-level=none +---- +ok + +raft-state +---- +1: StateFollower (Voter) Term:2 Lead:3 +2: StateFollower (Voter) Term:2 Lead:3 +3: StateLeader (Voter) Term:2 Lead:3 +4: StateFollower (Non-Voter) Term:2 Lead:3 + +# ForgetLeader shouldn't affect the election timeout: if a follower +# forgets the leader 1 tick before the election timeout fires, it +# will still campaign on the next tick. +set-randomized-election-timeout 2 timeout=3 +---- +ok + +tick-heartbeat 2 +---- +ok + +tick-heartbeat 2 +---- +ok + +forget-leader 2 +---- +INFO 2 forgetting leader 3 at term 2 + +tick-heartbeat 2 +---- +INFO 2 is starting a new election at term 2 +INFO 2 became candidate at term 3 +INFO 2 [logterm: 2, index: 12] sent MsgVote request to 1 at term 3 +INFO 2 [logterm: 2, index: 12] sent MsgVote request to 3 at term 3 + +stabilize log-level=none +---- +ok + +raft-state +---- +1: StateFollower (Voter) Term:3 Lead:2 +2: StateLeader (Voter) Term:3 Lead:2 +3: StateFollower (Voter) Term:3 Lead:2 +4: StateFollower (Non-Voter) Term:3 Lead:2 diff --git a/third_party/raft/testdata/forget_leader_prevote_checkquorum.txt b/third_party/raft/testdata/forget_leader_prevote_checkquorum.txt new file mode 100644 index 0000000000..9b3b80ffae --- /dev/null +++ b/third_party/raft/testdata/forget_leader_prevote_checkquorum.txt @@ -0,0 +1,235 @@ +# Tests that a follower with PreVote+CheckQuorum can forget the leader, allowing +# it to grant prevotes despite having heard from the leader recently. +# +# Also tests that forgetting the leader still won't grant prevotes to a +# replica that isn't up-to-date. +log-level none +---- +ok + +add-nodes 3 voters=(1,2,3) index=10 prevote=true checkquorum=true +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok + +log-level debug +---- +ok + +# If 3 attempts to campaign, 2 rejects it because it has a leader. +campaign 3 +---- +INFO 3 is starting a new election at term 1 +INFO 3 became pre-candidate at term 1 +INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 1 at term 1 +INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 2 at term 1 + +stabilize 3 +---- +> 3 handling Ready + Ready MustSync=false: + Lead:0 State:StatePreCandidate + Messages: + 3->1 MsgPreVote Term:2 Log:1/11 + 3->2 MsgPreVote Term:2 Log:1/11 + INFO 3 received MsgPreVoteResp from 3 at term 1 + INFO 3 has received 1 MsgPreVoteResp votes and 0 vote rejections + +deliver-msgs 1 2 +---- +3->1 MsgPreVote Term:2 Log:1/11 +INFO 1 [logterm: 1, index: 11, vote: 1] ignored MsgPreVote from 3 [logterm: 1, index: 11] at term 1: lease is not expired (remaining ticks: 3) +3->2 MsgPreVote Term:2 Log:1/11 +INFO 2 [logterm: 1, index: 11, vote: 1] ignored MsgPreVote from 3 [logterm: 1, index: 11] at term 1: lease is not expired (remaining ticks: 3) + +# Make 1 assert leadership over 3 again. +tick-heartbeat 1 +---- +ok + +stabilize +---- +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 2 receiving messages + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 3 receiving messages + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 + INFO 3 became follower at term 1 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgHeartbeatResp Term:1 Log:0/0 +> 3 handling Ready + Ready MustSync=false: + Lead:1 State:StateFollower + Messages: + 3->1 MsgHeartbeatResp Term:1 Log:0/0 +> 1 receiving messages + 2->1 MsgHeartbeatResp Term:1 Log:0/0 + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 + +# If 2 forgets the leader, then 3 can obtain prevotes and hold an election +# despite 2 having heard from the leader recently. +forget-leader 2 +---- +INFO 2 forgetting leader 1 at term 1 + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:0 +3: StateFollower (Voter) Term:1 Lead:1 + +campaign 3 +---- +INFO 3 is starting a new election at term 1 +INFO 3 became pre-candidate at term 1 +INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 1 at term 1 +INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 2 at term 1 + +stabilize 3 +---- +> 3 handling Ready + Ready MustSync=false: + Lead:0 State:StatePreCandidate + Messages: + 3->1 MsgPreVote Term:2 Log:1/11 + 3->2 MsgPreVote Term:2 Log:1/11 + INFO 3 received MsgPreVoteResp from 3 at term 1 + INFO 3 has received 1 MsgPreVoteResp votes and 0 vote rejections + +stabilize 2 +---- +> 2 handling Ready + Ready MustSync=false: + Lead:0 State:StateFollower +> 2 receiving messages + 3->2 MsgPreVote Term:2 Log:1/11 + INFO 2 [logterm: 1, index: 11, vote: 1] cast MsgPreVote for 3 [logterm: 1, index: 11] at term 1 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->3 MsgPreVoteResp Term:2 Log:0/0 + +stabilize 3 +---- +> 3 receiving messages + 2->3 MsgPreVoteResp Term:2 Log:0/0 + INFO 3 received MsgPreVoteResp from 2 at term 1 + INFO 3 has received 2 MsgPreVoteResp votes and 0 vote rejections + INFO 3 became candidate at term 2 + INFO 3 [logterm: 1, index: 11] sent MsgVote request to 1 at term 2 + INFO 3 [logterm: 1, index: 11] sent MsgVote request to 2 at term 2 +> 3 handling Ready + Ready MustSync=true: + Lead:0 State:StateCandidate + HardState Term:2 Vote:3 Commit:11 + Messages: + 3->1 MsgVote Term:2 Log:1/11 + 3->2 MsgVote Term:2 Log:1/11 + INFO 3 received MsgVoteResp from 3 at term 2 + INFO 3 has received 1 MsgVoteResp votes and 0 vote rejections + +stabilize log-level=none +---- +ok + +raft-state +---- +1: StateFollower (Voter) Term:2 Lead:3 +2: StateFollower (Voter) Term:2 Lead:3 +3: StateLeader (Voter) Term:2 Lead:3 + +# Test that forgetting the leader still won't grant prevotes if the candidate +# isn't up-to-date. We first replicate a proposal on 3 and 2. +propose 3 prop_1 +---- +ok + +stabilize 3 +---- +> 3 handling Ready + Ready MustSync=true: + Entries: + 2/13 EntryNormal "prop_1" + Messages: + 3->1 MsgApp Term:2 Log:2/12 Commit:12 Entries:[2/13 EntryNormal "prop_1"] + 3->2 MsgApp Term:2 Log:2/12 Commit:12 Entries:[2/13 EntryNormal "prop_1"] + +stabilize 2 +---- +> 2 receiving messages + 3->2 MsgApp Term:2 Log:2/12 Commit:12 Entries:[2/13 EntryNormal "prop_1"] +> 2 handling Ready + Ready MustSync=true: + Entries: + 2/13 EntryNormal "prop_1" + Messages: + 2->3 MsgAppResp Term:2 Log:0/13 + +forget-leader 2 +---- +INFO 2 forgetting leader 3 at term 2 + +# 1 is now behind on its log. It tries to campaign, but fails. +raft-log 1 +---- +1/11 EntryNormal "" +2/12 EntryNormal "" + +campaign 1 +---- +INFO 1 is starting a new election at term 2 +INFO 1 became pre-candidate at term 2 +INFO 1 [logterm: 2, index: 12] sent MsgPreVote request to 2 at term 2 +INFO 1 [logterm: 2, index: 12] sent MsgPreVote request to 3 at term 2 + +process-ready 1 +---- +Ready MustSync=false: +Lead:0 State:StatePreCandidate +Messages: +1->2 MsgPreVote Term:3 Log:2/12 +1->3 MsgPreVote Term:3 Log:2/12 +INFO 1 received MsgPreVoteResp from 1 at term 2 +INFO 1 has received 1 MsgPreVoteResp votes and 0 vote rejections + +stabilize 2 +---- +> 2 handling Ready + Ready MustSync=false: + Lead:0 State:StateFollower +> 2 receiving messages + 1->2 MsgPreVote Term:3 Log:2/12 + INFO 2 [logterm: 2, index: 13, vote: 3] rejected MsgPreVote from 1 [logterm: 2, index: 12] at term 2 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgPreVoteResp Term:2 Log:0/0 Rejected (Hint: 0) + +stabilize log-level=none +---- +ok + +raft-state +---- +1: StateFollower (Voter) Term:2 Lead:3 +2: StateFollower (Voter) Term:2 Lead:3 +3: StateLeader (Voter) Term:2 Lead:3 diff --git a/third_party/raft/testdata/forget_leader_read_only_lease_based.txt b/third_party/raft/testdata/forget_leader_read_only_lease_based.txt new file mode 100644 index 0000000000..4d4bd18836 --- /dev/null +++ b/third_party/raft/testdata/forget_leader_read_only_lease_based.txt @@ -0,0 +1,30 @@ +log-level none +---- +ok + +add-nodes 3 voters=(1,2,3) index=10 checkquorum=true read-only=lease-based +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok + +log-level debug +---- +ok + +# ForgetLeader fails with lease-based reads, as it's not safe. +forget-leader 2 +---- +ERROR ignoring MsgForgetLeader due to ReadOnlyLeaseBased + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 diff --git a/third_party/raft/testdata/heartbeat_resp_recovers_from_probing.txt b/third_party/raft/testdata/heartbeat_resp_recovers_from_probing.txt new file mode 100644 index 0000000000..e606a15577 --- /dev/null +++ b/third_party/raft/testdata/heartbeat_resp_recovers_from_probing.txt @@ -0,0 +1,90 @@ +# This test checks that if a fully caught-up follower transitions +# into StateProbe (for example due to a call to ReportUnreachable), the +# leader will react to a subsequent heartbeat response from the probing +# follower by sending an empty MsgApp, the response of which restores +# StateReplicate for the follower. In other words, we don't end up in +# a stable state with a fully caught up follower in StateProbe. + +# Turn off output during the setup of the test. +log-level none +---- +ok + +add-nodes 3 voters=(1,2,3) index=10 +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok + +log-level debug +---- +ok + +status 1 +---- +1: StateReplicate match=11 next=12 +2: StateReplicate match=11 next=12 +3: StateReplicate match=11 next=12 + +# On the first replica, report the second one as not reachable. +report-unreachable 1 2 +---- +DEBUG 1 failed to send message to 2 because it is unreachable [StateProbe match=11 next=12] + +status 1 +---- +1: StateReplicate match=11 next=12 +2: StateProbe match=11 next=12 +3: StateReplicate match=11 next=12 + +tick-heartbeat 1 +---- +ok + +# Heartbeat -> HeartbeatResp -> MsgApp -> MsgAppResp -> StateReplicate. +stabilize +---- +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 2 receiving messages + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 3 receiving messages + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgHeartbeatResp Term:1 Log:0/0 +> 3 handling Ready + Ready MustSync=false: + Messages: + 3->1 MsgHeartbeatResp Term:1 Log:0/0 +> 1 receiving messages + 2->1 MsgHeartbeatResp Term:1 Log:0/0 + 3->1 MsgHeartbeatResp Term:1 Log:0/0 +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/11 Commit:11 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/11 Commit:11 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgAppResp Term:1 Log:0/11 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/11 + +status 1 +---- +1: StateReplicate match=11 next=12 +2: StateReplicate match=11 next=12 +3: StateReplicate match=11 next=12 diff --git a/third_party/raft/testdata/lagging_commit.txt b/third_party/raft/testdata/lagging_commit.txt new file mode 100644 index 0000000000..8f8ba33635 --- /dev/null +++ b/third_party/raft/testdata/lagging_commit.txt @@ -0,0 +1,174 @@ +# This test demonstrates the effect of delayed commit on a follower node after a +# network hiccup between the leader and this follower. + +# Skip logging the boilerplate. Set up a raft group of 3 nodes, and elect node 1 +# as the leader. Nodes 2 and 3 are the followers. +log-level none +---- +ok + +add-nodes 3 voters=(1,2,3) index=10 +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok + +# Propose a couple of entries. +propose 1 data1 +---- +ok + +propose 1 data2 +---- +ok + +process-ready 1 +---- +ok + +# The interesting part starts below. +log-level debug +---- +ok + +deliver-msgs 2 3 +---- +1->2 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "data1"] +1->2 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "data2"] +1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "data1"] +1->3 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "data2"] + +process-ready 3 +---- +Ready MustSync=true: +Entries: +1/12 EntryNormal "data1" +1/13 EntryNormal "data2" +Messages: +3->1 MsgAppResp Term:1 Log:0/12 +3->1 MsgAppResp Term:1 Log:0/13 + +# Suppose there is a network blip which prevents the leader learning that the +# follower 3 has appended the proposed entries to the log. +deliver-msgs drop=(1) +---- +dropped: 3->1 MsgAppResp Term:1 Log:0/12 +dropped: 3->1 MsgAppResp Term:1 Log:0/13 + +# In the meantime, the entries are committed, and the leader sends the commit +# index to all the followers. +stabilize 1 2 +---- +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/12 EntryNormal "data1" + 1/13 EntryNormal "data2" + Messages: + 2->1 MsgAppResp Term:1 Log:0/12 + 2->1 MsgAppResp Term:1 Log:0/13 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/12 + 2->1 MsgAppResp Term:1 Log:0/13 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:13 + CommittedEntries: + 1/12 EntryNormal "data1" + 1/13 EntryNormal "data2" + Messages: + 1->2 MsgApp Term:1 Log:1/13 Commit:12 + 1->3 MsgApp Term:1 Log:1/13 Commit:12 + 1->2 MsgApp Term:1 Log:1/13 Commit:13 + 1->3 MsgApp Term:1 Log:1/13 Commit:13 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/13 Commit:12 + 1->2 MsgApp Term:1 Log:1/13 Commit:13 +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:13 + CommittedEntries: + 1/12 EntryNormal "data1" + 1/13 EntryNormal "data2" + Messages: + 2->1 MsgAppResp Term:1 Log:0/13 + 2->1 MsgAppResp Term:1 Log:0/13 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/13 + 2->1 MsgAppResp Term:1 Log:0/13 + +# The network blip prevents the follower 3 from learning that the previously +# appended entries are now committed. +deliver-msgs drop=(3) +---- +dropped: 1->3 MsgApp Term:1 Log:1/13 Commit:12 +dropped: 1->3 MsgApp Term:1 Log:1/13 Commit:13 + +# The network blip ends here. + +status 1 +---- +1: StateReplicate match=13 next=14 +2: StateReplicate match=13 next=14 +3: StateReplicate match=11 next=14 inflight=2 + +# The leader still observes that the entries are in-flight to the follower 3, +# since it hasn't heard from it. Nothing triggers updating the follower's +# commit index, so we have to wait up to the full heartbeat interval before +# the leader sends the commit index. +tick-heartbeat 1 +---- +ok + +# However, the leader does not push the real commit index to the follower 3. It +# cuts the commit index at the Progress.Match mark, because it thinks that it is +# unsafe to send a commit index higher than that. +process-ready 1 +---- +Ready MustSync=false: +Messages: +1->2 MsgHeartbeat Term:1 Log:0/0 Commit:13 +1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 + +# Since the heartbeat message does not bump the follower's commit index, it will +# take another roundtrip with the leader to update it. As such, the total time +# it takes for the follower to learn the commit index is: +# +# delay = HeartbeatInterval + 3/2 * RTT +# +# This is suboptimal. It could have taken HeartbeatInterval + 1/2 * RTT, if the +# leader sent the up-to-date commit index in the heartbeat message. +# +# See https://github.com/etcd-io/raft/issues/138 which aims to fix this. +stabilize 1 3 +---- +> 3 receiving messages + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 3 handling Ready + Ready MustSync=false: + Messages: + 3->1 MsgHeartbeatResp Term:1 Log:0/0 +> 1 receiving messages + 3->1 MsgHeartbeatResp Term:1 Log:0/0 +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->3 MsgApp Term:1 Log:1/13 Commit:13 +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/13 Commit:13 +> 3 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:13 + CommittedEntries: + 1/12 EntryNormal "data1" + 1/13 EntryNormal "data2" + Messages: + 3->1 MsgAppResp Term:1 Log:0/13 +> 1 receiving messages + 3->1 MsgAppResp Term:1 Log:0/13 diff --git a/third_party/raft/testdata/prevote.txt b/third_party/raft/testdata/prevote.txt new file mode 100644 index 0000000000..db763d3599 --- /dev/null +++ b/third_party/raft/testdata/prevote.txt @@ -0,0 +1,265 @@ +# Tests that PreVote prevents a node that is behind on the log from obtaining +# prevotes and calling an election. +# +# Also tests that a node that is up-to-date on its log can hold an election. +# Unlike the Raft thesis, the recent leader condition requires CheckQuorum +# and is not enforced with PreVote alone. + +log-level none +---- +ok + +add-nodes 3 voters=(1,2,3) index=10 prevote=true +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok + +log-level debug +---- +ok + +# Propose a command on 1 and replicate it to 2. +propose 1 prop_1 +---- +ok + +process-ready 1 +---- +Ready MustSync=true: +Entries: +1/12 EntryNormal "prop_1" +Messages: +1->2 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1"] +1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1"] + +deliver-msgs 2 +---- +1->2 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1"] + +process-ready 2 +---- +Ready MustSync=true: +Entries: +1/12 EntryNormal "prop_1" +Messages: +2->1 MsgAppResp Term:1 Log:0/12 + +# 3 is now behind on its log. Attempt to campaign. +raft-log 3 +---- +1/11 EntryNormal "" + +campaign 3 +---- +INFO 3 is starting a new election at term 1 +INFO 3 became pre-candidate at term 1 +INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 1 at term 1 +INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 2 at term 1 + +process-ready 3 +---- +Ready MustSync=false: +Lead:0 State:StatePreCandidate +Messages: +3->1 MsgPreVote Term:2 Log:1/11 +3->2 MsgPreVote Term:2 Log:1/11 +INFO 3 received MsgPreVoteResp from 3 at term 1 +INFO 3 has received 1 MsgPreVoteResp votes and 0 vote rejections + +deliver-msgs 1 2 +---- +2->1 MsgAppResp Term:1 Log:0/12 +3->1 MsgPreVote Term:2 Log:1/11 +INFO 1 [logterm: 1, index: 12, vote: 1] rejected MsgPreVote from 3 [logterm: 1, index: 11] at term 1 +3->2 MsgPreVote Term:2 Log:1/11 +INFO 2 [logterm: 1, index: 12, vote: 1] rejected MsgPreVote from 3 [logterm: 1, index: 11] at term 1 + +# 3 failed to campaign. Let the network stabilize. +stabilize +---- +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:12 + CommittedEntries: + 1/12 EntryNormal "prop_1" + Messages: + 1->2 MsgApp Term:1 Log:1/12 Commit:12 + 1->3 MsgApp Term:1 Log:1/12 Commit:12 + 1->3 MsgPreVoteResp Term:1 Log:0/0 Rejected (Hint: 0) +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->3 MsgPreVoteResp Term:1 Log:0/0 Rejected (Hint: 0) +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/12 Commit:12 +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1"] + INFO 3 became follower at term 1 + 1->3 MsgApp Term:1 Log:1/12 Commit:12 + 1->3 MsgPreVoteResp Term:1 Log:0/0 Rejected (Hint: 0) + 2->3 MsgPreVoteResp Term:1 Log:0/0 Rejected (Hint: 0) +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:12 + CommittedEntries: + 1/12 EntryNormal "prop_1" + Messages: + 2->1 MsgAppResp Term:1 Log:0/12 +> 3 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Vote:1 Commit:12 + Entries: + 1/12 EntryNormal "prop_1" + CommittedEntries: + 1/12 EntryNormal "prop_1" + Messages: + 3->1 MsgAppResp Term:1 Log:0/12 + 3->1 MsgAppResp Term:1 Log:0/12 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/12 + 3->1 MsgAppResp Term:1 Log:0/12 + 3->1 MsgAppResp Term:1 Log:0/12 + +# Let 2 campaign. It should succeed, since it's up-to-date on the log. +campaign 2 +---- +INFO 2 is starting a new election at term 1 +INFO 2 became pre-candidate at term 1 +INFO 2 [logterm: 1, index: 12] sent MsgPreVote request to 1 at term 1 +INFO 2 [logterm: 1, index: 12] sent MsgPreVote request to 3 at term 1 + +stabilize +---- +> 2 handling Ready + Ready MustSync=false: + Lead:0 State:StatePreCandidate + Messages: + 2->1 MsgPreVote Term:2 Log:1/12 + 2->3 MsgPreVote Term:2 Log:1/12 + INFO 2 received MsgPreVoteResp from 2 at term 1 + INFO 2 has received 1 MsgPreVoteResp votes and 0 vote rejections +> 1 receiving messages + 2->1 MsgPreVote Term:2 Log:1/12 + INFO 1 [logterm: 1, index: 12, vote: 1] cast MsgPreVote for 2 [logterm: 1, index: 12] at term 1 +> 3 receiving messages + 2->3 MsgPreVote Term:2 Log:1/12 + INFO 3 [logterm: 1, index: 12, vote: 1] cast MsgPreVote for 2 [logterm: 1, index: 12] at term 1 +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgPreVoteResp Term:2 Log:0/0 +> 3 handling Ready + Ready MustSync=false: + Messages: + 3->2 MsgPreVoteResp Term:2 Log:0/0 +> 2 receiving messages + 1->2 MsgPreVoteResp Term:2 Log:0/0 + INFO 2 received MsgPreVoteResp from 1 at term 1 + INFO 2 has received 2 MsgPreVoteResp votes and 0 vote rejections + INFO 2 became candidate at term 2 + INFO 2 [logterm: 1, index: 12] sent MsgVote request to 1 at term 2 + INFO 2 [logterm: 1, index: 12] sent MsgVote request to 3 at term 2 + 3->2 MsgPreVoteResp Term:2 Log:0/0 +> 2 handling Ready + Ready MustSync=true: + Lead:0 State:StateCandidate + HardState Term:2 Vote:2 Commit:12 + Messages: + 2->1 MsgVote Term:2 Log:1/12 + 2->3 MsgVote Term:2 Log:1/12 + INFO 2 received MsgVoteResp from 2 at term 2 + INFO 2 has received 1 MsgVoteResp votes and 0 vote rejections +> 1 receiving messages + 2->1 MsgVote Term:2 Log:1/12 + INFO 1 [term: 1] received a MsgVote message with higher term from 2 [term: 2] + INFO 1 became follower at term 2 + INFO 1 [logterm: 1, index: 12, vote: 0] cast MsgVote for 2 [logterm: 1, index: 12] at term 2 +> 3 receiving messages + 2->3 MsgVote Term:2 Log:1/12 + INFO 3 [term: 1] received a MsgVote message with higher term from 2 [term: 2] + INFO 3 became follower at term 2 + INFO 3 [logterm: 1, index: 12, vote: 0] cast MsgVote for 2 [logterm: 1, index: 12] at term 2 +> 1 handling Ready + Ready MustSync=true: + Lead:0 State:StateFollower + HardState Term:2 Vote:2 Commit:12 + Messages: + 1->2 MsgVoteResp Term:2 Log:0/0 +> 3 handling Ready + Ready MustSync=true: + Lead:0 State:StateFollower + HardState Term:2 Vote:2 Commit:12 + Messages: + 3->2 MsgVoteResp Term:2 Log:0/0 +> 2 receiving messages + 1->2 MsgVoteResp Term:2 Log:0/0 + INFO 2 received MsgVoteResp from 1 at term 2 + INFO 2 has received 2 MsgVoteResp votes and 0 vote rejections + INFO 2 became leader at term 2 + 3->2 MsgVoteResp Term:2 Log:0/0 +> 2 handling Ready + Ready MustSync=true: + Lead:2 State:StateLeader + Entries: + 2/13 EntryNormal "" + Messages: + 2->1 MsgApp Term:2 Log:1/12 Commit:12 Entries:[2/13 EntryNormal ""] + 2->3 MsgApp Term:2 Log:1/12 Commit:12 Entries:[2/13 EntryNormal ""] +> 1 receiving messages + 2->1 MsgApp Term:2 Log:1/12 Commit:12 Entries:[2/13 EntryNormal ""] +> 3 receiving messages + 2->3 MsgApp Term:2 Log:1/12 Commit:12 Entries:[2/13 EntryNormal ""] +> 1 handling Ready + Ready MustSync=true: + Lead:2 State:StateFollower + Entries: + 2/13 EntryNormal "" + Messages: + 1->2 MsgAppResp Term:2 Log:0/13 +> 3 handling Ready + Ready MustSync=true: + Lead:2 State:StateFollower + Entries: + 2/13 EntryNormal "" + Messages: + 3->2 MsgAppResp Term:2 Log:0/13 +> 2 receiving messages + 1->2 MsgAppResp Term:2 Log:0/13 + 3->2 MsgAppResp Term:2 Log:0/13 +> 2 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:2 Commit:13 + CommittedEntries: + 2/13 EntryNormal "" + Messages: + 2->1 MsgApp Term:2 Log:2/13 Commit:13 + 2->3 MsgApp Term:2 Log:2/13 Commit:13 +> 1 receiving messages + 2->1 MsgApp Term:2 Log:2/13 Commit:13 +> 3 receiving messages + 2->3 MsgApp Term:2 Log:2/13 Commit:13 +> 1 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:2 Commit:13 + CommittedEntries: + 2/13 EntryNormal "" + Messages: + 1->2 MsgAppResp Term:2 Log:0/13 +> 3 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:2 Commit:13 + CommittedEntries: + 2/13 EntryNormal "" + Messages: + 3->2 MsgAppResp Term:2 Log:0/13 +> 2 receiving messages + 1->2 MsgAppResp Term:2 Log:0/13 + 3->2 MsgAppResp Term:2 Log:0/13 diff --git a/third_party/raft/testdata/prevote_checkquorum.txt b/third_party/raft/testdata/prevote_checkquorum.txt new file mode 100644 index 0000000000..6db6662b03 --- /dev/null +++ b/third_party/raft/testdata/prevote_checkquorum.txt @@ -0,0 +1,345 @@ +# Tests that PreVote+CheckQuorum prevents a node from obtaining prevotes if +# voters have heard from a leader recently. Also tests that a node is able to +# obtain prevotes if the voter hasn't heard from the leader in the past election +# timeout interval, or if a quorum of voters are precandidates. + +log-level none +---- +ok + +add-nodes 3 voters=(1,2,3) index=10 prevote=true checkquorum=true +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok + +log-level debug +---- +ok + +# 2 should fail to campaign, leaving 1's leadership alone. +campaign 2 +---- +INFO 2 is starting a new election at term 1 +INFO 2 became pre-candidate at term 1 +INFO 2 [logterm: 1, index: 11] sent MsgPreVote request to 1 at term 1 +INFO 2 [logterm: 1, index: 11] sent MsgPreVote request to 3 at term 1 + +stabilize +---- +> 2 handling Ready + Ready MustSync=false: + Lead:0 State:StatePreCandidate + Messages: + 2->1 MsgPreVote Term:2 Log:1/11 + 2->3 MsgPreVote Term:2 Log:1/11 + INFO 2 received MsgPreVoteResp from 2 at term 1 + INFO 2 has received 1 MsgPreVoteResp votes and 0 vote rejections +> 1 receiving messages + 2->1 MsgPreVote Term:2 Log:1/11 + INFO 1 [logterm: 1, index: 11, vote: 1] ignored MsgPreVote from 2 [logterm: 1, index: 11] at term 1: lease is not expired (remaining ticks: 3) +> 3 receiving messages + 2->3 MsgPreVote Term:2 Log:1/11 + INFO 3 [logterm: 1, index: 11, vote: 1] ignored MsgPreVote from 2 [logterm: 1, index: 11] at term 1: lease is not expired (remaining ticks: 3) + +# If 2 hasn't heard from the leader in the past election timeout, it should +# grant prevotes, allowing 3 to hold an election. +set-randomized-election-timeout 2 timeout=5 +---- +ok + +tick-election 2 +---- +ok + +campaign 3 +---- +INFO 3 is starting a new election at term 1 +INFO 3 became pre-candidate at term 1 +INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 1 at term 1 +INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 2 at term 1 + +process-ready 3 +---- +Ready MustSync=false: +Lead:0 State:StatePreCandidate +Messages: +3->1 MsgPreVote Term:2 Log:1/11 +3->2 MsgPreVote Term:2 Log:1/11 +INFO 3 received MsgPreVoteResp from 3 at term 1 +INFO 3 has received 1 MsgPreVoteResp votes and 0 vote rejections + +deliver-msgs 2 +---- +3->2 MsgPreVote Term:2 Log:1/11 +INFO 2 [logterm: 1, index: 11, vote: 1] cast MsgPreVote for 3 [logterm: 1, index: 11] at term 1 + +process-ready 2 +---- +Ready MustSync=false: +Messages: +2->3 MsgPreVoteResp Term:2 Log:0/0 + +stabilize +---- +> 1 receiving messages + 3->1 MsgPreVote Term:2 Log:1/11 + INFO 1 [logterm: 1, index: 11, vote: 1] ignored MsgPreVote from 3 [logterm: 1, index: 11] at term 1: lease is not expired (remaining ticks: 3) +> 3 receiving messages + 2->3 MsgPreVoteResp Term:2 Log:0/0 + INFO 3 received MsgPreVoteResp from 2 at term 1 + INFO 3 has received 2 MsgPreVoteResp votes and 0 vote rejections + INFO 3 became candidate at term 2 + INFO 3 [logterm: 1, index: 11] sent MsgVote request to 1 at term 2 + INFO 3 [logterm: 1, index: 11] sent MsgVote request to 2 at term 2 +> 3 handling Ready + Ready MustSync=true: + Lead:0 State:StateCandidate + HardState Term:2 Vote:3 Commit:11 + Messages: + 3->1 MsgVote Term:2 Log:1/11 + 3->2 MsgVote Term:2 Log:1/11 + INFO 3 received MsgVoteResp from 3 at term 2 + INFO 3 has received 1 MsgVoteResp votes and 0 vote rejections +> 1 receiving messages + 3->1 MsgVote Term:2 Log:1/11 + INFO 1 [logterm: 1, index: 11, vote: 1] ignored MsgVote from 3 [logterm: 1, index: 11] at term 1: lease is not expired (remaining ticks: 3) +> 2 receiving messages + 3->2 MsgVote Term:2 Log:1/11 + INFO 2 [term: 1] received a MsgVote message with higher term from 3 [term: 2] + INFO 2 became follower at term 2 + INFO 2 [logterm: 1, index: 11, vote: 0] cast MsgVote for 3 [logterm: 1, index: 11] at term 2 +> 2 handling Ready + Ready MustSync=true: + Lead:0 State:StateFollower + HardState Term:2 Vote:3 Commit:11 + Messages: + 2->3 MsgVoteResp Term:2 Log:0/0 +> 3 receiving messages + 2->3 MsgVoteResp Term:2 Log:0/0 + INFO 3 received MsgVoteResp from 2 at term 2 + INFO 3 has received 2 MsgVoteResp votes and 0 vote rejections + INFO 3 became leader at term 2 +> 3 handling Ready + Ready MustSync=true: + Lead:3 State:StateLeader + Entries: + 2/12 EntryNormal "" + Messages: + 3->1 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] + 3->2 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] +> 1 receiving messages + 3->1 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] + INFO 1 [term: 1] received a MsgApp message with higher term from 3 [term: 2] + INFO 1 became follower at term 2 +> 2 receiving messages + 3->2 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] +> 1 handling Ready + Ready MustSync=true: + Lead:3 State:StateFollower + HardState Term:2 Commit:11 + Entries: + 2/12 EntryNormal "" + Messages: + 1->3 MsgAppResp Term:2 Log:0/12 +> 2 handling Ready + Ready MustSync=true: + Lead:3 State:StateFollower + Entries: + 2/12 EntryNormal "" + Messages: + 2->3 MsgAppResp Term:2 Log:0/12 +> 3 receiving messages + 1->3 MsgAppResp Term:2 Log:0/12 + 2->3 MsgAppResp Term:2 Log:0/12 +> 3 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:3 Commit:12 + CommittedEntries: + 2/12 EntryNormal "" + Messages: + 3->1 MsgApp Term:2 Log:2/12 Commit:12 + 3->2 MsgApp Term:2 Log:2/12 Commit:12 +> 1 receiving messages + 3->1 MsgApp Term:2 Log:2/12 Commit:12 +> 2 receiving messages + 3->2 MsgApp Term:2 Log:2/12 Commit:12 +> 1 handling Ready + Ready MustSync=false: + HardState Term:2 Commit:12 + CommittedEntries: + 2/12 EntryNormal "" + Messages: + 1->3 MsgAppResp Term:2 Log:0/12 +> 2 handling Ready + Ready MustSync=false: + HardState Term:2 Vote:3 Commit:12 + CommittedEntries: + 2/12 EntryNormal "" + Messages: + 2->3 MsgAppResp Term:2 Log:0/12 +> 3 receiving messages + 1->3 MsgAppResp Term:2 Log:0/12 + 2->3 MsgAppResp Term:2 Log:0/12 + +# Node 3 is now the leader. Even though the leader is active, nodes 1 and 2 can +# still win a prevote and election if they both explicitly campaign, since the +# PreVote+CheckQuorum recent leader condition only applies to follower voters. +# This is beneficial, because it allows a quorum of nodes to replace a leader +# when they have strong reason to believe that it's dead, despite having heard +# from it recently. +# +# We first let 1 lose an election, as we'd otherwise get a tie. +campaign 1 +---- +INFO 1 is starting a new election at term 2 +INFO 1 became pre-candidate at term 2 +INFO 1 [logterm: 2, index: 12] sent MsgPreVote request to 2 at term 2 +INFO 1 [logterm: 2, index: 12] sent MsgPreVote request to 3 at term 2 + +stabilize +---- +> 1 handling Ready + Ready MustSync=false: + Lead:0 State:StatePreCandidate + Messages: + 1->2 MsgPreVote Term:3 Log:2/12 + 1->3 MsgPreVote Term:3 Log:2/12 + INFO 1 received MsgPreVoteResp from 1 at term 2 + INFO 1 has received 1 MsgPreVoteResp votes and 0 vote rejections +> 2 receiving messages + 1->2 MsgPreVote Term:3 Log:2/12 + INFO 2 [logterm: 2, index: 12, vote: 3] ignored MsgPreVote from 1 [logterm: 2, index: 12] at term 2: lease is not expired (remaining ticks: 3) +> 3 receiving messages + 1->3 MsgPreVote Term:3 Log:2/12 + INFO 3 [logterm: 2, index: 12, vote: 3] ignored MsgPreVote from 1 [logterm: 2, index: 12] at term 2: lease is not expired (remaining ticks: 3) + +campaign 2 +---- +INFO 2 is starting a new election at term 2 +INFO 2 became pre-candidate at term 2 +INFO 2 [logterm: 2, index: 12] sent MsgPreVote request to 1 at term 2 +INFO 2 [logterm: 2, index: 12] sent MsgPreVote request to 3 at term 2 + +stabilize +---- +> 2 handling Ready + Ready MustSync=false: + Lead:0 State:StatePreCandidate + Messages: + 2->1 MsgPreVote Term:3 Log:2/12 + 2->3 MsgPreVote Term:3 Log:2/12 + INFO 2 received MsgPreVoteResp from 2 at term 2 + INFO 2 has received 1 MsgPreVoteResp votes and 0 vote rejections +> 1 receiving messages + 2->1 MsgPreVote Term:3 Log:2/12 + INFO 1 [logterm: 2, index: 12, vote: 0] cast MsgPreVote for 2 [logterm: 2, index: 12] at term 2 +> 3 receiving messages + 2->3 MsgPreVote Term:3 Log:2/12 + INFO 3 [logterm: 2, index: 12, vote: 3] ignored MsgPreVote from 2 [logterm: 2, index: 12] at term 2: lease is not expired (remaining ticks: 3) +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgPreVoteResp Term:3 Log:0/0 +> 2 receiving messages + 1->2 MsgPreVoteResp Term:3 Log:0/0 + INFO 2 received MsgPreVoteResp from 1 at term 2 + INFO 2 has received 2 MsgPreVoteResp votes and 0 vote rejections + INFO 2 became candidate at term 3 + INFO 2 [logterm: 2, index: 12] sent MsgVote request to 1 at term 3 + INFO 2 [logterm: 2, index: 12] sent MsgVote request to 3 at term 3 +> 2 handling Ready + Ready MustSync=true: + Lead:0 State:StateCandidate + HardState Term:3 Vote:2 Commit:12 + Messages: + 2->1 MsgVote Term:3 Log:2/12 + 2->3 MsgVote Term:3 Log:2/12 + INFO 2 received MsgVoteResp from 2 at term 3 + INFO 2 has received 1 MsgVoteResp votes and 0 vote rejections +> 1 receiving messages + 2->1 MsgVote Term:3 Log:2/12 + INFO 1 [term: 2] received a MsgVote message with higher term from 2 [term: 3] + INFO 1 became follower at term 3 + INFO 1 [logterm: 2, index: 12, vote: 0] cast MsgVote for 2 [logterm: 2, index: 12] at term 3 +> 3 receiving messages + 2->3 MsgVote Term:3 Log:2/12 + INFO 3 [logterm: 2, index: 12, vote: 3] ignored MsgVote from 2 [logterm: 2, index: 12] at term 2: lease is not expired (remaining ticks: 3) +> 1 handling Ready + Ready MustSync=true: + Lead:0 State:StateFollower + HardState Term:3 Vote:2 Commit:12 + Messages: + 1->2 MsgVoteResp Term:3 Log:0/0 +> 2 receiving messages + 1->2 MsgVoteResp Term:3 Log:0/0 + INFO 2 received MsgVoteResp from 1 at term 3 + INFO 2 has received 2 MsgVoteResp votes and 0 vote rejections + INFO 2 became leader at term 3 +> 2 handling Ready + Ready MustSync=true: + Lead:2 State:StateLeader + Entries: + 3/13 EntryNormal "" + Messages: + 2->1 MsgApp Term:3 Log:2/12 Commit:12 Entries:[3/13 EntryNormal ""] + 2->3 MsgApp Term:3 Log:2/12 Commit:12 Entries:[3/13 EntryNormal ""] +> 1 receiving messages + 2->1 MsgApp Term:3 Log:2/12 Commit:12 Entries:[3/13 EntryNormal ""] +> 3 receiving messages + 2->3 MsgApp Term:3 Log:2/12 Commit:12 Entries:[3/13 EntryNormal ""] + INFO 3 [term: 2] received a MsgApp message with higher term from 2 [term: 3] + INFO 3 became follower at term 3 +> 1 handling Ready + Ready MustSync=true: + Lead:2 State:StateFollower + Entries: + 3/13 EntryNormal "" + Messages: + 1->2 MsgAppResp Term:3 Log:0/13 +> 3 handling Ready + Ready MustSync=true: + Lead:2 State:StateFollower + HardState Term:3 Commit:12 + Entries: + 3/13 EntryNormal "" + Messages: + 3->2 MsgAppResp Term:3 Log:0/13 +> 2 receiving messages + 1->2 MsgAppResp Term:3 Log:0/13 + 3->2 MsgAppResp Term:3 Log:0/13 +> 2 handling Ready + Ready MustSync=false: + HardState Term:3 Vote:2 Commit:13 + CommittedEntries: + 3/13 EntryNormal "" + Messages: + 2->1 MsgApp Term:3 Log:3/13 Commit:13 + 2->3 MsgApp Term:3 Log:3/13 Commit:13 +> 1 receiving messages + 2->1 MsgApp Term:3 Log:3/13 Commit:13 +> 3 receiving messages + 2->3 MsgApp Term:3 Log:3/13 Commit:13 +> 1 handling Ready + Ready MustSync=false: + HardState Term:3 Vote:2 Commit:13 + CommittedEntries: + 3/13 EntryNormal "" + Messages: + 1->2 MsgAppResp Term:3 Log:0/13 +> 3 handling Ready + Ready MustSync=false: + HardState Term:3 Commit:13 + CommittedEntries: + 3/13 EntryNormal "" + Messages: + 3->2 MsgAppResp Term:3 Log:0/13 +> 2 receiving messages + 1->2 MsgAppResp Term:3 Log:0/13 + 3->2 MsgAppResp Term:3 Log:0/13 diff --git a/third_party/raft/testdata/probe_and_replicate.txt b/third_party/raft/testdata/probe_and_replicate.txt new file mode 100644 index 0000000000..d970183c67 --- /dev/null +++ b/third_party/raft/testdata/probe_and_replicate.txt @@ -0,0 +1,770 @@ +# This test creates a complete Raft log configuration and demonstrates how a +# leader probes and replicates to each of its followers. The log configuration +# constructed is almost[*] identical to the one present in Figure 7 of the raft +# paper (https://raft.github.io/raft.pdf), which looks like: +# +# 1 2 3 4 5 6 7 8 9 10 11 12 +# n1: [1][1][1][4][4][5][5][6][6][6] +# n2: [1][1][1][4][4][5][5][6][6] +# n3: [1][1][1][4] +# n4: [1][1][1][4][4][5][5][6][6][6][6] +# n5: [1][1][1][4][4][5][5][6][7][7][7][7] +# n6: [1][1][1][4][4][4][4] +# n7: [1][1][1][2][2][2][3][3][3][3][3] +# +# Once in this state, we then elect node 1 as the leader and stabilize the +# entire raft group. This demonstrates how a newly elected leader probes for +# matching indexes, overwrites conflicting entries, and catches up all +# followers. +# +# [*] the only differences are: +# 1. n5 is given a larger uncommitted log tail, which is used to demonstrate a +# follower-side probing optimization. +# 2. the log indexes are shifted by 10 in this test because add-nodes wants to +# start with an index > 1. +# + + +# Set up the log configuration. This is mostly unintersting, but the order of +# each leadership change and the nodes that are allowed to hear about them is +# very important. Most readers of this test can skip this section. +log-level none +---- +ok + +## Start with seven nodes. +add-nodes 7 voters=(1,2,3,4,5,6,7) index=10 +---- +ok + +## Create term 1 entries. +campaign 1 +---- +ok + +stabilize +---- +ok + +propose 1 prop_1_12 +---- +ok + +propose 1 prop_1_13 +---- +ok + +stabilize +---- +ok + +## Create term 2 entries. +campaign 2 +---- +ok + +stabilize 2 +---- +ok + +stabilize 6 +---- +ok + +stabilize 2 5 7 +---- +ok + +propose 2 prop_2_15 +---- +ok + +propose 2 prop_2_16 +---- +ok + +stabilize 2 7 +---- +ok + +deliver-msgs drop=(1,2,3,4,5,6,7) +---- +ok + +## Create term 3 entries. +campaign 7 +---- +ok + +stabilize 7 +---- +ok + +stabilize 1 2 3 4 5 6 +---- +ok + +stabilize 7 +---- +ok + +propose 7 prop_3_18 +---- +ok + +propose 7 prop_3_19 +---- +ok + +propose 7 prop_3_20 +---- +ok + +propose 7 prop_3_21 +---- +ok + +stabilize 7 +---- +ok + +deliver-msgs drop=(1,2,3,4,5,6,7) +---- +ok + +## Create term 4 entries. +campaign 6 +---- +ok + +stabilize 1 2 3 4 5 6 +---- +ok + +propose 6 prop_4_15 +---- +ok + +stabilize 1 2 4 5 6 +---- +ok + +propose 6 prop_4_16 +---- +ok + +propose 6 prop_4_17 +---- +ok + +stabilize 6 +---- +ok + +deliver-msgs drop=(1,2,3,4,5,6,7) +---- +ok + +## Create term 5 entries. +campaign 5 +---- +ok + +stabilize 1 2 4 5 +---- +ok + +propose 5 prop_5_17 +---- +ok + +stabilize 1 2 4 5 +---- +ok + +deliver-msgs drop=(1,2,3,4,5,6,7) +---- +ok + +## Create term 6 entries. +campaign 4 +---- +ok + +stabilize 1 2 4 5 +---- +ok + +propose 4 prop_6_19 +---- +ok + +stabilize 1 2 4 +---- +ok + +propose 4 prop_6_20 +---- +ok + +stabilize 1 4 +---- +ok + +propose 4 prop_6_21 +---- +ok + +stabilize 4 +---- +ok + +deliver-msgs drop=(1,2,3,4,5,6,7) +---- +ok + +## Create term 7 entries. +campaign 5 +---- +ok + +stabilize 5 +---- +ok + +stabilize 1 3 6 7 +---- +ok + +stabilize 5 +---- +ok + +propose 5 prop_7_20 +---- +ok + +propose 5 prop_7_21 +---- +ok + +propose 5 prop_7_22 +---- +ok + +stabilize 5 +---- +ok + +deliver-msgs drop=(1,2,3,4,5,6,7) +---- +ok + + +# Show the Raft log from each node. +log-level info +---- +ok + +raft-log 1 +---- +1/11 EntryNormal "" +1/12 EntryNormal "prop_1_12" +1/13 EntryNormal "prop_1_13" +4/14 EntryNormal "" +4/15 EntryNormal "prop_4_15" +5/16 EntryNormal "" +5/17 EntryNormal "prop_5_17" +6/18 EntryNormal "" +6/19 EntryNormal "prop_6_19" +6/20 EntryNormal "prop_6_20" + +raft-log 2 +---- +1/11 EntryNormal "" +1/12 EntryNormal "prop_1_12" +1/13 EntryNormal "prop_1_13" +4/14 EntryNormal "" +4/15 EntryNormal "prop_4_15" +5/16 EntryNormal "" +5/17 EntryNormal "prop_5_17" +6/18 EntryNormal "" +6/19 EntryNormal "prop_6_19" + +raft-log 3 +---- +1/11 EntryNormal "" +1/12 EntryNormal "prop_1_12" +1/13 EntryNormal "prop_1_13" +4/14 EntryNormal "" + +raft-log 4 +---- +1/11 EntryNormal "" +1/12 EntryNormal "prop_1_12" +1/13 EntryNormal "prop_1_13" +4/14 EntryNormal "" +4/15 EntryNormal "prop_4_15" +5/16 EntryNormal "" +5/17 EntryNormal "prop_5_17" +6/18 EntryNormal "" +6/19 EntryNormal "prop_6_19" +6/20 EntryNormal "prop_6_20" +6/21 EntryNormal "prop_6_21" + +raft-log 5 +---- +1/11 EntryNormal "" +1/12 EntryNormal "prop_1_12" +1/13 EntryNormal "prop_1_13" +4/14 EntryNormal "" +4/15 EntryNormal "prop_4_15" +5/16 EntryNormal "" +5/17 EntryNormal "prop_5_17" +6/18 EntryNormal "" +7/19 EntryNormal "" +7/20 EntryNormal "prop_7_20" +7/21 EntryNormal "prop_7_21" +7/22 EntryNormal "prop_7_22" + +raft-log 6 +---- +1/11 EntryNormal "" +1/12 EntryNormal "prop_1_12" +1/13 EntryNormal "prop_1_13" +4/14 EntryNormal "" +4/15 EntryNormal "prop_4_15" +4/16 EntryNormal "prop_4_16" +4/17 EntryNormal "prop_4_17" + +raft-log 7 +---- +1/11 EntryNormal "" +1/12 EntryNormal "prop_1_12" +1/13 EntryNormal "prop_1_13" +2/14 EntryNormal "" +2/15 EntryNormal "prop_2_15" +2/16 EntryNormal "prop_2_16" +3/17 EntryNormal "" +3/18 EntryNormal "prop_3_18" +3/19 EntryNormal "prop_3_19" +3/20 EntryNormal "prop_3_20" +3/21 EntryNormal "prop_3_21" + + +# Elect node 1 as leader and stabilize. +campaign 1 +---- +INFO 1 is starting a new election at term 7 +INFO 1 became candidate at term 8 +INFO 1 [logterm: 6, index: 20] sent MsgVote request to 2 at term 8 +INFO 1 [logterm: 6, index: 20] sent MsgVote request to 3 at term 8 +INFO 1 [logterm: 6, index: 20] sent MsgVote request to 4 at term 8 +INFO 1 [logterm: 6, index: 20] sent MsgVote request to 5 at term 8 +INFO 1 [logterm: 6, index: 20] sent MsgVote request to 6 at term 8 +INFO 1 [logterm: 6, index: 20] sent MsgVote request to 7 at term 8 + +## Get elected. +stabilize 1 +---- +> 1 handling Ready + Ready MustSync=true: + Lead:0 State:StateCandidate + HardState Term:8 Vote:1 Commit:18 + Messages: + 1->2 MsgVote Term:8 Log:6/20 + 1->3 MsgVote Term:8 Log:6/20 + 1->4 MsgVote Term:8 Log:6/20 + 1->5 MsgVote Term:8 Log:6/20 + 1->6 MsgVote Term:8 Log:6/20 + 1->7 MsgVote Term:8 Log:6/20 + INFO 1 received MsgVoteResp from 1 at term 8 + INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections + +stabilize 2 3 4 5 6 7 +---- +> 2 receiving messages + 1->2 MsgVote Term:8 Log:6/20 + INFO 2 [term: 6] received a MsgVote message with higher term from 1 [term: 8] + INFO 2 became follower at term 8 + INFO 2 [logterm: 6, index: 19, vote: 0] cast MsgVote for 1 [logterm: 6, index: 20] at term 8 +> 3 receiving messages + 1->3 MsgVote Term:8 Log:6/20 + INFO 3 [term: 7] received a MsgVote message with higher term from 1 [term: 8] + INFO 3 became follower at term 8 + INFO 3 [logterm: 4, index: 14, vote: 0] cast MsgVote for 1 [logterm: 6, index: 20] at term 8 +> 4 receiving messages + 1->4 MsgVote Term:8 Log:6/20 + INFO 4 [term: 6] received a MsgVote message with higher term from 1 [term: 8] + INFO 4 became follower at term 8 + INFO 4 [logterm: 6, index: 21, vote: 0] rejected MsgVote from 1 [logterm: 6, index: 20] at term 8 +> 5 receiving messages + 1->5 MsgVote Term:8 Log:6/20 + INFO 5 [term: 7] received a MsgVote message with higher term from 1 [term: 8] + INFO 5 became follower at term 8 + INFO 5 [logterm: 7, index: 22, vote: 0] rejected MsgVote from 1 [logterm: 6, index: 20] at term 8 +> 6 receiving messages + 1->6 MsgVote Term:8 Log:6/20 + INFO 6 [term: 7] received a MsgVote message with higher term from 1 [term: 8] + INFO 6 became follower at term 8 + INFO 6 [logterm: 4, index: 17, vote: 0] cast MsgVote for 1 [logterm: 6, index: 20] at term 8 +> 7 receiving messages + 1->7 MsgVote Term:8 Log:6/20 + INFO 7 [term: 7] received a MsgVote message with higher term from 1 [term: 8] + INFO 7 became follower at term 8 + INFO 7 [logterm: 3, index: 21, vote: 0] cast MsgVote for 1 [logterm: 6, index: 20] at term 8 +> 2 handling Ready + Ready MustSync=true: + Lead:0 State:StateFollower + HardState Term:8 Vote:1 Commit:18 + Messages: + 2->1 MsgVoteResp Term:8 Log:0/0 +> 3 handling Ready + Ready MustSync=true: + HardState Term:8 Vote:1 Commit:14 + Messages: + 3->1 MsgVoteResp Term:8 Log:0/0 +> 4 handling Ready + Ready MustSync=true: + Lead:0 State:StateFollower + HardState Term:8 Commit:18 + Messages: + 4->1 MsgVoteResp Term:8 Log:0/0 Rejected (Hint: 0) +> 5 handling Ready + Ready MustSync=true: + Lead:0 State:StateFollower + HardState Term:8 Commit:18 + Messages: + 5->1 MsgVoteResp Term:8 Log:0/0 Rejected (Hint: 0) +> 6 handling Ready + Ready MustSync=true: + HardState Term:8 Vote:1 Commit:15 + Messages: + 6->1 MsgVoteResp Term:8 Log:0/0 +> 7 handling Ready + Ready MustSync=true: + HardState Term:8 Vote:1 Commit:13 + Messages: + 7->1 MsgVoteResp Term:8 Log:0/0 + +stabilize 1 +---- +> 1 receiving messages + 2->1 MsgVoteResp Term:8 Log:0/0 + INFO 1 received MsgVoteResp from 2 at term 8 + INFO 1 has received 2 MsgVoteResp votes and 0 vote rejections + 3->1 MsgVoteResp Term:8 Log:0/0 + INFO 1 received MsgVoteResp from 3 at term 8 + INFO 1 has received 3 MsgVoteResp votes and 0 vote rejections + 4->1 MsgVoteResp Term:8 Log:0/0 Rejected (Hint: 0) + INFO 1 received MsgVoteResp rejection from 4 at term 8 + INFO 1 has received 3 MsgVoteResp votes and 1 vote rejections + 5->1 MsgVoteResp Term:8 Log:0/0 Rejected (Hint: 0) + INFO 1 received MsgVoteResp rejection from 5 at term 8 + INFO 1 has received 3 MsgVoteResp votes and 2 vote rejections + 6->1 MsgVoteResp Term:8 Log:0/0 + INFO 1 received MsgVoteResp from 6 at term 8 + INFO 1 has received 4 MsgVoteResp votes and 2 vote rejections + INFO 1 became leader at term 8 + 7->1 MsgVoteResp Term:8 Log:0/0 +> 1 handling Ready + Ready MustSync=true: + Lead:1 State:StateLeader + Entries: + 8/21 EntryNormal "" + Messages: + 1->2 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] + 1->3 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] + 1->4 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] + 1->5 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] + 1->6 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] + 1->7 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] + +## Recover each follower, one by one. +stabilize 1 2 +---- +> 2 receiving messages + 1->2 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] +> 2 handling Ready + Ready MustSync=false: + Lead:1 State:StateFollower + Messages: + 2->1 MsgAppResp Term:8 Log:6/20 Rejected (Hint: 19) +> 1 receiving messages + 2->1 MsgAppResp Term:8 Log:6/20 Rejected (Hint: 19) +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:8 Log:6/19 Commit:18 Entries:[ + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + ] +> 2 receiving messages + 1->2 MsgApp Term:8 Log:6/19 Commit:18 Entries:[ + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + ] +> 2 handling Ready + Ready MustSync=true: + Entries: + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:8 Log:0/21 +> 1 receiving messages + 2->1 MsgAppResp Term:8 Log:0/21 + +stabilize 1 3 +---- +> 3 receiving messages + 1->3 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] +> 3 handling Ready + Ready MustSync=false: + Lead:1 State:StateFollower + Messages: + 3->1 MsgAppResp Term:8 Log:4/20 Rejected (Hint: 14) +> 1 receiving messages + 3->1 MsgAppResp Term:8 Log:4/20 Rejected (Hint: 14) +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->3 MsgApp Term:8 Log:4/14 Commit:18 Entries:[ + 4/15 EntryNormal "prop_4_15" + 5/16 EntryNormal "" + 5/17 EntryNormal "prop_5_17" + 6/18 EntryNormal "" + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + ] +> 3 receiving messages + 1->3 MsgApp Term:8 Log:4/14 Commit:18 Entries:[ + 4/15 EntryNormal "prop_4_15" + 5/16 EntryNormal "" + 5/17 EntryNormal "prop_5_17" + 6/18 EntryNormal "" + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + ] +> 3 handling Ready + Ready MustSync=true: + HardState Term:8 Vote:1 Commit:18 + Entries: + 4/15 EntryNormal "prop_4_15" + 5/16 EntryNormal "" + 5/17 EntryNormal "prop_5_17" + 6/18 EntryNormal "" + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + CommittedEntries: + 4/15 EntryNormal "prop_4_15" + 5/16 EntryNormal "" + 5/17 EntryNormal "prop_5_17" + 6/18 EntryNormal "" + Messages: + 3->1 MsgAppResp Term:8 Log:0/21 +> 1 receiving messages + 3->1 MsgAppResp Term:8 Log:0/21 + +stabilize 1 4 +---- +> 4 receiving messages + 1->4 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] + INFO found conflict at index 21 [existing term: 6, conflicting term: 8] + INFO replace the unstable entries from index 21 +> 4 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + Entries: + 8/21 EntryNormal "" + Messages: + 4->1 MsgAppResp Term:8 Log:0/21 +> 1 receiving messages + 4->1 MsgAppResp Term:8 Log:0/21 +> 1 handling Ready + Ready MustSync=false: + HardState Term:8 Vote:1 Commit:21 + CommittedEntries: + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + Messages: + 1->2 MsgApp Term:8 Log:8/21 Commit:21 + 1->3 MsgApp Term:8 Log:8/21 Commit:21 + 1->4 MsgApp Term:8 Log:8/21 Commit:21 +> 4 receiving messages + 1->4 MsgApp Term:8 Log:8/21 Commit:21 +> 4 handling Ready + Ready MustSync=false: + HardState Term:8 Commit:21 + CommittedEntries: + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + Messages: + 4->1 MsgAppResp Term:8 Log:0/21 +> 1 receiving messages + 4->1 MsgAppResp Term:8 Log:0/21 + +stabilize 1 5 +---- +> 5 receiving messages + 1->5 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] +> 5 handling Ready + Ready MustSync=false: + Lead:1 State:StateFollower + Messages: + 5->1 MsgAppResp Term:8 Log:6/20 Rejected (Hint: 18) +> 1 receiving messages + 5->1 MsgAppResp Term:8 Log:6/20 Rejected (Hint: 18) +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->5 MsgApp Term:8 Log:6/18 Commit:21 Entries:[ + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + ] +> 5 receiving messages + 1->5 MsgApp Term:8 Log:6/18 Commit:21 Entries:[ + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + ] + INFO found conflict at index 19 [existing term: 7, conflicting term: 6] + INFO replace the unstable entries from index 19 +> 5 handling Ready + Ready MustSync=true: + HardState Term:8 Commit:21 + Entries: + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + CommittedEntries: + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + Messages: + 5->1 MsgAppResp Term:8 Log:0/21 +> 1 receiving messages + 5->1 MsgAppResp Term:8 Log:0/21 + +stabilize 1 6 +---- +> 6 receiving messages + 1->6 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] +> 6 handling Ready + Ready MustSync=false: + Lead:1 State:StateFollower + Messages: + 6->1 MsgAppResp Term:8 Log:4/20 Rejected (Hint: 17) +> 1 receiving messages + 6->1 MsgAppResp Term:8 Log:4/20 Rejected (Hint: 17) +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->6 MsgApp Term:8 Log:4/15 Commit:21 Entries:[ + 5/16 EntryNormal "" + 5/17 EntryNormal "prop_5_17" + 6/18 EntryNormal "" + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + ] +> 6 receiving messages + 1->6 MsgApp Term:8 Log:4/15 Commit:21 Entries:[ + 5/16 EntryNormal "" + 5/17 EntryNormal "prop_5_17" + 6/18 EntryNormal "" + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + ] + INFO found conflict at index 16 [existing term: 4, conflicting term: 5] + INFO replace the unstable entries from index 16 +> 6 handling Ready + Ready MustSync=true: + HardState Term:8 Vote:1 Commit:21 + Entries: + 5/16 EntryNormal "" + 5/17 EntryNormal "prop_5_17" + 6/18 EntryNormal "" + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + CommittedEntries: + 5/16 EntryNormal "" + 5/17 EntryNormal "prop_5_17" + 6/18 EntryNormal "" + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + Messages: + 6->1 MsgAppResp Term:8 Log:0/21 +> 1 receiving messages + 6->1 MsgAppResp Term:8 Log:0/21 + +stabilize 1 7 +---- +> 7 receiving messages + 1->7 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] +> 7 handling Ready + Ready MustSync=false: + Lead:1 State:StateFollower + Messages: + 7->1 MsgAppResp Term:8 Log:3/20 Rejected (Hint: 20) +> 1 receiving messages + 7->1 MsgAppResp Term:8 Log:3/20 Rejected (Hint: 20) +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->7 MsgApp Term:8 Log:1/13 Commit:21 Entries:[ + 4/14 EntryNormal "" + 4/15 EntryNormal "prop_4_15" + 5/16 EntryNormal "" + 5/17 EntryNormal "prop_5_17" + 6/18 EntryNormal "" + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + ] +> 7 receiving messages + 1->7 MsgApp Term:8 Log:1/13 Commit:21 Entries:[ + 4/14 EntryNormal "" + 4/15 EntryNormal "prop_4_15" + 5/16 EntryNormal "" + 5/17 EntryNormal "prop_5_17" + 6/18 EntryNormal "" + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + ] + INFO found conflict at index 14 [existing term: 2, conflicting term: 4] + INFO replace the unstable entries from index 14 +> 7 handling Ready + Ready MustSync=true: + HardState Term:8 Vote:1 Commit:21 + Entries: + 4/14 EntryNormal "" + 4/15 EntryNormal "prop_4_15" + 5/16 EntryNormal "" + 5/17 EntryNormal "prop_5_17" + 6/18 EntryNormal "" + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + CommittedEntries: + 4/14 EntryNormal "" + 4/15 EntryNormal "prop_4_15" + 5/16 EntryNormal "" + 5/17 EntryNormal "prop_5_17" + 6/18 EntryNormal "" + 6/19 EntryNormal "prop_6_19" + 6/20 EntryNormal "prop_6_20" + 8/21 EntryNormal "" + Messages: + 7->1 MsgAppResp Term:8 Log:0/21 +> 1 receiving messages + 7->1 MsgAppResp Term:8 Log:0/21 diff --git a/third_party/raft/testdata/replicate_pause.txt b/third_party/raft/testdata/replicate_pause.txt new file mode 100644 index 0000000000..d9cee59fe4 --- /dev/null +++ b/third_party/raft/testdata/replicate_pause.txt @@ -0,0 +1,190 @@ +# This test ensures that MsgApp stream to a follower is paused when the +# in-flight state exceeds the configured limits. This is a regression test for +# the issue fixed by https://github.com/etcd-io/etcd/pull/14633. + +# Turn off output during the setup of the test. +log-level none +---- +ok + +# Start with 3 nodes, with a limited in-flight capacity. +add-nodes 3 voters=(1,2,3) index=10 inflight=3 +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok + +# Propose 3 entries. +propose 1 prop_1_12 +---- +ok + +propose 1 prop_1_13 +---- +ok + +propose 1 prop_1_14 +---- +ok + +# Store entries and send proposals. +process-ready 1 +---- +ok + +# Re-enable log messages. +log-level debug +---- +ok + +# Expect that in-flight tracking to nodes 2 and 3 is saturated. +status 1 +---- +1: StateReplicate match=14 next=15 +2: StateReplicate match=11 next=15 paused inflight=3[full] +3: StateReplicate match=11 next=15 paused inflight=3[full] + +log-level none +---- +ok + +# Commit entries between nodes 1 and 2. +stabilize 1 2 +---- +ok + +log-level debug +---- +ok + +# Expect that the entries are committed and stored on nodes 1 and 2. +status 1 +---- +1: StateReplicate match=14 next=15 +2: StateReplicate match=14 next=15 +3: StateReplicate match=11 next=15 paused inflight=3[full] + +# Drop append messages to node 3. +deliver-msgs drop=3 +---- +dropped: 1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1_12"] +dropped: 1->3 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "prop_1_13"] +dropped: 1->3 MsgApp Term:1 Log:1/13 Commit:11 Entries:[1/14 EntryNormal "prop_1_14"] + + +# Repeat committing 3 entries. +propose 1 prop_1_15 +---- +ok + +propose 1 prop_1_16 +---- +ok + +propose 1 prop_1_17 +---- +ok + +# In-flight tracking to nodes 2 and 3 is saturated, but node 3 is behind. +status 1 +---- +1: StateReplicate match=14 next=15 +2: StateReplicate match=14 next=18 paused inflight=3[full] +3: StateReplicate match=11 next=15 paused inflight=3[full] + +log-level none +---- +ok + +# Commit entries between nodes 1 and 2 again. +stabilize 1 2 +---- +ok + +log-level debug +---- +ok + +# Expect that the entries are committed and stored only on nodes 1 and 2. +status 1 +---- +1: StateReplicate match=17 next=18 +2: StateReplicate match=17 next=18 +3: StateReplicate match=11 next=15 paused inflight=3[full] + +# Make a heartbeat roundtrip. +tick-heartbeat 1 +---- +ok + +stabilize 1 +---- +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:17 + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 + +stabilize 2 3 +---- +> 2 receiving messages + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:17 +> 3 receiving messages + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgHeartbeatResp Term:1 Log:0/0 +> 3 handling Ready + Ready MustSync=false: + Messages: + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + +# After handling heartbeat responses, node 1 sends an empty MsgApp to a +# throttled node 3 because it hasn't yet replied to a single MsgApp, and the +# in-flight tracker is still saturated. +stabilize 1 +---- +> 1 receiving messages + 2->1 MsgHeartbeatResp Term:1 Log:0/0 + 3->1 MsgHeartbeatResp Term:1 Log:0/0 +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->3 MsgApp Term:1 Log:1/14 Commit:17 + +# Node 3 finally receives a MsgApp, but there was a gap, so it rejects it. +stabilize 3 +---- +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/14 Commit:17 + DEBUG 3 [logterm: 0, index: 14] rejected MsgApp [logterm: 1, index: 14] from 1 +> 3 handling Ready + Ready MustSync=false: + Messages: + 3->1 MsgAppResp Term:1 Log:1/14 Rejected (Hint: 11) + +log-level none +---- +ok + +stabilize +---- +ok + +log-level debug +---- +ok + +# Eventually all nodes catch up on the committed state. +status 1 +---- +1: StateReplicate match=17 next=18 +2: StateReplicate match=17 next=18 +3: StateReplicate match=17 next=18 diff --git a/third_party/raft/testdata/single_node.txt b/third_party/raft/testdata/single_node.txt new file mode 100644 index 0000000000..f6aceb712f --- /dev/null +++ b/third_party/raft/testdata/single_node.txt @@ -0,0 +1,34 @@ +log-level info +---- +ok + +add-nodes 1 voters=(1) index=3 +---- +INFO 1 switched to configuration voters=(1) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1], term: 0, commit: 3, applied: 3, lastindex: 3, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 + +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Lead:0 State:StateCandidate + HardState Term:1 Vote:1 Commit:3 + INFO 1 received MsgVoteResp from 1 at term 1 + INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections + INFO 1 became leader at term 1 +> 1 handling Ready + Ready MustSync=true: + Lead:1 State:StateLeader + Entries: + 1/4 EntryNormal "" +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:4 + CommittedEntries: + 1/4 EntryNormal "" diff --git a/third_party/raft/testdata/slow_follower_after_compaction.txt b/third_party/raft/testdata/slow_follower_after_compaction.txt new file mode 100644 index 0000000000..0d3d48c8e6 --- /dev/null +++ b/third_party/raft/testdata/slow_follower_after_compaction.txt @@ -0,0 +1,121 @@ +# This is a regression test for https://github.com/etcd-io/raft/pull/31. + +# Turn off output during the setup of the test. +log-level none +---- +ok + +# Start with 3 nodes, with a limited in-flight capacity. +add-nodes 3 voters=(1,2,3) index=10 inflight=2 +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok + +# Propose 3 entries. +propose 1 prop_1_12 +---- +ok + +propose 1 prop_1_13 +---- +ok + +propose 1 prop_1_14 +---- +ok + +stabilize +---- +ok + +# Re-enable log messages. +log-level debug +---- +ok + +# All nodes up-to-date. +status 1 +---- +1: StateReplicate match=14 next=15 +2: StateReplicate match=14 next=15 +3: StateReplicate match=14 next=15 + +log-level none +---- +ok + +propose 1 prop_1_15 +---- +ok + +propose 1 prop_1_16 +---- +ok + +propose 1 prop_1_17 +---- +ok + +propose 1 prop_1_18 +---- +ok + +# Commit entries on nodes 1 and 2. +stabilize 1 2 +---- +ok + +log-level debug +---- +ok + +# Nodes 1 and 2 up-to-date, 3 is behind and MsgApp flow is throttled. +status 1 +---- +1: StateReplicate match=18 next=19 +2: StateReplicate match=18 next=19 +3: StateReplicate match=14 next=17 paused inflight=2[full] + +# Break the MsgApp flow from the leader to node 3. +deliver-msgs drop=3 +---- +dropped: 1->3 MsgApp Term:1 Log:1/14 Commit:14 Entries:[1/15 EntryNormal "prop_1_15"] +dropped: 1->3 MsgApp Term:1 Log:1/15 Commit:14 Entries:[1/16 EntryNormal "prop_1_16"] + +# Truncate the leader's log beyond node 3 log size. +compact 1 17 +---- +1/18 EntryNormal "prop_1_18" + +# Trigger a round of empty MsgApp "probe" from leader. It will reach node 3 +# which will reply with a rejection MsgApp because it sees a gap in the log. +# Node 1 will reset the MsgApp flow and send a snapshot to catch node 3 up. +tick-heartbeat 1 +---- +ok + +log-level none +---- +ok + +stabilize +---- +ok + +log-level debug +---- +ok + +# All nodes caught up. +status 1 +---- +1: StateReplicate match=18 next=19 +2: StateReplicate match=18 next=19 +3: StateReplicate match=18 next=19 diff --git a/third_party/raft/testdata/snapshot_succeed_via_app_resp.txt b/third_party/raft/testdata/snapshot_succeed_via_app_resp.txt new file mode 100644 index 0000000000..b1fc4144e7 --- /dev/null +++ b/third_party/raft/testdata/snapshot_succeed_via_app_resp.txt @@ -0,0 +1,147 @@ +# TestSnapshotSucceedViaAppResp regression tests the situation in which a snap- +# shot is sent to a follower at the most recent index (i.e. the snapshot index +# is the leader's last index is the committed index). In that situation, a bug +# in the past left the follower in probing status until the next log entry was +# committed. +# +# See https://github.com/etcd-io/etcd/pull/10308 for additional background. + +# Turn off output during the setup of the test. +log-level none +---- +ok + +# Start with two nodes, but the config already has a third. +add-nodes 2 voters=(1,2,3) index=10 +---- +ok + +campaign 1 +---- +ok + +# Fully replicate everything, including the leader's empty index. +stabilize +---- +ok + +compact 1 11 +---- +ok + +# Drop inflight messages to n3. +deliver-msgs drop=(3) +---- +ok + +# Show the Raft log messages from now on. +log-level debug +---- +ok + +status 1 +---- +1: StateReplicate match=11 next=12 +2: StateReplicate match=11 next=12 +3: StateProbe match=0 next=11 paused inactive + +# Add the node that will receive a snapshot (it has no state at all, does not +# even have a config). +add-nodes 1 +---- +INFO 3 switched to configuration voters=() +INFO 3 became follower at term 0 +INFO newRaft 3 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +# Time passes on the leader so that it will try the previously missing follower +# again. +tick-heartbeat 1 +---- +ok + +process-ready 1 +---- +Ready MustSync=false: +Messages: +1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 +1->3 MsgHeartbeat Term:1 Log:0/0 + +# Iterate until no more work is done by the new peer. It receives the heartbeat +# and responds. +stabilize 3 +---- +> 3 receiving messages + 1->3 MsgHeartbeat Term:1 Log:0/0 + INFO 3 [term: 0] received a MsgHeartbeat message with higher term from 1 [term: 1] + INFO 3 became follower at term 1 +> 3 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Commit:0 + Messages: + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + +# The leader in turn will realize that n3 needs a snapshot, which it initiates. +stabilize 1 +---- +> 1 receiving messages + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + DEBUG 1 [firstindex: 12, commit: 11] sent snapshot[index: 11, term: 1] to 3 [StateProbe match=0 next=11] + DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 next=12 paused pendingSnap=11] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->3 MsgSnap Term:1 Log:0/0 + Snapshot: Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + +status 1 +---- +1: StateReplicate match=11 next=12 +2: StateReplicate match=11 next=12 +3: StateSnapshot match=0 next=12 paused pendingSnap=11 + +# Follower applies the snapshot. Note how it reacts with a MsgAppResp upon completion. +# The snapshot fully catches the follower up (i.e. there are no more log entries it +# needs to apply after). The bug was that the leader failed to realize that the follower +# was now fully caught up. +stabilize 3 +---- +> 3 receiving messages + 1->3 MsgSnap Term:1 Log:0/0 + Snapshot: Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + INFO log [committed=0, applied=0, applying=0, unstable.offset=1, unstable.offsetInProgress=1, len(unstable.Entries)=0] starts to restore snapshot [index: 11, term: 1] + INFO 3 switched to configuration voters=(1 2 3) + INFO 3 [commit: 11, lastindex: 11, lastterm: 1] restored snapshot [index: 11, term: 1] + INFO 3 [commit: 11] restored snapshot [index: 11, term: 1] +> 3 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:11 + Snapshot Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + Messages: + 3->1 MsgAppResp Term:1 Log:0/11 + +# The MsgAppResp lets the leader move the follower back to replicating state. +# Leader sends another MsgAppResp, to communicate the updated commit index. +stabilize 1 +---- +> 1 receiving messages + 3->1 MsgAppResp Term:1 Log:0/11 + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=11 next=12 paused pendingSnap=11] + +status 1 +---- +1: StateReplicate match=11 next=12 +2: StateReplicate match=11 next=12 +3: StateReplicate match=11 next=12 + +# Let things settle. +stabilize +---- +> 2 receiving messages + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgHeartbeatResp Term:1 Log:0/0 +> 1 receiving messages + 2->1 MsgHeartbeatResp Term:1 Log:0/0 diff --git a/third_party/raft/testdata/snapshot_succeed_via_app_resp_behind.txt b/third_party/raft/testdata/snapshot_succeed_via_app_resp_behind.txt new file mode 100644 index 0000000000..3bab832d60 --- /dev/null +++ b/third_party/raft/testdata/snapshot_succeed_via_app_resp_behind.txt @@ -0,0 +1,167 @@ +# This is a variant of snapshot_succeed_via_app_resp in which the snapshot +# that is being sent is behind the PendingSnapshot index tracked by the leader. + +# Turn off output during the setup of the test. +log-level none +---- +ok + +# Start with three nodes, but the third is disconnected from the log. +add-nodes 2 voters=(1,2,3) index=10 +---- +ok + +add-nodes 1 voters=(1,2,3) index=5 +---- +ok + +# Elect 1 as leader. We first stabilize 3 to process the vote, then stabilize 1 +# and 2 to complete the leader election. We don't stabilize 3 after the +# election, so that it does not receive and process any MsgApp yet. +campaign 1 +---- +ok + +process-ready 1 +---- +ok + +stabilize 3 +---- +ok + +stabilize 1 2 +---- +ok + +log-level debug +---- +ok + +# We now have a leader at index 11 (it appended an empty entry when elected). 3 +# is still at index 5, and has not received any MsgApp from the leader yet. +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:0 + +status 1 +---- +1: StateReplicate match=11 next=12 +2: StateReplicate match=11 next=12 +3: StateProbe match=0 next=11 paused inactive + +raft-log 3 +---- +log is empty: first index=6, last index=5 + +# Send a manual snapshot from 1 to 3, which will be at index 11. This snapshot +# does not move 3 to StateSnapshot. +send-snapshot 1 3 +---- +1->3 MsgSnap Term:1 Log:0/0 + Snapshot: Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + +# Propose and commit an additional entry, which makes the leader's +# last index 12, beyond the snapshot it sent at index 11. +log-level none +---- +ok + +propose 1 "foo" +---- +ok + +stabilize 1 2 +---- +ok + +log-level debug +---- +ok + +status 1 +---- +1: StateReplicate match=12 next=13 +2: StateReplicate match=12 next=13 +3: StateProbe match=0 next=11 paused inactive + +# 3 now gets the first MsgApp the leader originally sent, trying to append entry +# 11 but this is rejected because the follower's log started at index 5. +deliver-msgs 3 type=MsgApp +---- +1->3 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] +DEBUG 3 [logterm: 0, index: 10] rejected MsgApp [logterm: 1, index: 10] from 1 + +# Note below that the RejectionHint is 5, which is below the first index 10 of the +# leader. Once the leader receives this, it will move 3 into StateSnapshot with +# PendingSnapshot=lastIndex=12. +process-ready 3 +---- +Ready MustSync=false: +Lead:1 State:StateFollower +Messages: +3->1 MsgAppResp Term:1 Log:1/10 Rejected (Hint: 5) + +# 3 receives and applies the snapshot, but doesn't respond with MsgAppResp yet. +deliver-msgs 3 +---- +1->3 MsgSnap Term:1 Log:0/0 + Snapshot: Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false +INFO log [committed=5, applied=5, applying=5, unstable.offset=6, unstable.offsetInProgress=6, len(unstable.Entries)=0] starts to restore snapshot [index: 11, term: 1] +INFO 3 switched to configuration voters=(1 2 3) +INFO 3 [commit: 11, lastindex: 11, lastterm: 1] restored snapshot [index: 11, term: 1] +INFO 3 [commit: 11] restored snapshot [index: 11, term: 1] + + +# 1 sees the MsgApp rejection and asks for a snapshot at index 12 (which is 1's +# current last index). +stabilize 1 +---- +> 1 receiving messages + 3->1 MsgAppResp Term:1 Log:1/10 Rejected (Hint: 5) + DEBUG 1 received MsgAppResp(rejected, hint: (index 5, term 1)) from 3 for index 10 + DEBUG 1 decreased progress of 3 to [StateProbe match=0 next=6] + DEBUG 1 [firstindex: 11, commit: 12] sent snapshot[index: 12, term: 1] to 3 [StateProbe match=0 next=6] + DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 next=13 paused pendingSnap=12] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->3 MsgSnap Term:1 Log:0/0 + Snapshot: Index:12 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + +# Drop the extra MsgSnap(index=12) that 1 just sent, to keep the test tidy. +deliver-msgs drop=(3) +---- +dropped: 1->3 MsgSnap Term:1 Log:0/0 + Snapshot: Index:12 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + +# 3 sends the affirmative MsgAppResp that resulted from applying the snapshot +# at index 11. +stabilize 3 +---- +> 3 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:11 + Snapshot Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + Messages: + 3->1 MsgAppResp Term:1 Log:0/11 + +stabilize 1 +---- +> 1 receiving messages + 3->1 MsgAppResp Term:1 Log:0/11 + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=11 next=13 paused pendingSnap=12] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->3 MsgApp Term:1 Log:1/11 Commit:12 Entries:[1/12 EntryNormal "\"foo\""] + +# 3 is in StateReplicate thanks to receiving the snapshot at index 11. +# This is despite its PendingSnapshot having been 12. +status 1 +---- +1: StateReplicate match=12 next=13 +2: StateReplicate match=12 next=13 +3: StateReplicate match=11 next=13 inflight=1 diff --git a/third_party/raft/tla/MCetcdraft.cfg b/third_party/raft/tla/MCetcdraft.cfg new file mode 100644 index 0000000000..31ad958462 --- /dev/null +++ b/third_party/raft/tla/MCetcdraft.cfg @@ -0,0 +1,69 @@ +\* Copyright 2024 The etcd Authors +\* +\* Licensed under the Apache License, Version 2.0 (the "License"); +\* you may not use this file except in compliance with the License. +\* You may obtain a copy of the License at +\* +\* http://www.apache.org/licenses/LICENSE-2.0 +\* +\* Unless required by applicable law or agreed to in writing, software +\* distributed under the License is distributed on an "AS IS" BASIS, +\* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +\* See the License for the specific language governing permissions and +\* limitations under the License. +\* +SPECIFICATION mc_etcdSpec + +CONSTANTS + s1 = 1 + s2 = 2 + s3 = 3 + s4 = 4 + s5 = 5 + + InitServer = {s1, s2, s3} + Server = {s1, s2, s3, s4} + + ReconfigurationLimit = 2 + MaxTermLimit = 10 + RequestLimit = 5 + + Timeout <- MCTimeout + Send <- MCSend + ClientRequest <- MCClientRequest + AddNewServer <- MCAddNewServer + DeleteServer <- MCDeleteServer + AddLearner <- MCAddLearner + + InitServerVars <- etcdInitServerVars + InitLogVars <- etcdInitLogVars + InitConfigVars <- etcdInitConfigVars + + Nil = 0 + + ValueEntry = "ValueEntry" + ConfigEntry = "ConfigEntry" + + Follower = "Follower" + Candidate = "Candidate" + Leader = "Leader" + RequestVoteRequest = "RequestVoteRequest" + RequestVoteResponse = "RequestVoteResponse" + AppendEntriesRequest = "AppendEntriesRequest" + AppendEntriesResponse = "AppendEntriesResponse" + +SYMMETRY Symmetry +VIEW View + +CHECK_DEADLOCK + FALSE + +INVARIANTS + LogInv + MoreThanOneLeaderInv + ElectionSafetyInv + LogMatchingInv + QuorumLogInv + MoreUpToDateCorrectInv + LeaderCompletenessInv + CommittedIsDurableInv \ No newline at end of file diff --git a/third_party/raft/tla/MCetcdraft.tla b/third_party/raft/tla/MCetcdraft.tla new file mode 100644 index 0000000000..5302c2bc9a --- /dev/null +++ b/third_party/raft/tla/MCetcdraft.tla @@ -0,0 +1,120 @@ +---------- MODULE MCetcdraft ---------- +\* Copyright 2024 The etcd Authors +\* +\* Licensed under the Apache License, Version 2.0 (the "License"); +\* you may not use this file except in compliance with the License. +\* You may obtain a copy of the License at +\* +\* http://www.apache.org/licenses/LICENSE-2.0 +\* +\* Unless required by applicable law or agreed to in writing, software +\* distributed under the License is distributed on an "AS IS" BASIS, +\* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +\* See the License for the specific language governing permissions and +\* limitations under the License. +\* + +EXTENDS etcdraft + +CONSTANT ReconfigurationLimit +ASSUME ReconfigurationLimit \in Nat + +CONSTANT MaxTermLimit +ASSUME MaxTermLimit \in Nat + +\* Limit on client requests +CONSTANT RequestLimit +ASSUME RequestLimit \in Nat + +etcd == INSTANCE etcdraft + +\* Application uses Node (instead of RawNode) will have multiple ConfigEntry entries appended to log in bootstrapping. +BootstrapLog == + LET prevConf(y) == IF Len(y) = 0 THEN {} ELSE y[Len(y)].value.newconf + IN FoldSeq(LAMBDA x, y: Append(y, [ term |-> 1, type |-> ConfigEntry, value |-> [ newconf |-> prevConf(y) \union {x}, learners |-> {} ] ]), <<>>, SetToSeq(InitServer)) + +\* etcd is bootstrapped in two ways. +\* 1. bootstrap a cluster for the first time: server vars are initialized with term 1 and pre-inserted log entries for initial configuration. +\* 2. adding a new member: server vars are initialized with all state 0 +\* 3. restarting an existing member: all states are loaded from durable storage +etcdInitServerVars == /\ currentTerm = [i \in Server |-> IF i \in InitServer THEN 1 ELSE 0] + /\ state = [i \in Server |-> Follower] + /\ votedFor = [i \in Server |-> Nil] +etcdInitLogVars == /\ log = [i \in Server |-> IF i \in InitServer THEN BootstrapLog ELSE <<>>] + /\ commitIndex = [i \in Server |-> IF i \in InitServer THEN Cardinality(InitServer) ELSE 0] +etcdInitConfigVars == /\ config = [i \in Server |-> [ jointConfig |-> IF i \in InitServer THEN <> ELSE <<{}, {}>>, learners |-> {}]] + /\ reconfigCount = 0 \* the bootstrap configuraitons are not counted + +\* This file controls the constants as seen below. +\* In addition to basic settings of how many nodes are to be model checked, +\* the model allows to place additional limitations on the state space of the program. + +\* Limit the # of reconfigurations to ReconfigurationLimit +MCAddNewServer(i, j) == + /\ reconfigCount < ReconfigurationLimit + /\ etcd!AddNewServer(i, j) +MCDeleteServer(i, j) == + /\ reconfigCount < ReconfigurationLimit + /\ etcd!DeleteServer(i, j) +MCAddLearner(i, j) == + /\ reconfigCount < ReconfigurationLimit + /\ etcd!AddLearner(i, j) + +\* Limit the terms that can be reached. Needs to be set to at least 3 to +\* evaluate all relevant states. If set to only 2, the candidate_quorum +\* constraint below is too restrictive. +MCTimeout(i) == + \* Limit the term of each server to reduce state space + /\ currentTerm[i] < MaxTermLimit + \* Limit max number of simultaneous candidates + \* We made several restrictions to the state space of Raft. However since we + \* made these restrictions, Deadlocks can occur at places that Raft would in + \* real-world deployments handle graciously. + \* One example of this is if a Quorum of nodes becomes Candidate but can not + \* timeout anymore since we constrained the terms. Then, an artificial Deadlock + \* is reached. We solve this below. If TermLimit is set to any number >2, this is + \* not an issue since breadth-first search will make sure that a similar + \* situation is simulated at term==1 which results in a term increase to 2. + /\ Cardinality({ s \in GetConfig(i) : state[s] = Candidate}) < 1 + /\ etcd!Timeout(i) + +\* Limit number of requests (new entries) that can be made +MCClientRequest(i, v) == + \* Allocation-free variant of Len(SelectSeq(log[i], LAMBDA e: e.contentType = TypeEntry)) < RequestLimit + /\ FoldSeq(LAMBDA e, count: IF e.type = ValueEntry THEN count + 1 ELSE count, 0, log[i]) < RequestLimit + /\ etcd!ClientRequest(i, v) + +\* Limit how many identical append entries messages each node can send to another +\* Limit number of duplicate messages sent to the same server +MCSend(msg) == + \* One AppendEntriesRequest per node-pair at a time: + \* a) No AppendEntries request from i to j. + /\ ~ \E n \in DOMAIN messages \union DOMAIN pendingMessages: + /\ n.mdest = msg.mdest + /\ n.msource = msg.msource + /\ n.mterm = msg.mterm + /\ n.mtype = AppendEntriesRequest + /\ msg.mtype = AppendEntriesRequest + \* b) No (corresponding) AppendEntries response from j to i. + /\ ~ \E n \in DOMAIN messages \union DOMAIN pendingMessages: + /\ n.mdest = msg.msource + /\ n.msource = msg.mdest + /\ n.mterm = msg.mterm + /\ n.mtype = AppendEntriesResponse + /\ msg.mtype = AppendEntriesRequest + /\ etcd!Send(msg) + +mc_etcdSpec == + /\ Init + /\ [][NextDynamic]_vars + +\* Symmetry set over possible servers. May dangerous and is only enabled +\* via the Symmetry option in cfg file. +Symmetry == Permutations(Server) + +\* Include all variables in the view, which is similar to defining no view. +View == << vars >> + +---- + +=================================== \ No newline at end of file diff --git a/third_party/raft/tla/README.md b/third_party/raft/tla/README.md new file mode 100644 index 0000000000..a29c4877e9 --- /dev/null +++ b/third_party/raft/tla/README.md @@ -0,0 +1,94 @@ +# TLA+ Specification and Trace Validation for Raft Library: A Brief Guide + +This document presents a brief guideline on how to validate the correctness of Raft library implemented in `etcd-io/raft`. A new TLA+ specification models the core algorithm of the library, including the distinctive behaviors like membership reconfiguration, that differentiate it from the classic Raft algorithm. The TLA+ specification serves dual purposes: it not only helps to verify the correctness of the model but also facilitates model-based trace validation. + +## Checking the Model with TLA+ Specifications + +The TLA+ specification can be verified using the TLC model checker. Here are a few methods: + +1. **TLA+ Toolbox**: This is an ideal tool for an in-depth study of the specification. For more information, refer to the [TLA+ Toolbox guideline](https://lamport.azurewebsites.net/tla/toolbox.html). + +2. **VSCode Plugin TLA+ Nightly**: This is a viable alternative to the TLA+ Toolbox, particularly for those accustomed to using VSCode. For more information, refer to the [VSCode Plugin TLA+ Nightly guideline](https://github.com/tlaplus/vscode-tlaplus/wiki). + +3. **CLI**: This is the best option for integration with existing test frameworks and automation. For more information, refer to the [CLI guideline](https://learntla.com/topics/cli.html). You can execute a typical command line to verify etcdraft.tla as shown below. Please ensure that tla2tools.jar and CommunityModules-deps.jar have been downloaded and are available in the current folder before proceeding. + + ```console + java -XX:+UseParallelGC -cp tla2tools.jar:CommunityModules-deps.jar tlc2.TLC -config MCetcdraft.cfg MCetcdraft.tla -lncheck final -fpmem 0.9 + ``` + +## TLA+ Model-Based Trace Validation + +The correctness of applications that implements a consensus algorithm is ensured by two factors: the correctness of the algorithm itself, and adherence of the implementation to algorithm specification. + +The first factor, the correctness of the algorithm, is assured through model checking the specification. The second one, adherence of the implementation to algorithm specification, is fortified through trace validation, which serves to bridge the gap between the model and its implementation. + +To ensure the robustness of trace validation, we initially inject multiple trace log points at specific sections in the code where state transitions occur (for example, SendAppendEntries, BecomeLeader, etc.). The trace validation specification leverages these traces as a state space constraint, guiding the state machine to traverse in a manner identical to that of the service. + +If a trace suggests a state or transition that the state machine can't accommodate, it indicates a discrepancy between the model and its implementation. In cases where the model has already been verified by the TLC model checker, it's more likely that any issues arise from the implementation rather than the model. In other cases where the code change reflects expected refactoring, we need to update the model accordingly and model checking it before validating traces with the new model. + +## Enable TLA+ Trace Validation + +To activate trace collection, build the application using the "with_tla" tag (`go build -tags=with_tla`). The `StartNode` and/or `RestartNode` should be invoked with `Config` with `TraceLogger` property set to an instance of `TraceLogger` implemented in the application. + +`Traceetcdraft.tla` expects a trace file in NDJSON format. Each entry must include an `event` field containing a `TracingEvent` object. Here's an example trace logger using zap.Logger. Note that sampling shall be disabled to ensure all traces are logged (see [Zap FAQ: Why sample application logs?](https://github.com/uber-go/zap/blob/master/FAQ.md#why-sample-application-logs)). + +```go +type TraceLogger interface { + TraceEvent(*TracingEvent) +} + +type MyTraceLogger struct{ + lg *zap.Logger +} + +func (t *MyTraceLogger) TraceEvent(ev *TracingEvent) { + t.lg.Debug("trace", zap.String("tag", "trace"), zap.Any("event", ev)) +} + +``` + +To start a three-node cluster with trace logger +```go +storage := raft.NewMemoryStorage() + c := &raft.Config{ + ID: 0x01, + ElectionTick: 10, + HeartbeatTick: 1, + Storage: storage, + MaxSizePerMsg: 4096, + MaxInflightMsgs: 256, + TraceLogger: &MyTraceLogger{}, + } + // Set peer list to the other nodes in the cluster. + // Note that they need to be started separately as well. + n := raft.StartNode(c, []raft.Peer{{ID: 0x02}, {ID: 0x03}}) +``` +**Note:** To preserve the causality of events across nodes, run all application instances on the same machine and store traces in the same file. This approach maintains the order of traces in all instances. + +## Model Checking TLA+ Spec +The TLA+ spec defines the desired behaviors of the model. To validate the correctness of the model, we need to make sure following commands run enough time (at least hours, depending on how confident we need) without failure. + +```console +./validate-model.sh -s ./MCetcdraft.tla -c ./MCetcdraft.cfg +``` + + +## Validate Collected Traces +With above example trace logger, validate.sh can be used to validate traces parallelly. + +```console +./validate.sh -s ./Traceetcdraft.tla -c ./Traceetcdraft.cfg /path/to/traces/*.ndjson +``` +By default validate.sh uses all CPU cores to validate the traces. You can also specify the maximum concurrent tasks by `-p` argument. + +Validating all traces in a file may take a long time. You can specify `MAX_TRACE` environment variable to limit the validation only on top `MAX_TRACE` traces in each file. + +## Known Issues +1. **Partially persisted log**. +etcdraft assumes atomic persisiting of states in Read action. This, however, may not apply in real-world application. For example, etcd may persist a prefix log when it crashes in the middle of saving data to disks. +We will introduce non-deterministic validation method in future to address this issue. + +2. **Long validation time**. +As aforementioned, trace validation may take long time when there are thousands of traces in a file. It typically takes a few minutes to validate 3000 traces. And the time increases non-linearly after that. +The workaround for this issue is to run validation in batches after all traces are collected. Each file may take long time but the overall validation time can be reasonable in this way. +All means to reduce trace amount would help to reduce validation time, e.g. lower QPS and larger heartbeat timeout. \ No newline at end of file diff --git a/third_party/raft/tla/Traceetcdraft.cfg b/third_party/raft/tla/Traceetcdraft.cfg new file mode 100644 index 0000000000..36e6c8aaf5 --- /dev/null +++ b/third_party/raft/tla/Traceetcdraft.cfg @@ -0,0 +1,75 @@ +\* Copyright 2024 The etcd Authors +\* +\* Licensed under the Apache License, Version 2.0 (the "License"); +\* you may not use this file except in compliance with the License. +\* You may obtain a copy of the License at +\* +\* http://www.apache.org/licenses/LICENSE-2.0 +\* +\* Unless required by applicable law or agreed to in writing, software +\* distributed under the License is distributed on an "AS IS" BASIS, +\* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +\* See the License for the specific language governing permissions and +\* limitations under the License. +\* + +\* A trace specification defines a set of one or more traces, wherein the value of some +\* variables in each state aligns with the values specified on the corresponding line +\* of the log or trace file. Commonly, the log file might be incomplete, failing to +\* provide the value of all variables. In such instances, the omitted values are +\* determined non-deterministically, adhering to the high-level specification's parameters. +\* Furthermore, a trace may not be complete such that it only matches a prefix of the log. +SPECIFICATION + TraceSpec + +VIEW + TraceView + +\* TLA has only limited support for hyperproperties. The following property is a poorman's +\* hyperproperty that asserts that TLC generated *at least one* trace that fully matches the +\* log file. +PROPERTIES + TraceMatched + +PROPERTIES + etcdSpec + +\* Checking for deadlocks during trace validation is disabled, as it may lead to false +\* counterexamples. A trace specification defines a set of traces, where at least one +\* trace is expected to match the log file in terms of variable values and length. +\* However, partial matches may occur where the trace cannot be extended to fully +\* correspond with the log file. In such cases, deadlock checking would report the first +\* of these traces. +CHECK_DEADLOCK + FALSE + +CONSTANTS + Nil = "0" + + ValueEntry = "ValueEntry" + ConfigEntry = "ConfigEntry" + + Follower = "StateFollower" + Candidate = "StateCandidate" + Leader = "StateLeader" + RequestVoteRequest = "RequestVoteRequest" + RequestVoteResponse = "RequestVoteResponse" + AppendEntriesRequest = "AppendEntriesRequest" + AppendEntriesResponse = "AppendEntriesResponse" + + InitServerVars <- TraceInitServerVars + InitLogVars <- TraceInitLogVars + InitConfigVars <- TraceInitConfigVars + + InitServer <- TraceInitServer + Server <- TraceServer + +INVARIANTS + LogInv + MoreThanOneLeaderInv + ElectionSafetyInv + LogMatchingInv + QuorumLogInv + MoreUpToDateCorrectInv + LeaderCompletenessInv + CommittedIsDurableInv \ No newline at end of file diff --git a/third_party/raft/tla/Traceetcdraft.tla b/third_party/raft/tla/Traceetcdraft.tla new file mode 100644 index 0000000000..ccbae2d3fe --- /dev/null +++ b/third_party/raft/tla/Traceetcdraft.tla @@ -0,0 +1,478 @@ +-------------------------------- MODULE Traceetcdraft ------------------------------- +\* Copyright 2024 The etcd Authors +\* +\* Licensed under the Apache License, Version 2.0 (the "License"); +\* you may not use this file except in compliance with the License. +\* You may obtain a copy of the License at +\* +\* http://www.apache.org/licenses/LICENSE-2.0 +\* +\* Unless required by applicable law or agreed to in writing, software +\* distributed under the License is distributed on an "AS IS" BASIS, +\* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +\* See the License for the specific language governing permissions and +\* limitations under the License. +\* + +EXTENDS etcdraft, Json, IOUtils, Sequences, TLC + +\* raft.pb.go enum MessageType +RaftMsgType == + "MsgApp" :> AppendEntriesRequest @@ "MsgAppResp" :> AppendEntriesResponse @@ + "MsgVote" :> RequestVoteRequest @@ "MsgVoteResp" :> RequestVoteResponse @@ + "MsgHeartbeat" :> AppendEntriesRequest @@ "MsgHeartbeatResp" :> AppendEntriesResponse @@ + "MsgSnap" :> AppendEntriesRequest + +RaftMsgSubtype == + "MsgHeartbeat" :> "heartbeat" @@ "MsgHeartbeatResp" :> "heartbeat" @@ + "MsgApp" :> "app" @@ "MsgAppResp" :> "app" @@ + "MsgSnap" :> "snapshot" + +------------------------------------------------------------------------------------- + +\* Trace validation has been designed for TLC running in default model-checking + \* mode, i.e., breadth-first search. +ASSUME TLCGet("config").mode = "bfs" + +JsonFile == + IF "JSON" \in DOMAIN IOEnv THEN IOEnv.JSON ELSE "./example.ndjson" + +OriginTraceLog == + \* Deserialize the System log as a sequence of records from the log file. + \* Run TLC with (assuming a suitable "tlc" shell alias): + \* $ JSON=../tests/raft_scenarios/4582.ndjson tlc -note Traceccfraft + \* Fall back to trace.ndjson if the JSON environment variable is not set. + SelectSeq(ndJsonDeserialize(JsonFile), LAMBDA l: "tag" \in DOMAIN l /\ l.tag = "trace") + +TraceLog == + TLCEval(IF "MAX_TRACE" \in DOMAIN IOEnv THEN SubSeq(OriginTraceLog, 1, atoi(IOEnv.MAX_TRACE)) ELSE OriginTraceLog) + +TraceServer == TLCEval(FoldSeq( + LAMBDA x, y: y \cup IF /\ x.event.name = "ChangeConf" + /\ "changes" \in DOMAIN x.event.prop.cc + /\ x.event.prop.cc.changes[1].action \in {"AddNewServer", "AddLearner"} + THEN {x.event.nid, x.event.prop.cc.changes[1].nid} + ELSE {x.event.nid}, + {}, TraceLog)) + +BootstrapLogIndicesForServer(i) == + LET + FirstBootstrapLogIndex == SelectInSeq(TraceLog, LAMBDA x: x.event.nid = i /\ x.event.name \in {"InitState", "BecomeFollower", "ApplyConfChange"}) + FirstNonBootstrapLogIndex == SelectInSeq(TraceLog, LAMBDA x: x.event.nid = i /\ x.event.name \notin {"InitState", "BecomeFollower", "ApplyConfChange"}) + LastBootstrapLogIndexUpperBound == IF FirstNonBootstrapLogIndex = 0 THEN Len(TraceLog) ELSE FirstNonBootstrapLogIndex-1 + IN + { k \in FirstBootstrapLogIndex..LastBootstrapLogIndexUpperBound: TraceLog[k].event.nid = i } + +BootstrapLogIndices == UNION { BootstrapLogIndicesForServer(i): i \in Server } + +LastBootstrapLog == [ i \in Server |-> TraceLog[Max(BootstrapLogIndicesForServer(i))] ] + +BootstrappedConfig(i) == + IF LastBootstrapLog[i].event.name = "ApplyConfChange" THEN + ToSet(LastBootstrapLog[i].event.prop.cc.newconf) + ELSE + ToSet(LastBootstrapLog[i].event.conf[1]) + +TraceInitServer == BootstrappedConfig(TraceLog[1].event.nid) +ASSUME TraceInitServer \subseteq TraceServer + +TraceInitServerVars == /\ currentTerm = [i \in Server |-> LastBootstrapLog[i].event.state.term] + /\ state = [i \in Server |-> LastBootstrapLog[i].event.role] + /\ votedFor = [i \in Server |-> LastBootstrapLog[i].event.state.vote] +TraceInitLogVars == /\ log = [i \in Server |-> [j \in 1..LastBootstrapLog[i].event.log |-> [ term |-> 1, type |-> "ConfigEntry", value |-> [newconf |-> BootstrappedConfig(i), learners |-> {}]]]] + /\ commitIndex = [i \in Server |-> LastBootstrapLog[i].event.state.commit] +TraceInitConfigVars == + /\ config = [i \in Server |-> [ jointConfig |-> <>, learners |-> {}] ] + /\ reconfigCount = 0 + + +------------------------------------------------------------------------------------- +ConfFromLog(l) == << ToSet(l.event.conf[1]), ToSet(l.event.conf[2]) >> + +OneMoreMessage(msg) == + \/ msg \notin DOMAIN pendingMessages /\ msg \in DOMAIN pendingMessages' /\ pendingMessages'[msg] = 1 + \/ msg \in DOMAIN pendingMessages /\ pendingMessages'[msg] = pendingMessages[msg] + 1 + +OneLessMessage(msg) == + \/ msg \in DOMAIN messages /\ messages[msg] = 1 /\ msg \notin DOMAIN messages' + \/ msg \in DOMAIN messages /\ messages'[msg] = messages[msg] - 1 + +------------------------------------------------------------------------------------- + +VARIABLE l +logline == TraceLog[l] +VARIABLE pl + + +TraceInit == + /\ l = 1 + /\ pl = 0 + /\ logline = TraceLog[l] + /\ Init + +StepToNextTrace == + /\ l' = l+1 + /\ pl' = l + /\ l % Max({1, Len(TraceLog) \div 100}) = 0 => PrintT(<< "Progress %:", (l * 100) \div Len(TraceLog)>>) + /\ l' > Len(TraceLog) => PrintT(<< "Progress %:", 100>>) + +StepToNextTraceIfMessageIsProcessed(msg) == + IF OneLessMessage(msg) + THEN StepToNextTrace + ELSE + /\ pl' = l + /\ UNCHANGED <> + +------------------------------------------------------------------------------------- + +LoglineIsEvent(e) == + /\ l <= Len(TraceLog) + /\ logline.event.name = e + +LoglineIsEvents(e) == + /\ l <= Len(TraceLog) + /\ logline.event.name \in e + +LoglineIsMessageEvent(e, i, j) == + /\ LoglineIsEvent(e) + /\ logline.event.msg.from = i + /\ logline.event.msg.to = j + +LoglineIsNodeEvent(e, i) == + /\ LoglineIsEvent(e) + /\ logline.event.nid = i + +LoglineIsAppendEntriesRequest(m) == + /\ m.mtype = AppendEntriesRequest + /\ m.mtype = RaftMsgType[logline.event.msg.type] + /\ m.msubtype = RaftMsgSubtype[logline.event.msg.type] + /\ m.mdest = logline.event.msg.to + /\ m.msource = logline.event.msg.from + /\ m.mterm = logline.event.msg.term + \* MsgSnap is equivalent to MsgApp except that it does not + \* have commit index. Snapshot message contains leader log prefix + \* up to a committed entry. That means the receiver can safely advance + \* its commit index at least to the last log entry in snapshot message. + \* Setting commit index in the MsgSnap message would become unnecessary. + \* So we can safely ignore checking this against the model. + /\ m.msubtype /= "snapshot" => m.mcommitIndex = logline.event.msg.commit + /\ m.msubtype /= "heartbeat" => /\ m.mprevLogTerm = logline.event.msg.logTerm + /\ m.mprevLogIndex = logline.event.msg.index + /\ Len(m.mentries) = logline.event.msg.entries + +LoglineIsAppendEntriesResponse(m) == + /\ m.mtype = AppendEntriesResponse + /\ m.mtype = RaftMsgType[logline.event.msg.type] + /\ m.msubtype = RaftMsgSubtype[logline.event.msg.type] + /\ m.mdest = logline.event.msg.to + /\ m.msource = logline.event.msg.from + /\ m.mterm = logline.event.msg.term + /\ m.msuccess = ~logline.event.msg.reject + /\ (\lnot logline.event.msg.reject /\ m.msubtype /= "heartbeat") => m.mmatchIndex = logline.event.msg.index + +LoglineIsRequestVoteRequest(m) == + /\ m.mtype = RequestVoteRequest + /\ m.mtype = RaftMsgType[logline.event.msg.type] + /\ m.mdest = logline.event.msg.to + /\ m.msource = logline.event.msg.from + /\ m.mterm = logline.event.msg.term + /\ m.mlastLogIndex = logline.event.msg.index + /\ m.mlastLogTerm = logline.event.msg.logTerm + +LoglineIsRequestVoteResponse(m) == + /\ m.mtype = RequestVoteResponse + /\ m.mtype = RaftMsgType[logline.event.msg.type] + /\ m.mdest = logline.event.msg.to + /\ m.msource = logline.event.msg.from + /\ m.mterm = logline.event.msg.term + /\ m.mvoteGranted = ~logline.event.msg.reject + +ValidatePreStates(i) == + \* only validate upon first visit of the logline + pl = l - 1 => /\ currentTerm[i] = logline.event.state.term + /\ state[i] = logline.event.role + /\ votedFor[i] = logline.event.state.vote + /\ Len(log[i]) = logline.event.log + /\ commitIndex[i] = logline.event.state.commit + /\ config[i].jointConfig = ConfFromLog(logline) + +ValidatePostStates(i) == + /\ currentTerm'[i] = logline.event.state.term + /\ state'[i] = logline.event.role + /\ votedFor'[i] = logline.event.state.vote + /\ Len(log'[i]) = logline.event.log + /\ commitIndex'[i] = logline.event.state.commit + /\ config'[i].jointConfig = ConfFromLog(logline) + +\* perform RequestVote transition if logline indicates so +ValidateAfterRequestVote(i, j) == + /\ ValidatePostStates(i) + /\ \E m \in DOMAIN pendingMessages': + /\ \/ LoglineIsRequestVoteRequest(m) + \/ /\ LoglineIsRequestVoteResponse(m) + /\ m.msource = m.mdest + /\ OneMoreMessage(m) + +RequestVoteIfLogged(i, j) == + /\ \/ LoglineIsMessageEvent("SendRequestVoteRequest", i, j) + \* etcd candidate sends MsgVoteResp to itself upon compain starting + \/ /\ LoglineIsMessageEvent("SendRequestVoteResponse", i, j) + /\ i = j + /\ RequestVote(i, j) + /\ ValidateAfterRequestVote(i, j) + +\* perform BecomeLeader transition if logline indicates so +ValidateAfterBecomeLeader(i) == + /\ ValidatePostStates(i) + /\ logline.event.role = "StateLeader" + /\ state'[i] = Leader + +BecomeLeaderIfLogged(i) == + /\ LoglineIsNodeEvent("BecomeLeader", i) + /\ BecomeLeader(i) + /\ ValidateAfterBecomeLeader(i) + +\* perform ClientRequest transition if logline indicates so +ClientRequestIfLogged(i, v) == + /\ LoglineIsNodeEvent("Replicate", i) + /\ ClientRequest(i, v) + +\* perform AdvanceCommitIndex transition if logline indicates so +ValidateAfterAdvanceCommitIndex(i) == + /\ ValidatePostStates(i) + /\ logline.event.role = "StateLeader" + /\ state[i] = Leader + +AdvanceCommitIndexIfLogged(i) == + /\ LoglineIsNodeEvent("Commit", i) + /\ AdvanceCommitIndex(i) + /\ ValidateAfterAdvanceCommitIndex(i) + +\* perform AppendEntries transition if logline indicates so +ValidateAfterAppendEntries(i, j) == + /\ ValidatePostStates(i) + /\ \E msg \in DOMAIN pendingMessages': + /\ LoglineIsAppendEntriesRequest(msg) + /\ OneMoreMessage(msg) + +ValidateAfterHeartbeat(i, j) == + /\ ValidatePostStates(i) + /\ \E msg \in DOMAIN pendingMessages': + /\ LoglineIsAppendEntriesRequest(msg) + /\ OneMoreMessage(msg) + +ValidateAfterAppendEntriesToSelf(i) == + /\ ValidatePostStates(i) + /\ \E msg \in DOMAIN pendingMessages': + /\ LoglineIsAppendEntriesResponse(msg) + /\ msg.msource = msg.mdest + \* There is now one more message of this type. + /\ OneMoreMessage(msg) + +AppendEntriesIfLogged(i, j, range) == + /\ LoglineIsMessageEvent("SendAppendEntriesRequest", i, j) + /\ logline.event.msg.type = "MsgApp" + /\ range[1] = logline.event.msg.index + 1 + /\ range[2] = range[1] + logline.event.msg.entries + /\ AppendEntries(i, j, range) + /\ ValidateAfterAppendEntries(i, j) + +HeartbeatIfLogged(i, j) == + /\ LoglineIsMessageEvent("SendAppendEntriesRequest", i, j) + /\ logline.event.msg.type = "MsgHeartbeat" + /\ Heartbeat(i, j) + /\ ValidateAfterAppendEntries(i, j) + +SendSnapshotIfLogged(i, j, index) == + /\ LoglineIsMessageEvent("SendAppendEntriesRequest", i, j) + /\ logline.event.msg.type = "MsgSnap" + /\ index = logline.event.msg.entries + /\ SendSnapshot(i, j, index) + /\ ValidateAfterAppendEntries(i, j) + +AppendEntriesToSelfIfLogged(i) == + /\ LoglineIsMessageEvent("SendAppendEntriesResponse", i, i) + /\ AppendEntriesToSelf(i) + /\ ValidateAfterAppendEntriesToSelf(i) + +ReceiveMessageTraceNames == { "ReceiveAppendEntriesRequest", "ReceiveAppendEntriesResponse", "ReceiveRequestVoteRequest", "ReceiveRequestVoteResponse", "ReceiveSnapshot" } +\* perform Receive transition if logline indicates so +LoglineIsReceivedMessage(m) == + \/ /\ LoglineIsEvent("ReceiveAppendEntriesRequest") + /\ LoglineIsAppendEntriesRequest(m) + \/ /\ LoglineIsEvent("ReceiveAppendEntriesResponse") + /\ LoglineIsAppendEntriesResponse(m) + \/ /\ LoglineIsEvent("ReceiveRequestVoteRequest") + /\ LoglineIsRequestVoteRequest(m) + \/ /\ LoglineIsEvent("ReceiveRequestVoteResponse") + /\ LoglineIsRequestVoteResponse(m) + \/ /\ LoglineIsEvent("ReceiveSnapshot") + /\ LoglineIsAppendEntriesRequest(m) + +ReceiveIfLogged(m) == + /\ LoglineIsReceivedMessage(m) + /\ ValidatePreStates(m.mdest) + /\ Receive(m) + +\* perform Timeout transition if logline indicates so +ValidateAfterTimeout(i) == + /\ logline.event.role = "StateCandidate" + /\ logline.event.nid = i + /\ state'[i] = Candidate + /\ currentTerm'[i] = logline.event.state.term + +TimeoutIfLogged(i) == + /\ LoglineIsNodeEvent("BecomeCandidate", i) + /\ Timeout(i) + /\ ValidateAfterTimeout(i) + +\* perform AddNewServer transition if logline indicates so +AddNewServerIfLogged(i, j) == + /\ LoglineIsNodeEvent("ChangeConf", i) + /\ Len(logline.event.prop.cc.changes) = 1 + /\ logline.event.prop.cc.changes[1].action = "AddNewServer" + /\ logline.event.prop.cc.changes[1].nid = j + /\ AddNewServer(i, j) + +\* perform AddLearner transition if logline indicates so +AddLearnerIfLogged(i, j) == + /\ LoglineIsNodeEvent("ChangeConf", i) + /\ Len(logline.event.prop.cc.changes) = 1 + /\ logline.event.prop.cc.changes[1].action = "AddLearner" + /\ logline.event.prop.cc.changes[1].nid = j + /\ AddLearner(i, j) + +\* perform DeleteServer transition if logline indicates so +DeleteServerIfLogged(i, j) == + /\ LoglineIsNodeEvent("ChangeConf", i) + /\ Len(logline.event.prop.cc.changes) = 1 + /\ logline.event.prop.cc.changes[1].action = "RemoveServer" + /\ logline.event.prop.cc.changes[1].nid = j + /\ DeleteServer(i, j) + +ApplySimpleConfChangeIfLogged(i) == + /\ LoglineIsNodeEvent("ApplyConfChange", i) + /\ ApplySimpleConfChange(i) + +ReadyIfLogged(i) == + /\ LoglineIsNodeEvent("Ready", i) + /\ Ready(i) + +RestartIfLogged(i) == + /\ LoglineIsNodeEvent("InitState", i) + /\ Restart(i) + /\ ValidatePostStates(i) + +LoglineIsBecomeFollowerInUpdateTermOrReturnToFollower == + /\ LoglineIsEvent("BecomeFollower") + /\ LET + k == SelectLastInSubSeq(TraceLog, 1, l-1, LAMBDA x: x.event.nid = logline.event.nid) + IN + /\ k > 0 + /\ \/ /\ TraceLog[k].event.name \in ReceiveMessageTraceNames + /\ TraceLog[k].event.state.term < TraceLog[k].event.msg.term + /\ TraceLog[k].event.msg.term = logline.event.state.term + \/ /\ TraceLog[k].event.name = "ReceiveAppendEntriesRequest" + /\ TraceLog[k].event.state.term = TraceLog[k].event.msg.term + /\ TraceLog[k].event.msg.term = logline.event.state.term + /\ TraceLog[k].event.role = Candidate + +StepDownToFollowerIfLogged(i) == + /\ LoglineIsNodeEvent("BecomeFollower", i) + /\ \lnot LoglineIsBecomeFollowerInUpdateTermOrReturnToFollower + /\ StepDownToFollower(i) + /\ ValidatePostStates(i) + +\* skip unused logs +SkipUnusedLogline == + /\ \/ /\ LoglineIsEvent("SendAppendEntriesResponse") + /\ logline.event.msg.from # logline.event.msg.to + \/ /\ LoglineIsEvent("SendRequestVoteResponse") + /\ logline.event.msg.from # logline.event.msg.to + \/ LoglineIsBecomeFollowerInUpdateTermOrReturnToFollower + \/ LoglineIsEvent("ReduceNextIndex") \* shall not be necessary when this is removed from raft + /\ UNCHANGED <> + +TraceNextNonReceiveActions == + /\ \/ /\ LoglineIsEvents({"SendRequestVoteRequest", "SendRequestVoteResponse"}) + /\ \E i,j \in Server : RequestVoteIfLogged(i, j) + \/ /\ LoglineIsEvent("BecomeLeader") + /\ \E i \in Server : BecomeLeaderIfLogged(i) + \/ /\ LoglineIsEvent("Replicate") + /\ \E i \in Server : ClientRequestIfLogged(i, 0) + \/ /\ LoglineIsEvent("Commit") + /\ \E i \in Server : AdvanceCommitIndexIfLogged(i) + \/ /\ LoglineIsEvent("SendAppendEntriesRequest") /\ logline.event.msg.type = "MsgApp" + /\ \E i,j \in Server : \E b,e \in matchIndex[i][j]+1..Len(log[i])+1 : AppendEntriesIfLogged(i, j, <>) + \/ /\ LoglineIsEvent("SendAppendEntriesResponse") + /\ \E i \in Server : AppendEntriesToSelfIfLogged(i) + \/ /\ LoglineIsEvent("SendAppendEntriesRequest") + /\ \E i,j \in Server : HeartbeatIfLogged(i, j) /\ logline.event.msg.type = "MsgHeartbeat" + \/ /\ LoglineIsEvent("SendAppendEntriesRequest") /\ logline.event.msg.type = "MsgSnap" + /\ \E i,j \in Server : \E index \in 1..commitIndex[i] : SendSnapshotIfLogged(i, j, index) + \/ /\ LoglineIsEvent("BecomeCandidate") + /\ \E i \in Server : TimeoutIfLogged(i) + \/ /\ LoglineIsEvent("ChangeConf") + /\ \E i,j \in Server: AddNewServerIfLogged(i, j) + \/ /\ LoglineIsEvent("ChangeConf") + /\ \E i,j \in Server: AddLearnerIfLogged(i, j) + \/ /\ LoglineIsEvent("ChangeConf") + /\ \E i,j \in Server: DeleteServerIfLogged(i, j) + \/ /\ LoglineIsEvent("ApplyConfChange") + /\ \E i \in Server: ApplySimpleConfChangeIfLogged(i) + \/ /\ LoglineIsEvent("Ready") + /\ \E i \in Server: ReadyIfLogged(i) + \/ /\ LoglineIsEvent("InitState") + /\ \E i \in Server: RestartIfLogged(i) + \/ /\ LoglineIsEvent("BecomeFollower") + /\ \E i \in Server: StepDownToFollowerIfLogged(i) + \/ SkipUnusedLogline + /\ StepToNextTrace + +TraceNextReceiveActions == + /\ LoglineIsEvents(ReceiveMessageTraceNames) + /\ \E m \in DOMAIN messages : + /\ ReceiveIfLogged(m) + /\ StepToNextTraceIfMessageIsProcessed(m) + +TraceNext == + \/ /\ l \in BootstrapLogIndices + /\ UNCHANGED <> + /\ StepToNextTrace + \/ /\ l \notin BootstrapLogIndices + /\ \/ TraceNextNonReceiveActions + \/ TraceNextReceiveActions + +TraceSpec == + TraceInit /\ [][TraceNext]_<> + +------------------------------------------------------------------------------------- + +TraceView == + \* A high-level state s can appear multiple times in a system trace. Including the + \* current level in TLC's view ensures that TLC will not stop model checking when s + \* appears the second time in the trace. Put differently, TraceView causes TLC to + \* consider s_i and s_j , where i and j are the positions of s in the trace, + \* to be different states. + <> + +------------------------------------------------------------------------------------- + +\* The property TraceMatched below will be violated if TLC runs with more than a single worker. +ASSUME TLCGet("config").worker = 1 + +TraceMatched == + \* We force TLC to check TraceMatched as a temporal property because TLC checks temporal + \* properties after generating all successor states of the current state, unlike + \* invariants that are checked after generating a successor state. + \* If the queue is empty after generating all successors of the current state, + \* and l is less than the length of the trace, then TLC failed to validate the trace. + \* + [](l <= Len(TraceLog) => [](TLCGet("queue") = 1 \/ l > Len(TraceLog))) + +etcd == INSTANCE etcdraft +etcdSpec == etcd!Init /\ [][etcd!NextDynamic]_etcd!vars + +================================================================================== + + diff --git a/third_party/raft/tla/etcdraft.cfg b/third_party/raft/tla/etcdraft.cfg new file mode 100644 index 0000000000..ae731380fe --- /dev/null +++ b/third_party/raft/tla/etcdraft.cfg @@ -0,0 +1,53 @@ +\* Copyright 2024 The etcd Authors +\* +\* Licensed under the Apache License, Version 2.0 (the "License"); +\* you may not use this file except in compliance with the License. +\* You may obtain a copy of the License at +\* +\* http://www.apache.org/licenses/LICENSE-2.0 +\* +\* Unless required by applicable law or agreed to in writing, software +\* distributed under the License is distributed on an "AS IS" BASIS, +\* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +\* See the License for the specific language governing permissions and +\* limitations under the License. +\* + +CONSTANTS + s1 = 1 + s2 = 2 + s3 = 3 + s4 = 4 + s5 = 5 + + InitServer = {s1, s2, s3} + Server = {s1, s2, s3} + + Nil = 0 + + ValueEntry = "ValueEntry" + ConfigEntry = "ConfigEntry" + + Follower = "Follower" + Candidate = "Candidate" + Leader = "Leader" + RequestVoteRequest = "RequestVoteRequest" + RequestVoteResponse = "RequestVoteResponse" + AppendEntriesRequest = "AppendEntriesRequest" + AppendEntriesResponse = "AppendEntriesResponse" + + +VIEW vars + +INIT Init +NEXT NextAsyncCrash + +\* Raft properties +INVARIANTS + LogInv + MoreThanOneLeaderInv + ElectionSafetyInv + LogMatchingInv + QuorumLogInv + MoreUpToDateCorrectInv + LeaderCompletenessInv diff --git a/third_party/raft/tla/etcdraft.tla b/third_party/raft/tla/etcdraft.tla new file mode 100644 index 0000000000..7828474506 --- /dev/null +++ b/third_party/raft/tla/etcdraft.tla @@ -0,0 +1,926 @@ +--------------------------------- MODULE etcdraft --------------------------------- +\* Copyright 2024 The etcd Authors +\* +\* Licensed under the Apache License, Version 2.0 (the "License"); +\* you may not use this file except in compliance with the License. +\* You may obtain a copy of the License at +\* +\* http://www.apache.org/licenses/LICENSE-2.0 +\* +\* Unless required by applicable law or agreed to in writing, software +\* distributed under the License is distributed on an "AS IS" BASIS, +\* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +\* See the License for the specific language governing permissions and +\* limitations under the License. +\* +\* +\* This is the formal specification for the Raft consensus algorithm. +\* +\* Copyright 2014 Diego Ongaro, 2015 Brandon Amos and Huanchen Zhang, +\* 2016 Daniel Ricketts, 2021 George Pîrlea and Darius Foo. +\* +\* This work is licensed under the Creative Commons Attribution-4.0 +\* International License https://creativecommons.org/licenses/by/4.0/ + +EXTENDS Naturals, Integers, Bags, FiniteSets, Sequences, SequencesExt, FiniteSetsExt, BagsExt, TLC + +\* The initial and global set of server IDs. +CONSTANTS InitServer, Server + +\* Log metadata to distinguish values from configuration changes. +CONSTANT ValueEntry, ConfigEntry + +\* Server states. +CONSTANTS + \* @type: Str; + Follower, + \* @type: Str; + Candidate, + \* @type: Str; + Leader + +\* A reserved value. +CONSTANTS + \* @type: Int; + Nil + +\* Message types: +CONSTANTS + \* @type: Str; + RequestVoteRequest, + \* @type: Str; + RequestVoteResponse, + \* @type: Str; + AppendEntriesRequest, + \* @type: Str; + AppendEntriesResponse + + +---- +\* Global variables + +\* A bag of records representing requests and responses sent from one server +\* to another. We differentiate between the message types to support Apalache. +VARIABLE + \* @typeAlias: ENTRY = [term: Int, value: Int]; + \* @typeAlias: LOGT = Seq(ENTRY); + \* @typeAlias: RVREQT = [mtype: Str, mterm: Int, mlastLogTerm: Int, mlastLogIndex: Int, msource: Int, mdest: Int]; + \* @typeAlias: RVRESPT = [mtype: Str, mterm: Int, mvoteGranted: Bool, msource: Int, mdest: Int ]; + \* @typeAlias: AEREQT = [mtype: Str, mterm: Int, mprevLogIndex: Int, mprevLogTerm: Int, mentries: LOGT, mcommitIndex: Int, msource: Int, mdest: Int ]; + \* @typeAlias: AERESPT = [mtype: Str, mterm: Int, msuccess: Bool, mmatchIndex: Int, msource: Int, mdest: Int ]; + \* @typeAlias: MSG = [ wrapped: Bool, mtype: Str, mterm: Int, msource: Int, mdest: Int, RVReq: RVREQT, RVResp: RVRESPT, AEReq: AEREQT, AEResp: AERESPT ]; + \* @type: MSG -> Int; + messages +VARIABLE + pendingMessages +messageVars == <> + +---- +\* The following variables are all per server (functions with domain Server). + +\* The server's term number. +VARIABLE + \* @type: Int -> Int; + currentTerm +\* The server's state (Follower, Candidate, or Leader). +VARIABLE + \* @type: Int -> Str; + state +\* The candidate the server voted for in its current term, or +\* Nil if it hasn't voted for any. +VARIABLE + \* @type: Int -> Int; + votedFor +serverVars == <> + +\* A Sequence of log entries. The index into this sequence is the index of the +\* log entry. Unfortunately, the Sequence module defines Head(s) as the entry +\* with index 1, so be careful not to use that! +VARIABLE + \* @type: Int -> [ entries: LOGT, len: Int ]; + log +\* The index of the latest entry in the log the state machine may apply. +VARIABLE + \* @type: Int -> Int; + commitIndex +logVars == <> + +\* The following variables are used only on candidates: +\* The set of servers from which the candidate has received a RequestVote +\* response in its currentTerm. +VARIABLE + \* @type: Int -> Set(Int); + votesResponded +\* The set of servers from which the candidate has received a vote in its +\* currentTerm. +VARIABLE + \* @type: Int -> Set(Int); + votesGranted +\* @type: Seq(Int -> Set(Int)); +candidateVars == <> + +\* The following variables are used only on leaders: +\* The latest entry that each follower has acknowledged is the same as the +\* leader's. This is used to calculate commitIndex on the leader. +VARIABLE + \* @type: Int -> (Int -> Int); + matchIndex +VARIABLE + pendingConfChangeIndex +leaderVars == <> + +\* @type: Int -> [jointConfig: Seq(Set(int)), learners: Set(int)] +VARIABLE + config +VARIABLE + reconfigCount + +configVars == <> + +VARIABLE + durableState + +\* End of per server variables. +---- + +\* All variables; used for stuttering (asserting state hasn't changed). +vars == <> + + +---- +\* Helpers + +\* The set of all quorums. This just calculates simple majorities, but the only +\* important property is that every quorum overlaps with every other. +Quorum(c) == {i \in SUBSET(c) : Cardinality(i) * 2 > Cardinality(c)} + +\* The term of the last entry in a log, or 0 if the log is empty. +\* @type: LOGT => Int; +LastTerm(xlog) == IF xlog = <<>> THEN 0 ELSE xlog[Len(xlog)].term + +\* Helper for Send and Reply. Given a message m and bag of messages, return a +\* new bag of messages with one more m in it. +\* @type: (MSG, MSG -> Int) => MSG -> Int; +WithMessage(m, msgs) == msgs (+) SetToBag({m}) + +\* Helper for Discard and Reply. Given a message m and bag of messages, return +\* a new bag of messages with one less m in it. +\* @type: (MSG, MSG -> Int) => MSG -> Int; +WithoutMessage(m, msgs) == msgs (-) SetToBag({m}) + +\* Add a message to the bag of pendingMessages. +SendDirect(m) == + pendingMessages' = WithMessage(m, pendingMessages) + +\* All pending messages sent from node i +PendingMessages(i) == + FoldBag(LAMBDA x, y: IF y.msource = i THEN BagAdd(x,y) ELSE x, EmptyBag, pendingMessages) + +\* Remove all messages in pendingMessages that were sent from node i +ClearPendingMessages(i) == + pendingMessages (-) PendingMessages(i) + +\* Move all messages which was sent from node i in pendingMessages to messages +SendPendingMessages(i) == + LET msgs == PendingMessages(i) + IN /\ messages' = msgs (+) messages + /\ pendingMessages' = pendingMessages (-) msgs + +\* Remove a message from the bag of messages. Used when a server is done +DiscardDirect(m) == + messages' = WithoutMessage(m, messages) + +\* Combination of Send and Discard +ReplyDirect(response, request) == + /\ pendingMessages' = WithMessage(response, pendingMessages) + /\ messages' = WithoutMessage(request, messages) + +\* Default: change when needed + Send(m) == SendDirect(m) + Reply(response, request) == ReplyDirect(response, request) + Discard(m) == DiscardDirect(m) + +MaxOrZero(s) == IF s = {} THEN 0 ELSE Max(s) + +GetJointConfig(i) == + config[i].jointConfig + +GetConfig(i) == + GetJointConfig(i)[1] + +GetOutgoingConfig(i) == + GetJointConfig(i)[2] + +IsJointConfig(i) == + /\ GetJointConfig(i)[2] # {} + +GetLearners(i) == + config[i].learners + +\* Apply conf change log entry to configuration +ApplyConfigUpdate(i, k) == + [config EXCEPT ![i]= [jointConfig |-> << log[i][k].value.newconf, {} >>, learners |-> log[i][k].value.learners]] + +CommitTo(i, c) == + commitIndex' = [commitIndex EXCEPT ![i] = Max({@, c})] + +CurrentLeaders == {i \in Server : state[i] = Leader} + +PersistState(i) == + durableState' = [durableState EXCEPT ![i] = [ + currentTerm |-> currentTerm[i], + votedFor |-> votedFor[i], + log |-> Len(log[i]), + commitIndex |-> commitIndex[i], + config |-> config[i] + ]] + +---- +\* Define initial values for all variables +InitMessageVars == /\ messages = EmptyBag + /\ pendingMessages = EmptyBag +InitServerVars == /\ currentTerm = [i \in Server |-> 0] + /\ state = [i \in Server |-> Follower] + /\ votedFor = [i \in Server |-> Nil] +InitCandidateVars == /\ votesResponded = [i \in Server |-> {}] + /\ votesGranted = [i \in Server |-> {}] +InitLeaderVars == /\ matchIndex = [i \in Server |-> [j \in Server |-> 0]] + /\ pendingConfChangeIndex = [i \in Server |-> 0] +InitLogVars == /\ log = [i \in Server |-> <<>>] + /\ commitIndex = [i \in Server |-> 0] +InitConfigVars == /\ config = [i \in Server |-> [ jointConfig |-> <>, learners |-> {}]] + /\ reconfigCount = 0 +InitDurableState == + durableState = [ i \in Server |-> [ + currentTerm |-> currentTerm[i], + votedFor |-> votedFor[i], + log |-> Len(log[i]), + commitIndex |-> commitIndex[i], + config |-> config[i] + ]] + +Init == /\ InitMessageVars + /\ InitServerVars + /\ InitCandidateVars + /\ InitLeaderVars + /\ InitLogVars + /\ InitConfigVars + /\ InitDurableState + +---- +\* Define state transitions + +\* Server i restarts from stable storage. +\* It loses everything but its currentTerm, commitIndex, votedFor, log, and config in durable state. +\* @type: Int => Bool; +Restart(i) == + /\ state' = [state EXCEPT ![i] = Follower] + /\ votesResponded' = [votesResponded EXCEPT ![i] = {}] + /\ votesGranted' = [votesGranted EXCEPT ![i] = {}] + /\ matchIndex' = [matchIndex EXCEPT ![i] = [j \in Server |-> 0]] + /\ pendingConfChangeIndex' = [pendingConfChangeIndex EXCEPT ![i] = 0] + /\ pendingMessages' = ClearPendingMessages(i) + /\ currentTerm' = [currentTerm EXCEPT ![i] = durableState[i].currentTerm] + /\ commitIndex' = [commitIndex EXCEPT ![i] = durableState[i].commitIndex] + /\ votedFor' = [votedFor EXCEPT ![i] = durableState[i].votedFor] + /\ log' = [log EXCEPT ![i] = SubSeq(@, 1, durableState[i].log)] + /\ config' = [config EXCEPT ![i] = durableState[i].config] + /\ UNCHANGED <> + +\* Server i times out and starts a new election. +\* @type: Int => Bool; +Timeout(i) == /\ state[i] \in {Follower, Candidate} + /\ i \in GetConfig(i) + /\ state' = [state EXCEPT ![i] = Candidate] + /\ currentTerm' = [currentTerm EXCEPT ![i] = currentTerm[i] + 1] + /\ votedFor' = [votedFor EXCEPT ![i] = i] + /\ votesResponded' = [votesResponded EXCEPT ![i] = {}] + /\ votesGranted' = [votesGranted EXCEPT ![i] = {}] + /\ UNCHANGED <> + +\* Candidate i sends j a RequestVote request. +\* @type: (Int, Int) => Bool; +RequestVote(i, j) == + /\ state[i] = Candidate + /\ j \in ((GetConfig(i) \union GetLearners(i)) \ votesResponded[i]) + /\ IF i # j + THEN Send([mtype |-> RequestVoteRequest, + mterm |-> currentTerm[i], + mlastLogTerm |-> LastTerm(log[i]), + mlastLogIndex |-> Len(log[i]), + msource |-> i, + mdest |-> j]) + ELSE Send([mtype |-> RequestVoteResponse, + mterm |-> currentTerm[i], + mvoteGranted |-> TRUE, + msource |-> i, + mdest |-> i]) + /\ UNCHANGED <> + +\* Leader i sends j an AppendEntries request containing entries in [b,e) range. +\* N.B. range is right open +\* @type: (Int, Int, <>, Int) => Bool; +AppendEntriesInRangeToPeer(subtype, i, j, range) == + /\ i /= j + /\ range[1] <= range[2] + /\ state[i] = Leader + /\ j \in GetConfig(i) \union GetLearners(i) + /\ LET + prevLogIndex == range[1] - 1 + \* The following upper bound on prevLogIndex is unnecessary + \* but makes verification substantially simpler. + prevLogTerm == IF prevLogIndex > 0 /\ prevLogIndex <= Len(log[i]) THEN + log[i][prevLogIndex].term + ELSE + 0 + \* Send the entries + lastEntry == Min({Len(log[i]), range[2]-1}) + entries == SubSeq(log[i], range[1], lastEntry) + commit == IF subtype = "heartbeat" THEN Min({commitIndex[i], matchIndex[i][j]}) ELSE Min({commitIndex[i], lastEntry}) + IN /\ Send( [mtype |-> AppendEntriesRequest, + msubtype |-> subtype, + mterm |-> currentTerm[i], + mprevLogIndex |-> prevLogIndex, + mprevLogTerm |-> prevLogTerm, + mentries |-> entries, + mcommitIndex |-> commit, + msource |-> i, + mdest |-> j]) + /\ UNCHANGED <> + +\* etcd leader sends MsgAppResp to itself immediately after appending log entry +AppendEntriesToSelf(i) == + /\ state[i] = Leader + /\ Send([mtype |-> AppendEntriesResponse, + msubtype |-> "app", + mterm |-> currentTerm[i], + msuccess |-> TRUE, + mmatchIndex |-> Len(log[i]), + msource |-> i, + mdest |-> i]) + /\ UNCHANGED <> + +AppendEntries(i, j, range) == + AppendEntriesInRangeToPeer("app", i, j, range) + +Heartbeat(i, j) == + \* heartbeat is equivalent to an append-entry request with 0 entry index 1 + AppendEntriesInRangeToPeer("heartbeat", i, j, <<1,1>>) + +SendSnapshot(i, j, index) == + AppendEntriesInRangeToPeer("snapshot", i, j, <<1,index+1>>) + +\* Candidate i transitions to leader. +\* @type: Int => Bool; +BecomeLeader(i) == + /\ state[i] = Candidate + /\ votesGranted[i] \in Quorum(GetConfig(i)) + /\ state' = [state EXCEPT ![i] = Leader] + /\ matchIndex' = [matchIndex EXCEPT ![i] = + [j \in Server |-> IF j = i THEN Len(log[i]) ELSE 0]] + /\ UNCHANGED <> + +Replicate(i, v, t) == + /\ t \in {ValueEntry, ConfigEntry} + /\ state[i] = Leader + /\ LET entry == [term |-> currentTerm[i], + type |-> t, + value |-> v] + newLog == Append(log[i], entry) + IN /\ log' = [log EXCEPT ![i] = newLog] + +\* Leader i receives a client request to add v to the log. +\* @type: (Int, Int) => Bool; +ClientRequest(i, v) == + /\ Replicate(i, [val |-> v], ValueEntry) + /\ UNCHANGED <> + +\* Leader i advances its commitIndex. +\* This is done as a separate step from handling AppendEntries responses, +\* in part to minimize atomic regions, and in part so that leaders of +\* single-server clusters are able to mark entries committed. +\* @type: Int => Bool; +AdvanceCommitIndex(i) == + /\ state[i] = Leader + /\ LET \* The set of servers that agree up through index. + Agree(index) == {k \in GetConfig(i) : matchIndex[i][k] >= index} + logSize == Len(log[i]) + \* logSize == MaxLogLength + \* The maximum indexes for which a quorum agrees + agreeIndexes == {index \in 1..logSize : + Agree(index) \in Quorum(GetConfig(i))} + \* New value for commitIndex'[i] + newCommitIndex == + IF /\ agreeIndexes /= {} + /\ log[i][Max(agreeIndexes)].term = currentTerm[i] + THEN + Max(agreeIndexes) + ELSE + commitIndex[i] + IN + /\ CommitTo(i, newCommitIndex) + /\ UNCHANGED <> + + +\* Leader i adds a new server j or promote learner j +AddNewServer(i, j) == + /\ state[i] = Leader + /\ j \notin GetConfig(i) + /\ ~IsJointConfig(i) + /\ IF pendingConfChangeIndex[i] = 0 THEN + /\ Replicate(i, [newconf |-> GetConfig(i) \union {j}, learners |-> GetLearners(i)], ConfigEntry) + /\ pendingConfChangeIndex' = [pendingConfChangeIndex EXCEPT ![i]=Len(log'[i])] + ELSE + /\ Replicate(i, <<>>, ValueEntry) + /\ UNCHANGED <> + /\ UNCHANGED <> + +\* Leader i adds a leaner j to the cluster. +AddLearner(i, j) == + /\ state[i] = Leader + /\ j \notin GetConfig(i) \union GetLearners(i) + /\ ~IsJointConfig(i) + /\ IF pendingConfChangeIndex[i] = 0 THEN + /\ Replicate(i, [newconf |-> GetConfig(i), learners |-> GetLearners(i) \union {j}], ConfigEntry) + /\ pendingConfChangeIndex' = [pendingConfChangeIndex EXCEPT ![i]=Len(log'[i])] + ELSE + /\ Replicate(i, <<>>, ValueEntry) + /\ UNCHANGED <> + /\ UNCHANGED <> + +\* Leader i removes a server j (possibly itself) from the cluster. +DeleteServer(i, j) == + /\ state[i] = Leader + /\ j \in GetConfig(i) \union GetLearners(i) + /\ ~IsJointConfig(i) + /\ IF pendingConfChangeIndex[i] = 0 THEN + /\ Replicate(i, [newconf |-> GetConfig(i) \ {j}, learners |-> GetLearners(i) \ {j}], ConfigEntry) + /\ pendingConfChangeIndex' = [pendingConfChangeIndex EXCEPT ![i]=Len(log'[i])] + ELSE + /\ Replicate(i, <<>>, ValueEntry) + /\ UNCHANGED <> + /\ UNCHANGED <> + +ApplySimpleConfChange(i) == + /\ ~IsJointConfig(i) + /\ LET k == SelectLastInSubSeq(log[i], 1, commitIndex[i], LAMBDA x: x.type = ConfigEntry) + IN + /\ k > 0 + /\ k <= commitIndex[i] + /\ config' = ApplyConfigUpdate(i, k) + /\ IF state[i] = Leader /\ pendingConfChangeIndex[i] >= k THEN + /\ reconfigCount' = reconfigCount + 1 + /\ pendingConfChangeIndex' = [pendingConfChangeIndex EXCEPT ![i] = 0] + ELSE UNCHANGED <> + /\ UNCHANGED <> + +Ready(i) == + /\ PersistState(i) + /\ SendPendingMessages(i) + /\ UNCHANGED <> + +BecomeFollowerOfTerm(i, t) == + /\ currentTerm' = [currentTerm EXCEPT ![i] = t] + /\ state' = [state EXCEPT ![i] = Follower] + /\ IF currentTerm[i] # t THEN + votedFor' = [votedFor EXCEPT ![i] = Nil] + ELSE + UNCHANGED <> + +StepDownToFollower(i) == + /\ state[i] \in {Leader, Candidate} + /\ BecomeFollowerOfTerm(i, currentTerm[i]) + /\ UNCHANGED <> + +---- +\* Message handlers +\* i = recipient, j = sender, m = message + +\* Server i receives a RequestVote request from server j with +\* m.mterm <= currentTerm[i]. +\* @type: (Int, Int, RVREQT) => Bool; +HandleRequestVoteRequest(i, j, m) == + LET logOk == \/ m.mlastLogTerm > LastTerm(log[i]) + \/ /\ m.mlastLogTerm = LastTerm(log[i]) + /\ m.mlastLogIndex >= Len(log[i]) + grant == /\ m.mterm = currentTerm[i] + /\ logOk + /\ votedFor[i] \in {Nil, j} + IN /\ m.mterm <= currentTerm[i] + /\ \/ grant /\ votedFor' = [votedFor EXCEPT ![i] = j] + \/ ~grant /\ UNCHANGED votedFor + /\ Reply([mtype |-> RequestVoteResponse, + mterm |-> currentTerm[i], + mvoteGranted |-> grant, + msource |-> i, + mdest |-> j], + m) + /\ UNCHANGED <> + +\* Server i receives a RequestVote response from server j with +\* m.mterm = currentTerm[i]. +\* @type: (Int, Int, RVRESPT) => Bool; +HandleRequestVoteResponse(i, j, m) == + \* This tallies votes even when the current state is not Candidate, but + \* they won't be looked at, so it doesn't matter. + /\ m.mterm = currentTerm[i] + /\ votesResponded' = [votesResponded EXCEPT ![i] = + votesResponded[i] \cup {j}] + /\ \/ /\ m.mvoteGranted + /\ votesGranted' = [votesGranted EXCEPT ![i] = + votesGranted[i] \cup {j}] + \/ /\ ~m.mvoteGranted + /\ UNCHANGED <> + /\ Discard(m) + /\ UNCHANGED <> + +\* @type: (Int, Int, AEREQT, Bool) => Bool; +RejectAppendEntriesRequest(i, j, m, logOk) == + /\ \/ m.mterm < currentTerm[i] + \/ /\ m.mterm = currentTerm[i] + /\ state[i] = Follower + /\ \lnot logOk + /\ Reply([mtype |-> AppendEntriesResponse, + msubtype |-> "app", + mterm |-> currentTerm[i], + msuccess |-> FALSE, + mmatchIndex |-> 0, + msource |-> i, + mdest |-> j], + m) + /\ UNCHANGED <> + +\* @type: (Int, MSG) => Bool; +ReturnToFollowerState(i, m) == + /\ m.mterm = currentTerm[i] + /\ state[i] = Candidate + /\ state' = [state EXCEPT ![i] = Follower] + /\ UNCHANGED <> + +HasNoConflict(i, index, ents) == + /\ index <= Len(log[i]) + 1 + /\ \A k \in 1..Len(ents): index + k - 1 <= Len(log[i]) => log[i][index+k-1].term = ents[k].term + +\* @type: (Int, Int, Int, AEREQT) => Bool; +AppendEntriesAlreadyDone(i, j, index, m) == + /\ \/ index <= commitIndex[i] + \/ /\ index > commitIndex[i] + /\ \/ m.mentries = << >> + \/ /\ m.mentries /= << >> + /\ m.mprevLogIndex + Len(m.mentries) <= Len(log[i]) + /\ HasNoConflict(i, index, m.mentries) + /\ IF index <= commitIndex[i] THEN + IF m.msubtype = "heartbeat" THEN CommitTo(i, m.mcommitIndex) ELSE UNCHANGED commitIndex + ELSE + CommitTo(i, Min({m.mcommitIndex, m.mprevLogIndex+Len(m.mentries)})) + /\ Reply([ mtype |-> AppendEntriesResponse, + msubtype |-> m.msubtype, + mterm |-> currentTerm[i], + msuccess |-> TRUE, + mmatchIndex |-> IF m.msubtype = "heartbeat" \/ index > commitIndex[i] THEN m.mprevLogIndex+Len(m.mentries) ELSE commitIndex[i], + msource |-> i, + mdest |-> j], + m) + /\ UNCHANGED <> + +\* @type: (Int, Int, AEREQT) => Bool; +ConflictAppendEntriesRequest(i, index, m) == + /\ m.mentries /= << >> + /\ index > commitIndex[i] + /\ ~HasNoConflict(i, index, m.mentries) + /\ log' = [log EXCEPT ![i] = SubSeq(@, 1, Len(@) - 1)] + /\ UNCHANGED <> + +\* @type: (Int, AEREQT) => Bool; +NoConflictAppendEntriesRequest(i, index, m) == + /\ m.mentries /= << >> + /\ index > commitIndex[i] + /\ HasNoConflict(i, index, m.mentries) + /\ log' = [log EXCEPT ![i] = @ \o SubSeq(m.mentries, Len(@)-index+2, Len(m.mentries))] + /\ UNCHANGED <> + +\* @type: (Int, Int, Bool, AEREQT) => Bool; +AcceptAppendEntriesRequest(i, j, logOk, m) == + \* accept request + /\ m.mterm = currentTerm[i] + /\ state[i] = Follower + /\ logOk + /\ LET index == m.mprevLogIndex + 1 + IN \/ AppendEntriesAlreadyDone(i, j, index, m) + \/ ConflictAppendEntriesRequest(i, index, m) + \/ NoConflictAppendEntriesRequest(i, index, m) + +\* Server i receives an AppendEntries request from server j with +\* m.mterm <= currentTerm[i]. This just handles m.entries of length 0 or 1, but +\* implementations could safely accept more by treating them the same as +\* multiple independent requests of 1 entry. +\* @type: (Int, Int, AEREQT) => Bool; +HandleAppendEntriesRequest(i, j, m) == + LET logOk == \/ m.mprevLogIndex = 0 + \/ /\ m.mprevLogIndex > 0 + /\ m.mprevLogIndex <= Len(log[i]) + /\ m.mprevLogTerm = log[i][m.mprevLogIndex].term + IN + /\ m.mterm <= currentTerm[i] + /\ \/ RejectAppendEntriesRequest(i, j, m, logOk) + \/ ReturnToFollowerState(i, m) + \/ AcceptAppendEntriesRequest(i, j, logOk, m) + /\ UNCHANGED <> + +\* Server i receives an AppendEntries response from server j with +\* m.mterm = currentTerm[i]. +\* @type: (Int, Int, AERESPT) => Bool; +HandleAppendEntriesResponse(i, j, m) == + /\ m.mterm = currentTerm[i] + /\ \/ /\ m.msuccess \* successful + /\ matchIndex' = [matchIndex EXCEPT ![i][j] = Max({@, m.mmatchIndex})] + /\ UNCHANGED <> + \/ /\ \lnot m.msuccess \* not successful + /\ UNCHANGED <> + /\ Discard(m) + /\ UNCHANGED <> + +\* Any RPC with a newer term causes the recipient to advance its term first. +\* @type: (Int, Int, MSG) => Bool; +UpdateTerm(i, j, m) == + /\ m.mterm > currentTerm[i] + /\ BecomeFollowerOfTerm(i, m.mterm) + \* messages is unchanged so m can be processed further. + /\ UNCHANGED <> + +\* Responses with stale terms are ignored. +\* @type: (Int, Int, MSG) => Bool; +DropStaleResponse(i, j, m) == + /\ m.mterm < currentTerm[i] + /\ Discard(m) + /\ UNCHANGED <> + +\* Receive a message. +ReceiveDirect(m) == + LET i == m.mdest + j == m.msource + IN \* Any RPC with a newer term causes the recipient to advance + \* its term first. Responses with stale terms are ignored. + \/ UpdateTerm(i, j, m) + \/ /\ m.mtype = RequestVoteRequest + /\ HandleRequestVoteRequest(i, j, m) + \/ /\ m.mtype = RequestVoteResponse + /\ \/ DropStaleResponse(i, j, m) + \/ HandleRequestVoteResponse(i, j, m) + \/ /\ m.mtype = AppendEntriesRequest + /\ HandleAppendEntriesRequest(i, j, m) + \/ /\ m.mtype = AppendEntriesResponse + /\ \/ DropStaleResponse(i, j, m) + \/ HandleAppendEntriesResponse(i, j, m) + +Receive(m) == ReceiveDirect(m) + +NextRequestVoteRequest == \E m \in DOMAIN messages : m.mtype = RequestVoteRequest /\ Receive(m) +NextRequestVoteResponse == \E m \in DOMAIN messages : m.mtype = RequestVoteResponse /\ Receive(m) +NextAppendEntriesRequest == \E m \in DOMAIN messages : m.mtype = AppendEntriesRequest /\ Receive(m) +NextAppendEntriesResponse == \E m \in DOMAIN messages : m.mtype = AppendEntriesResponse /\ Receive(m) + +\* End of message handlers. +---- +\* Network state transitions + +\* The network duplicates a message +\* @type: MSG => Bool; +DuplicateMessage(m) == + /\ m \in DOMAIN messages + /\ messages' = WithMessage(m, messages) + /\ UNCHANGED <> + +\* The network drops a message +\* @type: MSG => Bool; +DropMessage(m) == + \* Do not drop loopback messages + \* /\ m.msource /= m.mdest + /\ Discard(m) + /\ UNCHANGED <> + +---- + +\* Defines how the variables may transition. +NextAsync == + \/ \E i,j \in Server : RequestVote(i, j) + \/ \E i \in Server : BecomeLeader(i) + \/ \E i \in Server: ClientRequest(i, 0) + \/ \E i \in Server : AdvanceCommitIndex(i) + \/ \E i,j \in Server : \E b,e \in matchIndex[i][j]+1..Len(log[i])+1 : AppendEntries(i, j, <>) + \/ \E i \in Server : AppendEntriesToSelf(i) + \/ \E i,j \in Server : Heartbeat(i, j) + \/ \E i,j \in Server : \E index \in 1..commitIndex[i] : SendSnapshot(i, j, index) + \/ \E m \in DOMAIN messages : Receive(m) + \/ \E i \in Server : Timeout(i) + \/ \E i \in Server : Ready(i) + \/ \E i \in Server : StepDownToFollower(i) + +NextCrash == \E i \in Server : Restart(i) + +NextAsyncCrash == + \/ NextAsync + \/ NextCrash + +NextUnreliable == + \* Only duplicate once + \/ \E m \in DOMAIN messages : + /\ messages[m] = 1 + /\ DuplicateMessage(m) + \* Only drop if it makes a difference + \/ \E m \in DOMAIN messages : + /\ messages[m] = 1 + /\ DropMessage(m) + +\* Most pessimistic network model +Next == \/ NextAsync + \/ NextCrash + \/ NextUnreliable + +\* Membership changes +NextDynamic == + \/ Next + \/ \E i, j \in Server : AddNewServer(i, j) + \/ \E i, j \in Server : AddLearner(i, j) + \/ \E i, j \in Server : DeleteServer(i, j) + \/ \E i \in Server : ApplySimpleConfChange(i) + +\* The specification must start with the initial state and transition according +\* to Next. +Spec == Init /\ [][Next]_vars + +(***************************************************************************) +(* The main safety properties are below *) +(***************************************************************************) +---- + +ASSUME DistinctRoles == /\ Leader /= Candidate + /\ Candidate /= Follower + /\ Follower /= Leader + +ASSUME DistinctMessageTypes == /\ RequestVoteRequest /= AppendEntriesRequest + /\ RequestVoteRequest /= RequestVoteResponse + /\ RequestVoteRequest /= AppendEntriesResponse + /\ AppendEntriesRequest /= RequestVoteResponse + /\ AppendEntriesRequest /= AppendEntriesResponse + /\ RequestVoteResponse /= AppendEntriesResponse + +---- +\* Correctness invariants + +\* The prefix of the log of server i that has been committed +Committed(i) == SubSeq(log[i],1,commitIndex[i]) + +\* The current term of any server is at least the term +\* of any message sent by that server +\* @type: MSG => Bool; +MessageTermsLtCurrentTerm(m) == + m.mterm <= currentTerm[m.msource] + +\* Committed log entries should never conflict between servers +LogInv == + \A i, j \in Server : + \/ IsPrefix(Committed(i),Committed(j)) + \/ IsPrefix(Committed(j),Committed(i)) + +\* Note that LogInv checks for safety violations across space +\* This is a key safety invariant and should always be checked +THEOREM Spec => []LogInv + +\* There should not be more than one leader per term at the same time +\* Note that this does not rule out multiple leaders in the same term at different times +MoreThanOneLeaderInv == + \A i,j \in Server : + (/\ currentTerm[i] = currentTerm[j] + /\ state[i] = Leader + /\ state[j] = Leader) + => i = j + +\* A leader always has the greatest index for its current term +ElectionSafetyInv == + \A i \in Server : + state[i] = Leader => + \A j \in Server : + MaxOrZero({n \in DOMAIN log[i] : log[i][n].term = currentTerm[i]}) >= + MaxOrZero({n \in DOMAIN log[j] : log[j][n].term = currentTerm[i]}) + +\* Every (index, term) pair determines a log prefix +LogMatchingInv == + \A i, j \in Server : + \A n \in (1..Len(log[i])) \cap (1..Len(log[j])) : + log[i][n].term = log[j][n].term => + SubSeq(log[i],1,n) = SubSeq(log[j],1,n) + +\* When two candidates competes in a campaign, if a follower voted to +\* a candidate that did not win in the end, the follower's votedFor will +\* not reset nor change to the winner (the other candidate) because its +\* term remains same. This will violate this invariant. +\* +\* This invariant can be false because a server's votedFor is not reset +\* for messages with same term. Refer to the case below. +\* 1. This is a 3 node cluster with nodes A, B, and C. Let's assume they are all followers with same term 1 and log at beginning. +\* 2. Now B and C starts compaign and both become candidates of term 2. +\* 3. B requests vote to A and A grant it. Now A is a term 2 follower whose votedFor is B. +\* 4. A's response to B is lost. +\* 5. C requests vote to B and B grant it. Now B is a term 2 follower whose votedFor is C. +\* 6. C becomes leader of term 2. +\* 7. C replicates logs to A but not B. +\* 8. A's votedFor is not changed because the incoming messages has same term (see UpdateTerm and ReturnToFollowerState) +\* 9. Now the commited log in A will exceed B's log. The invariant is violated. +\* VotesGrantedInv == +\* \A i, j \in Server : +\* \* if i has voted for j +\* votedFor[i] = j => +\* IsPrefix(Committed(i), log[j]) + +\* All committed entries are contained in the log +\* of at least one server in every quorum +QuorumLogInv == + \A i \in Server : + \A S \in Quorum(GetConfig(i)) : + \E j \in S : + IsPrefix(Committed(i), log[j]) + +\* The "up-to-date" check performed by servers +\* before issuing a vote implies that i receives +\* a vote from j only if i has all of j's committed +\* entries +MoreUpToDateCorrectInv == + \A i, j \in Server : + (\/ LastTerm(log[i]) > LastTerm(log[j]) + \/ /\ LastTerm(log[i]) = LastTerm(log[j]) + /\ Len(log[i]) >= Len(log[j])) => + IsPrefix(Committed(j), log[i]) + +\* If a log entry is committed in a given term, then that +\* entry will be present in the logs of the leaders +\* for all higher-numbered terms +\* See: https://github.com/uwplse/verdi-raft/blob/master/raft/LeaderCompletenessInterface.v +LeaderCompletenessInv == + \A i \in Server : + LET committed == Committed(i) IN + \A idx \in 1..Len(committed) : + LET entry == log[i][idx] IN + \* if the entry is committed + \A l \in CurrentLeaders : + \* all leaders with higher-number terms + currentTerm[l] > entry.term => + \* have the entry at the same log position + log[l][idx] = entry + +\* Any entry committed by leader shall be persisted already +CommittedIsDurableInv == + \A i \in Server : + state[i] = Leader => commitIndex[i] <= durableState[i].log + +----- + + +=============================================================================== + + + +\* Changelog: +\* +\* 2023-11-23: +\* - Replace configuration actions by those in etcd implementation. +\* - Refactor spec structure to decouple core spec and model checker spec for +\* better readness and future update +\* - Remove unused actions and properties, e.g. wrapper +\* +\* 2021-04-??: +\* - Abandoned Apalache due to slowness and went back to TLC. There are remains +\* of the Apalache-specific annotations and message wrapping/unwrapping, but +\* those are not actually used. The annotations are no longer up-to-date. +\* +\* 2021-04-09: +\* - Added type annotations for Apalache symbolic model checker. As part of +\* this change, the message format was changed to a tagged union. +\* +\* 2016-09-09: +\* - Daniel Ricketts added the major safety invariants and proved them in +\* TLAPS. +\* +\* 2015-05-10: +\* - Add cluster membership changes as described in Section 4 of +\* Diego Ongaro. Consensus: Bridging theory and practice. +\* PhD thesis, Stanford University, 2014. +\* This introduces: InitServer, ValueEntry, ConfigEntry, CatchupRequest, +\* CatchupResponse, CheckOldConfig, GetMaxConfigIndex, +\* GetConfig (parameterized), AddNewServer, DeleteServer, +\* HandleCatchupRequest, HandleCatchupResponse, +\* HandleCheckOldConfig +\* +\* +\* 2014-12-02: +\* - Fix AppendEntries to only send one entry at a time, as originally +\* intended. Since SubSeq is inclusive, the upper bound of the range should +\* have been nextIndex, not nextIndex + 1. Thanks to Igor Kovalenko for +\* reporting the issue. +\* - Change matchIndex' to matchIndex (without the apostrophe) in +\* AdvanceCommitIndex. This apostrophe was not intentional and perhaps +\* confusing, though it makes no practical difference (matchIndex' equals +\* matchIndex). Thanks to Hugues Evrard for reporting the issue. +\* +\* 2014-07-06: +\* - Version from PhD dissertation diff --git a/third_party/raft/tla/example.ndjson b/third_party/raft/tla/example.ndjson new file mode 100644 index 0000000000..3fbd7e2c5f --- /dev/null +++ b/third_party/raft/tla/example.ndjson @@ -0,0 +1,4888 @@ +{"level":"debug","ts":1712329729.6024537,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"InitState","nid":"1","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329729.6026523,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"1","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"prop":{"cc":{}}}} +{"level":"debug","ts":1712329729.6027615,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"1","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329729.6028612,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"1","state":{"term":1,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329729.6029124,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"1","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[[],[]],"prop":{"cc":{"newconf":["1"]}}}} +{"level":"debug","ts":1712329729.6029756,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"1","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1"],[]],"prop":{"cc":{"newconf":["1","2"]}}}} +{"level":"debug","ts":1712329729.603029,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"1","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2"],[]],"prop":{"cc":{"newconf":["1","2","3"]}}}} +{"level":"debug","ts":1712329729.6030803,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"1","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3"],[]],"prop":{"cc":{"newconf":["1","2","3","4"]}}}} +{"level":"debug","ts":1712329729.6031656,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"1","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6033354,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"InitState","nid":"2","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329729.603347,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"prop":{"cc":{}}}} +{"level":"debug","ts":1712329729.6034048,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"2","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329729.6034226,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"2","state":{"term":1,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329729.6034346,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.6034617,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[[],[]],"prop":{"cc":{"newconf":["1"]}}}} +{"level":"debug","ts":1712329729.6035578,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1"],[]],"prop":{"cc":{"newconf":["1","2"]}}}} +{"level":"debug","ts":1712329729.6036708,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2"],[]],"prop":{"cc":{"newconf":["1","2","3"]}}}} +{"level":"debug","ts":1712329729.6037266,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3"],[]],"prop":{"cc":{"newconf":["1","2","3","4"]}}}} +{"level":"debug","ts":1712329729.6037762,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6038544,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"InitState","nid":"3","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329729.6038635,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"prop":{"cc":{}}}} +{"level":"debug","ts":1712329729.6039116,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"3","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329729.6039605,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"3","state":{"term":1,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329729.604047,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[[],[]],"prop":{"cc":{"newconf":["1"]}}}} +{"level":"debug","ts":1712329729.6041117,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1"],[]],"prop":{"cc":{"newconf":["1","2"]}}}} +{"level":"debug","ts":1712329729.604136,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.6041672,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2"],[]],"prop":{"cc":{"newconf":["1","2","3"]}}}} +{"level":"debug","ts":1712329729.6042209,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3"],[]],"prop":{"cc":{"newconf":["1","2","3","4"]}}}} +{"level":"debug","ts":1712329729.6042695,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6043477,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"InitState","nid":"4","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329729.6043582,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"4","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"prop":{"cc":{}}}} +{"level":"debug","ts":1712329729.6044211,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"4","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329729.6045005,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"1","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.604551,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"4","state":{"term":1,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329729.604564,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"1","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.604643,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"4","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[[],[]],"prop":{"cc":{"newconf":["1"]}}}} +{"level":"debug","ts":1712329729.6046493,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"1","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6046739,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.6047618,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"4","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1"],[]],"prop":{"cc":{"newconf":["1","2"]}}}} +{"level":"debug","ts":1712329729.6048722,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"4","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2"],[]],"prop":{"cc":{"newconf":["1","2","3"]}}}} +{"level":"debug","ts":1712329729.6051967,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6052165,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"1","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6053429,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6054454,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"4","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3"],[]],"prop":{"cc":{"newconf":["1","2","3","4"]}}}} +{"level":"debug","ts":1712329729.6054475,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"1","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6054597,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6055725,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6056206,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"4","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6057637,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"InitState","nid":"5","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329729.6057792,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"5","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"prop":{"cc":{}}}} +{"level":"debug","ts":1712329729.6058264,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6058288,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6058598,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.6059034,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6060448,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"5","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329729.6061022,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"5","state":{"term":1,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329729.6061196,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"5","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[[],[]],"prop":{"cc":{"newconf":["1"]}}}} +{"level":"debug","ts":1712329729.606306,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"5","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1"],[]],"prop":{"cc":{"newconf":["1","2"]}}}} +{"level":"debug","ts":1712329729.6063218,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6063397,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"5","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2"],[]],"prop":{"cc":{"newconf":["1","2","3"]}}}} +{"level":"debug","ts":1712329729.6064248,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6065383,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6066353,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"5","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3"],[]],"prop":{"cc":{"newconf":["1","2","3","4"]}}}} +{"level":"debug","ts":1712329729.6067195,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"5","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.606968,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.6069727,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"4","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6070786,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"4","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6071916,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"4","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.607335,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"4","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6074142,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"4","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6082566,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"5","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6084957,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"5","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6085846,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"5","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.6088376,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"5","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.608966,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"5","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329729.804121,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeCandidate","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateCandidate","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.804277,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendRequestVoteRequest","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateCandidate","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVote","term":2,"from":"2","to":"1","entries":0,"logTerm":1,"index":5,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8042963,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendRequestVoteResponse","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateCandidate","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVoteResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8043416,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendRequestVoteRequest","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateCandidate","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVote","term":2,"from":"2","to":"3","entries":0,"logTerm":1,"index":5,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8043556,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendRequestVoteRequest","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateCandidate","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVote","term":2,"from":"2","to":"4","entries":0,"logTerm":1,"index":5,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8043637,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendRequestVoteRequest","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateCandidate","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVote","term":2,"from":"2","to":"5","entries":0,"logTerm":1,"index":5,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8044193,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateCandidate","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8075442,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveRequestVoteResponse","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateCandidate","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVoteResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.810977,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveRequestVoteRequest","nid":"5","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVote","term":2,"from":"2","to":"5","entries":0,"logTerm":1,"index":5,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8110313,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveRequestVoteRequest","nid":"4","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVote","term":2,"from":"2","to":"4","entries":0,"logTerm":1,"index":5,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.811076,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"5","state":{"term":2,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.811199,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"4","state":{"term":2,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.81128,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendRequestVoteResponse","nid":"5","state":{"term":2,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVoteResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8113008,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.811333,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendRequestVoteResponse","nid":"4","state":{"term":2,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVoteResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8113415,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.814362,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveRequestVoteRequest","nid":"1","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVote","term":2,"from":"2","to":"1","entries":0,"logTerm":1,"index":5,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.814434,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"1","state":{"term":2,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8144512,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendRequestVoteResponse","nid":"1","state":{"term":2,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVoteResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8144534,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveRequestVoteResponse","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateCandidate","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVoteResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8144653,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8155646,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveRequestVoteRequest","nid":"3","state":{"term":1,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVote","term":2,"from":"2","to":"3","entries":0,"logTerm":1,"index":5,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8155932,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveRequestVoteResponse","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateCandidate","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVoteResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8156712,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"3","state":{"term":2,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8156888,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendRequestVoteResponse","nid":"3","state":{"term":2,"vote":"0","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVoteResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8156981,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.815723,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeLeader","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8157299,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8157358,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":6,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.815825,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":1,"index":5,"commit":5,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":5,"next":6}}} +{"level":"debug","ts":1712329729.815854,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":1,"index":5,"commit":5,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":5,"next":6}}} +{"level":"debug","ts":1712329729.815863,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":1,"index":5,"commit":5,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":5,"next":6}}} +{"level":"debug","ts":1712329729.8158698,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":1,"index":5,"commit":5,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":5,"next":6}}} +{"level":"debug","ts":1712329729.8160052,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8193276,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":6,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8193524,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8193848,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":1,"index":5,"commit":5,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8194098,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":6,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8195066,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8205228,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveRequestVoteResponse","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVoteResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8216653,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveRequestVoteResponse","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgVoteResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.823958,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8239806,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8239908,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8239973,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8240094,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.826303,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":1,"index":5,"commit":5,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8263268,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":6,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8263838,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8308992,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":6,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.830923,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8309326,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":6,"commit":5,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":6,"next":7}}} +{"level":"debug","ts":1712329729.8309789,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.833453,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":1,"index":5,"commit":5,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8334763,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":6,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8335602,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8346121,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":5},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":6,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8346322,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.834641,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":6,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":6,"next":7}}} +{"level":"debug","ts":1712329729.8346498,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":6,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":6,"next":7}}} +{"level":"debug","ts":1712329729.8346555,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8346598,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.83468,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8347638,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8358216,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8358383,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8372364,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.837261,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8372672,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8372865,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.837341,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":7,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":7,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8373535,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":7,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":6,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":6,"next":7}}} +{"level":"debug","ts":1712329729.837363,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":7,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":6,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":6,"next":7}}} +{"level":"debug","ts":1712329729.8374078,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":7,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8384821,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8385074,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8385572,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.839633,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8396595,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8396695,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.839673,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8396857,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":7,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":7,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8396993,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":7,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8397279,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8409016,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8422523,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":6,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8422756,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":6,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.842286,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8423038,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8423517,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":6,"commit":5,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8423796,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":6,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8423903,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8436613,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":7,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8436913,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":7,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":7,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":7,"next":8}}} +{"level":"debug","ts":1712329729.8437028,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":7,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8437102,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":7,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8437166,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":8,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":8,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8437238,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":8,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":7,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":7,"next":8}}} +{"level":"debug","ts":1712329729.8437314,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":8,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":7,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":7,"next":8}}} +{"level":"debug","ts":1712329729.8437333,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":6,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8437386,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":8,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8437448,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":9,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":9,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8437586,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":6,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8437736,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8437898,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":9,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":8,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":7,"next":8}}} +{"level":"debug","ts":1712329729.8437989,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":9,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":8,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":7,"next":8}}} +{"level":"debug","ts":1712329729.8450475,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":9,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8450778,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":10,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.845091,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":9,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":7,"next":8}}} +{"level":"debug","ts":1712329729.845102,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":9,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":7,"next":8}}} +{"level":"debug","ts":1712329729.8451118,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8451185,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8451188,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8451712,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8451805,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.845193,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8465173,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8477798,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":8,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8477988,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8478158,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8490832,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":6,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8491027,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":7,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":7,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8491118,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":7,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8491492,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":9,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8491693,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8492184,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":6,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8492303,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":7,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":7,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8492444,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":7,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8492494,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8503845,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":10,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8504064,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8504162,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8504229,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8504288,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":11,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8504372,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":10,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":10,"next":11}}} +{"level":"debug","ts":1712329729.850446,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":10,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":10,"next":11}}} +{"level":"debug","ts":1712329729.8515842,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":7,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8516066,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":6,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8516152,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8516316,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":5,"logTerm":2,"index":6,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":10,"next":11}}} +{"level":"debug","ts":1712329729.8516898,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8517056,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8517125,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":12,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.85172,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":11,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":10,"next":11}}} +{"level":"debug","ts":1712329729.8517282,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":11,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":10,"next":11}}} +{"level":"debug","ts":1712329729.8517349,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":11,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":10,"next":11}}} +{"level":"debug","ts":1712329729.8542674,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":7,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":7,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8542902,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":8,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":8,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.854303,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":8,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8543205,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":11,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8543518,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8543613,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8555193,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8555412,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":12,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":11,"next":12}}} +{"level":"debug","ts":1712329729.8555825,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":7,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":7,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8556042,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.855605,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":8,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":8,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8556159,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":8,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8568983,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":12,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8569198,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8569286,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8569362,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":13,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":13,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8569484,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":13,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":12,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":12,"next":13}}} +{"level":"debug","ts":1712329729.8569586,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":13,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":12,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":12,"next":13}}} +{"level":"debug","ts":1712329729.8569663,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":13,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":12,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":12,"next":13}}} +{"level":"debug","ts":1712329729.8569753,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":13,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8569963,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8581927,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":13,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8582163,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":13,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":0,"logTerm":2,"index":13,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":12,"next":13}}} +{"level":"debug","ts":1712329729.8594635,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":8,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":8,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8594873,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":9,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":9,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8594973,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":9,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.859506,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":13,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":13,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.859517,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":8,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8595235,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":13,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8595314,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":13,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.85954,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":8,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8595486,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":8,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8595839,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":14,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":14,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8596005,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":14,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":13,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":13,"next":14}}} +{"level":"debug","ts":1712329729.8596103,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":14,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":13,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":13,"next":14}}} +{"level":"debug","ts":1712329729.859618,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":14,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":13,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":13,"next":14}}} +{"level":"debug","ts":1712329729.8596268,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":14,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.859632,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":15,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8596406,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":14,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":13,"next":14}}} +{"level":"debug","ts":1712329729.8596523,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":14,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":13,"next":14}}} +{"level":"debug","ts":1712329729.8596592,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":14,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":13,"next":14}}} +{"level":"debug","ts":1712329729.8607712,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":6,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8608568,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":8,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8620672,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":6,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.862094,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":9,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":7,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8621035,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":9,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":7,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8621113,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":9,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.863351,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.86338,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":7,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8634071,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8634167,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8634248,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":8,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":10,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8634467,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":8,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":10,"commit":0,"vote":"0","reject":true,"rejectHint":8}}} +{"level":"debug","ts":1712329729.8634555,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":8,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8646705,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":9,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":9,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.864714,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":10,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8647535,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":6,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8647568,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.866037,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":8,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":8,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8660629,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":9,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":9,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.866075,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":9,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8660944,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":11,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8661053,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":11,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.8661132,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.866133,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":14,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8661404,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8674521,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":15,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.867479,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.867493,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8675036,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8675137,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.867522,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8675387,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8675485,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":15,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8675566,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":16,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":16,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8675675,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":16,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":15,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":15,"next":16}}} +{"level":"debug","ts":1712329729.8675795,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":16,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":15,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":15,"next":16}}} +{"level":"debug","ts":1712329729.8675904,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":16,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":15,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":15,"next":16}}} +{"level":"debug","ts":1712329729.867602,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":16,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8676116,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":17,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":17,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.867622,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":17,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":16,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":15,"next":16}}} +{"level":"debug","ts":1712329729.867635,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":17,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":16,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":15,"next":16}}} +{"level":"debug","ts":1712329729.8676465,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":17,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":16,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":15,"next":16}}} +{"level":"debug","ts":1712329729.8676627,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":17,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8676715,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":18,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8676858,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":17,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":15,"next":16}}} +{"level":"debug","ts":1712329729.8676965,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":17,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":15,"next":16}}} +{"level":"debug","ts":1712329729.867708,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":17,"commit":6,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":15,"next":16}}} +{"level":"debug","ts":1712329729.8677306,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":9,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":9,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.867741,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":10,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8677516,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8677616,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8677776,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8677926,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8690317,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8690786,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8703184,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":6},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":8,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8703423,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.870351,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":0,"logTerm":2,"index":18,"commit":7,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":15,"next":16}}} +{"level":"debug","ts":1712329729.8703601,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":18,"commit":7,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":15,"next":16}}} +{"level":"debug","ts":1712329729.870368,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":18,"commit":7,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":15,"next":16}}} +{"level":"debug","ts":1712329729.8704343,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8717284,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":16,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8717518,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8717747,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":5,"logTerm":2,"index":6,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.871778,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.871785,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":11,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8719194,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8737142,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8737378,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8737912,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8738048,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":17,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8738234,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8750587,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":18,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8750796,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8750904,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8751113,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":12,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8751147,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":12,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8751237,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":12,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.8751428,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8751523,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":10,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8751543,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":12,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.8751616,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8751843,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":11,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8751998,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.876302,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":11,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8763247,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":12,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8763342,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":7,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8763502,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8775113,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":9,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8775313,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8775408,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":18,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.877547,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":19,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":19,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8775566,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":19,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":18,"commit":7,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":18,"next":19}}} +{"level":"debug","ts":1712329729.8775656,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":19,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":18,"commit":7,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":18,"next":19}}} +{"level":"debug","ts":1712329729.877574,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":19,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":18,"commit":7,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":18,"next":19}}} +{"level":"debug","ts":1712329729.8788848,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":12,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.879026,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":12,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.879052,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":13,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8790612,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8790865,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":13,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.8791041,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8791127,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":11,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8791149,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":7},"role":"StateLeader","log":19,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":8,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8791246,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":19,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8791277,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":5,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":1,"index":5,"commit":5,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.879144,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":11,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.879145,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":6,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8791537,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8791587,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":19,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":0,"logTerm":2,"index":19,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":18,"next":19}}} +{"level":"debug","ts":1712329729.879172,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":19,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":19,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":18,"next":19}}} +{"level":"debug","ts":1712329729.8791893,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":19,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":19,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":18,"next":19}}} +{"level":"debug","ts":1712329729.8792017,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":19,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8803384,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":12,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8803673,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":12,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.8803813,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8804252,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":14,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8804405,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":14,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.880443,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":19,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":7,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8804562,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8804712,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":19,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8804836,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":20,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":20,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8804953,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":20,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":19,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":18,"next":19}}} +{"level":"debug","ts":1712329729.8805134,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":20,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":19,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":18,"next":19}}} +{"level":"debug","ts":1712329729.8805294,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":20,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":19,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":18,"next":19}}} +{"level":"debug","ts":1712329729.8817146,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":20,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":19,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.881738,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":20,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8817816,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":20,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8829036,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":20,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":10,"commit":0,"vote":"0","reject":true,"rejectHint":8}}} +{"level":"debug","ts":1712329729.8829257,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":20,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":12,"logTerm":2,"index":8,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":19,"next":20}}} +{"level":"debug","ts":1712329729.8840663,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":20,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":10,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8840842,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":20,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8840916,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":20,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8840976,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":21,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":21,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8841105,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":21,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":20,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":19,"next":20}}} +{"level":"debug","ts":1712329729.884119,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":21,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":20,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":19,"next":20}}} +{"level":"debug","ts":1712329729.885252,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":21,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8852828,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":21,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":21,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":19,"next":20}}} +{"level":"debug","ts":1712329729.8852978,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":21,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8853068,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":22,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":22,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8853168,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":22,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":21,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":19,"next":20}}} +{"level":"debug","ts":1712329729.8853257,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":22,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":21,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":19,"next":20}}} +{"level":"debug","ts":1712329729.886511,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":17,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.886529,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":17,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.8865428,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8865523,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":22,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":20,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8865592,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":22,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.886567,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":22,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8865728,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":23,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":23,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8865795,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":23,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":22,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.886585,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":0,"logTerm":2,"index":13,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8865898,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":23,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":22,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.8866088,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":13,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.88662,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8866422,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":14,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.886675,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":14,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.8867254,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.887744,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":15,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8877633,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":15,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.8877742,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":13,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8877807,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":13,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.8877885,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":23,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8877969,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":13,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8877988,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":23,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":15,"logTerm":2,"index":8,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.8878078,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":23,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8878157,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":13,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.887837,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":24,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8878465,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":23,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.8878548,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":23,"commit":8,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.8890862,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8891118,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8891225,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":8},"role":"StateLeader","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":10,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8891277,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":16,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8891354,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":16,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.889136,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":16,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.889147,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8891518,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":16,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.8891585,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8891692,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":24,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8891761,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":0,"logTerm":2,"index":24,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.8891854,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":14,"logTerm":2,"index":10,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.889193,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":24,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.8892014,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":24,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8892083,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":25,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8892145,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":24,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.8892214,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":24,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.8892298,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":24,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.8903058,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":17,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8903253,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":17,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.8903375,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8903458,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8903549,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":20,"logTerm":1,"index":5,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.8903646,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.890367,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8903718,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.890378,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8903968,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8904064,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.891692,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8917212,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":14,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8917484,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":14,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.8917577,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8917716,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":11,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.891809,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":15,"logTerm":2,"index":10,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.8918245,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.891831,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":26,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":26,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8918383,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":26,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":25,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.8918455,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":26,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":25,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.8918533,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":26,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.891865,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":27,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8918717,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":26,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.8918793,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":26,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":20,"next":21}}} +{"level":"debug","ts":1712329729.8932278,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8932545,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8932643,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.893322,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":21,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8933465,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8944004,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":22,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8944228,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8944404,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":18,"commit":7,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8944554,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":18,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.8944645,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8956134,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":17,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8956356,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":17,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.8956447,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8956676,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":23,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.895696,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.895703,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":15,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8957138,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":15,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.8957736,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8971643,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":15,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8971918,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":15,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.897201,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.897229,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":24,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8972375,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8984673,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":25,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.89849,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8984995,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.898509,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":16,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8985162,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8985171,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":16,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.898528,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8985376,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8985472,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8985543,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8985825,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8985946,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.8997824,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":18,"commit":7,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8998094,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":18,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.8998203,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.899831,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":9,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.899846,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":19,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8998537,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":0,"logTerm":2,"index":18,"commit":7,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.8998613,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":18,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.8998647,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":19,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.8998766,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.89988,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.901199,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9012215,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":0,"logTerm":2,"index":27,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":25,"next":26}}} +{"level":"debug","ts":1712329729.9012418,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":19,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9012666,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":19,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9012792,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9025075,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":19,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9025242,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":0,"logTerm":2,"index":19,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9025323,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":19,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.902543,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":19,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.902546,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":12,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9025586,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9025602,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9025934,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9038093,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":26,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9038353,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9038372,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":18,"commit":7,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9038649,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":18,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.90388,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9051864,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":18,"commit":7,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.90521,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":18,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.905225,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9052374,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":27,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.905245,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9052532,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9052596,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.905265,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.905271,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9052725,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":18,"commit":7,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9052923,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":18,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9053025,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9065485,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":12,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.90658,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":27,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9065878,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":28,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":28,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9065967,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":28,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":27,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":27,"next":28}}} +{"level":"debug","ts":1712329729.9066048,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":28,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":27,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":27,"next":28}}} +{"level":"debug","ts":1712329729.9066107,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":19,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9066136,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":28,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.906634,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":19,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.906646,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9078395,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":20,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9078603,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":20,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9078684,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.907881,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":28,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":11,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.907896,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":19,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9079237,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":19,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.907968,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9091322,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":28,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":13,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9091632,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":28,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":16,"logTerm":2,"index":12,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":27,"next":28}}} +{"level":"debug","ts":1712329729.90918,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":28,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9091904,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9092014,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":28,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":27,"next":28}}} +{"level":"debug","ts":1712329729.9105551,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":12,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.9105828,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":19,"logTerm":2,"index":10,"commit":10,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":27,"next":28}}} +{"level":"debug","ts":1712329729.9118795,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":22,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.911906,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":22,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.911916,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9119265,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":10,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":12,"logTerm":2,"index":8,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9119334,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":28,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.911943,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":29,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.911958,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.911967,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":8},"role":"StateFollower","log":20,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":20,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9119673,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.911974,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9119785,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":8},"role":"StateFollower","log":20,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9119859,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":0,"logTerm":2,"index":24,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9120443,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":24,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.91206,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.912083,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":29,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9134505,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":24,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9134684,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":21,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.913483,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":24,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9135032,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":21,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9135034,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":10},"role":"StateLeader","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":11,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9135222,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9135556,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":29,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9135797,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":18,"logTerm":2,"index":11,"commit":11,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":28,"next":29}}} +{"level":"debug","ts":1712329729.9146812,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":14,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9147205,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":21,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9147437,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":21,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9160724,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":20,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9160926,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":20,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9161015,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9161367,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":23,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9161458,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":8},"role":"StateFollower","log":20,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":24,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9161577,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":23,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.916164,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":8},"role":"StateFollower","log":20,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":24,"commit":0,"vote":"0","reject":true,"rejectHint":20}}} +{"level":"debug","ts":1712329729.9161675,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9161813,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":8},"role":"StateFollower","log":20,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9161816,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9161906,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":29,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9174008,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":8},"role":"StateFollower","log":20,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":15,"logTerm":2,"index":8,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9174209,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":8},"role":"StateFollower","log":23,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":23,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9174488,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":22,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9174523,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":8},"role":"StateFollower","log":23,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.917457,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":22,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9174888,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":11,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.9175453,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":29,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9175563,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":29,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.917568,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":30,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":30,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.917576,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":30,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":29,"commit":11,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":29,"next":30}}} +{"level":"debug","ts":1712329729.9186504,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":8},"role":"StateFollower","log":23,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9186687,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":24,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9186723,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":30,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":12,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.9186792,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":24,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9186857,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9187207,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":23,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.918747,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":23,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.918794,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.919851,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":30,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":12,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9198816,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":30,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":19,"logTerm":2,"index":11,"commit":11,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":29,"next":30}}} +{"level":"debug","ts":1712329729.9198925,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":30,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.919899,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":31,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":31,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9199388,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.921106,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9211545,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":31,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":17,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9224162,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":21,"commit":8,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.922434,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":21,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9224448,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.922455,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":31,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":6,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.922477,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":31,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9224882,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":31,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":25,"logTerm":2,"index":6,"commit":11,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":29,"next":30}}} +{"level":"debug","ts":1712329729.9225047,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":31,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9225118,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":32,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":32,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9225187,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":32,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":31,"commit":11,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":29,"next":30}}} +{"level":"debug","ts":1712329729.9235837,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":32,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":17,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.9236014,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":32,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.923627,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.923647,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9237196,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9247918,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":32,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":13,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9260027,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":32,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":14,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.9273932,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":32,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":30,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.927422,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":32,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9274335,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":24,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9274418,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":24,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.927452,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9274611,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":8},"role":"StateFollower","log":23,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":14,"logTerm":2,"index":10,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.927469,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":24,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9274766,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9288194,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":32,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":31,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.928843,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":32,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9304004,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":32,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":32,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9304228,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":32,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.930432,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":32,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9304411,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9304438,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":33,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":33,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9304526,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":33,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":32,"commit":11,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":32,"next":33}}} +{"level":"debug","ts":1712329729.9304647,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9304774,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9304817,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":33,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9304888,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":34,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":34,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.930496,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":34,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":33,"commit":11,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":32,"next":33}}} +{"level":"debug","ts":1712329729.9304967,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":25,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9305513,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":34,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9305632,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":34,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9305787,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":25,"commit":0,"vote":"0","reject":true,"rejectHint":24}}} +{"level":"debug","ts":1712329729.9306061,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":35,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":35,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9306185,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":35,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":34,"commit":11,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":32,"next":33}}} +{"level":"debug","ts":1712329729.9306307,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":35,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9306378,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":35,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.930643,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":35,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9306498,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":35,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9307172,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":35,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9307497,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":36,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":36,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.93077,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":36,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":35,"commit":11,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":32,"next":33}}} +{"level":"debug","ts":1712329729.9307919,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ChangeConf","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":36,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"changes":[{"nid":"1","action":"RemoveServer"}]}}}} +{"level":"debug","ts":1712329729.930818,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":37,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":37,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9308913,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":37,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":36,"commit":11,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":32,"next":33}}} +{"level":"debug","ts":1712329729.9309165,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":37,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.930926,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":38,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9309337,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":37,"commit":11,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":32,"next":33}}} +{"level":"debug","ts":1712329729.931964,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9319854,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9319942,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9320037,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":15,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9333992,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":13,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9347565,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":26,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.934787,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":26,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9348245,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":33,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9348512,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.935962,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":25,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9359753,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":34,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9359837,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":25,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9359932,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9360337,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9372754,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9373157,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":27,"logTerm":2,"index":11,"commit":11,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":34,"next":35}}} +{"level":"debug","ts":1712329729.9373384,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9384534,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":16,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.939802,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":35,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.939826,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9410675,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":36,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.941094,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.942232,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":37,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9422565,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9435167,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":38,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9435413,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9449873,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":16,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9461808,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":13,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.9462106,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.94622,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":39,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9462287,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":38,"commit":11,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":38,"next":39}}} +{"level":"debug","ts":1712329729.9462397,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9462473,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9462535,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9462597,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9462655,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.947381,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":19,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9485335,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9485612,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":29,"logTerm":2,"index":10,"commit":11,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":38,"next":39}}} +{"level":"debug","ts":1712329729.9485896,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":26,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9486358,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":26,"commit":0,"vote":"0","reject":true,"rejectHint":24}}} +{"level":"debug","ts":1712329729.9486387,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":11,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":15,"logTerm":2,"index":10,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9486475,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9486644,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":25,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.948675,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9498277,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":16,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.9509983,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":6},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9510221,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9510348,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9510424,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":15,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9522786,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":39,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9523003,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9523098,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9536645,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":14,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9548428,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":18,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9561734,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9561977,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":27,"logTerm":2,"index":12,"commit":11,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":39,"next":40}}} +{"level":"debug","ts":1712329729.9562364,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9562602,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9563046,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9574916,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":15,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.958603,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":17,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.960019,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9600444,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":39,"commit":11,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":39,"next":40}}} +{"level":"debug","ts":1712329729.9613605,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":18,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.962753,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":27,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9627767,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":27,"commit":0,"vote":"0","reject":true,"rejectHint":24}}} +{"level":"debug","ts":1712329729.962787,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9628015,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":1,"logTerm":2,"index":27,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9628253,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":27,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9628363,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9628394,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":18,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.9640346,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":19,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.964059,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9652534,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":19,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9665353,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":18,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9680743,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":0,"logTerm":2,"index":27,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9680977,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":27,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9681077,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9681234,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":19,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9681587,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9681673,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9682055,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9682167,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9694827,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":18,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.9706435,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":18,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9707057,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9741852,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":19,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9755552,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":20,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9778748,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9779003,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9779377,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9789996,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":24,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":19,"logTerm":2,"index":10,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9790452,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9790635,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":29,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9812374,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":28,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.981266,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9813023,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":29,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9840262,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":19,"commit":0,"vote":"0","reject":true,"rejectHint":10}}} +{"level":"debug","ts":1712329729.9851496,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":20,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9851933,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9852042,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9852111,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9852228,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.9852448,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9863443,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":11},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":20,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329729.986374,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9863858,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":19,"logTerm":2,"index":20,"commit":12,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":39,"next":40}}} +{"level":"debug","ts":1712329729.9863968,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":39,"commit":12,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":39,"next":40}}} +{"level":"debug","ts":1712329729.986418,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.9891076,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":22,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329729.9903457,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":21,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329729.9917958,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":24,"commit":0,"vote":"0","reject":true,"rejectHint":20}}} +{"level":"debug","ts":1712329729.9919214,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":19,"logTerm":2,"index":20,"commit":12,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":39,"next":40}}} +{"level":"debug","ts":1712329729.99204,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329729.994418,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":24,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329730.0039198,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0039508,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.00396,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0039666,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0039768,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0064855,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":12,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":16,"logTerm":2,"index":12,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.006789,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":28,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":28,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.006833,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":28,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0087452,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":5},"role":"StateFollower","log":6,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":20,"logTerm":1,"index":5,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0087843,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0088038,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":25,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0088203,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":39,"commit":12,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":39,"next":40}}} +{"level":"debug","ts":1712329730.0088358,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0088427,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":40,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":40,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0088503,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":40,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":39,"commit":12,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":39,"next":40}}} +{"level":"debug","ts":1712329730.0088596,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0088654,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":40,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.009982,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":40,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":22,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329730.012094,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":40,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":40,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.012129,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":40,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0121946,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":40,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0122116,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":41,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.012223,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":40,"commit":12,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":40,"next":41}}} +{"level":"debug","ts":1712329730.0122328,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0145335,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":41,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0145562,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0156803,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":23,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.015704,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0157154,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":18,"logTerm":2,"index":23,"commit":12,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":41,"next":42}}} +{"level":"debug","ts":1712329730.0157323,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0221472,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":21,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329730.0240393,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0241435,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0243242,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.02453,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0245762,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0277073,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":24,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329730.0297756,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":19,"logTerm":2,"index":11,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0298438,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":41,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0298629,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":30,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":30,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0298827,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":42,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0298963,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":30,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0299027,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":41,"commit":12,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":41,"next":42}}} +{"level":"debug","ts":1712329730.0299199,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":41,"commit":12,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":41,"next":42}}} +{"level":"debug","ts":1712329730.029933,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0323539,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":42,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0323815,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.03586,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":23,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329730.037301,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":23,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329730.0438726,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0439022,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0439117,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0439181,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.043925,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0493195,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":30,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":18,"logTerm":2,"index":11,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0493512,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":30,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.049377,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":30,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0528576,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":21,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329730.0529053,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":30,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":29,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0529358,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":30,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":30,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0529943,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":30,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0540273,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":24,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329730.0647194,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0647616,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":42,"commit":12,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":42,"next":43}}} +{"level":"debug","ts":1712329730.064775,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0647871,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0647936,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0647995,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0648499,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0706935,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":24,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.070725,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0718458,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0718718,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":30,"logTerm":2,"index":12,"commit":12,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":42,"next":43}}} +{"level":"debug","ts":1712329730.0719216,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0751898,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":2,"index":24,"commit":0,"vote":"0","reject":true,"rejectHint":11}}} +{"level":"debug","ts":1712329730.0764136,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":25,"logTerm":2,"index":6,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0769558,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":10,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.07699,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0805485,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":26,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329730.0840778,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0841064,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0841153,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0841217,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0841289,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0875635,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":25,"commit":0,"vote":"0","reject":true,"rejectHint":24}}} +{"level":"debug","ts":1712329730.087606,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":18,"logTerm":2,"index":24,"commit":12,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":42,"next":43}}} +{"level":"debug","ts":1712329730.0876715,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.0902855,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.090399,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0904484,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":29,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.095986,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":25,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329730.09725,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.0972886,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":31,"logTerm":2,"index":11,"commit":12,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":42,"next":43}}} +{"level":"debug","ts":1712329730.0973518,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.100629,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":31,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.100665,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":31,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.1006875,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.104139,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1041803,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1041949,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.104202,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1043787,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1076548,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":30,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1076834,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":30,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1077082,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":30,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.110478,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":28,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1105115,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":28,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1105208,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":30,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":27,"logTerm":2,"index":11,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.110535,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":28,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1105635,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":26,"commit":0,"vote":"0","reject":true,"rejectHint":24}}} +{"level":"debug","ts":1712329730.110574,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":38,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1106484,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1127887,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":33,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1130319,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":33,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.1130471,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1153145,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1153803,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1153803,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1154034,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1154113,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.115442,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":29,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1227453,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":12},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":25,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1227822,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1228034,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":17,"logTerm":2,"index":25,"commit":24,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":42,"next":43}}} +{"level":"debug","ts":1712329730.1228132,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":42,"commit":24,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":42,"next":43}}} +{"level":"debug","ts":1712329730.1228714,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1239724,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.124015,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.124036,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.124051,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1240718,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1244974,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1245403,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1245506,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1314118,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":35,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1314595,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":35,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.1314707,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1336691,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":28,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1337159,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":28,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1337261,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":28,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1439888,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1440187,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1440332,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.14404,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1440508,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.145101,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":29,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":29,"logTerm":2,"index":10,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1451278,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":39,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.145152,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1478364,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1478772,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.148028,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":27,"commit":0,"vote":"0","reject":true,"rejectHint":24}}} +{"level":"debug","ts":1712329730.148224,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.14948,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1495202,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":30,"logTerm":2,"index":12,"commit":24,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":42,"next":43}}} +{"level":"debug","ts":1712329730.1507897,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":28,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":27,"logTerm":2,"index":12,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.150792,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1508212,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":39,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1508439,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.150859,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":42,"commit":24,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":42,"next":43}}} +{"level":"debug","ts":1712329730.1508682,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.151916,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1519415,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1519647,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.152025,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":27,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329730.153087,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":34,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1531126,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":34,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.1531267,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1543255,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.154349,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1543596,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1560493,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.156088,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1560993,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":42,"commit":24,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":42,"next":43}}} +{"level":"debug","ts":1712329730.1561086,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":42,"commit":24,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":42,"next":43}}} +{"level":"debug","ts":1712329730.156117,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.157135,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":36,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1571786,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":36,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.1571894,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1592624,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1592863,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1593082,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1593473,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.159372,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1605248,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":2,"index":27,"commit":0,"vote":"0","reject":true,"rejectHint":12}}} +{"level":"debug","ts":1712329730.1605704,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.160594,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1606233,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1606467,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":39,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1606724,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":39,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.160682,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1639836,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1640131,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1640213,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1640275,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1640766,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.166941,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":24},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1669724,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.166993,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":14,"logTerm":2,"index":29,"commit":25,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.1670084,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":25,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.167017,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":43,"commit":25,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.1670773,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1682365,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1682684,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":25,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.1683555,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1715295,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.171587,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":32,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1716118,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":32,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.1716344,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1726766,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":37,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1727126,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":37,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.1727417,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1756213,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.175648,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1756694,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.178336,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1783695,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1783922,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1841495,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1841855,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.184195,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.184206,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1842523,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1852224,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":19,"logTerm":2,"index":20,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1852624,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":39,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1852722,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1876135,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":10,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.187648,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1876712,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1887293,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1887708,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1887815,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1918602,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":19,"logTerm":2,"index":20,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1918945,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":39,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1919153,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1923106,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":39,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1923563,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":39,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.192366,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1933784,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":38,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.193399,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":38,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.1934114,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1950119,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1950445,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1950717,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1955826,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.195623,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1956325,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.1990595,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.19909,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.1991405,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2039332,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2039628,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2039719,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2039785,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.203991,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2066345,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":25},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":28,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.206663,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":28},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2066734,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":28},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":15,"logTerm":2,"index":28,"commit":28,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.2066832,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":28},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":28,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.2066917,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":28},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":28,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.2066984,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":28},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":43,"commit":28,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.206746,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":28},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.208751,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2088034,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2088258,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2114697,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2114944,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2115178,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.212602,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":39,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2126343,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":39,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.2126613,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2152765,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2153141,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.21533,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2154903,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":18,"logTerm":2,"index":23,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2155192,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":41,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":41,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.215552,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":41,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2183,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":41,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2183256,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":41,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2183354,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":41,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2226987,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2227354,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2227578,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2239163,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":28},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":28,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.223944,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":28},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":28,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.223953,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":28},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2239587,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":28},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2239704,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":28},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.22465,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2246666,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2246864,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.226191,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":28},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":30,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2262146,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2262247,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":0,"logTerm":2,"index":43,"commit":29,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.2262342,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":29,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.2262425,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":29,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.2262487,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":43,"commit":29,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.2262933,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2279365,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":41,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":41,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.227974,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":42,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2279959,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2286,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2286224,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2286515,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.22965,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":39,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2297559,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":39,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.2297702,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.232648,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":25,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2326798,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2337961,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2338414,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.233842,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2338521,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2338657,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2338877,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.235013,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.236087,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.236108,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2361279,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2392118,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":30,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2392557,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":40,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.239286,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":40,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.2393067,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2439322,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":28,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2439604,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2439685,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2439747,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2439814,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2442174,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":41,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.24423,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":41,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.2442482,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2476213,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2476528,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2476735,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.255439,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2554731,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.255537,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.258383,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2584164,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2584383,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2628267,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":42,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2628598,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":42,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2628813,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.263888,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":28,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.26392,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2639296,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2639358,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2639532,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2640462,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.264054,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.264071,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2665014,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":39,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":30,"logTerm":2,"index":12,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2665293,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":42,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2665508,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2671733,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2672355,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2672603,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2702668,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":18,"logTerm":2,"index":24,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2703462,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":42,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.270359,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2714605,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.271488,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2715092,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2777264,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2777832,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2778115,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2778187,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2778614,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.277877,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2805965,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2806346,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":29,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.2806427,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2806582,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2806673,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2806947,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2826135,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":11},"role":"StateFollower","log":38,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":31,"logTerm":2,"index":11,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2826443,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":42,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2826555,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.284027,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":28,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2840564,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2840652,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.284072,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2840788,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2848191,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2848608,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2848723,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2849288,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":11,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.284956,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.284977,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2889836,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":38,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.289013,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.289033,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2890618,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2890837,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2901742,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2901952,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":29,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.2902064,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.292359,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":10,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2937405,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2937737,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":0,"logTerm":2,"index":43,"commit":29,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.293786,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2949574,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2949784,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":29,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.295104,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.2973373,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.297365,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2973871,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.299632,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":17,"logTerm":2,"index":25,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.2996652,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":42,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.300872,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3009005,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3009114,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3035173,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":12,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3035471,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.303571,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"1","state":{"term":2,"vote":"2","commit":12},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3038807,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"1","entries":0,"logTerm":0,"index":0,"commit":28,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3039033,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3039122,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3039181,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.303927,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3050997,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3051326,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3051953,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3062449,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":35,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.3062708,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":18,"logTerm":2,"index":25,"commit":29,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.307435,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"1","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.307463,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":0,"logTerm":2,"index":43,"commit":29,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.3074844,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.308653,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3086865,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.308697,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3087037,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3087347,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":18,"logTerm":2,"index":25,"commit":29,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.308835,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.310081,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":29},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":39,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3101027,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3101118,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"1","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.3101213,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.3101215,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3101287,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.31014,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.31017,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3102012,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3113694,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["2","3","4","5"]}}}} +{"level":"debug","ts":1712329730.3114707,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3126454,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3146994,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3147237,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3147452,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3157802,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":31,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.3181527,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":42,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3181822,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":42,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.3181922,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.318215,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3182514,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":18,"logTerm":2,"index":25,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.3182666,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3238702,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3238926,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3239005,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3239071,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3260033,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":42,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3260427,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":42,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3260524,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3281913,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":42,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3282146,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.328224,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.331512,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3315485,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.3315606,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3346112,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3346338,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.3346443,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.337181,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":33,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.3390584,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3390827,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":18,"logTerm":2,"index":25,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.3390934,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3395548,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":42,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":14,"logTerm":2,"index":29,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3395853,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3396103,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3396194,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.339626,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.339641,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3402493,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3402686,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.3403044,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3440368,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.344065,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3440733,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3440857,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3473847,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3474078,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.348441,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3484669,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3485181,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.350992,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":42,"commit":24,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3510141,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":42,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.3510387,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":32,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.351045,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.351098,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":24},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3511252,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3511355,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3529632,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":39,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.354991,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3550184,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3550422,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3560805,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.356125,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.3561394,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3584702,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":39,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3639903,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.364023,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3640325,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3640401,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3745673,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3747487,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.374756,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3748171,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3748522,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3748572,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3748906,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.37491,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.374938,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3780687,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":34,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.3801951,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3802316,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.380245,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3838944,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3839185,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3839276,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3839355,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3895,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":39,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3931725,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.3932045,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.3932176,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.3943384,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":37,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.395523,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":36,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.4038975,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.403927,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.403936,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4039435,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.40618,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4062204,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4062748,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.4072814,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4073174,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.40733,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.4097311,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.409782,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4097931,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.4142382,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":43,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.414264,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":43,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.414316,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.417842,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4178655,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.417894,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.4243724,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":38,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.424466,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4244843,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4244964,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4245088,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.4277766,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":39,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.436551,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":28,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4365778,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":28},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4366045,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":28},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.4439087,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4439392,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4439523,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4439604,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.4476588,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":28,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4476945,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":28},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4477208,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":28},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.449376,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4494658,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":18,"logTerm":2,"index":25,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.4494796,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.4517357,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4517584,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.451775,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.4543595,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.454386,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4544535,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.4557257,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":41,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4557538,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.4604442,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":28},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4604695,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":28},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4604795,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":28},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.463131,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4631598,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.4631808,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.4644508,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4644732,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4644818,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4644892,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.4677637,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4679012,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":18,"logTerm":2,"index":25,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.4679172,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.4707758,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":28},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4708035,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4708278,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.4758477,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":28},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4758756,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4758978,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.481219,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":39,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.48386,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4838858,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4838939,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.4839015,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5038815,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5039675,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5039775,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5039866,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5147502,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":43,"commit":28,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.514778,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":43,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.5148377,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5161457,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":28,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5162566,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5166304,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5197797,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":42,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5198097,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5222933,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5223222,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5223315,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5223324,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5223498,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":6,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5223577,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5223765,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.522379,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.522386,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.522392,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5223987,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ChangeConf","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":43,"conf":[["2","3","4","5"],[]],"prop":{"cc":{"changes":[{"nid":"4","action":"RemoveServer"}]}}}} +{"level":"debug","ts":1712329730.5224051,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5224068,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5224147,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.5224257,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.5224338,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5237148,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5237434,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5237474,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.523758,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":39,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.5237713,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.523793,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":43,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5238185,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":43,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.5250201,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5250504,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5250654,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5250704,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":10},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5250778,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.5250926,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.52512,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.525147,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.526229,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5262551,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":43,"next":44}}} +{"level":"debug","ts":1712329730.5262673,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5262706,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5262842,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5263035,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.526304,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5263236,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.527432,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5274525,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5274637,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5274706,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.527478,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.527481,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5274878,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5274947,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5275085,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5275116,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5275466,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5275664,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5286622,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5286815,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5286875,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5287018,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.528708,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.528715,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5287218,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5287247,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5287285,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.529888,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.529907,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5299172,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.529925,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5299318,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5299451,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5299554,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5299652,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.529966,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.529975,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.531265,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5312858,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5312965,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5312994,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5313036,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5313249,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["2","3","4","5"]}}}} +{"level":"debug","ts":1712329730.531326,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5313437,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":25},"role":"StateFollower","log":25,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":18,"logTerm":2,"index":25,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5313718,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5313816,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5325038,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":42,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5325277,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5325358,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5325422,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5325499,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["2","3","4","5"]}}}} +{"level":"debug","ts":1712329730.5325584,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":18,"logTerm":2,"index":25,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5325606,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5325658,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5337512,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":18,"logTerm":2,"index":25,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5337706,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5337842,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5337868,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5337934,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5338054,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5338194,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.533822,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5338438,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5350628,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.535081,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5351117,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5351274,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":40,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.53513,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.535138,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.535165,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5351918,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5363376,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5363624,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5363736,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5363896,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5363963,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.536403,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5364096,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5364125,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":18,"logTerm":2,"index":25,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5364187,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5364265,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5375614,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":42,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5375993,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5376139,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.537618,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5376296,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5376418,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5376546,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5391133,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.539136,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5391648,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5391998,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5392032,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5392241,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5392334,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5392659,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5392787,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5405889,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5406098,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.540629,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5406303,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5406368,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5406458,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5406537,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5406759,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5406842,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5407374,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5419,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5419188,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5419273,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5419285,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":18,"logTerm":2,"index":25,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5419345,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5419436,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5419478,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5419614,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.543169,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.543173,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5431862,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5431957,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":38,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.543198,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5432081,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5432117,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5432281,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5432434,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5443451,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.54436,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5443683,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5443835,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5443923,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5444002,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":41,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.5444405,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5444572,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5457206,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":38},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":42,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5457413,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.545761,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":42,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5457623,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5457706,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":42,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5457816,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5457833,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.545789,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5458,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5458136,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5458431,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5471017,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.547123,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5471394,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5471544,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5471592,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.547162,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5471675,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5471754,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":42,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5471785,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5471854,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":42,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5471923,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":42,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5471973,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.548405,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5484242,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5484326,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5484915,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5485415,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.548547,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5485575,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5485637,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.548573,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":42,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5498922,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":1,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5499167,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5499191,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5499337,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5499375,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5499432,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":42,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.549946,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.549959,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":18,"logTerm":2,"index":25,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5499659,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5499763,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5510654,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":43,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5510976,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":42,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5511098,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5511189,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5511312,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.551154,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5511644,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.552518,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.552526,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5525424,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":42,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5525494,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.552563,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5535798,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5536048,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":42,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5536146,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5536351,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.553643,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.554877,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.554902,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":42,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5549135,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.55593,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.555964,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":42,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.556988,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5570178,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5581899,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":42,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.5592883,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":42,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5604753,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":42},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5605032,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5605257,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5605352,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.561633,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.561658,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5616698,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5625892,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.562617,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5626438,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5628195,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5628366,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.563886,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.564974,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5649981,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.56501,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5650167,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5650232,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5650349,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5660803,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":42,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.5672452,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5683095,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5683331,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.569484,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.569507,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5705776,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.570602,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5706148,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5716276,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5716572,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5726736,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5726967,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5737398,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5737636,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5737748,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5748727,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.574897,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.574922,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5749328,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5761898,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":43,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.5772536,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.577407,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5774207,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.578565,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5796957,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.579736,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5809689,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5820858,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.582109,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5837214,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5837522,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5848906,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":43,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.5859327,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.585957,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5869737,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.586996,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.587007,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5870147,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.58702,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5870264,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5881557,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5881827,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5892274,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5892515,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5892694,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5892787,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5892928,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5904253,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5904555,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5914752,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5926576,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5937095,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.593737,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5949368,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5949585,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.5949688,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5959811,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5974572,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5987828,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.5988264,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.598845,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.5999694,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6011448,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6011713,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6022332,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6022627,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.603372,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.603397,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6034086,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6045961,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6057036,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6057346,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6070323,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6070645,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.607073,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6070797,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6070864,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6081653,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.608187,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6093385,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6093636,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6103761,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6104052,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6109064,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.610925,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.610951,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6114771,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6115038,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6127295,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.612751,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6138742,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6138992,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.614668,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":42,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6147492,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6147668,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6149912,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6150296,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.615801,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6158278,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6158404,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6161244,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.616146,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.616168,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6162329,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.616259,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6172767,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6173005,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6184251,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":43,"commit":0,"vote":"0","reject":true,"rejectHint":25}}} +{"level":"debug","ts":1712329730.6194901,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.619516,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.619527,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":15,"logTerm":2,"index":29,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6207647,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6207888,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.621921,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6219451,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.623127,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.623149,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6243663,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6243806,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":42,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6244037,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6244287,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6256135,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6256409,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6256585,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6256664,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6256719,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.625678,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6268065,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6268325,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6279876,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":42,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6280155,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6280246,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6280286,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6291516,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6291811,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6301928,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6312315,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6312544,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6312637,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6322775,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6322956,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6333468,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6334171,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6334324,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6344457,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6344714,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.634925,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":38},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":42,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6349502,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.634974,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6355832,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.635606,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6366348,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.636656,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.636666,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6378317,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.637854,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.639067,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.639092,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6402266,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6402516,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6412702,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6412938,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6423647,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6423852,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6434405,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6445448,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.644583,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6456,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6456246,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.645691,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6457164,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6457248,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6457322,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6467445,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.646768,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6478448,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":43},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.647874,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6478841,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6478934,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6479013,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6479099,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6492276,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6492522,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6492805,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.649338,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6493762,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","4","5"],[]],"prop":{"cc":{"newconf":["2","3","5"]}}}} +{"level":"debug","ts":1712329730.6494439,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.6507058,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":42,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6507366,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6517837,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":42,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6518188,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.651837,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.652344,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6544755,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":42,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.654506,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6545308,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6549535,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6549914,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.655048,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.6565602,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6565857,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6566107,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6590025,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":42,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6590326,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6590433,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6612406,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":42},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6612718,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6622884,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.662315,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6623256,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6639755,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6640074,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.664018,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.666409,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6664345,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6664464,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6709669,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6710048,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6710186,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.6711977,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6712227,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.671239,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6730845,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6731095,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6731207,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.676449,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6764743,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6764865,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.676506,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.676533,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6765733,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6787198,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.678763,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.6787834,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.6839712,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6840355,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6840563,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.6933563,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.693415,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6934545,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.6978414,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6978714,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.6979032,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.703619,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7036762,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.7036912,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.7047114,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7047355,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.704745,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.7109704,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.711007,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.7110214,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.7164083,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7164447,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.716523,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.7238815,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.723923,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.723933,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.727049,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7270856,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7271225,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.733535,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7335658,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7336018,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.740319,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7403543,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.7403684,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.7443314,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.744399,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.744414,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.7529595,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7530582,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7531533,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.7604105,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7641113,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.764153,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7641723,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.7694287,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7694778,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7695837,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7695882,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7696564,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7696807,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7698007,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7716079,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":42,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7716382,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7716465,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.771675,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7716832,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.7716985,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7716997,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7717085,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.7717266,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7717428,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.7730913,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.773117,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.773136,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":42,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7731369,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7731442,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7731547,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.77317,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7731948,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7742977,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7743154,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7743244,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.774333,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7743397,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.774342,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7743487,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7743556,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7743645,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.775808,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7758338,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7758522,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.775864,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7758708,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7758825,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7759109,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.77597,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.777064,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7771058,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7771926,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7772155,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7772295,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.777245,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.777247,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":42,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7772663,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.777271,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.778502,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7786005,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7786431,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7786756,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7786953,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7787182,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.778752,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.7787616,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7800016,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7800264,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7800467,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7800477,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.780056,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7800686,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7800713,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.780091,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.7810614,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":38,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.78108,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.781093,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7811224,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.78113,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.781151,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.781174,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7822826,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7823067,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7823212,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.782328,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7823403,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7823486,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7823713,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7823956,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.783505,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.783528,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7835488,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.783556,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7835658,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7835867,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7836027,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7847848,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7848117,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7848196,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7848275,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.784842,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7848482,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":19,"logTerm":2,"index":25,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7848642,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7848804,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.784893,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7859042,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7859292,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.785938,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7859406,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7859488,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.7859573,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.785966,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7859743,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7859802,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7859807,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7872865,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7873068,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.7873187,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7873254,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7873335,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7873504,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7873566,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7885826,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7886066,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7886286,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":25,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7886517,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7886574,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7886684,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7886696,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.78989,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7899375,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":29},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7899594,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7899728,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7899787,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7899942,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.790006,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.790018,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.7900586,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.7911656,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7911878,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7912033,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":29,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7912104,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7912133,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7912192,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.791246,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.791257,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7923968,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7924197,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.7924387,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7924454,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7924528,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7924557,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.792462,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7936246,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7936454,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.793651,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7936552,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"4","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7936625,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7936893,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7937086,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.7937338,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["2","3","4","5"]}}}} +{"level":"debug","ts":1712329730.7949364,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.79496,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.794982,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7949903,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.7950358,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7950487,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7950606,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.79508,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.7962747,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"4","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"4","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7963047,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"4","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"4","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7963285,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"4","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.7963448,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7963703,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7963989,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7964005,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.796423,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.796435,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.797838,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.797861,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.797871,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.79789,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.797893,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7979174,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7979212,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"4","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"prop":{"cc":{"newconf":["2","3","5"]}}}} +{"level":"debug","ts":1712329730.7993321,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":15,"logTerm":2,"index":29,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.799355,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7993703,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7993817,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7994041,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.7994134,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.8004887,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8005095,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8005276,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.800529,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"prop":{"cc":{"newconf":["2","3","5"]}}}} +{"level":"debug","ts":1712329730.8006046,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8006327,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.8018527,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8018796,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8018892,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8019032,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8019338,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.8030612,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8030787,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.8030946,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8031008,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8031182,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.804255,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.804282,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8042927,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.804308,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.8043106,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.804331,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8058286,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8058655,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.80587,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8058786,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.805895,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8059022,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8059084,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.8059232,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8059516,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.8070881,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8071122,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.8071458,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8085809,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8086789,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8087115,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8087218,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.8087447,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8087645,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.810075,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.810097,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8112957,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8113203,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.811371,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.8125885,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8126197,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8138435,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8138723,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.813882,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.815334,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8153586,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8164282,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8164613,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8164885,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.8239017,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8254857,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8255126,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.827768,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8277986,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.8278131,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.836052,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8439,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8439429,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8439527,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.8484037,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.850943,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8638802,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8639278,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.863939,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.884043,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8840868,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.8840969,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.905415,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9054513,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9054632,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.905523,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9055238,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.905578,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9055989,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.905608,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.906637,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9066656,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.906675,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.906687,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.906692,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.906699,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9067056,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.907863,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9078844,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9079065,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9079137,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9079354,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9091117,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9091406,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9091508,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9091558,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.909179,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9101746,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9102008,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9102154,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9102185,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9102418,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9102669,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9115722,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9115927,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9116154,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.912909,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9129457,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.912975,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9141448,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9141662,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9141755,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9142308,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.9154456,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9154646,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9154716,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9166088,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.916635,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.916647,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9166677,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.9166925,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.917744,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9187748,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9187956,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.9201617,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9201965,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9212728,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9225516,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9235775,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9246032,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9256449,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.92567,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.9256833,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9256907,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9269545,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9269757,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9280052,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9291396,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9302745,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9316168,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9327364,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9327774,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.9327908,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9338021,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.934928,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9353821,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.935405,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9354284,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9360182,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9360368,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.9360478,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.93715,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9381895,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9382126,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.9392397,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":29,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9392607,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9403071,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9413342,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.942415,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9435747,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9446485,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":43,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9446695,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9446821,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.945798,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9458327,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.9468699,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9469025,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9469128,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9469206,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9480984,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9481297,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.949148,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9491718,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.9501944,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9513621,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9513974,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.952686,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":43},"role":"StateFollower","log":43,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.952707,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9527388,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]]}} +{"level":"debug","ts":1712329730.9527557,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.952783,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.953852,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9538887,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.9539073,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.953917,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","4","5"],[]],"prop":{"cc":{"newconf":["2","3","5"]}}}} +{"level":"debug","ts":1712329730.9554918,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.95557,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9568636,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9568958,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9580736,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.959267,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9592874,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.9592984,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9593053,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.959311,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":45,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":45,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.959326,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":45,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.9605193,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9605591,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":45,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9616053,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9616315,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":45,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9616423,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":45,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":2,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.963136,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":45,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.963164,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":45,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":45,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.9643137,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":45,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9643316,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":45,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":45,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.964342,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":45,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.964354,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":46,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9643624,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":45,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.9643712,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9643767,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.964383,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9655483,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9655755,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":46,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.965601,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.96683,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9668577,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9668665,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":46,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.9680018,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9680293,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":46,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":44,"next":45}}} +{"level":"debug","ts":1712329730.9691954,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9692183,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9692278,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.969248,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":45,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.969275,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9704282,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9704473,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.970456,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9705153,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":46,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9705377,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.971693,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9730198,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9730408,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":3,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":46,"next":47}}} +{"level":"debug","ts":1712329730.9741669,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9741917,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":3,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":46,"next":47}}} +{"level":"debug","ts":1712329730.9753635,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.975431,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":46,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9754431,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":47,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":47,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9754534,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":47,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":46,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":46,"next":47}}} +{"level":"debug","ts":1712329730.9766874,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9767427,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":47,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9767635,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":47,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":47,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":46,"next":47}}} +{"level":"debug","ts":1712329730.9768097,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":47,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9778554,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9790213,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":47,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":47,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9790444,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":47,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9790783,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":47,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.979099,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":48,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":48,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9791136,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":48,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":47,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":47,"next":48}}} +{"level":"debug","ts":1712329730.9791238,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":48,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9803023,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":48,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9803295,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.98035,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.980351,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9803584,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.981399,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":48,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":48,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.981425,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":48,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9814718,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9827826,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":48,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9828038,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":48,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":5,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":48,"next":49}}} +{"level":"debug","ts":1712329730.9828148,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":48,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9828207,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":49,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":49,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.982823,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9828281,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":49,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":48,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":48,"next":49}}} +{"level":"debug","ts":1712329730.9828362,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":49,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.982852,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.982867,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9840112,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.984048,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9840772,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9841266,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9852228,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9853141,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":49,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":49,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9853404,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":49,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9864807,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":49,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":44,"commit":0,"vote":"0","reject":true,"rejectHint":43}}} +{"level":"debug","ts":1712329730.9877949,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":49,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9878156,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":49,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":6,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":49,"next":50}}} +{"level":"debug","ts":1712329730.987867,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":49,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9878774,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":49,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9878836,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":50,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":50,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9878907,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":50,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":49,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":49,"next":50}}} +{"level":"debug","ts":1712329730.987898,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":50,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9879036,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":51,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":51,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9879105,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":51,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":50,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":49,"next":50}}} +{"level":"debug","ts":1712329730.9879186,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":51,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9879243,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":51,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9891396,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":51,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9891927,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9920337,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9929497,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":51,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":50,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9931235,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":51,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9953425,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":51,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":51,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9960964,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":51,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.996207,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":51,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9962683,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":52,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9963503,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":51,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":51,"next":52}}} +{"level":"debug","ts":1712329730.997594,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.99764,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9977086,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":9,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":51,"next":52}}} +{"level":"debug","ts":1712329730.9977465,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":52,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9978247,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329730.9978483,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9989238,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9989734,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329730.9989843,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.000065,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":52,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.000089,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":52,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0001073,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":52,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.000113,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0001142,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":53,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":53,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0001256,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":53,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":52,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":52,"next":53}}} +{"level":"debug","ts":1712329731.0001411,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.000162,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0001624,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":53,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0001793,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.001427,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0014524,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.001465,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0014818,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0027316,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":53,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":53,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0028071,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.002867,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":53,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0028768,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":53,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0028825,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":54,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":54,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0028899,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":54,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":53,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":53,"next":54}}} +{"level":"debug","ts":1712329731.0029058,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":54,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0054877,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":54,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":54,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0055087,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":54,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0055177,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":54,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.005524,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":55,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":55,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0055308,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":55,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":54,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":54,"next":55}}} +{"level":"debug","ts":1712329731.0055408,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":55,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0055478,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":55,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0055547,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":55,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0067358,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.006767,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":55,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0067682,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0067804,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":55,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":12,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":54,"next":55}}} +{"level":"debug","ts":1712329731.0068007,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0068114,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.008059,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":55,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":55,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0080855,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":55,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.008097,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":55,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0081646,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0094666,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":45,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0094967,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":45,"commit":0,"vote":"0","reject":true,"rejectHint":44}}} +{"level":"debug","ts":1712329731.0095017,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":55,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0095053,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0095196,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":56,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":56,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0095282,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":56,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":55,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":55,"next":56}}} +{"level":"debug","ts":1712329731.009555,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":56,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.010679,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0118732,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":56,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":56,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0118902,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":56,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0118976,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0131874,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":56,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.013209,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":57,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":57,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0132189,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":57,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":56,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":56,"next":57}}} +{"level":"debug","ts":1712329731.0132294,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":57,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0132704,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0133052,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0146885,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":45,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0147278,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":45,"commit":0,"vote":"0","reject":true,"rejectHint":44}}} +{"level":"debug","ts":1712329731.014742,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0147638,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0147743,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0159855,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":57,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":57,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0160089,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":57,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0160294,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0160532,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":45,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":45,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0172853,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":45,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":45,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0173037,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":46,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0173452,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":46,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0185525,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":46,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0185902,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":57,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0185988,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":58,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":58,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0186074,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":58,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":57,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":57,"next":58}}} +{"level":"debug","ts":1712329731.018616,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":58,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0186296,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ChangeConf","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":58,"conf":[["2","3","5"],[]],"prop":{"cc":{"changes":[{"nid":"5","action":"RemoveServer"}]}}}} +{"level":"debug","ts":1712329731.0186448,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":59,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":59,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.018652,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":59,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":58,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":57,"next":58}}} +{"level":"debug","ts":1712329731.0209296,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":59,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":58,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0209515,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":59,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.02096,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":59,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0234494,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":59,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":59,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0234942,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":59,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0235393,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":59,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0235827,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":60,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.023593,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":59,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":59,"next":60}}} +{"level":"debug","ts":1712329731.0236082,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0258622,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":60,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0258894,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.025902,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.025909,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0259163,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0259266,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":2,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0259497,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0259879,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0270567,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0270808,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0270934,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.028754,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0287774,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":60,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":60,"next":61}}} +{"level":"debug","ts":1712329731.0287938,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":46,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":46,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0288033,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":47,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":47,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0288105,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":47,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0299845,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.030006,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0300443,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":16,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":60,"next":61}}} +{"level":"debug","ts":1712329731.030115,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0318346,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":47,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":46,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0319116,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":47,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":46,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0331812,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":47,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":46,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0332067,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":47,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":46,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0343692,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":47,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":47,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0343943,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":48,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":48,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0344052,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":48,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0365486,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":48,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":46,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0365734,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":48,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":46,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0365841,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":48,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0401998,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.040236,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":60,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":60,"next":61}}} +{"level":"debug","ts":1712329731.0402873,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.04031,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":60,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0403168,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":61,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":61,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0403237,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":61,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":60,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":60,"next":61}}} +{"level":"debug","ts":1712329731.040332,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":61,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":60,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":60,"next":61}}} +{"level":"debug","ts":1712329731.0414028,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":61,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.044332,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":61,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":61,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0443556,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":61,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0444074,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":61,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0444183,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":61,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0444431,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":61,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0455585,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":48,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":47,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.045584,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":48,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":47,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0456128,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":48,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.04689,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":61,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0469186,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":62,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0469291,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":61,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":61,"next":62}}} +{"level":"debug","ts":1712329731.0469387,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":61,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":61,"next":62}}} +{"level":"debug","ts":1712329731.0469472,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":62,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.049955,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":62,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0500114,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":62,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0516813,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0517125,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0517387,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0539527,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":62,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0539944,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":63,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":63,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0540109,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":63,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":62,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":62,"next":63}}} +{"level":"debug","ts":1712329731.0540214,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":63,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":62,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":62,"next":63}}} +{"level":"debug","ts":1712329731.0540342,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":63,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0563169,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":63,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":63,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.056349,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":63,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.056359,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0563607,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":63,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0563674,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":64,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":64,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0563858,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":64,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":63,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":63,"next":64}}} +{"level":"debug","ts":1712329731.0563965,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":64,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":63,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":63,"next":64}}} +{"level":"debug","ts":1712329731.0564132,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":64,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.05642,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.056431,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0588255,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":64,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":64,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0588489,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":64,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0631294,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":48,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":48,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0631666,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":49,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":49,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0631907,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":49,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.064047,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":64,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0640872,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":64,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.064102,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":64,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.069504,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0695534,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.069564,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0759907,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":5,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.076032,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0760643,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0841832,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":64,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0842106,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":64,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0842235,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":64,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0887153,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":49,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":49,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0887651,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":50,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":50,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.089967,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":50,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":50,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0900576,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":51,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":51,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0900722,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":51,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.0962467,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0962913,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0963018,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.097434,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":3,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0974638,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.0974767,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1042354,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":64,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1042747,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":64,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.104285,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":64,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.105972,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":64,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1059961,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":65,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":65,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1060073,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":65,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":64,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":64,"next":65}}} +{"level":"debug","ts":1712329731.1060178,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":65,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":64,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":64,"next":65}}} +{"level":"debug","ts":1712329731.1060636,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":65,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.109051,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":65,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":65,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1090875,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":65,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.109841,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":65,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.109889,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":66,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":66,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1099079,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":66,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":65,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":65,"next":66}}} +{"level":"debug","ts":1712329731.1099284,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":66,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":65,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":65,"next":66}}} +{"level":"debug","ts":1712329731.111591,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":66,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.111625,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":66,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1116345,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":66,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1116393,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":51,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1116416,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":67,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":67,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1116502,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":67,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":66,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":65,"next":66}}} +{"level":"debug","ts":1712329731.1116605,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":67,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":66,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":65,"next":66}}} +{"level":"debug","ts":1712329731.1116881,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":51,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1117368,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":51,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1141183,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":67,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":66,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1141722,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":67,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1142173,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":67,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1153674,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":67,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1153903,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":67,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":67,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":66,"next":67}}} +{"level":"debug","ts":1712329731.1166275,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":67,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":67,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1166544,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":67,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1167064,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":67,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1194777,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1198685,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1198993,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1219318,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":67,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1219692,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":68,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":68,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1219862,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":68,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":67,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":67,"next":68}}} +{"level":"debug","ts":1712329731.1219966,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":68,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":67,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":67,"next":68}}} +{"level":"debug","ts":1712329731.1220088,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":68,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1245315,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":68,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":68,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1245596,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":68,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1245928,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":68,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1246016,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":68,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.124614,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":68,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.126106,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1261568,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1262093,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1272361,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":68,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1315207,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":51,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":53,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1315718,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":51,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":53,"commit":0,"vote":"0","reject":true,"rejectHint":51}}} +{"level":"debug","ts":1712329731.1315823,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":51,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1326706,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":68,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1327167,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":69,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":69,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1327283,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":69,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":68,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":68,"next":69}}} +{"level":"debug","ts":1712329731.1327386,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":69,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":68,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":68,"next":69}}} +{"level":"debug","ts":1712329731.132788,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":69,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1338418,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":51,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":52,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.133877,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":51,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":52,"commit":0,"vote":"0","reject":true,"rejectHint":51}}} +{"level":"debug","ts":1712329731.1338904,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":51,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1344523,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":9,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1344745,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1345372,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1355965,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":69,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":69,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1356235,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":69,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1373148,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":69,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.13905,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":51,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":51,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.139087,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":52,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1391106,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.140317,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":69,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.140361,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":70,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":70,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.140377,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":70,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":69,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":69,"next":70}}} +{"level":"debug","ts":1712329731.140404,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":70,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":69,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":69,"next":70}}} +{"level":"debug","ts":1712329731.140418,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":70,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1427302,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":70,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":70,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1427577,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":70,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1441493,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":70,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1441786,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":70,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1441896,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":70,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.145359,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":70,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1453815,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":71,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":71,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.145391,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":71,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":70,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":70,"next":71}}} +{"level":"debug","ts":1712329731.1454003,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":71,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":70,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":70,"next":71}}} +{"level":"debug","ts":1712329731.1454203,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":71,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.146487,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":55,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1465292,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":55,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.1465394,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.14805,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":71,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":71,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1480734,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":71,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1528704,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":54,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1529028,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":54,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.1529286,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1569774,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":71,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":45,"commit":0,"vote":"0","reject":true,"rejectHint":44}}} +{"level":"debug","ts":1712329731.157024,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":71,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":71,"next":72}}} +{"level":"debug","ts":1712329731.157037,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":71,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1613102,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":71,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1613503,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":72,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":72,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1613617,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":72,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":71,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":71,"next":72}}} +{"level":"debug","ts":1712329731.1613717,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":72,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1638365,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1638749,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":72,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":72,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1638846,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1638966,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.163905,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":72,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1639316,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":72,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1639526,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":72,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1639783,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":72,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1661942,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.166204,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":72,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1662428,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1662984,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1663556,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":73,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.166383,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":72,"commit":44,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":72,"next":73}}} +{"level":"debug","ts":1712329731.1664104,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1690989,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":73,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.169124,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1733518,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.173385,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1734085,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1783273,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":56,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.178373,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":56,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.1783834,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1839154,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1839566,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1839657,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.190298,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":44},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":46,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1903565,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1903691,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":46,"commit":46,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":73,"next":74}}} +{"level":"debug","ts":1712329731.1903806,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":73,"commit":46,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":73,"next":74}}} +{"level":"debug","ts":1712329731.1904285,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.1916823,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1950674,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":3,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1952903,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.1953182,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2039123,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":46,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2039409,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2039945,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2103975,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":73,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2105415,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":74,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2105577,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":73,"commit":46,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":73,"next":74}}} +{"level":"debug","ts":1712329731.2105687,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":73,"commit":46,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":73,"next":74}}} +{"level":"debug","ts":1712329731.210579,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":74,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2134266,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":74,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2134755,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":74,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2156506,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":6,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2157142,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2157292,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2239096,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":46,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2239861,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2241144,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":74,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.232537,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":57,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2325904,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":57,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.232602,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.235898,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2362263,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2362554,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2397475,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2397861,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":74,"commit":46,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":74,"next":75}}} +{"level":"debug","ts":1712329731.239795,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":12,"logTerm":2,"index":43,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2398021,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":74,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2398238,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2398348,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2398436,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":58,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2398915,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":58,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.239942,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2409644,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":43,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2409933,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.241004,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":59,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2410114,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":59,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.2410202,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2410417,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":74,"commit":46,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":74,"next":75}}} +{"level":"debug","ts":1712329731.2410924,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":74,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2411027,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":75,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2411106,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":74,"commit":46,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":74,"next":75}}} +{"level":"debug","ts":1712329731.2411187,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":74,"commit":46,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":74,"next":75}}} +{"level":"debug","ts":1712329731.2421427,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2421696,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2421806,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":45,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.242184,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":60,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2421927,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":60,"commit":0,"vote":"0","reject":true,"rejectHint":44}}} +{"level":"debug","ts":1712329731.2422001,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":75,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.242207,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":76,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":76,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2422144,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":76,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":75,"commit":46,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":74,"next":75}}} +{"level":"debug","ts":1712329731.242223,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":76,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":75,"commit":46,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":74,"next":75}}} +{"level":"debug","ts":1712329731.243374,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":44,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":16,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2434154,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":60,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":60,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2434392,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":60,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2434478,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":60,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.2434585,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":76,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":45,"commit":0,"vote":"0","reject":true,"rejectHint":44}}} +{"level":"debug","ts":1712329731.2434704,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.24482,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":60,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2448227,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":46},"role":"StateLeader","log":76,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":47,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2448454,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":60,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.2448618,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":76,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2448692,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":76,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":76,"commit":47,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":74,"next":75}}} +{"level":"debug","ts":1712329731.244877,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":76,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":76,"commit":47,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":74,"next":75}}} +{"level":"debug","ts":1712329731.2449372,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":60,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":60,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2449667,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":61,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":61,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2460716,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":61,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":62,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2460938,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":61,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":62,"commit":0,"vote":"0","reject":true,"rejectHint":61}}} +{"level":"debug","ts":1712329731.2461057,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":61,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2461138,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":61,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.2461195,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":76,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":46,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2472527,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":76,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2472758,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":76,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2473023,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":76,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":47,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2473068,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2473087,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":76,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2473178,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":61,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":63,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2473333,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2473514,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":61,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":63,"commit":0,"vote":"0","reject":true,"rejectHint":61}}} +{"level":"debug","ts":1712329731.248496,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":61,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":61,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2485192,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":62,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2485304,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2485423,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":62,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2485654,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":62,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.248595,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":76,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":46,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2497394,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2497644,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.249782,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.249789,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2498002,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.249801,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":76,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2498307,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":76,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.249838,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":77,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":77,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2498462,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":77,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":76,"commit":47,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":74,"next":75}}} +{"level":"debug","ts":1712329731.2498565,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":77,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":76,"commit":47,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":74,"next":75}}} +{"level":"debug","ts":1712329731.2498708,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":77,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2498763,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":78,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":78,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2498884,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":78,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":77,"commit":47,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":74,"next":75}}} +{"level":"debug","ts":1712329731.2498953,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":78,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":77,"commit":47,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":74,"next":75}}} +{"level":"debug","ts":1712329731.2509885,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":78,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":75,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.251012,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":78,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.251031,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2510529,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2510645,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":63,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2510808,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":63,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.252288,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2522957,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":78,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":76,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2523098,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2523208,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2523258,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":78,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2523336,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.252336,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":78,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2523432,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":78,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2523594,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2523737,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":78,"commit":47,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":76,"next":77}}} +{"level":"debug","ts":1712329731.252393,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":78,"commit":47,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":76,"next":77}}} +{"level":"debug","ts":1712329731.2534306,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.25345,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2534692,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":47},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":48,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.253477,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":48},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2534838,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":48},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":79,"commit":48,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":76,"next":77}}} +{"level":"debug","ts":1712329731.2534926,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":48},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":79,"commit":48,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":76,"next":77}}} +{"level":"debug","ts":1712329731.253507,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":64,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2535288,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":64,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.2546341,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":64,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.254653,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2546585,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":64,"commit":0,"vote":"0","reject":true,"rejectHint":62}}} +{"level":"debug","ts":1712329731.2546668,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":48},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":46,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2547061,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.255813,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":48},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2558372,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":65,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2558506,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":65,"commit":0,"vote":"0","reject":true,"rejectHint":62}}} +{"level":"debug","ts":1712329731.2558775,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":65,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2558994,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":65,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.2570782,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":66,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2570987,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":66,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.257111,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":48},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.257136,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2571638,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2571805,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2584932,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":48},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":77,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2585478,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":67,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.258557,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":48},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2585847,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":68,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2585866,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":67,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.2585948,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":68,"commit":0,"vote":"0","reject":true,"rejectHint":62}}} +{"level":"debug","ts":1712329731.2596242,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":48},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":78,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.25965,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":48},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2596655,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":48},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2596657,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":66,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2596903,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2596974,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":66,"commit":0,"vote":"0","reject":true,"rejectHint":62}}} +{"level":"debug","ts":1712329731.2596984,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2597055,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2610066,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":48},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":47,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2610335,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":67,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2610414,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":67,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2610435,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":67,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.261064,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":67,"commit":0,"vote":"0","reject":true,"rejectHint":62}}} +{"level":"debug","ts":1712329731.2621877,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":71,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.262203,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":71,"commit":0,"vote":"0","reject":true,"rejectHint":62}}} +{"level":"debug","ts":1712329731.262222,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":48},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2622316,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":69,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2622557,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":69,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.263395,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":68,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2634134,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":68,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.2634528,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":48},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":49,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2634602,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2634706,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.263481,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":49},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2634912,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":49},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":79,"commit":49,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":78,"next":79}}} +{"level":"debug","ts":1712329731.2635074,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":49},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":79,"commit":49,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":78,"next":79}}} +{"level":"debug","ts":1712329731.2635176,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.264613,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":70,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2646346,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":70,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.2646453,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":72,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2646484,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":49},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2646515,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":72,"commit":0,"vote":"0","reject":true,"rejectHint":62}}} +{"level":"debug","ts":1712329731.2658017,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2658207,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2658317,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":49},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2658389,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":49},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2658415,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":69,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2658608,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":69,"commit":0,"vote":"0","reject":true,"rejectHint":62}}} +{"level":"debug","ts":1712329731.2671769,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2672036,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2672129,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2672443,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":49},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":50,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2672665,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":52,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":44,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.267269,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":50},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2672782,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":50},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":79,"commit":50,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.2673235,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":50},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":79,"commit":50,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.2673514,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":71,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":71,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2673936,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":50},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":50,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2674103,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":50},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2674892,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":50},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2686715,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":50},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2687023,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":71,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2687232,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":70,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2687237,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":71,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.26874,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":71,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2687578,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":70,"commit":0,"vote":"0","reject":true,"rejectHint":62}}} +{"level":"debug","ts":1712329731.2700276,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2700498,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2700615,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":71,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":46,"commit":46,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2700732,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":50},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":51,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2700844,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":73,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":73,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2700856,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.270096,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":79,"commit":51,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.2701051,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":79,"commit":51,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.2712178,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":73,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.271232,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":73,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2712483,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2712567,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.271271,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2712781,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2725422,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":1,"logTerm":2,"index":73,"commit":46,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.272563,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":73,"commit":0,"vote":"0","reject":true,"rejectHint":62}}} +{"level":"debug","ts":1712329731.2725732,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2725801,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":73,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":73,"commit":46,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2725835,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2726026,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":74,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2739003,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":73,"commit":46,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2739213,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":2,"index":73,"commit":0,"vote":"0","reject":true,"rejectHint":62}}} +{"level":"debug","ts":1712329731.2739313,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2739406,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":46,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2739623,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":79,"commit":51,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.273981,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.274007,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2740304,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.275237,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2752628,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.275278,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2752821,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2753441,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":46,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2753756,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.276602,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":76,"commit":47,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.276623,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":76,"commit":0,"vote":"0","reject":true,"rejectHint":74}}} +{"level":"debug","ts":1712329731.276631,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.276648,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.276655,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":79,"commit":51,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.2778006,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":53,"commit":0,"vote":"0","reject":true,"rejectHint":51}}} +{"level":"debug","ts":1712329731.2778258,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":28,"logTerm":2,"index":51,"commit":51,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.2778404,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":76,"commit":47,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2778568,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":76,"commit":0,"vote":"0","reject":true,"rejectHint":74}}} +{"level":"debug","ts":1712329731.277867,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.279207,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2792652,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.280377,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":51},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":52,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.280399,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2804084,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":52,"commit":52,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.2804177,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":79,"commit":52,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.2814968,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":54,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.2815168,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":52,"commit":52,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.2828164,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":55,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.2840624,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":52,"commit":0,"vote":"0","reject":true,"rejectHint":51}}} +{"level":"debug","ts":1712329731.2840846,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":52,"commit":52,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.2841096,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":74,"commit":46,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2841654,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":74,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.284177,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2852821,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":75,"commit":46,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.285302,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":75,"commit":0,"vote":"0","reject":true,"rejectHint":74}}} +{"level":"debug","ts":1712329731.2853167,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2853243,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":79,"commit":52,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.2853298,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2853332,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2866454,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2866688,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":52,"commit":52,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.2867296,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":52,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2867448,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2878246,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":74,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":74,"commit":46,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2878468,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":75,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2878563,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2878726,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.28907,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":56,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.289096,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2902951,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2903333,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":46},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":47,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2903426,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2903948,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2914805,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2915022,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":79,"commit":52,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.2927809,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.292803,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.294125,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2941608,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":44,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.294184,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.2941935,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"5","state":{"term":2,"vote":"2","commit":44},"role":"StateFollower","log":62,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.2953484,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":57,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.296737,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":59,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.2980068,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":60,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.2992663,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":58,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.3006277,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3006551,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":52,"commit":52,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.3018918,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3019154,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":52,"commit":52,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.3030407,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":52},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":61,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3030636,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.3030725,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":79,"commit":61,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.3044295,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.304471,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"5","entries":0,"logTerm":2,"index":79,"commit":61,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.304533,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.3057716,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"5","to":"2","entries":0,"logTerm":0,"index":44,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3057942,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":52,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3058016,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"5","entries":0,"logTerm":0,"index":0,"commit":61,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.306992,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":60,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.3071046,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3","5"],[]],"prop":{"cc":{"newconf":["2","3"]}}}} +{"level":"debug","ts":1712329731.3071842,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.308645,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":61,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.3086698,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.308684,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":78,"commit":47,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.308692,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":78,"commit":0,"vote":"0","reject":true,"rejectHint":75}}} +{"level":"debug","ts":1712329731.308704,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.309832,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":62,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.3109868,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.311004,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":52,"commit":61,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":79,"next":80}}} +{"level":"debug","ts":1712329731.311051,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":79,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.311062,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":80,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3110938,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3121657,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":63,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.313356,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":80,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3133779,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.313387,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":77,"commit":47,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3133948,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":77,"commit":0,"vote":"0","reject":true,"rejectHint":75}}} +{"level":"debug","ts":1712329731.3134086,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.3144565,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":65,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.3155484,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3155715,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":28,"logTerm":2,"index":52,"commit":61,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":80,"next":81}}} +{"level":"debug","ts":1712329731.3155828,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.316594,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":64,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.3186548,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3186789,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":28,"logTerm":2,"index":52,"commit":61,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":80,"next":81}}} +{"level":"debug","ts":1712329731.3186898,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3228462,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":66,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.3239505,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3239982,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":28,"logTerm":2,"index":52,"commit":61,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":80,"next":81}}} +{"level":"debug","ts":1712329731.3240118,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.32402,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":52,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3250365,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.327227,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":67,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.3357046,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":79,"commit":48,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.335732,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":79,"commit":0,"vote":"0","reject":true,"rejectHint":75}}} +{"level":"debug","ts":1712329731.3357706,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.3439848,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":52,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.344013,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3460774,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":69,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.3526204,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3526661,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":79,"commit":51,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.352671,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":28,"logTerm":2,"index":52,"commit":61,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":80,"next":81}}} +{"level":"debug","ts":1712329731.352686,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3526947,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ChangeConf","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":80,"conf":[["2","3"],[]],"prop":{"cc":{"changes":[{"nid":"6","action":"AddNewServer"}]}}}} +{"level":"debug","ts":1712329731.3527026,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3527315,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":79,"commit":0,"vote":"0","reject":true,"rejectHint":75}}} +{"level":"debug","ts":1712329731.3527443,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.3539155,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":79,"commit":49,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3539338,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":79,"commit":0,"vote":"0","reject":true,"rejectHint":75}}} +{"level":"debug","ts":1712329731.353961,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":61},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":71,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3539922,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":71},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3540092,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":71},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":10,"logTerm":2,"index":71,"commit":71,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":80,"next":81}}} +{"level":"debug","ts":1712329731.355078,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":79,"commit":50,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3550951,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":79,"commit":0,"vote":"0","reject":true,"rejectHint":75}}} +{"level":"debug","ts":1712329731.3551176,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":71},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":67,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.3551717,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":71},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3562446,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":47},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":50,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.356262,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":50},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.356265,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":71},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":74,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3562706,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":50},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.356291,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3563018,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":74,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":80,"next":81}}} +{"level":"debug","ts":1712329731.3574407,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":68,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.357477,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":50},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":79,"commit":51,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3574893,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":50},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":79,"commit":0,"vote":"0","reject":true,"rejectHint":75}}} +{"level":"debug","ts":1712329731.358618,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":50},"role":"StateFollower","log":75,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":28,"logTerm":2,"index":51,"commit":51,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3586433,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":51},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3586528,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":51},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.3586695,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.358692,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3587358,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3598418,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3598814,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":51},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":52,"commit":52,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3599355,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":74,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.360016,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3611786,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":52,"commit":52,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3612092,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3612192,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.361226,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3612485,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":74,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.3623705,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":70,"commit":0,"vote":"0","reject":true,"rejectHint":52}}} +{"level":"debug","ts":1712329731.3624055,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":52,"commit":52,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3624282,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3635545,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3635793,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":74,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.363584,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":52,"commit":52,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.363591,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3636346,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3647385,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3647604,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":74,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.3647914,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":52,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.364817,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3661196,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":76,"commit":0,"vote":"0","reject":true,"rejectHint":74}}} +{"level":"debug","ts":1712329731.3661346,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":52,"commit":52,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3661416,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":7,"logTerm":2,"index":74,"commit":74,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.3662405,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3662612,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.3662775,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":74,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.36731,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":52,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.36733,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3673444,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":73,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3673518,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.368507,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":52,"commit":52,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3685322,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3685596,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":74,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3686006,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.368612,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":7,"logTerm":2,"index":74,"commit":74,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.3709655,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":52},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":27,"logTerm":2,"index":52,"commit":61,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3709922,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":61},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3709984,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":76,"commit":0,"vote":"0","reject":true,"rejectHint":74}}} +{"level":"debug","ts":1712329731.371025,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":7,"logTerm":2,"index":74,"commit":74,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.3710368,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3722606,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":75,"commit":0,"vote":"0","reject":true,"rejectHint":74}}} +{"level":"debug","ts":1712329731.3722932,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":61},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":28,"logTerm":2,"index":52,"commit":61,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3723025,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":61},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":61,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3736246,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":61},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":28,"logTerm":2,"index":52,"commit":61,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3736484,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":61},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":61,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3736603,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":61},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]]}} +{"level":"debug","ts":1712329731.3736627,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":74},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":75,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3736806,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":75},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3736885,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":75},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":6,"logTerm":2,"index":75,"commit":75,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.3748033,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":75},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3748257,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":75},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":75,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.3748388,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":75},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3748512,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":61},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":52,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.374871,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":61},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3749254,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":2,"vote":"2","commit":61},"role":"StateFollower","log":79,"conf":[["2","3","5"],[]],"prop":{"cc":{"newconf":["2","3"]}}}} +{"level":"debug","ts":1712329731.3760352,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":75},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":77,"commit":0,"vote":"0","reject":true,"rejectHint":75}}} +{"level":"debug","ts":1712329731.3760767,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":75},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":6,"logTerm":2,"index":75,"commit":75,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.3761504,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":61},"role":"StateFollower","log":79,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":28,"logTerm":2,"index":52,"commit":61,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3761868,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":61},"role":"StateFollower","log":79,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":61,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3775563,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":61},"role":"StateFollower","log":79,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":52,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.377578,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":75},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":78,"commit":0,"vote":"0","reject":true,"rejectHint":75}}} +{"level":"debug","ts":1712329731.3775818,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":61},"role":"StateFollower","log":79,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3776069,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":75},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3776388,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":61},"role":"StateFollower","log":79,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.378769,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":75},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":79,"commit":0,"vote":"0","reject":true,"rejectHint":75}}} +{"level":"debug","ts":1712329731.3788445,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":61},"role":"StateFollower","log":79,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":10,"logTerm":2,"index":71,"commit":71,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3788722,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":71},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3799784,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":75},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":79,"commit":0,"vote":"0","reject":true,"rejectHint":75}}} +{"level":"debug","ts":1712329731.3800673,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":71},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":28,"logTerm":2,"index":52,"commit":61,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3800864,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":71},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":71,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3811712,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":75},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.381198,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3811996,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":71},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":74,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3812075,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":2,"logTerm":2,"index":79,"commit":79,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.3812187,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3812287,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3812423,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3824537,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":74,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3824806,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3825357,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":79,"commit":0,"vote":"0","reject":true,"rejectHint":75}}} +{"level":"debug","ts":1712329731.38372,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":74,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.383743,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3837676,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":79,"commit":0,"vote":"0","reject":true,"rejectHint":75}}} +{"level":"debug","ts":1712329731.3849545,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.384976,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":79,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.3849869,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.384998,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":74,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3850052,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3863811,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":74,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.386406,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.386434,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3876848,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":74,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3877075,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3877108,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":79,"commit":0,"vote":"0","reject":true,"rejectHint":75}}} +{"level":"debug","ts":1712329731.388956,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3889995,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":7,"logTerm":2,"index":74,"commit":74,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3890204,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3890421,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3904612,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":7,"logTerm":2,"index":74,"commit":74,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3904839,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3904903,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3905175,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":79,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.391614,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":74},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":75,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3916311,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":75},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.391679,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.392805,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":75},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":7,"logTerm":2,"index":74,"commit":74,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.392826,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":75},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":75,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3928404,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":75},"role":"StateFollower","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3928456,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3928764,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":79,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.3929267,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3939767,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":75},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":6,"logTerm":2,"index":75,"commit":75,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3939922,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":75},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3940125,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.396297,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":61,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3962984,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":75},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":2,"logTerm":2,"index":79,"commit":79,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3970437,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3985233,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3986468,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":79,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.398656,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":6,"logTerm":2,"index":75,"commit":75,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.398744,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3987894,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.3998442,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3998795,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":79,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.3999677,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":79,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.3999949,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4014175,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":79,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4014182,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4014935,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4016395,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.4016414,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.4028318,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4048426,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":61,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.404896,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":79,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4049234,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4049876,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.4059196,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":61,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4059436,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":79,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4059525,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.4084027,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":79,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4085517,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":79},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4092185,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4093497,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.4093854,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.4094107,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":81,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.4104924,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.410532,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.412281,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":79,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4123855,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4124594,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4124916,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.4126468,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3"],[]],"prop":{"cc":{"newconf":["2","3","6"]}}}} +{"level":"debug","ts":1712329731.4129565,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4129727,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":80,"commit":81,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.4140253,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4151702,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":71,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4164321,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4175198,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.418699,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4198115,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4215841,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":75,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4227269,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4239888,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":79,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.424014,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4240322,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4241252,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4241314,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.424153,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":81,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4241626,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.425199,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":79},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":81,"commit":81,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4252367,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4252503,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":79,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4252994,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":81,"conf":[["2","3"],[]]}} +{"level":"debug","ts":1712329731.4263935,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4264057,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":81,"conf":[["2","3"],[]],"prop":{"cc":{"newconf":["2","3","6"]}}}} +{"level":"debug","ts":1712329731.4264154,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4265563,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"InitState","nid":"6","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.4265726,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"6","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"prop":{"cc":{}}}} +{"level":"debug","ts":1712329731.4267373,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"6","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.427848,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4290712,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4302175,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4312832,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":80,"commit":81,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4313905,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.431443,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"6","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.43149,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":80,"commit":0,"vote":"0","reject":true,"rejectHint":0}}} +{"level":"debug","ts":1712329731.431663,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.432567,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4337568,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4348996,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4349232,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":81,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4349322,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":82,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":82,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4349408,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":82,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":81,"commit":81,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.434946,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4349704,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4350162,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.4360878,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":81,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4361084,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":82,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4361198,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":82,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":82,"commit":81,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":81,"next":82}}} +{"level":"debug","ts":1712329731.4361346,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":82,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4372764,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.4373038,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":82,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.438623,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":81,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4386377,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.438646,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":81,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4386668,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":82,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":82,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4387085,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":82,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4388237,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":82,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4388378,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":83,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":83,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.438854,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":83,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":82,"commit":81,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":82,"next":83}}} +{"level":"debug","ts":1712329731.4398334,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":81,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":81,"commit":81,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4398696,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":82,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":82,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4398859,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":83,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":80,"commit":0,"vote":"0","reject":true,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4398994,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":83,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":83,"logTerm":0,"index":0,"commit":81,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":82,"next":83}}} +{"level":"debug","ts":1712329731.43991,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":83,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.440991,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":82,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":82,"commit":81,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.441029,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":83,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.441086,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":83,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":83,"logTerm":0,"index":0,"commit":81,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":82,"next":83}}} +{"level":"debug","ts":1712329731.441102,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":83,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4411082,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":84,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":84,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4411151,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":84,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":83,"commit":81,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":82,"next":83}}} +{"level":"debug","ts":1712329731.44112,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":82,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":82,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.442336,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":82,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":82,"commit":81,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4423602,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":83,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":83,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4423692,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":83,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4423826,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":84,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":83,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4424078,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":84,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4424174,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":84,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4435866,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":83,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4436195,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.4449513,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":84,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":84,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4449732,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":84,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4449818,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":84,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4449875,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":85,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":85,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4449944,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":85,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":84,"commit":81,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":84,"next":85}}} +{"level":"debug","ts":1712329731.445011,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":85,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4450173,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":86,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":86,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.445024,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":86,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":85,"commit":81,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":84,"next":85}}} +{"level":"debug","ts":1712329731.445034,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":86,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":81,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4450395,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":86,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4450464,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":86,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.445052,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":86,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4450567,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":87,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":87,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4450626,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":87,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":86,"commit":81,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":84,"next":85}}} +{"level":"debug","ts":1712329731.4463775,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":83,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":83,"commit":81,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.446405,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":84,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":84,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4464238,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":84,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4475482,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":84,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4475944,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.4475975,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":87,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":85,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4476142,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":87,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4491036,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":87,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":86,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4491835,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":87,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4499269,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.449961,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4503117,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":87,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4503357,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":87,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":87,"commit":81,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":86,"next":87}}} +{"level":"debug","ts":1712329731.4510827,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":83,"logTerm":0,"index":0,"commit":81,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4511225,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":81},"role":"StateFollower","log":83,"conf":[[],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":83,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4511347,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":84,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":84,"commit":81,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.451136,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":81},"role":"StateFollower","log":83,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.4511669,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":85,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":85,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4512012,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":85,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4522822,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":81},"role":"StateLeader","log":87,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":82,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4523036,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":82},"role":"StateLeader","log":87,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.452313,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":82},"role":"StateLeader","log":87,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":87,"commit":82,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":86,"next":87}}} +{"level":"debug","ts":1712329731.4523258,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":85,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":81,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4523425,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":85,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.452346,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"6","state":{"term":2,"vote":"0","commit":81},"role":"StateFollower","log":83,"conf":[[],[]],"prop":{"cc":{"newconf":["1"]}}}} +{"level":"debug","ts":1712329731.452357,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":85,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4524183,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"6","state":{"term":2,"vote":"0","commit":81},"role":"StateFollower","log":83,"conf":[["1"],[]],"prop":{"cc":{"newconf":["1","2"]}}}} +{"level":"debug","ts":1712329731.4524422,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"6","state":{"term":2,"vote":"0","commit":81},"role":"StateFollower","log":83,"conf":[["1","2"],[]],"prop":{"cc":{"newconf":["1","2","3"]}}}} +{"level":"debug","ts":1712329731.4525023,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"6","state":{"term":2,"vote":"0","commit":81},"role":"StateFollower","log":83,"conf":[["1","2","3"],[]],"prop":{"cc":{"newconf":["1","2","3","4"]}}}} +{"level":"debug","ts":1712329731.4525888,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"6","state":{"term":2,"vote":"0","commit":81},"role":"StateFollower","log":83,"conf":[["1","2","3","4"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329731.4526582,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"6","state":{"term":2,"vote":"0","commit":81},"role":"StateFollower","log":83,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["2","3","4","5"]}}}} +{"level":"debug","ts":1712329731.4527478,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"6","state":{"term":2,"vote":"0","commit":81},"role":"StateFollower","log":83,"conf":[["2","3","4","5"],[]],"prop":{"cc":{"newconf":["2","3","5"]}}}} +{"level":"debug","ts":1712329731.4528365,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"6","state":{"term":2,"vote":"0","commit":81},"role":"StateFollower","log":83,"conf":[["2","3","5"],[]],"prop":{"cc":{"newconf":["2","3"]}}}} +{"level":"debug","ts":1712329731.4529245,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"6","state":{"term":2,"vote":"0","commit":81},"role":"StateFollower","log":83,"conf":[["2","3"],[]],"prop":{"cc":{"newconf":["2","3","6"]}}}} +{"level":"debug","ts":1712329731.4535744,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":82},"role":"StateLeader","log":87,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":82,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4535997,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":82},"role":"StateLeader","log":87,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.45361,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ChangeConf","nid":"2","state":{"term":2,"vote":"2","commit":82},"role":"StateLeader","log":87,"conf":[["2","3","6"],[]],"prop":{"cc":{"changes":[{"nid":"7","action":"AddNewServer"}]}}}} +{"level":"debug","ts":1712329731.453619,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":82},"role":"StateLeader","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":88,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4536262,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":82},"role":"StateLeader","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":87,"commit":82,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":86,"next":87}}} +{"level":"debug","ts":1712329731.4536397,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":82},"role":"StateLeader","log":88,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4536464,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":82},"role":"StateLeader","log":89,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":89,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4536524,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":82},"role":"StateLeader","log":89,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":88,"commit":82,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":86,"next":87}}} +{"level":"debug","ts":1712329731.4550455,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":81},"role":"StateFollower","log":83,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":83,"logTerm":0,"index":0,"commit":81,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4551804,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":82},"role":"StateLeader","log":89,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":83,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.455605,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":81},"role":"StateFollower","log":83,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4556262,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":81},"role":"StateFollower","log":83,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4556403,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":83},"role":"StateLeader","log":89,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4556477,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":83},"role":"StateLeader","log":89,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":89,"commit":83,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":86,"next":87}}} +{"level":"debug","ts":1712329731.4563134,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":85,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":85,"commit":81,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4563742,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":86,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":86,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4564354,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":86,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4574544,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":83},"role":"StateLeader","log":89,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":84,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.457478,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":89,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4574876,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":89,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":89,"commit":84,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":86,"next":87}}} +{"level":"debug","ts":1712329731.4588594,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":89,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":87,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4588814,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":89,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.458891,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":89,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4590027,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":89,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4590175,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":90,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":90,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.459044,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":90,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":89,"commit":84,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":87,"next":88}}} +{"level":"debug","ts":1712329731.4602168,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":86,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":87,"commit":81,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4602416,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":86,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":87,"commit":0,"vote":"0","reject":true,"rejectHint":86}}} +{"level":"debug","ts":1712329731.4602642,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":90,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":81,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4602883,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":90,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4603007,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":90,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":9,"logTerm":2,"index":81,"commit":84,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":87,"next":88}}} +{"level":"debug","ts":1712329731.4617739,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":86,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":87,"commit":82,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4617772,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":90,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":88,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4618177,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":90,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4618332,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":86,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":87,"commit":0,"vote":"0","reject":true,"rejectHint":86}}} +{"level":"debug","ts":1712329731.4629369,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":86,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":86,"commit":81,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.462961,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":87,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":87,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4629717,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":87,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4629815,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":90,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":89,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.46301,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":90,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4630587,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":90,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4630682,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":91,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4630759,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":90,"commit":84,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":89,"next":90}}} +{"level":"debug","ts":1712329731.4630837,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":90,"commit":84,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":89,"next":90}}} +{"level":"debug","ts":1712329731.4630923,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":91,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.463098,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":91,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4631033,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":92,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.463109,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":91,"commit":84,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":89,"next":90}}} +{"level":"debug","ts":1712329731.4631228,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":91,"commit":84,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":89,"next":90}}} +{"level":"debug","ts":1712329731.4641545,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":87,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":88,"commit":82,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.464205,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":87,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":88,"commit":0,"vote":"0","reject":true,"rejectHint":87}}} +{"level":"debug","ts":1712329731.4642096,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":83,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4642396,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":87,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4642467,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4654884,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":90,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.465501,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.466661,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":91,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.466679,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":81},"role":"StateFollower","log":87,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":87,"commit":82,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4666808,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.466702,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":88,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4667122,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.468053,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":84},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":86,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.468076,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4681077,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":92,"commit":86,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":91,"next":92}}} +{"level":"debug","ts":1712329731.4681175,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":92,"commit":86,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":91,"next":92}}} +{"level":"debug","ts":1712329731.468138,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4693274,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":81},"role":"StateFollower","log":83,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":9,"logTerm":2,"index":81,"commit":84,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.469352,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":90,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":90,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4693627,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":90,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4693906,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":85,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.469407,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":86,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4694133,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":83,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4694188,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":89,"commit":84,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4694443,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":89,"commit":0,"vote":"0","reject":true,"rejectHint":88}}} +{"level":"debug","ts":1712329731.4694688,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4706464,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":89,"commit":83,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.470668,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":89,"commit":0,"vote":"0","reject":true,"rejectHint":88}}} +{"level":"debug","ts":1712329731.4706917,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":92,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4707165,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4717102,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":90,"commit":84,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4717453,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":90,"commit":0,"vote":"0","reject":true,"rejectHint":88}}} +{"level":"debug","ts":1712329731.471756,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4728882,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4729054,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":92,"commit":86,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.4729156,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4729242,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":89,"commit":84,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4729578,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":89,"commit":0,"vote":"0","reject":true,"rejectHint":88}}} +{"level":"debug","ts":1712329731.4729698,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4741704,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":90,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":91,"commit":84,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4741905,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":90,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":91,"commit":0,"vote":"0","reject":true,"rejectHint":90}}} +{"level":"debug","ts":1712329731.4742005,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":90,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.47421,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4742174,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":92,"commit":86,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.4753559,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":87,"commit":0,"vote":"0","reject":true,"rejectHint":86}}} +{"level":"debug","ts":1712329731.4753788,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":6,"logTerm":2,"index":86,"commit":86,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.476587,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":88,"commit":0,"vote":"0","reject":true,"rejectHint":87}}} +{"level":"debug","ts":1712329731.4766161,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":90,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":90,"commit":84,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4766371,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":91,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.476658,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":92,"commit":86,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4766812,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":92,"commit":0,"vote":"0","reject":true,"rejectHint":88}}} +{"level":"debug","ts":1712329731.4767237,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4777548,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":87,"commit":0,"vote":"0","reject":true,"rejectHint":86}}} +{"level":"debug","ts":1712329731.477773,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4777915,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":92,"commit":86,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4777992,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":92,"commit":0,"vote":"0","reject":true,"rejectHint":88}}} +{"level":"debug","ts":1712329731.4778483,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":83,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4778614,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.477883,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4791832,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":91,"commit":84,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4792087,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":91,"commit":0,"vote":"0","reject":true,"rejectHint":88}}} +{"level":"debug","ts":1712329731.4792192,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4792442,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":86},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":87,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.479262,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":87},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.479272,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":87},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":5,"logTerm":2,"index":87,"commit":87,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.4792821,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":87},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":92,"commit":87,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.4804733,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":87},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":88,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4804928,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":88},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.480501,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":88},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":92,"commit":88,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.480572,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":88},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":92,"commit":88,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.481802,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":88},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":89,"commit":0,"vote":"0","reject":true,"rejectHint":88}}} +{"level":"debug","ts":1712329731.4818294,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":88},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":4,"logTerm":2,"index":88,"commit":88,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.4832501,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":88},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":89,"commit":0,"vote":"0","reject":true,"rejectHint":88}}} +{"level":"debug","ts":1712329731.4832745,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":88},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4832957,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":92,"commit":86,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4833014,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":82},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":86,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4833093,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":86},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.483319,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":92,"commit":0,"vote":"0","reject":true,"rejectHint":91}}} +{"level":"debug","ts":1712329731.4833615,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":86},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4845262,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":92,"commit":86,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4845557,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":92,"commit":0,"vote":"0","reject":true,"rejectHint":91}}} +{"level":"debug","ts":1712329731.4845655,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4846697,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":88},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":90,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4846847,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4846923,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":92,"commit":90,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.485759,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":92,"commit":0,"vote":"0","reject":true,"rejectHint":88}}} +{"level":"debug","ts":1712329731.4869673,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":89,"commit":0,"vote":"0","reject":true,"rejectHint":88}}} +{"level":"debug","ts":1712329731.4870036,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6"],[]],"prop":{"cc":{"newconf":["2","3","6","7"]}}}} +{"level":"debug","ts":1712329731.4870687,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.4870808,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"7","entries":1,"logTerm":2,"index":91,"commit":90,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.487155,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":88,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.487182,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":90,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4871955,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"InitState","nid":"7","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.4872043,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"7","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4872174,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"7","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"prop":{"cc":{}}}} +{"level":"debug","ts":1712329731.4872875,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"7","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.487323,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.4884138,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":90,"commit":0,"vote":"0","reject":true,"rejectHint":88}}} +{"level":"debug","ts":1712329731.4884214,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":86},"role":"StateFollower","log":88,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":6,"logTerm":2,"index":86,"commit":86,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4884605,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":86},"role":"StateFollower","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":92,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4884722,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":86},"role":"StateFollower","log":92,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4896548,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":86},"role":"StateFollower","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":92,"commit":88,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4896748,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":88},"role":"StateFollower","log":92,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":92,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4897017,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":88},"role":"StateFollower","log":92,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4897048,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4897108,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":91,"commit":0,"vote":"0","reject":true,"rejectHint":88}}} +{"level":"debug","ts":1712329731.4897275,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"7","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"7","entries":1,"logTerm":2,"index":91,"commit":90,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4897962,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"7","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.4898071,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"7","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgAppResp","term":2,"from":"7","to":"2","entries":0,"logTerm":0,"index":91,"commit":0,"vote":"0","reject":true,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4899182,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"7","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.4909356,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":2,"vote":"2","commit":88},"role":"StateFollower","log":92,"conf":[["2","3","6"],[]],"prop":{"cc":{"newconf":["2","3","6","7"]}}}} +{"level":"debug","ts":1712329731.4910052,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"7","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.4910183,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":91,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4910574,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.4922988,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":91,"commit":0,"vote":"0","reject":true,"rejectHint":90}}} +{"level":"debug","ts":1712329731.4923317,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":88},"role":"StateFollower","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":4,"logTerm":2,"index":88,"commit":88,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.49236,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":88},"role":"StateFollower","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":92,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.492369,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":88},"role":"StateFollower","log":92,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.4923844,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":92,"commit":87,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4923913,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":92,"commit":0,"vote":"0","reject":true,"rejectHint":91}}} +{"level":"debug","ts":1712329731.4936397,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":92,"commit":90,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4936638,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":92,"commit":0,"vote":"0","reject":true,"rejectHint":91}}} +{"level":"debug","ts":1712329731.4936745,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":92,"commit":0,"vote":"0","reject":true,"rejectHint":88}}} +{"level":"debug","ts":1712329731.4936752,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4948883,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.494911,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":92,"commit":90,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.494923,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.4949327,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":92,"commit":88,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4949346,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":92,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.4949405,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":93,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4949474,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":92,"commit":90,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.4949567,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":92,"commit":0,"vote":"0","reject":true,"rejectHint":91}}} +{"level":"debug","ts":1712329731.4949718,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.494983,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":88},"role":"StateFollower","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":5,"logTerm":2,"index":87,"commit":87,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4950063,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":88},"role":"StateFollower","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":88,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4950156,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":88},"role":"StateFollower","log":92,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.4962268,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":88},"role":"StateFollower","log":92,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.496262,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":92,"commit":0,"vote":"0","reject":true,"rejectHint":91}}} +{"level":"debug","ts":1712329731.496326,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":2,"logTerm":2,"index":91,"commit":90,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.4974134,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":84},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":90,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4974346,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":90},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4974446,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":90},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.4974504,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4974678,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"7","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"7","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4974723,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":5,"logTerm":2,"index":88,"commit":90,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.4974742,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"7","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"7","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4974802,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"7","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.4987118,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":90},"role":"StateLeader","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":92,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4987328,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":93,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.498741,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":92,"commit":91,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.4987783,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"6","state":{"term":2,"vote":"0","commit":90},"role":"StateFollower","log":91,"conf":[["2","3","6"],[]],"prop":{"cc":{"newconf":["2","3","6","7"]}}}} +{"level":"debug","ts":1712329731.498802,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":93,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.4988427,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ChangeConf","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":93,"conf":[["2","3","6","7"],[]],"prop":{"cc":{"changes":[{"nid":"7","action":"RemoveServer"}]}}}} +{"level":"debug","ts":1712329731.4988532,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":94,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":94,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4988618,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":94,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":93,"commit":91,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.4988642,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":90},"role":"StateFollower","log":91,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.4988692,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":94,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.4988809,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":95,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":95,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.4988892,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":95,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":94,"commit":91,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":92,"next":93}}} +{"level":"debug","ts":1712329731.501478,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":88},"role":"StateFollower","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":88,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5014982,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":88},"role":"StateFollower","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5015073,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":88},"role":"StateFollower","log":92,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5015204,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":95,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":93,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5015273,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":95,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5015426,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":95,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5015597,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":96,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":96,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.501567,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":96,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":95,"commit":91,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":93,"next":94}}} +{"level":"debug","ts":1712329731.5015798,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":96,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5015857,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":96,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.501591,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":97,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.501605,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":96,"commit":91,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":93,"next":94}}} +{"level":"debug","ts":1712329731.5028663,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":90},"role":"StateFollower","log":91,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":92,"commit":90,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5028887,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":90},"role":"StateFollower","log":91,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":92,"commit":0,"vote":"0","reject":true,"rejectHint":91}}} +{"level":"debug","ts":1712329731.5029023,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":90},"role":"StateFollower","log":91,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5029235,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":92,"commit":0,"vote":"0","reject":true,"rejectHint":91}}} +{"level":"debug","ts":1712329731.5040698,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":88},"role":"StateFollower","log":92,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":5,"logTerm":2,"index":88,"commit":90,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5040963,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":90},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":93,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.504107,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":90},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5041203,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":92,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5052817,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":92,"commit":0,"vote":"0","reject":true,"rejectHint":91}}} +{"level":"debug","ts":1712329731.506475,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":92,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5077379,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"7","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5077617,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"7","entries":6,"logTerm":2,"index":91,"commit":91,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":93,"next":94}}} +{"level":"debug","ts":1712329731.5091114,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"7","to":"2","entries":0,"logTerm":0,"index":91,"commit":0,"vote":"0","reject":true,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5091536,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"7","entries":97,"logTerm":0,"index":0,"commit":91,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":93,"next":94}}} +{"level":"debug","ts":1712329731.5104449,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":90},"role":"StateFollower","log":91,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":2,"logTerm":2,"index":91,"commit":90,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.510468,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":90},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":93,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5104775,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":90},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5104868,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":90},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":92,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5104935,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":93,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5105004,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5105133,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":94,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5105424,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5117142,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":95,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.511743,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5117598,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":90},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":92,"commit":90,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5117917,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":90},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":93,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5118015,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":94,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.511824,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":90},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.511824,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":94,"commit":0,"vote":"0","reject":true,"rejectHint":93}}} +{"level":"debug","ts":1712329731.5118423,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5130417,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":96,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5130591,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.513067,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5130734,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":97,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.513079,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":98,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":98,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5130868,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":98,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":97,"commit":91,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":96,"next":97}}} +{"level":"debug","ts":1712329731.5131006,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":98,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5131073,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":98,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5131128,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":98,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"7","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.514271,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":98,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":92,"commit":0,"vote":"0","reject":true,"rejectHint":91}}} +{"level":"debug","ts":1712329731.5142918,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":98,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5142999,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5143106,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":98,"commit":91,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":96,"next":97}}} +{"level":"debug","ts":1712329731.5143166,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":93,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5143342,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":94,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.514343,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5154893,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":92,"commit":0,"vote":"0","reject":true,"rejectHint":91}}} +{"level":"debug","ts":1712329731.516917,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":95,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5169494,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":95,"commit":0,"vote":"0","reject":true,"rejectHint":94}}} +{"level":"debug","ts":1712329731.5169613,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5169716,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"7","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"7","entries":97,"logTerm":0,"index":0,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5169837,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":97,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.516994,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[[],[]],"msg":{"type":"MsgAppResp","term":2,"from":"7","to":"2","entries":0,"logTerm":0,"index":97,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5170457,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5171049,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.5182123,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[[],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"7","entries":6,"logTerm":2,"index":91,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5182438,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[[],[]],"msg":{"type":"MsgAppResp","term":2,"from":"7","to":"2","entries":0,"logTerm":0,"index":97,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5182617,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[[],[]],"prop":{"cc":{"newconf":["1"]}}}} +{"level":"debug","ts":1712329731.5182798,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":88,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5183737,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[["1"],[]],"prop":{"cc":{"newconf":["1","2"]}}}} +{"level":"debug","ts":1712329731.5184958,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5186026,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[["1","2"],[]],"prop":{"cc":{"newconf":["1","2","3"]}}}} +{"level":"debug","ts":1712329731.518736,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[["1","2","3"],[]],"prop":{"cc":{"newconf":["1","2","3","4"]}}}} +{"level":"debug","ts":1712329731.5187793,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[["1","2","3","4"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329731.518817,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["2","3","4","5"]}}}} +{"level":"debug","ts":1712329731.519007,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[["2","3","4","5"],[]],"prop":{"cc":{"newconf":["2","3","5"]}}}} +{"level":"debug","ts":1712329731.5191102,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[["2","3","5"],[]],"prop":{"cc":{"newconf":["2","3"]}}}} +{"level":"debug","ts":1712329731.5192475,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[["2","3"],[]],"prop":{"cc":{"newconf":["2","3","6"]}}}} +{"level":"debug","ts":1712329731.5193527,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[["2","3","6"],[]],"prop":{"cc":{"newconf":["2","3","6","7"]}}}} +{"level":"debug","ts":1712329731.5194862,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5197198,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":92,"commit":0,"vote":"0","reject":true,"rejectHint":91}}} +{"level":"debug","ts":1712329731.5209277,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.520962,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":99,"commit":91,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":97,"next":98}}} +{"level":"debug","ts":1712329731.5222795,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":98,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5223024,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5223143,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5223222,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5234601,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":98,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5234785,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":98,"commit":0,"vote":"0","reject":true,"rejectHint":94}}} +{"level":"debug","ts":1712329731.5234983,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5235205,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5235426,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"7","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5235643,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"7","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5236232,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.524823,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":90},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5248313,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5248497,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5248592,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":8,"logTerm":2,"index":91,"commit":91,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":99,"next":100}}} +{"level":"debug","ts":1712329731.5248618,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5248883,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":97,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.524917,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":97,"commit":0,"vote":"0","reject":true,"rejectHint":94}}} +{"level":"debug","ts":1712329731.524993,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5261908,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":93,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.526212,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.526223,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":96,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5262275,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5262346,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5262442,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":96,"commit":0,"vote":"0","reject":true,"rejectHint":94}}} +{"level":"debug","ts":1712329731.5262458,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"7","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5262554,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5274162,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":91},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":93,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5274324,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":93},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5274403,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":93},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":99,"commit":93,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":99,"next":100}}} +{"level":"debug","ts":1712329731.527461,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":93},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":6,"logTerm":2,"index":93,"commit":93,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":99,"next":100}}} +{"level":"debug","ts":1712329731.528627,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":93},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":93,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5297654,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":93},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":94,"commit":0,"vote":"0","reject":true,"rejectHint":93}}} +{"level":"debug","ts":1712329731.5297897,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":93},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":6,"logTerm":2,"index":93,"commit":93,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":99,"next":100}}} +{"level":"debug","ts":1712329731.5319602,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":93},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":93,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5320277,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":93},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5332444,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":93},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":94,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5332932,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":93},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.533306,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":93},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":5,"logTerm":2,"index":94,"commit":93,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":99,"next":100}}} +{"level":"debug","ts":1712329731.5344603,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":93},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"7","to":"2","entries":0,"logTerm":0,"index":97,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5344822,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.534491,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":99,"commit":94,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":99,"next":100}}} +{"level":"debug","ts":1712329731.5345006,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":99,"commit":94,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":99,"next":100}}} +{"level":"debug","ts":1712329731.534514,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"7","entries":2,"logTerm":2,"index":97,"commit":94,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":99,"next":100}}} +{"level":"debug","ts":1712329731.5345814,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5357447,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":99,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5357757,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":99,"commit":0,"vote":"0","reject":true,"rejectHint":94}}} +{"level":"debug","ts":1712329731.5358078,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":95,"commit":0,"vote":"0","reject":true,"rejectHint":94}}} +{"level":"debug","ts":1712329731.5358102,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.535854,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":5,"logTerm":2,"index":94,"commit":94,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":99,"next":100}}} +{"level":"debug","ts":1712329731.535859,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"7","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5359,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"7","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5359123,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"7","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":97,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.535918,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6","7"],[]],"prop":{"cc":{"newconf":["2","3","6"]}}}} +{"level":"debug","ts":1712329731.536027,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5375733,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":91},"role":"StateFollower","log":94,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":6,"logTerm":2,"index":93,"commit":93,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.537599,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":97,"commit":0,"vote":"0","reject":true,"rejectHint":94}}} +{"level":"debug","ts":1712329731.5376005,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":93},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.537614,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":93},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5389326,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":98,"commit":0,"vote":"0","reject":true,"rejectHint":94}}} +{"level":"debug","ts":1712329731.5390177,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.540062,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":93},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.540112,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5401363,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":93},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5401502,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":93},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5401652,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":99,"commit":94,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":99,"next":100}}} +{"level":"debug","ts":1712329731.5413723,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5413978,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":5,"logTerm":2,"index":94,"commit":94,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":99,"next":100}}} +{"level":"debug","ts":1712329731.5414705,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5428462,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":96,"commit":0,"vote":"0","reject":true,"rejectHint":94}}} +{"level":"debug","ts":1712329731.5429082,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":99,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.54292,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":100,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.54293,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":99,"commit":94,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":99,"next":100}}} +{"level":"debug","ts":1712329731.5429473,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5440266,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":99,"commit":0,"vote":"0","reject":true,"rejectHint":94}}} +{"level":"debug","ts":1712329731.5440483,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":93,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":8,"logTerm":2,"index":91,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5440567,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5440915,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5441005,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":93},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":99,"commit":94,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.544107,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.545222,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":99,"commit":93,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5452428,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5452545,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.545268,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":91},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":99,"commit":94,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5452893,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.54532,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":100,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5453384,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.545346,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":94,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5453641,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":93,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5464728,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":5,"logTerm":2,"index":94,"commit":93,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5464916,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.546508,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"prop":{"cc":{"newconf":["2","3","6"]}}}} +{"level":"debug","ts":1712329731.546513,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":94},"role":"StateLeader","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5465362,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5465467,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":99,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":100,"next":101}}} +{"level":"debug","ts":1712329731.5465622,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":100,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":100,"next":101}}} +{"level":"debug","ts":1712329731.5465748,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5466106,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":91,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5466323,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.547923,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":5,"logTerm":2,"index":94,"commit":94,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5479374,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":6,"logTerm":2,"index":93,"commit":93,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5479484,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5479574,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":94,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5492618,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":5,"logTerm":2,"index":94,"commit":94,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.549314,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5493345,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.549342,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":100,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":100,"next":101}}} +{"level":"debug","ts":1712329731.5493534,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5493546,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":99,"commit":94,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.54936,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":101,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5493846,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5494008,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]]}} +{"level":"debug","ts":1712329731.5494199,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":100,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":100,"next":101}}} +{"level":"debug","ts":1712329731.549431,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":100,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":100,"next":101}}} +{"level":"debug","ts":1712329731.5505931,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5506184,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":101,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5506287,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":101,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5506358,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":101,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.550642,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":102,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":102,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5506496,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":102,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":101,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":100,"next":101}}} +{"level":"debug","ts":1712329731.5506632,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":102,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":101,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":100,"next":101}}} +{"level":"debug","ts":1712329731.5506723,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":102,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5506785,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":103,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":103,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5506856,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":103,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":102,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":100,"next":101}}} +{"level":"debug","ts":1712329731.5506861,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":94,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5506926,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":103,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":102,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":100,"next":101}}} +{"level":"debug","ts":1712329731.5507092,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.55072,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5507252,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6","7"],[]],"prop":{"cc":{"newconf":["2","3","6"]}}}} +{"level":"debug","ts":1712329731.5519788,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5532987,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":100,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5533261,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":100,"commit":0,"vote":"0","reject":true,"rejectHint":99}}} +{"level":"debug","ts":1712329731.5533571,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":103,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":101,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.553367,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":103,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5533745,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":103,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5533805,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":103,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.553386,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":104,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":104,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5533931,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":104,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":103,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":101,"next":102}}} +{"level":"debug","ts":1712329731.5534017,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":104,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":103,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":101,"next":102}}} +{"level":"debug","ts":1712329731.5546486,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":104,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5546873,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":93,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5547075,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5547173,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5561526,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":104,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":102,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5561714,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":99,"commit":94,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.556191,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":104,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5562336,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":100,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5562565,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5576108,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":101,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5576332,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":101,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.5576448,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":100,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5576513,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":100,"commit":0,"vote":"0","reject":true,"rejectHint":99}}} +{"level":"debug","ts":1712329731.5576582,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5576658,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":104,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":103,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5576773,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":104,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5589955,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":104,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5590284,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":104,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":104,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":103,"next":104}}} +{"level":"debug","ts":1712329731.5590513,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":104,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5590668,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":105,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":105,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5590792,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":105,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":104,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":103,"next":104}}} +{"level":"debug","ts":1712329731.559086,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":102,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.559104,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":105,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":104,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":103,"next":104}}} +{"level":"debug","ts":1712329731.5591152,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":102,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.5591266,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":105,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5604587,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":105,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.56048,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":94},"role":"StateFollower","log":99,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":99,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5605052,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":100,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5605218,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":94},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":100,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5605419,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":101,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5605605,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5617776,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":102,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.561804,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":102,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.561819,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5618315,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":105,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":104,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.56186,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":105,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5630763,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":103,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5631003,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":103,"commit":0,"vote":"0","reject":true,"rejectHint":101}}} +{"level":"debug","ts":1712329731.5631135,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":101,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.563118,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5631344,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":101,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.5631385,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":105,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":105,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5631657,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":105,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.563214,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":105,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5632243,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":106,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":106,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.563232,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":106,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":105,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":105,"next":106}}} +{"level":"debug","ts":1712329731.56324,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":106,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":105,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":105,"next":106}}} +{"level":"debug","ts":1712329731.563248,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":106,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.563253,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":107,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":107,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.563272,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":107,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":106,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":105,"next":106}}} +{"level":"debug","ts":1712329731.5632794,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":107,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":106,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":105,"next":106}}} +{"level":"debug","ts":1712329731.563288,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":107,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.563294,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":108,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":108,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5633314,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":108,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":107,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":105,"next":106}}} +{"level":"debug","ts":1712329731.5633445,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":108,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":107,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":105,"next":106}}} +{"level":"debug","ts":1712329731.5633535,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":108,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5643258,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":100,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5643494,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":100,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5643837,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5644298,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":108,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":94,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.565429,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":104,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5654466,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":104,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.5655186,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":108,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":106,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.565542,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":108,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5666869,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":108,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":107,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5667083,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":108,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5667243,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":106,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5667255,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":106,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5667343,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":106,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.5667832,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":106,"commit":0,"vote":"0","reject":true,"rejectHint":101}}} +{"level":"debug","ts":1712329731.5678706,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":108,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":108,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5679014,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":108,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5679157,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":103,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5679402,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":103,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.5679407,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":104,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.567951,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5679927,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":104,"commit":0,"vote":"0","reject":true,"rejectHint":101}}} +{"level":"debug","ts":1712329731.5680523,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5691018,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":104,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5691257,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":104,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.5691383,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5691469,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5691926,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":108,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.570392,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":108,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.571867,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":105,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5718896,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":105,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.5718985,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5719092,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":105,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5719137,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":108,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5719218,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":108,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":108,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":108,"next":109}}} +{"level":"debug","ts":1712329731.5719285,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":105,"commit":0,"vote":"0","reject":true,"rejectHint":101}}} +{"level":"debug","ts":1712329731.571938,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5719955,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":108,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5720232,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":109,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":109,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.572036,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":109,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":108,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":108,"next":109}}} +{"level":"debug","ts":1712329731.5720682,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":109,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":108,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":108,"next":109}}} +{"level":"debug","ts":1712329731.5720782,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":109,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5720978,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":109,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5721047,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":109,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.57319,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":109,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5748017,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":109,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":109,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5748236,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":109,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5748503,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":107,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5748599,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":107,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.5748641,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":107,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.574867,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5749135,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":107,"commit":0,"vote":"0","reject":true,"rejectHint":101}}} +{"level":"debug","ts":1712329731.5749218,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.576051,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5760722,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":109,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.576081,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":109,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5760875,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":110,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5760965,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":109,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":109,"next":110}}} +{"level":"debug","ts":1712329731.5761065,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":109,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":109,"next":110}}} +{"level":"debug","ts":1712329731.5761204,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.577588,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":99,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5788164,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":100,"commit":0,"vote":"0","reject":true,"rejectHint":99}}} +{"level":"debug","ts":1712329731.5788474,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":11,"logTerm":2,"index":99,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":109,"next":110}}} +{"level":"debug","ts":1712329731.5788598,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":108,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.578881,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":108,"commit":0,"vote":"0","reject":true,"rejectHint":101}}} +{"level":"debug","ts":1712329731.57913,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5800498,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.580086,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":11,"logTerm":2,"index":99,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":109,"next":110}}} +{"level":"debug","ts":1712329731.5800972,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5801148,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5801485,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5802267,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.581408,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":100,"commit":0,"vote":"0","reject":true,"rejectHint":99}}} +{"level":"debug","ts":1712329731.5814376,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":11,"logTerm":2,"index":99,"commit":99,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":109,"next":110}}} +{"level":"debug","ts":1712329731.5825186,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":99},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":100,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.582541,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":100},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.582557,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":100},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":10,"logTerm":2,"index":100,"commit":100,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":109,"next":110}}} +{"level":"debug","ts":1712329731.5838308,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5838494,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5838623,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":100},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":110,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.583869,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":100},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5838785,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":100},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5838807,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":108,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5838845,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":100},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":100,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5839067,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":108,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.5839162,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5852888,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":100},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":101,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.5853066,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":100},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":10,"logTerm":2,"index":100,"commit":100,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":110,"next":111}}} +{"level":"debug","ts":1712329731.585317,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":100},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5853283,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":101,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":11,"logTerm":2,"index":99,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.585336,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":110,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.585343,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5863922,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":108,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5864117,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":108,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5864277,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":100},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":102,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.5864685,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":109,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5864918,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":109,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.5865378,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5875711,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":100},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":101,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.5876236,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":109,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.587649,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":110,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5877652,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5887756,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":100},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":101,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.588799,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5888085,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":9,"logTerm":2,"index":101,"commit":101,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":110,"next":111}}} +{"level":"debug","ts":1712329731.5888228,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":99},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":100,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5888598,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":100},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5888782,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":100},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5900202,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":104,"commit":0,"vote":"0","reject":true,"rejectHint":101}}} +{"level":"debug","ts":1712329731.5900502,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":9,"logTerm":2,"index":101,"commit":101,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":110,"next":111}}} +{"level":"debug","ts":1712329731.590065,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":110,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5900733,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":111,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":111,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.590081,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":111,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5900862,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":112,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":112,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5900936,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":112,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5900986,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":112,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5901039,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":113,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":113,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5911372,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":100},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":11,"logTerm":2,"index":99,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5911708,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":100},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":100,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5911796,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":100},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.591191,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":113,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":103,"commit":0,"vote":"0","reject":true,"rejectHint":101}}} +{"level":"debug","ts":1712329731.5912378,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5923667,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":113,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":111,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5923903,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":113,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5924013,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":100},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":10,"logTerm":2,"index":100,"commit":100,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.592409,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":100},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":110,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5924227,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":100},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.593591,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":100},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":9,"logTerm":2,"index":101,"commit":101,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5936184,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":101},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":110,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5936482,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":113,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":112,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.593662,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":101},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5936747,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":113,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5936859,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":113,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.593692,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":114,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":114,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.594926,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":114,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":104,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.594951,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":114,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.594959,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":115,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":115,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5949671,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":115,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.594976,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":101},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":10,"logTerm":2,"index":100,"commit":100,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5949807,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5949876,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5949986,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":101},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":101,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5950198,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":101},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5950227,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5961368,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":115,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":103,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.5961556,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":115,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5961618,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":116,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5976007,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5976236,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":100,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.5976446,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.5976539,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":16,"logTerm":2,"index":100,"commit":101,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":112,"next":113}}} +{"level":"debug","ts":1712329731.5990725,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":102,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.5991004,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":16,"logTerm":2,"index":100,"commit":101,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":112,"next":113}}} +{"level":"debug","ts":1712329731.600545,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":100,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6005814,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6006136,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":16,"logTerm":2,"index":100,"commit":101,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":112,"next":113}}} +{"level":"debug","ts":1712329731.6016326,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":106,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.601671,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":16,"logTerm":2,"index":100,"commit":101,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":112,"next":113}}} +{"level":"debug","ts":1712329731.6026888,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":104,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.6038246,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":107,"commit":0,"vote":"0","reject":true,"rejectHint":101}}} +{"level":"debug","ts":1712329731.6051748,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":106,"commit":0,"vote":"0","reject":true,"rejectHint":101}}} +{"level":"debug","ts":1712329731.6069252,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":101},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":9,"logTerm":2,"index":101,"commit":101,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6069624,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":113,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.606987,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6069903,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":100,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":11,"logTerm":2,"index":99,"commit":99,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6070173,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":101},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":110,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.607026,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":101},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6070378,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":110,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.607097,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6081002,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":114,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6081223,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6091805,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":115,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.609205,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6092145,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.609222,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":100,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6092465,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":101,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6092567,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ChangeConf","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":116,"conf":[["2","3","6"],[]],"prop":{"cc":{"changes":[{"nid":"8","action":"AddNewServer"}]}}}} +{"level":"debug","ts":1712329731.60928,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":117,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":117,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6102967,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":117,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":105,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.610316,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":117,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6103237,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6113887,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":116,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6114216,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6114354,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.612916,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":105,"commit":0,"vote":"0","reject":true,"rejectHint":101}}} +{"level":"debug","ts":1712329731.6129568,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":99},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":16,"logTerm":2,"index":100,"commit":101,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6129813,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":116,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6141016,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":117,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6141238,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6141524,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":16,"logTerm":2,"index":100,"commit":101,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.614177,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":101,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6142247,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.615136,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6151583,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6162384,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":107,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.6174536,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":108,"commit":0,"vote":"0","reject":true,"rejectHint":101}}} +{"level":"debug","ts":1712329731.6194286,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":101},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":110,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6194577,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.619477,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":8,"logTerm":2,"index":110,"commit":110,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.6194975,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":16,"logTerm":2,"index":100,"commit":101,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6195216,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":101,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.619531,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.620647,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":16,"logTerm":2,"index":100,"commit":101,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6206691,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":101,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6207407,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6207666,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.620792,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":118,"commit":110,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.620804,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6217422,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":101},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":101,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6217709,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":101},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6217804,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":101},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6218674,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6218858,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":18,"logTerm":2,"index":100,"commit":110,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.6229534,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":108,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.6242752,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":101},"role":"StateFollower","log":110,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":8,"logTerm":2,"index":110,"commit":110,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.624298,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":110},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.624308,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":110},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6243196,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":109,"commit":0,"vote":"0","reject":true,"rejectHint":100}}} +{"level":"debug","ts":1712329731.624321,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":100,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6243439,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6243763,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6254416,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":108,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6254647,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6266716,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":100,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6266959,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":100,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6267045,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":110,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6280763,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6281147,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":118,"commit":110,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.6292107,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":110,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6303356,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6303608,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":18,"logTerm":2,"index":100,"commit":110,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.6322474,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":110,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6322808,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":110},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":118,"commit":110,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6322834,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":18,"logTerm":2,"index":100,"commit":110,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6322932,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":101,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6323087,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":110},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6323364,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":110},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6323638,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6332748,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":101,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6333,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6344717,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":110,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6355822,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":110,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6356254,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":110},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":110,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6356266,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6356378,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":8,"logTerm":2,"index":110,"commit":110,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.6356533,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":110},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.63567,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.636706,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":110},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":118,"commit":110,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.63671,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":110,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6367269,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":110},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6367362,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":110},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6384907,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":101,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6385183,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":100,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6385455,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6385636,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6396475,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":110},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":116,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6396706,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":116},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6396787,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":116},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":118,"commit":116,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.639688,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":116},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":118,"commit":116,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.6407502,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":116},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":101,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6418583,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":116},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6418798,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6418898,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":118,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.641899,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":118,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.64299,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":101,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6430469,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6445642,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.64459,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":118,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.6445966,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":18,"logTerm":2,"index":100,"commit":110,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6446195,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":101,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.64463,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.6456475,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6456845,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6"],[]],"prop":{"cc":{"newconf":["2","3","6","8"]}}}} +{"level":"debug","ts":1712329731.6459055,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6459818,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":1,"logTerm":2,"index":117,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.6472645,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6473083,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":101},"role":"StateFollower","log":116,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":8,"logTerm":2,"index":110,"commit":110,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6473272,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":110},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6486447,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":101,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6486723,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":116,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6486802,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6486876,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6486945,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6487005,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":110},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":118,"commit":116,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.648708,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":110},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":118,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6487231,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":116},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6487432,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":116},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.648754,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6487613,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.649882,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6499178,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":116},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":118,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6499276,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6499925,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]],"prop":{"cc":{"newconf":["2","3","6","8"]}}}} +{"level":"debug","ts":1712329731.6500335,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]]}} +{"level":"debug","ts":1712329731.651012,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6510968,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6"],[]],"prop":{"cc":{"newconf":["2","3","6","8"]}}}} +{"level":"debug","ts":1712329731.6512299,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"InitState","nid":"8","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.6512444,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"8","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"prop":{"cc":{}}}} +{"level":"debug","ts":1712329731.6512742,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"8","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.652551,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":101,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.65264,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6538935,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6539204,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":118,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.6539338,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6551085,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6551304,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.655139,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6551552,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6551778,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6563199,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6642263,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6642535,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6642623,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6643128,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6737719,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":118,"commit":116,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.673795,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.673816,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.67386,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6738775,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":118,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6738966,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"8","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.6739151,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6739373,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":118,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6739466,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6739528,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":119,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":119,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6739619,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":119,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":118,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.6739666,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6739712,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":119,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":118,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.6739807,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":119,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6740513,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.6749868,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6750124,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6750224,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6750274,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":116,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.675045,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":119,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6750548,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6750658,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":1,"logTerm":2,"index":117,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.675073,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":117,"commit":0,"vote":"0","reject":true,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6761787,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":119,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6761992,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":119,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":119,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.6762097,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":119,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6762154,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":120,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6762228,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":119,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.67623,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":119,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":118,"next":119}}} +{"level":"debug","ts":1712329731.676249,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6762512,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":118,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6762593,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6762688,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6762793,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.6775186,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":119,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.677544,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":120,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6775534,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":120,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6775663,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6775763,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.678741,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":118,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.67877,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":119,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":119,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.678782,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":119,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6787846,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":118,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":118,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.678808,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":119,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":119,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.678821,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":119,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.678838,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6788661,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":120,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":119,"next":120}}} +{"level":"debug","ts":1712329731.6802385,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":120,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6802576,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":120,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6802664,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":120,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6802728,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":120,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6802871,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":121,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6802945,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":120,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":120,"next":121}}} +{"level":"debug","ts":1712329731.6803024,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":120,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":120,"next":121}}} +{"level":"debug","ts":1712329731.6803112,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":119,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6815593,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":117,"commit":0,"vote":"0","reject":true,"rejectHint":0}}} +{"level":"debug","ts":1712329731.681612,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":121,"logTerm":0,"index":0,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":120,"next":121}}} +{"level":"debug","ts":1712329731.6816168,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":119,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":119,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6816397,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":120,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.68164,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.6816528,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6816607,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":121,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6829066,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":119,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":120,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.682929,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":119,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":120,"commit":0,"vote":"0","reject":true,"rejectHint":119}}} +{"level":"debug","ts":1712329731.6829383,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":119,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6829934,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":119,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6830184,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":121,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6830277,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ChangeConf","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":121,"conf":[["2","3","6","8"],[]],"prop":{"cc":{"changes":[{"nid":"9","action":"AddNewServer"}]}}}} +{"level":"debug","ts":1712329731.6830359,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":122,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":122,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6830435,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":122,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":121,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":120,"next":121}}} +{"level":"debug","ts":1712329731.683051,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":122,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":121,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":120,"next":121}}} +{"level":"debug","ts":1712329731.684317,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":122,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":121,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6843429,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":122,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.684353,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":122,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6843657,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":122,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6843724,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":122,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6843967,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":122,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6854162,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":119,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":119,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.68544,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":120,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6854494,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6854615,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":122,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6854882,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":122,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":122,"logTerm":0,"index":0,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":121,"next":122}}} +{"level":"debug","ts":1712329731.6868193,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":122,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":122,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6868503,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":122,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6868734,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":122,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.686887,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":122,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6868935,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":123,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":123,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6869009,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":123,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":122,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":122,"next":123}}} +{"level":"debug","ts":1712329731.6869085,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":123,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":122,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":122,"next":123}}} +{"level":"debug","ts":1712329731.6869159,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":123,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6869211,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":124,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":124,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6869273,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":124,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":123,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":122,"next":123}}} +{"level":"debug","ts":1712329731.6869335,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":124,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":123,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":122,"next":123}}} +{"level":"debug","ts":1712329731.6869402,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":124,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6869452,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6869543,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":124,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":122,"next":123}}} +{"level":"debug","ts":1712329731.686964,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":124,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":122,"next":123}}} +{"level":"debug","ts":1712329731.688254,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6882746,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6882842,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6882956,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":121,"logTerm":0,"index":0,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6883006,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.68831,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":125,"commit":118,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":122,"next":123}}} +{"level":"debug","ts":1712329731.6883247,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[[],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":121,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6883383,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.6894445,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.689693,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6897902,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[[],[]],"prop":{"cc":{"newconf":["1"]}}}} +{"level":"debug","ts":1712329731.689945,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["1"],[]],"prop":{"cc":{"newconf":["1","2"]}}}} +{"level":"debug","ts":1712329731.690003,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["1","2"],[]],"prop":{"cc":{"newconf":["1","2","3"]}}}} +{"level":"debug","ts":1712329731.6900342,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["1","2","3"],[]],"prop":{"cc":{"newconf":["1","2","3","4"]}}}} +{"level":"debug","ts":1712329731.6901834,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["1","2","3","4"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329731.6903136,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["2","3","4","5"]}}}} +{"level":"debug","ts":1712329731.6904287,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","4","5"],[]],"prop":{"cc":{"newconf":["2","3","5"]}}}} +{"level":"debug","ts":1712329731.6905303,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","5"],[]],"prop":{"cc":{"newconf":["2","3"]}}}} +{"level":"debug","ts":1712329731.6909306,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3"],[]],"prop":{"cc":{"newconf":["2","3","6"]}}}} +{"level":"debug","ts":1712329731.6909664,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":119,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6909878,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":119,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6910384,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6"],[]],"prop":{"cc":{"newconf":["2","3","6","7"]}}}} +{"level":"debug","ts":1712329731.6910648,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","7"],[]],"prop":{"cc":{"newconf":["2","3","6"]}}}} +{"level":"debug","ts":1712329731.6910946,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6911347,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6"],[]],"prop":{"cc":{"newconf":["2","3","6","8"]}}}} +{"level":"debug","ts":1712329731.6911376,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":118},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":120,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6911664,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.691176,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":125,"commit":119,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":122,"next":123}}} +{"level":"debug","ts":1712329731.691186,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":125,"commit":119,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":122,"next":123}}} +{"level":"debug","ts":1712329731.69264,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":121,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.692665,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":121,"commit":0,"vote":"0","reject":true,"rejectHint":120}}} +{"level":"debug","ts":1712329731.6926908,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.692693,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":123,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6927013,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6927094,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6927338,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6927452,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6939137,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":124,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6939464,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6940184,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":121,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6940567,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":121,"commit":0,"vote":"0","reject":true,"rejectHint":120}}} +{"level":"debug","ts":1712329731.695196,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":124,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6952224,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":120,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6952307,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":124,"commit":0,"vote":"0","reject":true,"rejectHint":120}}} +{"level":"debug","ts":1712329731.6952436,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":122,"logTerm":0,"index":0,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6952538,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6952548,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":121,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6952746,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6952968,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6953099,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6953235,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6953492,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6954033,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.696509,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":122,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6965444,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":122,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.6965687,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":120,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":120,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6965768,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":119,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6965964,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":121,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6978347,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6978807,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":125,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6979058,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":125,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.6979063,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":124,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6979382,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":124,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.6979523,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.6990962,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6991215,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":125,"logTerm":0,"index":0,"commit":119,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.6991603,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6991875,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.6992054,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7005634,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":123,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7005877,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":123,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.7005982,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7006147,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":125,"commit":119,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7006218,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":125,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.7006228,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":120,"commit":0,"vote":"0","reject":true,"rejectHint":119}}} +{"level":"debug","ts":1712329731.7006326,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.70065,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":6,"logTerm":2,"index":119,"commit":119,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7017813,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7018175,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":122,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7018352,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":122,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.701882,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7029555,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7029748,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":125,"commit":119,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7041118,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":119,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7041283,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.70414,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":6,"logTerm":2,"index":119,"commit":119,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7041588,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7041678,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":123,"commit":118,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.70417,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":125,"commit":119,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.704175,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":123,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.704187,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7041962,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":125,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.7042053,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7055664,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":119},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":121,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.705606,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7056231,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":125,"commit":120,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7056396,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":125,"commit":120,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7056572,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":4,"logTerm":2,"index":121,"commit":120,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7067468,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.706769,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":125,"commit":120,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7067852,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7068007,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":120,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7068083,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":119,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7068138,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":120,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7079587,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":121,"commit":0,"vote":"0","reject":true,"rejectHint":120}}} +{"level":"debug","ts":1712329731.7079923,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":6,"logTerm":2,"index":119,"commit":120,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7093034,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":120,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7093258,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7093358,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":5,"logTerm":2,"index":120,"commit":120,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7105474,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7117875,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":124,"commit":0,"vote":"0","reject":true,"rejectHint":120}}} +{"level":"debug","ts":1712329731.7118192,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":5,"logTerm":2,"index":120,"commit":120,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7131772,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":125,"commit":119,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7132242,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":125,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.7132566,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7132876,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":125,"logTerm":0,"index":0,"commit":119,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7133067,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":6,"logTerm":2,"index":119,"commit":119,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7133067,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7133245,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7133503,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":119},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7133896,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":121,"commit":0,"vote":"0","reject":true,"rejectHint":120}}} +{"level":"debug","ts":1712329731.7145085,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":119},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":6,"logTerm":2,"index":119,"commit":119,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7145326,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":119},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7145426,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":124,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.7145457,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":119},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.714568,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":5,"logTerm":2,"index":120,"commit":120,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7145858,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7159526,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7159755,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":125,"commit":120,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7160075,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":4,"logTerm":2,"index":121,"commit":120,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7160304,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7160406,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7160437,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":125,"commit":120,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7160578,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":125,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.7160661,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7172453,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.717265,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":5,"logTerm":2,"index":120,"commit":120,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.718475,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":122,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.7198894,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":120},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":121,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7199147,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7199218,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":120,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7199378,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":4,"logTerm":2,"index":121,"commit":121,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7199492,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7199607,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":125,"commit":121,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.719968,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.719969,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":119},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":119,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7199762,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.719995,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":119},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.721145,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":119},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":125,"commit":120,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7211673,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.72118,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":125,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.721188,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":4,"logTerm":2,"index":121,"commit":121,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7223542,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":125,"commit":120,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7223828,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7223892,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":121,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7223926,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7224102,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":125,"commit":120,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7224174,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7224472,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7224648,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":4,"logTerm":2,"index":121,"commit":121,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7237713,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":123,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.7238002,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":5,"logTerm":2,"index":120,"commit":120,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7238097,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.723818,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.724922,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":125,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.7249467,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":4,"logTerm":2,"index":121,"commit":121,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.726434,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":5,"logTerm":2,"index":120,"commit":120,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.726457,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.726467,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7264783,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":118},"role":"StateFollower","log":121,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":5,"logTerm":2,"index":120,"commit":120,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7264798,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":122,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.7264855,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7278435,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":125,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.7278697,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":121,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7278848,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":121,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7278907,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":121,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.727898,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7278984,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":120,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7279177,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":6,"logTerm":2,"index":119,"commit":120,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7279236,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.727925,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":120,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7279341,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7292414,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":4,"logTerm":2,"index":121,"commit":121,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.729262,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7292745,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7292874,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":123,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.7293055,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":5,"logTerm":2,"index":120,"commit":120,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7293162,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.729342,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.730477,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":125,"commit":121,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7305005,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7305098,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.730533,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7305548,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7316926,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":125,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.7329147,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":121,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.732937,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7329514,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7329607,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7329729,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":120},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":4,"logTerm":2,"index":121,"commit":121,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.73298,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.732987,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7342856,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7343073,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":125,"commit":121,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.734318,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7355242,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":2,"index":125,"commit":0,"vote":"0","reject":true,"rejectHint":121}}} +{"level":"debug","ts":1712329731.7355692,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":121,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7355945,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7356086,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7368865,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":118,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.738147,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":4,"logTerm":2,"index":121,"commit":121,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7381709,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7381802,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.738202,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7393537,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.739399,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":4,"logTerm":2,"index":121,"commit":121,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.73941,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.740591,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":121,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.740616,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7406251,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7406313,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":121},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7406533,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.740662,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":125,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7406707,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":125,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.741926,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7419453,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":4,"logTerm":2,"index":121,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7419558,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7431896,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":120,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7432518,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8"],[]],"prop":{"cc":{"newconf":["2","3","6","8","9"]}}}} +{"level":"debug","ts":1712329731.7433724,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7433932,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"9","entries":1,"logTerm":2,"index":124,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7434905,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"InitState","nid":"9","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.7435062,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"9","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"prop":{"cc":{}}}} +{"level":"debug","ts":1712329731.7435882,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"9","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.7444947,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":125,"commit":121,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.744517,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7445278,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7445364,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":125,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7445421,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7445493,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.744562,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7445881,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":121,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7446008,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7446072,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7446132,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"9","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7457116,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"prop":{"cc":{"newconf":["2","3","6","8","9"]}}}} +{"level":"debug","ts":1712329731.7457495,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7457833,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.745814,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7470448,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":125,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7470975,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7471073,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7471123,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7471163,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7483068,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"prop":{"cc":{"newconf":["2","3","6","8","9"]}}}} +{"level":"debug","ts":1712329731.7483258,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":121},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":4,"logTerm":2,"index":121,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7483468,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7483556,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]]}} +{"level":"debug","ts":1712329731.7483735,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7483883,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7498758,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7499006,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.749916,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7499504,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.749977,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8"],[]],"prop":{"cc":{"newconf":["2","3","6","8","9"]}}}} +{"level":"debug","ts":1712329731.7499907,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7500045,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7500083,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7511718,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"9","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"9","entries":1,"logTerm":2,"index":124,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7512434,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"9","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.7512543,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"9","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgAppResp","term":2,"from":"9","to":"2","entries":0,"logTerm":0,"index":124,"commit":0,"vote":"0","reject":true,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7512658,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7512937,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7513034,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":125,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7524369,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"9","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"9","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7524583,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"9","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"9","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7524774,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7524834,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"9","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.7534974,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7535143,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":125,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7535214,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":126,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":126,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7535312,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":126,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":125,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7535405,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":126,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":125,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.753549,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":126,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":1,"logTerm":2,"index":125,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7535572,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":126,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7535632,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":127,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7535708,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":126,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.753582,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":126,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.7535887,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":1,"logTerm":2,"index":126,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.753596,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":127,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.754922,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7549446,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":127,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":125,"next":126}}} +{"level":"debug","ts":1712329731.75609,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.756106,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":125,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7561252,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":125,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7561324,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":126,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":126,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7561514,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":126,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":126,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7561615,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":126,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7561824,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":126,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7571592,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":126,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7571828,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":127,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7571933,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":126,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":126,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.757201,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":127,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7574937,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.758343,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":126,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":126,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7583654,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":127,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.758377,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":127,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7583847,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":127,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7583969,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Replicate","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":127,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7584028,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":128,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7584102,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":127,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":127,"next":128}}} +{"level":"debug","ts":1712329731.7584188,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":127,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":127,"next":128}}} +{"level":"debug","ts":1712329731.7584262,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":1,"logTerm":2,"index":127,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":127,"next":128}}} +{"level":"debug","ts":1712329731.759447,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.759516,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":121,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.759541,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7595525,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7605915,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.761419,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":125,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":1,"logTerm":2,"index":125,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7614477,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":126,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":126,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7614574,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":126,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7616336,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7628617,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7628884,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":128,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":127,"next":128}}} +{"level":"debug","ts":1712329731.7639046,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7639327,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":126,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":1,"logTerm":2,"index":126,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7639563,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":127,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7639673,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7649405,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.765061,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":128,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":127,"next":128}}} +{"level":"debug","ts":1712329731.7650807,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":125,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7651124,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.765123,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7667952,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7669365,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":128,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":127,"next":128}}} +{"level":"debug","ts":1712329731.7669766,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7669895,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.767001,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7670076,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"9","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7670271,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ChangeConf","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":128,"conf":[["2","3","6","8","9"],[]],"prop":{"cc":{"changes":[{"nid":"10","action":"AddNewServer"}]}}}} +{"level":"debug","ts":1712329731.7670388,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.767052,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":128,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":127,"next":128}}} +{"level":"debug","ts":1712329731.7670605,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":128,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":127,"next":128}}} +{"level":"debug","ts":1712329731.7670712,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":1,"logTerm":2,"index":128,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":127,"next":128}}} +{"level":"debug","ts":1712329731.7670796,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7683744,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7694314,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":128,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7694566,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7708068,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"2","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.770838,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.770844,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":128,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7708716,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":2,"index":128,"commit":0,"vote":"0","reject":true,"rejectHint":127}}} +{"level":"debug","ts":1712329731.7718637,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7718945,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":128,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7719054,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":128,"commit":0,"vote":"0","reject":true,"rejectHint":127}}} +{"level":"debug","ts":1712329731.7719138,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":1,"logTerm":2,"index":127,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7719297,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7719388,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":128,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7719755,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":128,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7728972,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"9","to":"2","entries":0,"logTerm":0,"index":124,"commit":0,"vote":"0","reject":true,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7729242,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"9","entries":129,"logTerm":0,"index":0,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.7730203,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":1,"logTerm":2,"index":128,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7730384,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.77305,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7734592,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":127,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7734811,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":128,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7735038,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":128,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7743967,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":127,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":127,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.774448,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":128,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7744675,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":128,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.774486,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7745998,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"9","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"9","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.774634,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"9","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"9","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7746572,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"9","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.7755213,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":126,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.775547,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7756855,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":128,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7757144,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":128,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7757242,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":128,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7766037,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":128,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.77663,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7766404,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7766492,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":127,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7766604,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":127,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7766678,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7766783,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7766871,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":129,"commit":125,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.7767365,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7767594,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7767787,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":128,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7776527,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.777681,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7777023,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":125},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":127,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7777176,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":126},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.77774,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":126},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":129,"commit":126,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.77776,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":126},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":129,"commit":126,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.777778,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":126},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":129,"commit":126,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.7777975,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7778084,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.777824,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7778575,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7787995,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":126},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":126,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7788236,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":126},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7788363,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":128,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":1,"logTerm":2,"index":128,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.778844,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7788568,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.77989,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":126},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"9","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7799168,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":126},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"9","entries":129,"logTerm":0,"index":0,"commit":126,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.7809339,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":126},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":127,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.780958,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7809677,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":129,"commit":127,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.7809777,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":129,"commit":127,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.7809858,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":129,"commit":127,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.7820005,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.783173,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":129,"commit":126,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7832015,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":126},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.783212,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":126},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7832217,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":129,"commit":126,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7832284,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":126},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7832303,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":127,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7832353,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":126},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.783244,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":129,"commit":126,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7832599,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7832706,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7832837,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":126},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7832918,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":126},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.78431,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":126},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":129,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7843375,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":126},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7843497,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":126},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7843566,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":126,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7855995,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":2,"index":128,"commit":0,"vote":"0","reject":true,"rejectHint":127}}} +{"level":"debug","ts":1712329731.7856274,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":2,"logTerm":2,"index":127,"commit":127,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.7867799,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":128,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7868083,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7868237,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":126},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":129,"commit":127,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7868319,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7868392,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.788588,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7886112,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":126},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":129,"commit":127,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7886157,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7886307,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":127,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.78864,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":127,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7886472,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":127,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7886481,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"9","state":{"term":2,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"9","entries":129,"logTerm":0,"index":0,"commit":125,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7886539,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"9","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7886658,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7886772,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7886772,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[[],[]],"msg":{"type":"MsgAppResp","term":2,"from":"9","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7887018,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.790041,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[[],[]],"prop":{"cc":{"newconf":["1"]}}}} +{"level":"debug","ts":1712329731.7900677,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":2,"index":128,"commit":0,"vote":"0","reject":true,"rejectHint":127}}} +{"level":"debug","ts":1712329731.7900925,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7901268,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["1"],[]],"prop":{"cc":{"newconf":["1","2"]}}}} +{"level":"debug","ts":1712329731.7901514,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["1","2"],[]],"prop":{"cc":{"newconf":["1","2","3"]}}}} +{"level":"debug","ts":1712329731.7901733,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["1","2","3"],[]],"prop":{"cc":{"newconf":["1","2","3","4"]}}}} +{"level":"debug","ts":1712329731.7901926,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["1","2","3","4"],[]],"prop":{"cc":{"newconf":["1","2","3","4","5"]}}}} +{"level":"debug","ts":1712329731.79021,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["1","2","3","4","5"],[]],"prop":{"cc":{"newconf":["2","3","4","5"]}}}} +{"level":"debug","ts":1712329731.7902274,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","4","5"],[]],"prop":{"cc":{"newconf":["2","3","5"]}}}} +{"level":"debug","ts":1712329731.7903354,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","5"],[]],"prop":{"cc":{"newconf":["2","3"]}}}} +{"level":"debug","ts":1712329731.790394,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3"],[]],"prop":{"cc":{"newconf":["2","3","6"]}}}} +{"level":"debug","ts":1712329731.790427,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6"],[]],"prop":{"cc":{"newconf":["2","3","6","7"]}}}} +{"level":"debug","ts":1712329731.7904487,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","7"],[]],"prop":{"cc":{"newconf":["2","3","6"]}}}} +{"level":"debug","ts":1712329731.7906199,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6"],[]],"prop":{"cc":{"newconf":["2","3","6","8"]}}}} +{"level":"debug","ts":1712329731.7906857,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8"],[]],"prop":{"cc":{"newconf":["2","3","6","8","9"]}}}} +{"level":"debug","ts":1712329731.791384,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7925622,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"9","entries":129,"logTerm":0,"index":0,"commit":126,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7925835,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"9","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7925847,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.792611,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":2,"logTerm":2,"index":127,"commit":127,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.7926352,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7936616,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":127},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":128,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7936833,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":128},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7936924,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":128},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":129,"commit":128,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.793707,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":128},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":129,"commit":128,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.7937214,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":126},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":0,"logTerm":2,"index":129,"commit":127,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7937286,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7937355,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7948413,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":128},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"9","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.795221,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":128},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"9","entries":129,"logTerm":0,"index":0,"commit":128,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.7952545,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":128},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7961817,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":127,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7962074,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7962427,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7962558,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":127,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7962673,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7962859,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.796432,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":128},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":128,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7974732,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":128},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7975028,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":128},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":129,"commit":128,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.7986207,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":2,"logTerm":2,"index":127,"commit":127,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.79865,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7986586,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":129,"commit":128,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.79866,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7986705,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":128},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":127,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7986782,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":128},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7986946,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":128},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7986996,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":127,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7987065,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":128},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7987537,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7987797,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7998257,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":128},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7998526,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.7998638,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":129,"commit":129,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.7998698,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":129,"commit":128,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.7998738,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":129,"commit":129,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.7998948,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":128},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.799913,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":128},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.801362,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":128,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8013837,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.8013928,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":128,"commit":129,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.8014066,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":2,"logTerm":2,"index":127,"commit":127,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8014145,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8014216,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.8015177,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"9","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8015387,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"9","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8015602,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.8026357,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8026645,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.8026738,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.8036895,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.803914,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":128},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":129,"commit":128,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8039374,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":128},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8039465,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":128},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.8047178,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8047483,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9"],[]],"prop":{"cc":{"newconf":["2","3","6","8","9","10"]}}}} +{"level":"debug","ts":1712329731.8048117,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]]}} +{"level":"debug","ts":1712329731.8048255,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"10","entries":1,"logTerm":2,"index":128,"commit":129,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.8062289,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.807316,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8091223,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":127},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"6","entries":1,"logTerm":2,"index":128,"commit":129,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.809147,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8091583,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"6","state":{"term":2,"vote":"0","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.8091705,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"9","entries":129,"logTerm":0,"index":0,"commit":128,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.809174,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8091772,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"9","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.809227,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.8101854,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"6","state":{"term":2,"vote":"0","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"prop":{"cc":{"newconf":["2","3","6","8","9","10"]}}}} +{"level":"debug","ts":1712329731.8103173,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8103397,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]]}} +{"level":"debug","ts":1712329731.8103485,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":129,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8103614,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":129,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8103693,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"8","entries":0,"logTerm":0,"index":0,"commit":129,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8103747,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"9","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8103812,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"10","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8109384,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"8","state":{"term":2,"vote":"0","commit":128},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"8","entries":0,"logTerm":2,"index":129,"commit":129,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8109615,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"8","state":{"term":2,"vote":"0","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8109715,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"8","state":{"term":2,"vote":"0","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.8115761,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"9","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8115985,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Commit","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]]}} +{"level":"debug","ts":1712329731.8116074,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesRequest","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"9","entries":0,"logTerm":2,"index":129,"commit":129,"vote":"0","reject":false,"rejectHint":0},"prop":{"match":129,"next":130}}} +{"level":"debug","ts":1712329731.811994,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"8","state":{"term":2,"vote":"0","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"prop":{"cc":{"newconf":["2","3","6","8","9","10"]}}}} +{"level":"debug","ts":1712329731.8120272,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":128},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"3","entries":0,"logTerm":2,"index":129,"commit":129,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8120496,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8120604,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.8127367,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"9","to":"2","entries":0,"logTerm":0,"index":125,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8127642,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]]}} +{"level":"debug","ts":1712329731.8134768,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"3","state":{"term":2,"vote":"2","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"prop":{"cc":{"newconf":["2","3","6","8","9","10"]}}}} +{"level":"debug","ts":1712329731.813846,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8150158,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8160515,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.816073,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"9","state":{"term":2,"vote":"0","commit":125},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgApp","term":2,"from":"2","to":"9","entries":0,"logTerm":2,"index":129,"commit":129,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8161044,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"9","state":{"term":2,"vote":"0","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"9","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8161683,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"9","state":{"term":2,"vote":"0","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]]}} +{"level":"debug","ts":1712329731.8171096,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8171842,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"3","state":{"term":2,"vote":"2","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"3","entries":0,"logTerm":0,"index":0,"commit":129,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.817209,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"3","state":{"term":2,"vote":"2","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8172202,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"Ready","nid":"3","state":{"term":2,"vote":"2","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9","10"],[]]}} +{"level":"debug","ts":1712329731.8173785,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"9","state":{"term":2,"vote":"0","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9"],[]],"prop":{"cc":{"newconf":["2","3","6","8","9","10"]}}}} +{"level":"debug","ts":1712329731.8174837,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"InitState","nid":"10","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.8174965,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ApplyConfChange","nid":"10","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]],"prop":{"cc":{}}}} +{"level":"debug","ts":1712329731.8175213,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"BecomeFollower","nid":"10","state":{"term":0,"vote":"0","commit":0},"role":"StateFollower","log":0,"conf":[[],[]]}} +{"level":"debug","ts":1712329731.8188226,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgAppResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":129,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8199763,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"8","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8200097,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesRequest","nid":"6","state":{"term":2,"vote":"0","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgHeartbeat","term":2,"from":"2","to":"6","entries":0,"logTerm":0,"index":0,"commit":129,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8200195,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"SendAppendEntriesResponse","nid":"6","state":{"term":2,"vote":"0","commit":129},"role":"StateFollower","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"6","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} +{"level":"debug","ts":1712329731.8212218,"caller":"rafttest/trace_logger.go:42","msg":"trace","tag":"trace","event":{"name":"ReceiveAppendEntriesResponse","nid":"2","state":{"term":2,"vote":"2","commit":129},"role":"StateLeader","log":129,"conf":[["2","3","6","8","9","10"],[]],"msg":{"type":"MsgHeartbeatResp","term":2,"from":"3","to":"2","entries":0,"logTerm":0,"index":0,"commit":0,"vote":"0","reject":false,"rejectHint":0}}} diff --git a/third_party/raft/tla/validate-model.sh b/third_party/raft/tla/validate-model.sh new file mode 100644 index 0000000000..2c0fd7d1d0 --- /dev/null +++ b/third_party/raft/tla/validate-model.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +WORKDIR="$(mktemp -d)" +TOOLDIR="${WORKDIR}/tool" +STATEDIR="${WORKDIR}/state" +FAILFAST=false +PARALLEL=$(nproc) + +function show_usage { + echo "usage: validate-model.sh -s -c ">&2 +} + +function install_tlaplus { + echo -n "Downloading TLA+ tools ... " + wget -qN https://nightly.tlapl.us/dist/tla2tools.jar -P ${TOOLDIR} + wget -qN https://github.com/tlaplus/CommunityModules/releases/latest/download/CommunityModules-deps.jar -P ${TOOLDIR} + echo "done." +} + +function validate { + local spec=${1} + local config=${2} + local tooldir=${3} + local statedir=${4} + + set -o pipefail + java -XX:+UseParallelGC -cp ${tooldir}/tla2tools.jar:${tooldir}/CommunityModules-deps.jar tlc2.TLC -config "${config}" "${spec}" -lncheck final -metadir "${statedir}" -fpmem 0.9 +} + +while getopts :hs:c:p: flag +do + case "${flag}" in + s) SPEC=${OPTARG};; + c) CONFIG=${OPTARG};; + h|*) show_usage; exit 1;; + esac +done + +if [ ! "$SPEC" ] || [ ! "$CONFIG" ] +then + show_usage + exit 1 +fi + +echo "spec: ${SPEC}" +echo "config: ${CONFIG}" + +install_tlaplus + +validate $SPEC $CONFIG $TOOLDIR $STATEDIR + diff --git a/third_party/raft/tla/validate.sh b/third_party/raft/tla/validate.sh new file mode 100644 index 0000000000..9de02548e0 --- /dev/null +++ b/third_party/raft/tla/validate.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash + +WORKDIR="$(mktemp -d)" +TOOLDIR="${WORKDIR}/tool" +STATEDIR="${WORKDIR}/state" +WORKDIR="$(mktemp -d)" +TOOLDIR="${WORKDIR}/tool" +STATEDIR="${WORKDIR}/state" +FAILFAST=false +PARALLEL=$(nproc) + +function show_usage { + echo "usage: validate.sh [-p ] -s -c ">&2 +} + +function install_tlaplus { + echo -n "Downloading TLA+ tools ... " + wget -qN https://nightly.tlapl.us/dist/tla2tools.jar -P ${TOOLDIR} + wget -qN https://github.com/tlaplus/CommunityModules/releases/latest/download/CommunityModules-deps.jar -P ${TOOLDIR} + echo "done." +} +PARALLEL=$(nproc) + +function preprocess_log { + local trace=${1} + sed -i -E 's/^[^{]+//' ${trace} + sort -t":" -k 3 ${trace} -o ${trace} +} + +function colored_text { + color=$1 + text=$2 + + case $color in + "red") + echo $"\033[0;31m$text\033[0m" + ;; + + "green") + echo "\033[0;32m$text\033[0m" + ;; + + *) + echo text + ;; + esac +} + +function validate { + local id=${1} + local trace=${2} + local spec=${3} + local config=${4} + local tooldir=${5} + local statedir=${6} + local name=$(basename $trace .ndjson) + local id=${1} + local trace=${2} + local spec=${3} + local config=${4} + local tooldir=${5} + local statedir=${6} + local name=$(basename $trace .ndjson) + + preprocess_log $trace + + set -o pipefail + env JSON="${trace}" java -XX:+UseParallelGC -cp ${tooldir}/tla2tools.jar:${tooldir}/CommunityModules-deps.jar tlc2.TLC -config "${config}" "${spec}" -lncheck final -metadir "${statedir}" -fpmem 0.9 | sed -nuE "s/<<\"Progress %:\", ([0-9]+)>>$/${id} \1/p" + + if [ "$?" -ne 0 ]; then + echo ${id} -1 + fi +} + +function show_progress { + p=0 + r=100 + total=${#trace_files[@]} + pending=0 + running=0 + for i in ${!trace_files[@]}; do + if [ "${result[$i]}" = "success" ] && [ "${printed[$i]}" = "" ]; then + p=$((p+100)) + echo -e ${trace_files[$i]} "-" $(colored_text "green" "PASS") "\033[0K" + printed[$i]="true" + elif [ "${result[$i]}" = "fail" ] && [ "${printed[$i]}" = "" ]; then + p=$((p+100)) + echo -e ${trace_files[$i]} "-" $(colored_text "red" "FAIL") "\033[0K" + printed[$i]="true" + elif [ "${result[$i]}" = "" ]; then + p=$((p+progress[$i])) + if [ $((progress[$i])) -gt 0 ]; then + running=$((running+1)) + else + pending=$((pending+1)) + fi + else + p=$((p+100)) + fi + done + if [ $((running+pending)) -gt 0 ]; then + bar_len=$(($(tput cols) - 45)) + p=$((p/total)) + p_bar=$((p*bar_len/100)) + r_bar=$((bar_len-p_bar)) + completed=$(printf "%${p_bar}s" | tr " " "#") + remaining=$(printf "%${r_bar}s" | tr " " ".") + + echo -ne " [${completed}${remaining}] running: ${running} pending: $((pending)) progress: ${p}%\r" + fi +} + +while getopts :hs:c:p: flag +do + case "${flag}" in + s) SPEC=${OPTARG};; + c) CONFIG=${OPTARG};; + p) PARALLEL=${OPTARG};; + h|*) show_usage; exit 1;; + p) PARALLEL=${OPTARG};; + h|*) show_usage; exit 1;; + esac +done + +shift $(($OPTIND -1)) +trace_files=("$@") +printed=() +progress=() +result=() + +if [ ! "$SPEC" ] || [ ! "$CONFIG" ] || [ ! "$trace_files" ] +then + show_usage + exit 1 +fi + +echo "spec: ${SPEC}" +echo "config: ${CONFIG}" +if [ ! "$SPEC" ] || [ ! "$CONFIG" ] || [ ! "$trace_files" ] +then + show_usage + exit 1 +fi + +echo "spec: ${SPEC}" +echo "config: ${CONFIG}" + +install_tlaplus + +total=${#trace_files[@]} +SECONDS=0 + +export -f validate +export -f preprocess_log +while read line; do + tokens=(${line// / }) + id=${tokens[0]} + p=${tokens[1]} + progress[$id]=$p + if [ "${p}" = 100 ] && [ "${result[$id]}" = "" ]; then + result[$id]="success" + show_progress + elif [ "${p}" = -1 ] && [ "${result[$id]}" = "" ]; then + result[$id]="fail" + show_progress + fi + + if [ "${SECONDS}" -ge 2 ]; then + show_progress + SECONDS=0 + fi +done < \ +<(for i in ${!trace_files[@]}; do + progress[$i]=0 + echo $i ${trace_files[$i]} +done | \ +xargs -I{} -P ${PARALLEL} bash -c 'validate $@' _ {} $SPEC $CONFIG $TOOLDIR $STATEDIR) + + +passed=0 +for s in ${result[@]}; do + if [ "$s" = "success" ]; then + passed=$((passed+1)) + fi +done + +echo -e "$passed of $total trace(s) passed" + +if [ $passed -lt $total ]; then + exit 1 +fi diff --git a/third_party/raft/tracker/inflights.go b/third_party/raft/tracker/inflights.go new file mode 100644 index 0000000000..cb091e542a --- /dev/null +++ b/third_party/raft/tracker/inflights.go @@ -0,0 +1,143 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tracker + +// inflight describes an in-flight MsgApp message. +type inflight struct { + index uint64 // the index of the last entry inside the message + bytes uint64 // the total byte size of the entries in the message +} + +// Inflights limits the number of MsgApp (represented by the largest index +// contained within) sent to followers but not yet acknowledged by them. Callers +// use Full() to check whether more messages can be sent, call Add() whenever +// they are sending a new append, and release "quota" via FreeLE() whenever an +// ack is received. +type Inflights struct { + // the starting index in the buffer + start int + + count int // number of inflight messages in the buffer + bytes uint64 // number of inflight bytes + + size int // the max number of inflight messages + maxBytes uint64 // the max total byte size of inflight messages + + // buffer is a ring buffer containing info about all in-flight messages. + buffer []inflight +} + +// NewInflights sets up an Inflights that allows up to size inflight messages, +// with the total byte size up to maxBytes. If maxBytes is 0 then there is no +// byte size limit. The maxBytes limit is soft, i.e. we accept a single message +// that brings it from size < maxBytes to size >= maxBytes. +func NewInflights(size int, maxBytes uint64) *Inflights { + return &Inflights{ + size: size, + maxBytes: maxBytes, + } +} + +// Clone returns an *Inflights that is identical to but shares no memory with +// the receiver. +func (in *Inflights) Clone() *Inflights { + ins := *in + ins.buffer = append([]inflight(nil), in.buffer...) + return &ins +} + +// Add notifies the Inflights that a new message with the given index and byte +// size is being dispatched. Full() must be called prior to Add() to verify that +// there is room for one more message, and consecutive calls to Add() must +// provide a monotonic sequence of indexes. +func (in *Inflights) Add(index, bytes uint64) { + if in.Full() { + panic("cannot add into a Full inflights") + } + next := in.start + in.count + size := in.size + if next >= size { + next -= size + } + if next >= len(in.buffer) { + in.grow() + } + in.buffer[next] = inflight{index: index, bytes: bytes} + in.count++ + in.bytes += bytes +} + +// grow the inflight buffer by doubling up to inflights.size. We grow on demand +// instead of preallocating to inflights.size to handle systems which have +// thousands of Raft groups per process. +func (in *Inflights) grow() { + newSize := len(in.buffer) * 2 + if newSize == 0 { + newSize = 1 + } else if newSize > in.size { + newSize = in.size + } + newBuffer := make([]inflight, newSize) + copy(newBuffer, in.buffer) + in.buffer = newBuffer +} + +// FreeLE frees the inflights smaller or equal to the given `to` flight. +func (in *Inflights) FreeLE(to uint64) { + if in.count == 0 || to < in.buffer[in.start].index { + // out of the left side of the window + return + } + + idx := in.start + var i int + var bytes uint64 + for i = 0; i < in.count; i++ { + if to < in.buffer[idx].index { // found the first large inflight + break + } + bytes += in.buffer[idx].bytes + + // increase index and maybe rotate + size := in.size + if idx++; idx >= size { + idx -= size + } + } + // free i inflights and set new start index + in.count -= i + in.bytes -= bytes + in.start = idx + if in.count == 0 { + // inflights is empty, reset the start index so that we don't grow the + // buffer unnecessarily. + in.start = 0 + } +} + +// Full returns true if no more messages can be sent at the moment. +func (in *Inflights) Full() bool { + return in.count == in.size || (in.maxBytes != 0 && in.bytes >= in.maxBytes) +} + +// Count returns the number of inflight messages. +func (in *Inflights) Count() int { return in.count } + +// reset frees all inflights. +func (in *Inflights) reset() { + in.start = 0 + in.count = 0 + in.bytes = 0 +} diff --git a/third_party/raft/tracker/inflights_test.go b/third_party/raft/tracker/inflights_test.go new file mode 100644 index 0000000000..397bc324e6 --- /dev/null +++ b/third_party/raft/tracker/inflights_test.go @@ -0,0 +1,256 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tracker + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInflightsAdd(t *testing.T) { + // no rotating case + in := &Inflights{ + size: 10, + buffer: make([]inflight, 10), + } + + for i := 0; i < 5; i++ { + in.Add(uint64(i), uint64(100+i)) + } + + wantIn := &Inflights{ + start: 0, + count: 5, + bytes: 510, + size: 10, + buffer: inflightsBuffer( + // ↓------------ + []uint64{0, 1, 2, 3, 4, 0, 0, 0, 0, 0}, + []uint64{100, 101, 102, 103, 104, 0, 0, 0, 0, 0}), + } + require.Equal(t, wantIn, in) + + for i := 5; i < 10; i++ { + in.Add(uint64(i), uint64(100+i)) + } + + wantIn2 := &Inflights{ + start: 0, + count: 10, + bytes: 1045, + size: 10, + buffer: inflightsBuffer( + // ↓--------------------------- + []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + []uint64{100, 101, 102, 103, 104, 105, 106, 107, 108, 109}), + } + require.Equal(t, wantIn2, in) + + // rotating case + in2 := &Inflights{ + start: 5, + size: 10, + buffer: make([]inflight, 10), + } + + for i := 0; i < 5; i++ { + in2.Add(uint64(i), uint64(100+i)) + } + + wantIn21 := &Inflights{ + start: 5, + count: 5, + bytes: 510, + size: 10, + buffer: inflightsBuffer( + // ↓------------ + []uint64{0, 0, 0, 0, 0, 0, 1, 2, 3, 4}, + []uint64{0, 0, 0, 0, 0, 100, 101, 102, 103, 104}), + } + require.Equal(t, wantIn21, in2) + + for i := 5; i < 10; i++ { + in2.Add(uint64(i), uint64(100+i)) + } + + wantIn22 := &Inflights{ + start: 5, + count: 10, + bytes: 1045, + size: 10, + buffer: inflightsBuffer( + // -------------- ↓------------ + []uint64{5, 6, 7, 8, 9, 0, 1, 2, 3, 4}, + []uint64{105, 106, 107, 108, 109, 100, 101, 102, 103, 104}), + } + require.Equal(t, wantIn22, in2) +} + +func TestInflightFreeTo(t *testing.T) { + // no rotating case + in := NewInflights(10, 0) + for i := 0; i < 10; i++ { + in.Add(uint64(i), uint64(100+i)) + } + + in.FreeLE(0) + + wantIn0 := &Inflights{ + start: 1, + count: 9, + bytes: 945, + size: 10, + buffer: inflightsBuffer( + // ↓------------------------ + []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + []uint64{100, 101, 102, 103, 104, 105, 106, 107, 108, 109}), + } + require.Equal(t, wantIn0, in) + + in.FreeLE(4) + + wantIn := &Inflights{ + start: 5, + count: 5, + bytes: 535, + size: 10, + buffer: inflightsBuffer( + // ↓------------ + []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + []uint64{100, 101, 102, 103, 104, 105, 106, 107, 108, 109}), + } + require.Equal(t, wantIn, in) + + in.FreeLE(8) + + wantIn2 := &Inflights{ + start: 9, + count: 1, + bytes: 109, + size: 10, + buffer: inflightsBuffer( + // ↓ + []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + []uint64{100, 101, 102, 103, 104, 105, 106, 107, 108, 109}), + } + require.Equal(t, wantIn2, in) + + // rotating case + for i := 10; i < 15; i++ { + in.Add(uint64(i), uint64(100+i)) + } + + in.FreeLE(12) + + wantIn3 := &Inflights{ + start: 3, + count: 2, + bytes: 227, + size: 10, + buffer: inflightsBuffer( + // ↓----- + []uint64{10, 11, 12, 13, 14, 5, 6, 7, 8, 9}, + []uint64{110, 111, 112, 113, 114, 105, 106, 107, 108, 109}), + } + require.Equal(t, wantIn3, in) + + in.FreeLE(14) + + wantIn4 := &Inflights{ + start: 0, + count: 0, + size: 10, + buffer: inflightsBuffer( + // ↓ + []uint64{10, 11, 12, 13, 14, 5, 6, 7, 8, 9}, + []uint64{110, 111, 112, 113, 114, 105, 106, 107, 108, 109}), + } + require.Equal(t, wantIn4, in) +} + +func TestInflightsFull(t *testing.T) { + for _, tc := range []struct { + name string + size int + maxBytes uint64 + fullAt int + freeLE uint64 + againAt int + }{ + {name: "always-full", size: 0, fullAt: 0}, + {name: "single-entry", size: 1, fullAt: 1, freeLE: 1, againAt: 2}, + {name: "single-entry-overflow", size: 1, maxBytes: 10, fullAt: 1, freeLE: 1, againAt: 2}, + {name: "multi-entry", size: 15, fullAt: 15, freeLE: 6, againAt: 22}, + {name: "slight-overflow", size: 8, maxBytes: 400, fullAt: 4, freeLE: 2, againAt: 7}, + {name: "exact-max-bytes", size: 8, maxBytes: 406, fullAt: 4, freeLE: 3, againAt: 8}, + {name: "larger-overflow", size: 15, maxBytes: 408, fullAt: 5, freeLE: 1, againAt: 6}, + } { + t.Run(tc.name, func(t *testing.T) { + in := NewInflights(tc.size, tc.maxBytes) + + addUntilFull := func(begin, end int) { + for i := begin; i < end; i++ { + require.False(t, in.Full(), "full at %d, want %d", i, end) + in.Add(uint64(i), uint64(100+i)) + } + require.True(t, in.Full(), "not full at %d", end) + } + + addUntilFull(0, tc.fullAt) + in.FreeLE(tc.freeLE) + addUntilFull(tc.fullAt, tc.againAt) + + defer func() { + assert.NotNil(t, recover(), "Add() did not panic") + }() + in.Add(100, 1024) + }) + } +} + +func TestInflightsReset(t *testing.T) { + in := NewInflights(10, 1000) + // Imitate a semi-realistic flow during which the inflight tracker is + // periodically reset to empty. Byte usage must not "leak" across resets. + index := uint64(0) + for epoch := 0; epoch < 100; epoch++ { + in.reset() + // Add 5 messages. They should not max out the limit yet. + for i := 0; i < 5; i++ { + require.False(t, in.Full()) + index++ + in.Add(index, 16) + } + // Ack all but last 2 indices. + in.FreeLE(index - 2) + require.False(t, in.Full()) + require.Equal(t, 2, in.Count()) + } + in.FreeLE(index) + require.Equal(t, 0, in.Count()) +} + +func inflightsBuffer(indices []uint64, sizes []uint64) []inflight { + if len(indices) != len(sizes) { + panic("len(indices) != len(sizes)") + } + buffer := make([]inflight, 0, len(indices)) + for i, idx := range indices { + buffer = append(buffer, inflight{index: idx, bytes: sizes[i]}) + } + return buffer +} diff --git a/third_party/raft/tracker/progress.go b/third_party/raft/tracker/progress.go new file mode 100644 index 0000000000..5716661c86 --- /dev/null +++ b/third_party/raft/tracker/progress.go @@ -0,0 +1,314 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tracker + +import ( + "fmt" + "slices" + "strings" +) + +// Progress represents a follower’s progress in the view of the leader. Leader +// maintains progresses of all followers, and sends entries to the follower +// based on its progress. +// +// NB(tbg): Progress is basically a state machine whose transitions are mostly +// strewn around `*raft.raft`. Additionally, some fields are only used when in a +// certain State. All of this isn't ideal. +type Progress struct { + // Match is the index up to which the follower's log is known to match the + // leader's. + Match uint64 + // Next is the log index of the next entry to send to this follower. All + // entries with indices in (Match, Next) interval are already in flight. + // + // Invariant: 0 <= Match < Next. + // NB: it follows that Next >= 1. + // + // In StateSnapshot, Next == PendingSnapshot + 1. + Next uint64 + + // sentCommit is the highest commit index in flight to the follower. + // + // Generally, it is monotonic, but con regress in some cases, e.g. when + // converting to `StateProbe` or when receiving a rejection from a follower. + // + // In StateSnapshot, sentCommit == PendingSnapshot == Next-1. + sentCommit uint64 + + // State defines how the leader should interact with the follower. + // + // When in StateProbe, leader sends at most one replication message + // per heartbeat interval. It also probes actual progress of the follower. + // + // When in StateReplicate, leader optimistically increases next + // to the latest entry sent after sending replication message. This is + // an optimized state for fast replicating log entries to the follower. + // + // When in StateSnapshot, leader should have sent out snapshot + // before and stops sending any replication message. + State StateType + + // PendingSnapshot is used in StateSnapshot and tracks the last index of the + // leader at the time at which it realized a snapshot was necessary. This + // matches the index in the MsgSnap message emitted from raft. + // + // While there is a pending snapshot, replication to the follower is paused. + // The follower will transition back to StateReplicate if the leader + // receives an MsgAppResp from it that reconnects the follower to the + // leader's log (such an MsgAppResp is emitted when the follower applies a + // snapshot). It may be surprising that PendingSnapshot is not taken into + // account here, but consider that complex systems may delegate the sending + // of snapshots to alternative datasources (i.e. not the leader). In such + // setups, it is difficult to manufacture a snapshot at a particular index + // requested by raft and the actual index may be ahead or behind. This + // should be okay, as long as the snapshot allows replication to resume. + // + // The follower will transition to StateProbe if ReportSnapshot is called on + // the leader; if SnapshotFinish is passed then PendingSnapshot becomes the + // basis for the next attempt to append. In practice, the first mechanism is + // the one that is relevant in most cases. However, if this MsgAppResp is + // lost (fallible network) then the second mechanism ensures that in this + // case the follower does not erroneously remain in StateSnapshot. + PendingSnapshot uint64 + + // RecentActive is true if the progress is recently active. Receiving any messages + // from the corresponding follower indicates the progress is active. + // RecentActive can be reset to false after an election timeout. + // This is always true on the leader. + RecentActive bool + + // MsgAppFlowPaused is used when the MsgApp flow to a node is throttled. This + // happens in StateProbe, or StateReplicate with saturated Inflights. In both + // cases, we need to continue sending MsgApp once in a while to guarantee + // progress, but we only do so when MsgAppFlowPaused is false (it is reset on + // receiving a heartbeat response), to not overflow the receiver. See + // IsPaused(). + MsgAppFlowPaused bool + + // Inflights is a sliding window for the inflight messages. + // Each inflight message contains one or more log entries. + // The max number of entries per message is defined in raft config as MaxSizePerMsg. + // Thus inflight effectively limits both the number of inflight messages + // and the bandwidth each Progress can use. + // When inflights is Full, no more message should be sent. + // When a leader sends out a message, the index of the last + // entry should be added to inflights. The index MUST be added + // into inflights in order. + // When a leader receives a reply, the previous inflights should + // be freed by calling inflights.FreeLE with the index of the last + // received entry. + Inflights *Inflights + + // IsLearner is true if this progress is tracked for a learner. + IsLearner bool +} + +// ResetState moves the Progress into the specified State, resetting MsgAppFlowPaused, +// PendingSnapshot, and Inflights. +func (pr *Progress) ResetState(state StateType) { + pr.MsgAppFlowPaused = false + pr.PendingSnapshot = 0 + pr.State = state + pr.Inflights.reset() +} + +// BecomeProbe transitions into StateProbe. Next is reset to Match+1 or, +// optionally and if larger, the index of the pending snapshot. +func (pr *Progress) BecomeProbe() { + // If the original state is StateSnapshot, progress knows that + // the pending snapshot has been sent to this peer successfully, then + // probes from pendingSnapshot + 1. + if pr.State == StateSnapshot { + pendingSnapshot := pr.PendingSnapshot + pr.ResetState(StateProbe) + pr.Next = max(pr.Match+1, pendingSnapshot+1) + } else { + pr.ResetState(StateProbe) + pr.Next = pr.Match + 1 + } + pr.sentCommit = min(pr.sentCommit, pr.Next-1) +} + +// BecomeReplicate transitions into StateReplicate, resetting Next to Match+1. +func (pr *Progress) BecomeReplicate() { + pr.ResetState(StateReplicate) + pr.Next = pr.Match + 1 +} + +// BecomeSnapshot moves the Progress to StateSnapshot with the specified pending +// snapshot index. +func (pr *Progress) BecomeSnapshot(snapshoti uint64) { + pr.ResetState(StateSnapshot) + pr.PendingSnapshot = snapshoti + pr.Next = snapshoti + 1 + pr.sentCommit = snapshoti +} + +// SentEntries updates the progress on the given number of consecutive entries +// being sent in a MsgApp, with the given total bytes size, appended at log +// indices >= pr.Next. +// +// Must be used with StateProbe or StateReplicate. +func (pr *Progress) SentEntries(entries int, bytes uint64) { + switch pr.State { + case StateReplicate: + if entries > 0 { + pr.Next += uint64(entries) + pr.Inflights.Add(pr.Next-1, bytes) + } + // If this message overflows the in-flights tracker, or it was already full, + // consider this message being a probe, so that the flow is paused. + pr.MsgAppFlowPaused = pr.Inflights.Full() + case StateProbe: + // TODO(pavelkalinnikov): this condition captures the previous behaviour, + // but we should set MsgAppFlowPaused unconditionally for simplicity, because any + // MsgApp in StateProbe is a probe, not only non-empty ones. + if entries > 0 { + pr.MsgAppFlowPaused = true + } + default: + panic(fmt.Sprintf("sending append in unhandled state %s", pr.State)) + } +} + +// CanBumpCommit returns true if sending the given commit index can potentially +// advance the follower's commit index. +func (pr *Progress) CanBumpCommit(index uint64) bool { + // Sending the given commit index may bump the follower's commit index up to + // Next-1 in normal operation, or higher in some rare cases. Allow sending a + // commit index eagerly only if we haven't already sent one that bumps the + // follower's commit all the way to Next-1. + return index > pr.sentCommit && pr.sentCommit < pr.Next-1 +} + +// SentCommit updates the sentCommit. +func (pr *Progress) SentCommit(commit uint64) { + pr.sentCommit = commit +} + +// MaybeUpdate is called when an MsgAppResp arrives from the follower, with the +// index acked by it. The method returns false if the given n index comes from +// an outdated message. Otherwise it updates the progress and returns true. +func (pr *Progress) MaybeUpdate(n uint64) bool { + if n <= pr.Match { + return false + } + pr.Match = n + pr.Next = max(pr.Next, n+1) // invariant: Match < Next + pr.MsgAppFlowPaused = false + return true +} + +// MaybeDecrTo adjusts the Progress to the receipt of a MsgApp rejection. The +// arguments are the index of the append message rejected by the follower, and +// the hint that we want to decrease to. +// +// Rejections can happen spuriously as messages are sent out of order or +// duplicated. In such cases, the rejection pertains to an index that the +// Progress already knows were previously acknowledged, and false is returned +// without changing the Progress. +// +// If the rejection is genuine, Next is lowered sensibly, and the Progress is +// cleared for sending log entries. +func (pr *Progress) MaybeDecrTo(rejected, matchHint uint64) bool { + if pr.State == StateReplicate { + // The rejection must be stale if the progress has matched and "rejected" + // is smaller than "match". + if rejected <= pr.Match { + return false + } + // Directly decrease next to match + 1. + // + // TODO(tbg): why not use matchHint if it's larger? + pr.Next = pr.Match + 1 + // Regress the sentCommit since it unlikely has been applied. + pr.sentCommit = min(pr.sentCommit, pr.Next-1) + return true + } + + // The rejection must be stale if "rejected" does not match next - 1. This + // is because non-replicating followers are probed one entry at a time. + // The check is a best effort assuming message reordering is rare. + if pr.Next-1 != rejected { + return false + } + + pr.Next = max(min(rejected, matchHint+1), pr.Match+1) + // Regress the sentCommit since it unlikely has been applied. + pr.sentCommit = min(pr.sentCommit, pr.Next-1) + pr.MsgAppFlowPaused = false + return true +} + +// IsPaused returns whether sending log entries to this node has been throttled. +// This is done when a node has rejected recent MsgApps, is currently waiting +// for a snapshot, or has reached the MaxInflightMsgs limit. In normal +// operation, this is false. A throttled node will be contacted less frequently +// until it has reached a state in which it's able to accept a steady stream of +// log entries again. +func (pr *Progress) IsPaused() bool { + switch pr.State { + case StateProbe: + return pr.MsgAppFlowPaused + case StateReplicate: + return pr.MsgAppFlowPaused + case StateSnapshot: + return true + default: + panic("unexpected state") + } +} + +func (pr *Progress) String() string { + var buf strings.Builder + fmt.Fprintf(&buf, "%s match=%d next=%d", pr.State, pr.Match, pr.Next) + if pr.IsLearner { + fmt.Fprint(&buf, " learner") + } + if pr.IsPaused() { + fmt.Fprint(&buf, " paused") + } + if pr.PendingSnapshot > 0 { + fmt.Fprintf(&buf, " pendingSnap=%d", pr.PendingSnapshot) + } + if !pr.RecentActive { + fmt.Fprint(&buf, " inactive") + } + if n := pr.Inflights.Count(); n > 0 { + fmt.Fprintf(&buf, " inflight=%d", n) + if pr.Inflights.Full() { + fmt.Fprint(&buf, "[full]") + } + } + return buf.String() +} + +// ProgressMap is a map of *Progress. +type ProgressMap map[uint64]*Progress + +// String prints the ProgressMap in sorted key order, one Progress per line. +func (m ProgressMap) String() string { + ids := make([]uint64, 0, len(m)) + for k := range m { + ids = append(ids, k) + } + slices.Sort(ids) + var buf strings.Builder + for _, id := range ids { + fmt.Fprintf(&buf, "%d: %s\n", id, m[id]) + } + return buf.String() +} diff --git a/third_party/raft/tracker/progress_test.go b/third_party/raft/tracker/progress_test.go new file mode 100644 index 0000000000..49dedb536b --- /dev/null +++ b/third_party/raft/tracker/progress_test.go @@ -0,0 +1,211 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tracker + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestProgressString(t *testing.T) { + ins := NewInflights(1, 0) + ins.Add(123, 1) + pr := &Progress{ + Match: 1, + Next: 2, + State: StateSnapshot, + PendingSnapshot: 123, + RecentActive: false, + MsgAppFlowPaused: true, + IsLearner: true, + Inflights: ins, + } + const exp = `StateSnapshot match=1 next=2 learner paused pendingSnap=123 inactive inflight=1[full]` + assert.Equal(t, exp, pr.String()) +} + +func TestProgressIsPaused(t *testing.T) { + tests := []struct { + state StateType + paused bool + + w bool + }{ + {StateProbe, false, false}, + {StateProbe, true, true}, + {StateReplicate, false, false}, + {StateReplicate, true, true}, + {StateSnapshot, false, true}, + {StateSnapshot, true, true}, + } + for i, tt := range tests { + p := &Progress{ + State: tt.state, + MsgAppFlowPaused: tt.paused, + Inflights: NewInflights(256, 0), + } + assert.Equal(t, tt.w, p.IsPaused(), i) + } +} + +// TestProgressResume ensures that MaybeUpdate and MaybeDecrTo will reset +// MsgAppFlowPaused. +func TestProgressResume(t *testing.T) { + p := &Progress{ + Next: 2, + MsgAppFlowPaused: true, + } + p.MaybeDecrTo(1, 1) + assert.False(t, p.MsgAppFlowPaused) + p.MsgAppFlowPaused = true + p.MaybeUpdate(2) + assert.False(t, p.MsgAppFlowPaused) +} + +func TestProgressBecomeProbe(t *testing.T) { + match := uint64(1) + tests := []struct { + p *Progress + wnext uint64 + }{ + { + &Progress{State: StateReplicate, Match: match, Next: 5, Inflights: NewInflights(256, 0)}, + 2, + }, + { + // snapshot finish + &Progress{State: StateSnapshot, Match: match, Next: 5, PendingSnapshot: 10, Inflights: NewInflights(256, 0)}, + 11, + }, + { + // snapshot failure + &Progress{State: StateSnapshot, Match: match, Next: 5, PendingSnapshot: 0, Inflights: NewInflights(256, 0)}, + 2, + }, + } + for i, tt := range tests { + tt.p.BecomeProbe() + assert.Equal(t, StateProbe, tt.p.State, i) + assert.Equal(t, match, tt.p.Match, i) + assert.Equal(t, tt.wnext, tt.p.Next, i) + } +} + +func TestProgressBecomeReplicate(t *testing.T) { + p := &Progress{State: StateProbe, Match: 1, Next: 5, Inflights: NewInflights(256, 0)} + p.BecomeReplicate() + assert.Equal(t, StateReplicate, p.State) + assert.Equal(t, uint64(1), p.Match) + assert.Equal(t, p.Match+1, p.Next) +} + +func TestProgressBecomeSnapshot(t *testing.T) { + p := &Progress{State: StateProbe, Match: 1, Next: 5, Inflights: NewInflights(256, 0)} + p.BecomeSnapshot(10) + assert.Equal(t, StateSnapshot, p.State) + assert.Equal(t, uint64(1), p.Match) + assert.Equal(t, uint64(10), p.PendingSnapshot) +} + +func TestProgressUpdate(t *testing.T) { + prevM, prevN := uint64(3), uint64(5) + tests := []struct { + update uint64 + + wm uint64 + wn uint64 + wok bool + }{ + {prevM - 1, prevM, prevN, false}, // do not decrease match, next + {prevM, prevM, prevN, false}, // do not decrease next + {prevM + 1, prevM + 1, prevN, true}, // increase match, do not decrease next + {prevM + 2, prevM + 2, prevN + 1, true}, // increase match, next + } + for i, tt := range tests { + p := &Progress{ + Match: prevM, + Next: prevN, + } + assert.Equal(t, tt.wok, p.MaybeUpdate(tt.update), i) + assert.Equal(t, tt.wm, p.Match, i) + assert.Equal(t, tt.wn, p.Next, i) + } +} + +func TestProgressMaybeDecr(t *testing.T) { + tests := []struct { + state StateType + m uint64 + n uint64 + rejected uint64 + last uint64 + + w bool + wn uint64 + }{ + { + // state replicate and rejected is not greater than match + StateReplicate, 5, 10, 5, 5, false, 10, + }, + { + // state replicate and rejected is not greater than match + StateReplicate, 5, 10, 4, 4, false, 10, + }, + { + // state replicate and rejected is greater than match + // directly decrease to match+1 + StateReplicate, 5, 10, 9, 9, true, 6, + }, + { + // next-1 != rejected is always false + StateProbe, 0, 0, 0, 0, false, 0, + }, + { + // next-1 != rejected is always false + StateProbe, 0, 10, 5, 5, false, 10, + }, + { + // next>1 = decremented by 1 + StateProbe, 0, 10, 9, 9, true, 9, + }, + { + // next>1 = decremented by 1 + StateProbe, 0, 2, 1, 1, true, 1, + }, + { + // next<=1 = reset to 1 + StateProbe, 0, 1, 0, 0, true, 1, + }, + { + // decrease to min(rejected, last+1) + StateProbe, 0, 10, 9, 2, true, 3, + }, + { + // rejected < 1, reset to 1 + StateProbe, 0, 10, 9, 0, true, 1, + }, + } + for i, tt := range tests { + p := &Progress{ + State: tt.state, + Match: tt.m, + Next: tt.n, + } + assert.Equal(t, tt.w, p.MaybeDecrTo(tt.rejected, tt.last), i) + assert.Equal(t, tt.m, p.Match, i) + assert.Equal(t, tt.wn, p.Next, i) + } +} diff --git a/third_party/raft/tracker/state.go b/third_party/raft/tracker/state.go new file mode 100644 index 0000000000..7dbdd63fa6 --- /dev/null +++ b/third_party/raft/tracker/state.go @@ -0,0 +1,42 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tracker + +// StateType is the state of a tracked follower. +type StateType uint64 + +const ( + // StateProbe indicates a follower whose last index isn't known. Such a + // follower is "probed" (i.e. an append sent periodically) to narrow down + // its last index. In the ideal (and common) case, only one round of probing + // is necessary as the follower will react with a hint. Followers that are + // probed over extended periods of time are often offline. + StateProbe StateType = iota + // StateReplicate is the state steady in which a follower eagerly receives + // log entries to append to its log. + StateReplicate + // StateSnapshot indicates a follower that needs log entries not available + // from the leader's Raft log. Such a follower needs a full snapshot to + // return to StateReplicate. + StateSnapshot +) + +var prstmap = [...]string{ + "StateProbe", + "StateReplicate", + "StateSnapshot", +} + +func (st StateType) String() string { return prstmap[st] } diff --git a/third_party/raft/tracker/tracker.go b/third_party/raft/tracker/tracker.go new file mode 100644 index 0000000000..ddf15bae9e --- /dev/null +++ b/third_party/raft/tracker/tracker.go @@ -0,0 +1,281 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tracker + +import ( + "fmt" + "slices" + "strings" + + "go.etcd.io/raft/v3/quorum" + pb "go.etcd.io/raft/v3/raftpb" +) + +// Config reflects the configuration tracked in a ProgressTracker. +type Config struct { + Voters quorum.JointConfig + // AutoLeave is true if the configuration is joint and a transition to the + // incoming configuration should be carried out automatically by Raft when + // this is possible. If false, the configuration will be joint until the + // application initiates the transition manually. + AutoLeave bool + // Learners is a set of IDs corresponding to the learners active in the + // current configuration. + // + // Invariant: Learners and Voters does not intersect, i.e. if a peer is in + // either half of the joint config, it can't be a learner; if it is a + // learner it can't be in either half of the joint config. This invariant + // simplifies the implementation since it allows peers to have clarity about + // its current role without taking into account joint consensus. + Learners map[uint64]struct{} + // When we turn a voter into a learner during a joint consensus transition, + // we cannot add the learner directly when entering the joint state. This is + // because this would violate the invariant that the intersection of + // voters and learners is empty. For example, assume a Voter is removed and + // immediately re-added as a learner (or in other words, it is demoted): + // + // Initially, the configuration will be + // + // voters: {1 2 3} + // learners: {} + // + // and we want to demote 3. Entering the joint configuration, we naively get + // + // voters: {1 2} & {1 2 3} + // learners: {3} + // + // but this violates the invariant (3 is both voter and learner). Instead, + // we get + // + // voters: {1 2} & {1 2 3} + // learners: {} + // next_learners: {3} + // + // Where 3 is now still purely a voter, but we are remembering the intention + // to make it a learner upon transitioning into the final configuration: + // + // voters: {1 2} + // learners: {3} + // next_learners: {} + // + // Note that next_learners is not used while adding a learner that is not + // also a voter in the joint config. In this case, the learner is added + // right away when entering the joint configuration, so that it is caught up + // as soon as possible. + LearnersNext map[uint64]struct{} +} + +func (c Config) String() string { + var buf strings.Builder + fmt.Fprintf(&buf, "voters=%s", c.Voters) + if c.Learners != nil { + fmt.Fprintf(&buf, " learners=%s", quorum.MajorityConfig(c.Learners).String()) + } + if c.LearnersNext != nil { + fmt.Fprintf(&buf, " learners_next=%s", quorum.MajorityConfig(c.LearnersNext).String()) + } + if c.AutoLeave { + fmt.Fprint(&buf, " autoleave") + } + return buf.String() +} + +// Clone returns a copy of the Config that shares no memory with the original. +func (c *Config) Clone() Config { + clone := func(m map[uint64]struct{}) map[uint64]struct{} { + if m == nil { + return nil + } + mm := make(map[uint64]struct{}, len(m)) + for k := range m { + mm[k] = struct{}{} + } + return mm + } + return Config{ + Voters: quorum.JointConfig{clone(c.Voters[0]), clone(c.Voters[1])}, + Learners: clone(c.Learners), + LearnersNext: clone(c.LearnersNext), + } +} + +// ProgressTracker tracks the currently active configuration and the information +// known about the nodes and learners in it. In particular, it tracks the match +// index for each peer which in turn allows reasoning about the committed index. +type ProgressTracker struct { + Config + + Progress ProgressMap + + Votes map[uint64]bool + + MaxInflight int + MaxInflightBytes uint64 +} + +// MakeProgressTracker initializes a ProgressTracker. +func MakeProgressTracker(maxInflight int, maxBytes uint64) ProgressTracker { + p := ProgressTracker{ + MaxInflight: maxInflight, + MaxInflightBytes: maxBytes, + Config: Config{ + Voters: quorum.JointConfig{ + quorum.MajorityConfig{}, + nil, // only populated when used + }, + Learners: nil, // only populated when used + LearnersNext: nil, // only populated when used + }, + Votes: map[uint64]bool{}, + Progress: map[uint64]*Progress{}, + } + return p +} + +// ConfState returns a ConfState representing the active configuration. +func (p *ProgressTracker) ConfState() *pb.ConfState { + return &pb.ConfState{ + Voters: p.Voters[0].Slice(), + VotersOutgoing: p.Voters[1].Slice(), + Learners: quorum.MajorityConfig(p.Learners).Slice(), + LearnersNext: quorum.MajorityConfig(p.LearnersNext).Slice(), + AutoLeave: new(p.AutoLeave), + } +} + +// IsSingleton returns true if (and only if) there is only one voting member +// (i.e. the leader) in the current configuration. +func (p *ProgressTracker) IsSingleton() bool { + return len(p.Voters[0]) == 1 && len(p.Voters[1]) == 0 +} + +type matchAckIndexer map[uint64]*Progress + +var _ quorum.AckedIndexer = matchAckIndexer(nil) + +// AckedIndex implements IndexLookuper. +func (l matchAckIndexer) AckedIndex(id uint64) (quorum.Index, bool) { + pr, ok := l[id] + if !ok { + return 0, false + } + return quorum.Index(pr.Match), true +} + +// Committed returns the largest log index known to be committed based on what +// the voting members of the group have acknowledged. +func (p *ProgressTracker) Committed() uint64 { + return uint64(p.Voters.CommittedIndex(matchAckIndexer(p.Progress))) +} + +// Visit invokes the supplied closure for all tracked progresses in stable order. +func (p *ProgressTracker) Visit(f func(id uint64, pr *Progress)) { + n := len(p.Progress) + // We need to sort the IDs and don't want to allocate since this is hot code. + // The optimization here mirrors that in `(MajorityConfig).CommittedIndex`, + // see there for details. + var sl [7]uint64 + var ids []uint64 + if len(sl) >= n { + ids = sl[:n] + } else { + ids = make([]uint64, n) + } + for id := range p.Progress { + n-- + ids[n] = id + } + slices.Sort(ids) + for _, id := range ids { + f(id, p.Progress[id]) + } +} + +// QuorumActive returns true if the quorum is active from the view of the local +// raft state machine. Otherwise, it returns false. +func (p *ProgressTracker) QuorumActive() bool { + votes := map[uint64]bool{} + p.Visit(func(id uint64, pr *Progress) { + if pr.IsLearner { + return + } + votes[id] = pr.RecentActive + }) + + return p.Voters.VoteResult(votes) == quorum.VoteWon +} + +// VoterNodes returns a sorted slice of voters. +func (p *ProgressTracker) VoterNodes() []uint64 { + m := p.Voters.IDs() + nodes := make([]uint64, 0, len(m)) + for id := range m { + nodes = append(nodes, id) + } + slices.Sort(nodes) + return nodes +} + +// LearnerNodes returns a sorted slice of learners. +func (p *ProgressTracker) LearnerNodes() []uint64 { + if len(p.Learners) == 0 { + return nil + } + nodes := make([]uint64, 0, len(p.Learners)) + for id := range p.Learners { + nodes = append(nodes, id) + } + slices.Sort(nodes) + return nodes +} + +// ResetVotes prepares for a new round of vote counting via recordVote. +func (p *ProgressTracker) ResetVotes() { + p.Votes = map[uint64]bool{} +} + +// RecordVote records that the node with the given id voted for this Raft +// instance if v == true (and declined it otherwise). +func (p *ProgressTracker) RecordVote(id uint64, v bool) { + _, ok := p.Votes[id] + if !ok { + p.Votes[id] = v + } +} + +// TallyVotes returns the number of granted and rejected Votes, and whether the +// election outcome is known. +func (p *ProgressTracker) TallyVotes() (granted int, rejected int, _ quorum.VoteResult) { + // Make sure to populate granted/rejected correctly even if the Votes slice + // contains members no longer part of the configuration. This doesn't really + // matter in the way the numbers are used (they're informational), but might + // as well get it right. + for id, pr := range p.Progress { + if pr.IsLearner { + continue + } + v, voted := p.Votes[id] + if !voted { + continue + } + if v { + granted++ + } else { + rejected++ + } + } + result := p.Voters.VoteResult(p.Votes) + return granted, rejected, result +} diff --git a/third_party/raft/types.go b/third_party/raft/types.go new file mode 100644 index 0000000000..908edcbfa6 --- /dev/null +++ b/third_party/raft/types.go @@ -0,0 +1,106 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "fmt" + + pb "go.etcd.io/raft/v3/raftpb" +) + +// entryID uniquely identifies a raft log entry. +// +// Every entry is associated with a leadership term which issued this entry and +// initially appended it to the log. There can only be one leader at any term, +// and a leader never issues two entries with the same index. +type entryID struct { + term uint64 + index uint64 +} + +// pbEntryID returns the ID of the given pb.Entry. +func pbEntryID(entry *pb.Entry) entryID { + return entryID{term: entry.GetTerm(), index: entry.GetIndex()} +} + +// logSlice describes a correct slice of a raft log. +// +// Every log slice is considered in a context of a specific leader term. This +// term does not necessarily match entryID.term of the entries, since a leader +// log contains both entries from its own term, and some earlier terms. +// +// Two slices with a matching logSlice.term are guaranteed to be consistent, +// i.e. they never contain two different entries at the same index. The reverse +// is not true: two slices with different logSlice.term may contain both +// matching and mismatching entries. Specifically, logs at two different leader +// terms share a common prefix, after which they *permanently* diverge. +// +// A well-formed logSlice conforms to raft safety properties. It provides the +// following guarantees: +// +// 1. entries[i].Index == prev.index + 1 + i, +// 2. prev.term <= entries[0].Term, +// 3. entries[i-1].Term <= entries[i].Term, +// 4. entries[len-1].Term <= term. +// +// Property (1) means the slice is contiguous. Properties (2) and (3) mean that +// the terms of the entries in a log never regress. Property (4) means that a +// leader log at a specific term never has entries from higher terms. +// +// Users of this struct can assume the invariants hold true. Exception is the +// "gateway" code that initially constructs logSlice, such as when its content +// is sourced from a message that was received via transport, or from Storage, +// or in a test code that manually hard-codes this struct. In these cases, the +// invariants should be validated using the valid() method. +type logSlice struct { + // term is the leader term containing the given entries in its log. + term uint64 + // prev is the ID of the entry immediately preceding the entries. + prev entryID + // entries contains the consecutive entries representing this slice. + entries []*pb.Entry +} + +// lastIndex returns the index of the last entry in this log slice. Returns +// prev.index if there are no entries. +func (s logSlice) lastIndex() uint64 { + return s.prev.index + uint64(len(s.entries)) +} + +// lastEntryID returns the ID of the last entry in this log slice, or prev if +// there are no entries. +func (s logSlice) lastEntryID() entryID { + if ln := len(s.entries); ln != 0 { + return pbEntryID(s.entries[ln-1]) + } + return s.prev +} + +// valid returns nil iff the logSlice is a well-formed log slice. See logSlice +// comment for details on what constitutes a valid raft log slice. +func (s logSlice) valid() error { + prev := s.prev + for i := range s.entries { + id := pbEntryID(s.entries[i]) + if id.term < prev.term || id.index != prev.index+1 { + return fmt.Errorf("leader term %d: entries %+v and %+v not consistent", s.term, prev, id) + } + prev = id + } + if s.term < prev.term { + return fmt.Errorf("leader term %d: entry %+v has a newer term", s.term, prev) + } + return nil +} diff --git a/third_party/raft/types_test.go b/third_party/raft/types_test.go new file mode 100644 index 0000000000..c449f74ee5 --- /dev/null +++ b/third_party/raft/types_test.go @@ -0,0 +1,94 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "testing" + + "github.com/stretchr/testify/require" + + pb "go.etcd.io/raft/v3/raftpb" +) + +func TestEntryID(t *testing.T) { + // Some obvious checks first. + require.Equal(t, entryID{term: 5, index: 10}, entryID{term: 5, index: 10}) + require.NotEqual(t, entryID{term: 4, index: 10}, entryID{term: 5, index: 10}) + require.NotEqual(t, entryID{term: 5, index: 9}, entryID{term: 5, index: 10}) + + for _, tt := range []struct { + entry *pb.Entry + want entryID + }{ + {entry: &pb.Entry{}, want: entryID{term: 0, index: 0}}, + {entry: &pb.Entry{Term: new(uint64(1)), Index: new(uint64(2)), Data: []byte("data")}, want: entryID{term: 1, index: 2}}, + {entry: &pb.Entry{Term: new(uint64(10)), Index: new(uint64(123))}, want: entryID{term: 10, index: 123}}, + } { + require.Equal(t, tt.want, pbEntryID(tt.entry)) + } +} + +func TestLogSlice(t *testing.T) { + id := func(index, term uint64) entryID { + return entryID{term: term, index: index} + } + e := func(index, term uint64) *pb.Entry { + return &pb.Entry{Term: new(term), Index: new(index)} + } + for _, tt := range []struct { + term uint64 + prev entryID + entries []*pb.Entry + + notOk bool + last entryID + }{ + // Empty "dummy" slice, starting at (0, 0) origin of the log. + {last: id(0, 0)}, + // Empty slice with a given prev ID. Valid only if term >= prev.term. + {prev: id(123, 10), notOk: true}, + {term: 9, prev: id(123, 10), notOk: true}, + {term: 10, prev: id(123, 10), last: id(123, 10)}, + {term: 11, prev: id(123, 10), last: id(123, 10)}, + // A single entry. + {term: 0, entries: []*pb.Entry{e(1, 1)}, notOk: true}, + {term: 1, entries: []*pb.Entry{e(1, 1)}, last: id(1, 1)}, + {term: 2, entries: []*pb.Entry{e(1, 1)}, last: id(1, 1)}, + // Multiple entries. + {term: 2, entries: []*pb.Entry{e(2, 1), e(3, 1), e(4, 2)}, notOk: true}, + {term: 1, prev: id(1, 1), entries: []*pb.Entry{e(2, 1), e(3, 1), e(4, 2)}, notOk: true}, + {term: 2, prev: id(1, 1), entries: []*pb.Entry{e(2, 1), e(3, 1), e(4, 2)}, last: id(4, 2)}, + // First entry inconsistent with prev. + {term: 10, prev: id(123, 5), entries: []*pb.Entry{e(111, 5)}, notOk: true}, + {term: 10, prev: id(123, 5), entries: []*pb.Entry{e(124, 4)}, notOk: true}, + {term: 10, prev: id(123, 5), entries: []*pb.Entry{e(234, 6)}, notOk: true}, + {term: 10, prev: id(123, 5), entries: []*pb.Entry{e(124, 6)}, last: id(124, 6)}, + // Inconsistent entries. + {term: 10, prev: id(12, 2), entries: []*pb.Entry{e(13, 2), e(12, 2)}, notOk: true}, + {term: 10, prev: id(12, 2), entries: []*pb.Entry{e(13, 2), e(15, 2)}, notOk: true}, + {term: 10, prev: id(12, 2), entries: []*pb.Entry{e(13, 2), e(14, 1)}, notOk: true}, + {term: 10, prev: id(12, 2), entries: []*pb.Entry{e(13, 2), e(14, 3)}, last: id(14, 3)}, + } { + t.Run("", func(t *testing.T) { + s := logSlice{term: tt.term, prev: tt.prev, entries: tt.entries} + require.Equal(t, tt.notOk, s.valid() != nil) + if !tt.notOk { + last := s.lastEntryID() + require.Equal(t, tt.last, last) + require.Equal(t, last.index, s.lastIndex()) + } + }) + } +} diff --git a/third_party/raft/util.go b/third_party/raft/util.go new file mode 100644 index 0000000000..0be56ba656 --- /dev/null +++ b/third_party/raft/util.go @@ -0,0 +1,343 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "bytes" + "fmt" + "strings" + + "google.golang.org/protobuf/proto" + + pb "go.etcd.io/raft/v3/raftpb" +) + +func (st StateType) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf("%q", st.String())), nil +} + +var isLocalMsg = [...]bool{ + pb.MsgHup: true, + pb.MsgBeat: true, + pb.MsgUnreachable: true, + pb.MsgSnapStatus: true, + pb.MsgCheckQuorum: true, + pb.MsgStorageAppend: true, + pb.MsgStorageAppendResp: true, + pb.MsgStorageApply: true, + pb.MsgStorageApplyResp: true, +} + +var isResponseMsg = [...]bool{ + pb.MsgAppResp: true, + pb.MsgVoteResp: true, + pb.MsgHeartbeatResp: true, + pb.MsgUnreachable: true, + pb.MsgReadIndexResp: true, + pb.MsgPreVoteResp: true, + pb.MsgStorageAppendResp: true, + pb.MsgStorageApplyResp: true, +} + +func isMsgInArray(msgt pb.MessageType, arr []bool) bool { + i := int(msgt) + return i < len(arr) && arr[i] +} + +func IsLocalMsg(msgt pb.MessageType) bool { + return isMsgInArray(msgt, isLocalMsg[:]) +} + +func IsResponseMsg(msgt pb.MessageType) bool { + return isMsgInArray(msgt, isResponseMsg[:]) +} + +func IsLocalMsgTarget(id uint64) bool { + return id == LocalAppendThread || id == LocalApplyThread +} + +// voteResponseType maps vote and prevote message types to their corresponding responses. +func voteRespMsgType(msgt pb.MessageType) pb.MessageType { + switch msgt { + case pb.MsgVote: + return pb.MsgVoteResp + case pb.MsgPreVote: + return pb.MsgPreVoteResp + default: + panic(fmt.Sprintf("not a vote message: %s", msgt)) + } +} + +func DescribeHardState(hs *pb.HardState) string { + var buf strings.Builder + fmt.Fprintf(&buf, "Term:%d", hs.GetTerm()) + if hs.GetVote() != 0 { + fmt.Fprintf(&buf, " Vote:%d", hs.GetVote()) + } + fmt.Fprintf(&buf, " Commit:%d", hs.GetCommit()) + return buf.String() +} + +func DescribeSoftState(ss SoftState) string { + return fmt.Sprintf("Lead:%d State:%s", ss.Lead, ss.RaftState) +} + +func DescribeConfState(state *pb.ConfState) string { + return fmt.Sprintf( + "Voters:%v VotersOutgoing:%v Learners:%v LearnersNext:%v AutoLeave:%v", + state.Voters, state.VotersOutgoing, state.Learners, state.LearnersNext, state.GetAutoLeave(), + ) +} + +func DescribeSnapshot(snap *pb.Snapshot) string { + m := snap.GetMetadata() + return fmt.Sprintf("Index:%d Term:%d ConfState:%s", m.GetIndex(), m.GetTerm(), DescribeConfState(m.GetConfState())) +} + +func DescribeReady(rd Ready, f EntryFormatter) string { + var buf strings.Builder + if rd.SoftState != nil { + fmt.Fprint(&buf, DescribeSoftState(*rd.SoftState)) + buf.WriteByte('\n') + } + if !IsEmptyHardState(rd.HardState) { + fmt.Fprintf(&buf, "HardState %s", DescribeHardState(rd.HardState)) + buf.WriteByte('\n') + } + if len(rd.ReadStates) > 0 { + fmt.Fprintf(&buf, "ReadStates %v\n", rd.ReadStates) + } + if len(rd.Entries) > 0 { + buf.WriteString("Entries:\n") + fmt.Fprint(&buf, DescribeEntries(rd.Entries, f)) + } + if !IsEmptySnap(rd.Snapshot) { + fmt.Fprintf(&buf, "Snapshot %s\n", DescribeSnapshot(rd.Snapshot)) + } + if len(rd.CommittedEntries) > 0 { + buf.WriteString("CommittedEntries:\n") + fmt.Fprint(&buf, DescribeEntries(rd.CommittedEntries, f)) + } + if len(rd.Messages) > 0 { + buf.WriteString("Messages:\n") + for _, msg := range rd.Messages { + fmt.Fprint(&buf, DescribeMessage(msg, f)) + buf.WriteByte('\n') + } + } + if buf.Len() > 0 { + return fmt.Sprintf("Ready MustSync=%t:\n%s", rd.MustSync, buf.String()) + } + return "" +} + +// EntryFormatter can be implemented by the application to provide human-readable formatting +// of entry data. Nil is a valid EntryFormatter and will use a default format. +type EntryFormatter func([]byte) string + +// DescribeMessage returns a concise human-readable description of a +// Message for debugging. +func DescribeMessage(m *pb.Message, f EntryFormatter) string { + return describeMessageWithIndent("", m, f) +} + +func describeMessageWithIndent(indent string, m *pb.Message, f EntryFormatter) string { + var buf bytes.Buffer + fmt.Fprintf(&buf, "%s%s->%s %v Term:%d Log:%d/%d", indent, + describeTarget(m.GetFrom()), describeTarget(m.GetTo()), m.GetType(), m.GetTerm(), m.GetLogTerm(), m.GetIndex()) + if m.GetReject() { + fmt.Fprintf(&buf, " Rejected (Hint: %d)", m.GetRejectHint()) + } + if m.GetCommit() != 0 { + fmt.Fprintf(&buf, " Commit:%d", m.GetCommit()) + } + if m.GetVote() != 0 { + fmt.Fprintf(&buf, " Vote:%d", m.GetVote()) + } + if ents := m.GetEntries(); len(ents) == 1 { + fmt.Fprintf(&buf, " Entries:[%s]", DescribeEntry(ents[0], f)) + } else if len(ents) > 1 { + fmt.Fprint(&buf, " Entries:[") + for _, e := range ents { + fmt.Fprintf(&buf, "\n%s ", indent) + buf.WriteString(DescribeEntry(e, f)) + } + fmt.Fprintf(&buf, "\n%s]", indent) + } + if s := m.GetSnapshot(); s != nil && !IsEmptySnap(s) { + fmt.Fprintf(&buf, "\n%s Snapshot: %s", indent, DescribeSnapshot(s)) + } + if len(m.GetResponses()) > 0 { + fmt.Fprintf(&buf, " Responses:[") + for _, r := range m.GetResponses() { + buf.WriteString("\n") + buf.WriteString(describeMessageWithIndent(indent+" ", r, f)) + } + fmt.Fprintf(&buf, "\n%s]", indent) + } + return buf.String() +} + +func describeTarget(id uint64) string { + switch id { + case None: + return "None" + case LocalAppendThread: + return "AppendThread" + case LocalApplyThread: + return "ApplyThread" + default: + return fmt.Sprintf("%x", id) + } +} + +// DescribeEntry returns a concise human-readable description of an +// Entry for debugging. +func DescribeEntry(e *pb.Entry, f EntryFormatter) string { + if f == nil { + f = func(data []byte) string { return fmt.Sprintf("%q", data) } + } + + formatConfChange := func(cc pb.ConfChangeI) string { + // TODO(tbg): give the EntryFormatter a type argument so that it gets + // a chance to expose the Context. + return pb.ConfChangesToString(cc.AsV2().Changes) + } + + var formatted string + switch e.GetType() { + case pb.EntryNormal: + formatted = f(e.GetData()) + case pb.EntryConfChange: + cc := &pb.ConfChange{} + if err := proto.Unmarshal(e.GetData(), cc); err != nil { + formatted = err.Error() + } else { + formatted = formatConfChange(cc) + } + case pb.EntryConfChangeV2: + cc := &pb.ConfChangeV2{} + if err := proto.Unmarshal(e.GetData(), cc); err != nil { + formatted = err.Error() + } else { + formatted = formatConfChange(cc) + } + } + if formatted != "" { + formatted = " " + formatted + } + return fmt.Sprintf("%d/%d %s%s", e.GetTerm(), e.GetIndex(), e.GetType(), formatted) +} + +// DescribeConfChange returns a deterministic, human-readable representation of +// a ConfChangeI. It avoids using the proto text format (which adds random extra +// spaces via detrand, producing unstable output across architectures/builds). +func DescribeConfChange(cc pb.ConfChangeI) string { + cv2 := cc.AsV2() + var b strings.Builder + fmt.Fprintf(&b, "transition:%v", cv2.GetTransition()) + for _, c := range cv2.GetChanges() { + fmt.Fprintf(&b, " changes:{type:%v node_id:%d}", c.GetType(), c.GetNodeId()) + } + if len(cv2.Context) > 0 { + fmt.Fprintf(&b, " context:%q", cv2.Context) + } + return b.String() +} + +// DescribeEntries calls DescribeEntry for each Entry, adding a newline to +// each. +func DescribeEntries(ents []*pb.Entry, f EntryFormatter) string { + var buf bytes.Buffer + for _, e := range ents { + _, _ = buf.WriteString(DescribeEntry(e, f) + "\n") + } + return buf.String() +} + +// entryEncodingSize represents the protocol buffer encoding size of one or more +// entries. +type entryEncodingSize uint64 + +func entsSize(ents []*pb.Entry) entryEncodingSize { + var size entryEncodingSize + for _, ent := range ents { + size += entryEncodingSize(proto.Size(ent)) + } + return size +} + +// limitSize returns the longest prefix of the given entries slice, such that +// its total byte size does not exceed maxSize. Always returns a non-empty slice +// if the input is non-empty, so, as an exception, if the size of the first +// entry exceeds maxSize, a non-empty slice with just this entry is returned. +func limitSize(ents []*pb.Entry, maxSize entryEncodingSize) []*pb.Entry { + if len(ents) == 0 { + return ents + } + size := entryEncodingSize(proto.Size(ents[0])) + for limit := 1; limit < len(ents); limit++ { + size += entryEncodingSize(proto.Size(ents[limit])) + if size > maxSize { + return ents[:limit] + } + } + return ents +} + +// entryPayloadSize represents the size of one or more entries' payloads. +// Notably, it does not depend on its Index or Term. Entries with empty +// payloads, like those proposed after a leadership change, are considered +// to be zero size. +type entryPayloadSize uint64 + +// payloadSize is the size of the payload of the provided entry. +func payloadSize(e *pb.Entry) entryPayloadSize { + return entryPayloadSize(len(e.GetData())) +} + +// payloadsSize is the size of the payloads of the provided entries. +func payloadsSize(ents []*pb.Entry) entryPayloadSize { + var s entryPayloadSize + for _, e := range ents { + s += payloadSize(e) + } + return s +} + +func assertConfStatesEquivalent(l Logger, cs1, cs2 *pb.ConfState) { + err := cs1.Equivalent(cs2) + if err == nil { + return + } + l.Panic(err) +} + +// extend appends vals to the given dst slice. It differs from the standard +// slice append only in the way it allocates memory. If cap(dst) is not enough +// for appending the values, precisely size len(dst)+len(vals) is allocated. +// +// Use this instead of standard append in situations when this is the last +// append to dst, so there is no sense in allocating more than needed. +func extend(dst, vals []*pb.Entry) []*pb.Entry { + need := len(dst) + len(vals) + if need <= cap(dst) { + return append(dst, vals...) // does not allocate + } + buf := make([]*pb.Entry, need, need) // allocates precisely what's needed + copy(buf, dst) + copy(buf[len(dst):], vals) + return buf +} diff --git a/third_party/raft/util_test.go b/third_party/raft/util_test.go new file mode 100644 index 0000000000..ce0971e888 --- /dev/null +++ b/third_party/raft/util_test.go @@ -0,0 +1,152 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "fmt" + "math" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + pb "go.etcd.io/raft/v3/raftpb" +) + +var testFormatter EntryFormatter = func(data []byte) string { + return strings.ToUpper(string(data)) +} + +func TestDescribeEntry(t *testing.T) { + entry := &pb.Entry{ + Term: new(uint64(1)), + Index: new(uint64(2)), + Type: pb.EntryNormal.Enum(), + Data: []byte("hello\x00world"), + } + require.Equal(t, `1/2 EntryNormal "hello\x00world"`, DescribeEntry(entry, nil)) + require.Equal(t, "1/2 EntryNormal HELLO\x00WORLD", DescribeEntry(entry, testFormatter)) +} + +func TestLimitSize(t *testing.T) { + ents := []*pb.Entry{{Index: new(uint64(4)), Term: new(uint64(4))}, {Index: new(uint64(5)), Term: new(uint64(5))}, {Index: new(uint64(6)), Term: new(uint64(6))}} + prefix := func(size int) []*pb.Entry { + return append([]*pb.Entry{}, ents[:size]...) // protect the original slice + } + for _, tt := range []struct { + maxSize uint64 + want []*pb.Entry + }{ + {math.MaxUint64, prefix(len(ents))}, // all entries are returned + // Even if maxSize is zero, the first entry should be returned. + {0, prefix(1)}, + // Limit to 2. + {uint64(proto.Size(ents[0]) + proto.Size(ents[1])), prefix(2)}, + {uint64(proto.Size(ents[0]) + proto.Size(ents[1]) + proto.Size(ents[2])/2), prefix(2)}, + {uint64(proto.Size(ents[0]) + proto.Size(ents[1]) + proto.Size(ents[2]) - 1), prefix(2)}, + // All. + {uint64(proto.Size(ents[0]) + proto.Size(ents[1]) + proto.Size(ents[2])), prefix(3)}, + } { + t.Run("", func(t *testing.T) { + got := limitSize(ents, entryEncodingSize(tt.maxSize)) + require.Equal(t, tt.want, got) + size := entsSize(got) + require.True(t, len(got) == 1 || size <= entryEncodingSize(tt.maxSize)) + }) + } +} + +func TestIsLocalMsg(t *testing.T) { + tests := []struct { + msgt pb.MessageType + isLocal bool + }{ + {pb.MsgHup, true}, + {pb.MsgBeat, true}, + {pb.MsgUnreachable, true}, + {pb.MsgSnapStatus, true}, + {pb.MsgCheckQuorum, true}, + {pb.MsgTransferLeader, false}, + {pb.MsgProp, false}, + {pb.MsgApp, false}, + {pb.MsgAppResp, false}, + {pb.MsgVote, false}, + {pb.MsgVoteResp, false}, + {pb.MsgSnap, false}, + {pb.MsgHeartbeat, false}, + {pb.MsgHeartbeatResp, false}, + {pb.MsgTimeoutNow, false}, + {pb.MsgReadIndex, false}, + {pb.MsgReadIndexResp, false}, + {pb.MsgPreVote, false}, + {pb.MsgPreVoteResp, false}, + {pb.MsgStorageAppend, true}, + {pb.MsgStorageAppendResp, true}, + {pb.MsgStorageApply, true}, + {pb.MsgStorageApplyResp, true}, + } + + for _, tt := range tests { + t.Run(fmt.Sprint(tt.msgt), func(t *testing.T) { + require.Equal(t, tt.isLocal, IsLocalMsg(tt.msgt)) + }) + } +} + +func TestIsResponseMsg(t *testing.T) { + tests := []struct { + msgt pb.MessageType + isResponse bool + }{ + {pb.MsgHup, false}, + {pb.MsgBeat, false}, + {pb.MsgUnreachable, true}, + {pb.MsgSnapStatus, false}, + {pb.MsgCheckQuorum, false}, + {pb.MsgTransferLeader, false}, + {pb.MsgProp, false}, + {pb.MsgApp, false}, + {pb.MsgAppResp, true}, + {pb.MsgVote, false}, + {pb.MsgVoteResp, true}, + {pb.MsgSnap, false}, + {pb.MsgHeartbeat, false}, + {pb.MsgHeartbeatResp, true}, + {pb.MsgTimeoutNow, false}, + {pb.MsgReadIndex, false}, + {pb.MsgReadIndexResp, true}, + {pb.MsgPreVote, false}, + {pb.MsgPreVoteResp, true}, + {pb.MsgStorageAppend, false}, + {pb.MsgStorageAppendResp, true}, + {pb.MsgStorageApply, false}, + {pb.MsgStorageApplyResp, true}, + } + + for i, tt := range tests { + got := IsResponseMsg(tt.msgt) + assert.Equal(t, tt.isResponse, got, "#%d", i) + } +} + +// TestPayloadSizeOfEmptyEntry ensures that payloadSize of empty entry is always zero. +// This property is important because new leaders append an empty entry to their log, +// and we don't want this to count towards the uncommitted log quota. +func TestPayloadSizeOfEmptyEntry(t *testing.T) { + e := &pb.Entry{Data: nil} + require.Equal(t, 0, int(payloadSize(e))) +} From 0409a4a2eadc98fbd0f55b67247431aff896a644 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Fri, 23 Oct 2026 22:13:10 -0400 Subject: [PATCH 004/103] Make consensus effects wait for explicit durable completions. Wrap RawNode in a seeded single-threaded input and effect contract so hosts cannot release dependent messages or authority reads before persistence and application. Bound read contexts, reject peer-injected control messages, stop on storage or schema failures, and pin election replay with focused quorum and transfer smoke tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/raftcore/README.md | 112 ++++ internal/raftcore/core.go | 485 +++++++++++++++++ internal/raftcore/core_test.go | 630 +++++++++++++++++++++++ internal/raftcore/logger.go | 19 + internal/raftcore/random.go | 30 ++ internal/raftcore/testdata/replay.sha256 | 1 + internal/raftcore/types.go | 159 ++++++ 7 files changed, 1436 insertions(+) create mode 100644 internal/raftcore/README.md create mode 100644 internal/raftcore/core.go create mode 100644 internal/raftcore/core_test.go create mode 100644 internal/raftcore/logger.go create mode 100644 internal/raftcore/random.go create mode 100644 internal/raftcore/testdata/replay.sha256 create mode 100644 internal/raftcore/types.go diff --git a/internal/raftcore/README.md b/internal/raftcore/README.md new file mode 100644 index 0000000000..c5a7c3c947 --- /dev/null +++ b/internal/raftcore/README.md @@ -0,0 +1,112 @@ +# Deterministic consensus boundary + +`raftcore` is an internal development seam shared by native and WASI hosts. It +has no production routing, goroutine, filesystem, timer, network, or ambient +random input. It owns RawNode and a copied in-memory view of recovered log +bytes. It imports P01's canonical command decoder, not directory/HTTP/SQLite. +Both targets must build through the root module's patched Raft replacement; +see `third_party/raft-patch/README.md` for pinned source and licenses. + +## One call, explicit effects + +Construct `New(Config, RecoveredState)`, then `Step(Poll{})` for initial work. +For fresh bootstrap every actor supplies the same voter set on empty storage; +this is not three singleton bootstraps. The host must authenticate the identity, +protocol, manifest, node ID and generation before constructing a member. IDs +cannot be reused. An empty bootstrap supports a not-yet-admitted learner; this +low-level mechanism does not grant enrollment or choose production topology. + +Each Step returns diagnostic status and at most one new persistence batch. +Handle effects as follows: + +1. Atomically persist all `Persist` entries, hard state and snapshot pointers, + replacing conflicting suffixes. Await durability, then submit + `Persisted{BatchID}`. Even commit-only hard state is durable. An empty batch + is a dependency barrier and needs no physical transaction. +2. Only that completion releases `Send` messages and `Apply` work. Apply + entries in index order. Configuration entries have already been passed to + RawNode; the effect's complete ConfState must be persisted alongside the + applied index, application rows, visible version, logical time and dedup + results. A snapshot install must restore its application state before its + suffix. Empty normal entries also advance the applied index. +3. After that application transaction is durable, submit `Applied{BatchID}`. + The core advances RawNode only here (or immediately after persistence when + there was no application work). The completion may return another Persist. + Client mutation results belong to the durable application transaction, not + successful proposal admission. +4. Serve an authority read only from a `Reads` barrier after the host's durable + applied index reaches it. Reads use safe quorum ReadIndex, with a committed + current-term entry, rather than a lease or remembered leader. Barriers that + run ahead of a size-limited apply batch are retained. Cancel timed-out reads + explicitly; there are at most 128 pending contexts per handle. A cancelled + submitted context keeps its admission slot until its quorum round completes; + pre-current-term reads stay in the bounded wrapper map until ready to send. + +The host serializes calls. While a batch is pending, ordinary input returns +`ErrBusy` without being consumed: retain and retry it, with transport admission +bounds. Completions must match the pending batch and phase. Batch IDs belong +only to one handle lifetime; discard old-generation callbacks after recovery. +On storage failure send `StorageFailed{BatchID}` or drop the handle. No further +participation is permitted; retain the store and recover deliberately. Unknown +committed command schemas/operations stop participation rather than becoming +successful no-ops. Invariant panics are caught and stop the handle with a +constant diagnostic; no command bytes are logged. + +## Host and later packet responsibilities + +`PeerMessage` accepts only network Raft replication/election types. It rejects +local triggers, storage completions, forwarded proposals, peer authority reads, +and peer-initiated transfers. P07 authenticates original envelope signatures, +cluster/protocol, enrollment and revocation generations before this call. Local +`Propose` validates canonical schema/cluster/size but does not authenticate the +signature or apply policy. P09 must recheck committed authorization and produce +application results. Normal commands are bounded by the shared 64 KiB batch +budget; uncommitted payload is bounded at 1 MiB and inflight append batches at 8. +Transport chunking/snapshot admission remain P05/P07. + +`Configure` and `Transfer` are mechanisms for authorized host intents, not +public APIs. Transfer requires a caught-up incoming voter outside joint state. +Request admission is not transfer completion: the host must observe the target +as leader with a current-term commit and a new read barrier. P12/P13 own the +registry, single membership intent, joint swaps, configuration revision, +owner/persistent-storage eligibility, fallback and policy. No topology or +leadership-reacquisition policy is implemented here. + +P04 supplies actual durable storage. Recovered suffixes must be contiguous and +include uncommitted entries; ConfState is the most recent DURABLY APPLIED one. +The snapshot supplied at construction must already be fully installed in the +application (Applied >= its included index). If a crash left a snapshot pointer +without its application transaction, recovery must complete that installation +before constructing the core, or retain the preceding installed generation. +The host validates checksums and cluster/schema manifests; core structural +validation is not a substitute. A recovered core starts as a follower. Every +new handle receives explicit election seed input; it does not restore volatile +leadership, timers or RNG state. Replay records each such seed. + +The current in-memory log view is not a persistent database or a complete +production memory policy. P05 must bound it through explicit snapshot/cache and +compaction effects, stream snapshot data and preserve two durable generations. +Do not use this package to load a full application identity into wasm: P01 rows +and P09 paged apply remain outside the consensus cache. Quiescence scheduling, +clock samples, retry deadlines and application logical time are explicit host +inputs in later packets. `NeedNextTick` currently means every non-stopped handle +needs ticks; the safe quiescence protocol belongs to P14. + +## Replay contract + +Election timeout is 6–11 ticks, with a heartbeat every tick. Hosts currently +schedule 5-second ticks (P14); no wall duration is read here. SplitMix64's exact +64-bit constants, overflow and rejection sampling are specified in random.go; +seed zero is valid. Every timeout reset consumes this per-node provider and +never upstream global randomness. RNG state/draws appear only in diagnostics. +Wire protobuf marshaling is deterministic. RawNode already orders meaningful +peer traversal; preserve its per-peer message order. Completed read contexts +sort before emission. P03 versions the binary ABI and checks actual wasm/native +parity; a JSON encoding of Go diagnostic structs is not that ABI. + +Focused proof: `go test -p=1 ./internal/raftcore -run '^TestR' -count=1` at the +wave's nice/CPU/pressure limits. The tests cover R3/R4/R5/R6/R12/R14 mechanisms, +not the later simulator, persistent power-loss, joint-swap, cross-runtime or +real-client release proofs. The initial trace SHA-256 is a regression fixture; +SplitMix64 vectors are independently published values. Full fault checking is +P06 and the actual Worker stop/go measurement is P03. diff --git a/internal/raftcore/core.go b/internal/raftcore/core.go new file mode 100644 index 0000000000..79fef56913 --- /dev/null +++ b/internal/raftcore/core.go @@ -0,0 +1,485 @@ +package raftcore + +import ( + "bytes" + "encoding/binary" + "slices" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "go.etcd.io/raft/v3" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +var wire = proto.MarshalOptions{Deterministic: true} + +// storage is an explicit in-memory view of recovered and completed durable +// bytes. No host callback or filesystem read occurs while stepping consensus. +// P05 will bound retained history by snapshots; this is not a durable store. +type storage struct { + *raft.MemoryStorage + conf *pb.ConfState +} + +func (s *storage) InitialState() (*pb.HardState, *pb.ConfState, error) { + h, _, err := s.MemoryStorage.InitialState() + return h, s.conf, err +} + +type pending struct { + id uint64 + ready raft.Ready + persisted bool + index uint64 +} +type readRequest struct { + id string + term uint64 + submitted bool + cancelled bool +} + +// Core owns no goroutines or ambient clocks. A host serializes calls and +// retries ErrBusy inputs after completing the outstanding batch; nothing is +// queued implicitly. Storage failure or an unknown committed command stops the +// handle. Recovery always starts as a follower from supplied durable bytes. +type Core struct { + cfg Config + node *raft.RawNode + store *storage + rng splitMix64 + applied uint64 + serial uint64 + readSerial uint64 + reads map[string]readRequest + barriers map[string]uint64 + pending *pending + stopped bool +} + +func clone[T proto.Message](v T) T { + if !v.ProtoReflect().IsValid() { + return v + } + return proto.Clone(v).(T) +} +func copyEntries(entries []*pb.Entry) []*pb.Entry { + if entries == nil { + return nil + } + out := make([]*pb.Entry, len(entries)) + for i, e := range entries { + out[i] = clone(e) + } + return out +} + +// New refuses inconsistent recovery rather than manufacturing missing committed +// bytes. The caller validates durable checksums and cluster/format manifests +// before supplying this state; those storage formats belong to P04/P05. +func New(cfg Config, recovered RecoveredState) (core *Core, err error) { + defer func() { + if recover() != nil { + core, err = nil, ErrConfig + } + }() + if cfg.Protocol != ProtocolVersion { + return nil, ErrProtocol + } + if cfg.ClusterUUID == "" || cfg.NodeID == 0 || raft.IsLocalMsgTarget(cfg.NodeID) { + return nil, ErrConfig + } + snapshot := clone(recovered.Snapshot) + snapIndex := snapshot.GetMetadata().GetIndex() + last := snapIndex + lastTerm := snapshot.GetMetadata().GetTerm() + for _, e := range recovered.Entries { + if e == nil || last == ^uint64(0) || e.GetIndex() != last+1 || e.GetTerm() == 0 || e.GetTerm() < lastTerm { + return nil, ErrConfig + } + last = e.GetIndex() + lastTerm = e.GetTerm() + } + h := recovered.HardState + if recovered.Applied < snapIndex || recovered.Applied > h.GetCommit() || h.GetCommit() > last || h.GetCommit() < snapIndex || h.GetTerm() < lastTerm { + return nil, ErrConfig + } + if len(cfg.Bootstrap) != 0 && (last != 0 || recovered.Applied != 0 || !raft.IsEmptyHardState(h) || recovered.ConfState != nil) { + return nil, ErrConfig + } + conf := clone(recovered.ConfState) + if conf == nil { + conf = clone(snapshot.GetMetadata().GetConfState()) + } + if conf == nil { + conf = &pb.ConfState{} + } + c := &Core{cfg: cfg, rng: splitMix64{state: cfg.Seed}, applied: recovered.Applied, reads: make(map[string]readRequest), barriers: make(map[string]uint64)} + c.store = &storage{MemoryStorage: raft.NewMemoryStorage(), conf: conf} + if !raft.IsEmptySnap(snapshot) { + if err := c.store.ApplySnapshot(snapshot); err != nil { + return nil, ErrConfig + } + } + if err := c.store.Append(copyEntries(recovered.Entries)); err != nil { + return nil, ErrConfig + } + if h != nil { + if err := c.store.SetHardState(clone(h)); err != nil { + return nil, ErrConfig + } + } + c.node, err = raft.NewRawNode(&raft.Config{ + ID: cfg.NodeID, ElectionTick: ElectionTicks, HeartbeatTick: HeartbeatTicks, + Storage: c.store, Applied: recovered.Applied, ElectionRandom: c.rng.bounded, + PreVote: true, CheckQuorum: true, ReadOnlyOption: raft.ReadOnlySafe, + DisableProposalForwarding: true, AsyncStorageWrites: false, + MaxSizePerMsg: MaxMessageBytes, MaxCommittedSizePerReady: MaxMessageBytes, + MaxUncommittedEntriesSize: MaxUncommittedBytes, MaxInflightMsgs: MaxInflightBatches, + MaxInflightBytes: MaxInflightBatches * MaxMessageBytes, + StepDownOnRemoval: true, Logger: silentLogger{}, + }) + if err != nil { + return nil, err + } + if len(cfg.Bootstrap) != 0 { + ids := slices.Clone(cfg.Bootstrap) + slices.Sort(ids) + peers := make([]raft.Peer, len(ids)) + for i, id := range ids { + if id == 0 || raft.IsLocalMsgTarget(id) || (i > 0 && ids[i-1] == id) { + return nil, ErrConfig + } + peers[i] = raft.Peer{ID: id} + } + if err := c.node.Bootstrap(peers); err != nil { + return nil, ErrConfig + } + } + return c, nil +} + +func (c *Core) status() Status { + s := c.node.BasicStatus() + return Status{NodeID: c.cfg.NodeID, Leader: s.Lead, Term: s.GetTerm(), Commit: s.GetCommit(), Applied: c.applied, Role: s.RaftState, RNGState: c.rng.state, RNGDraws: c.rng.draws, Stopped: c.stopped} +} +func (c *Core) leader() bool { return c.node.BasicStatus().RaftState == raft.StateLeader } + +// Step is synchronous. Effects contain independent bytes, not aliases into +// RawNode. Peer authentication and membership generations are host obligations; +// this boundary additionally forbids all network-injected local control verbs. +func (c *Core) Step(input Input) (effects Effects, err error) { + defer func() { + if recover() != nil { + c.stopped = true + effects = Effects{} + err = ErrStopped + } + effects.Status = c.status() + effects.NeedNextTick = !c.stopped + }() + if c.stopped { + return effects, ErrStopped + } + switch in := input.(type) { + case StorageFailed: + if c.pending == nil || in.BatchID != c.pending.id { + return effects, ErrCompletion + } + c.stopped = true + return effects, ErrStopped + case Persisted: + p := c.pending + if p == nil || in.BatchID != p.id || p.persisted { + return effects, ErrCompletion + } + rd := p.ready + if !raft.IsEmptySnap(rd.Snapshot) { + if err := c.store.ApplySnapshot(clone(rd.Snapshot)); err != nil { + c.stopped = true + return effects, ErrStopped + } + c.store.conf = clone(rd.Snapshot.GetMetadata().GetConfState()) + } + if err := c.store.Append(copyEntries(rd.Entries)); err != nil { + c.stopped = true + return effects, ErrStopped + } + if !raft.IsEmptyHardState(rd.HardState) { + if err := c.store.SetHardState(clone(rd.HardState)); err != nil { + c.stopped = true + return effects, ErrStopped + } + } + p.persisted = true + return c.release() + case Applied: + p := c.pending + if p == nil || in.BatchID != p.id || !p.persisted || p.index == 0 { + return effects, ErrCompletion + } + c.applied = p.index + c.finish(&effects) + return effects, c.collect(&effects) + } + if c.pending != nil { + return effects, ErrBusy + } + switch in := input.(type) { + case Poll: + case Tick: + c.node.Tick() + case PeerMessage: + m := &pb.Message{} + if len(in.Bytes) > MaxUncommittedBytes || proto.Unmarshal(in.Bytes, m) != nil || !c.validPeer(m) { + return effects, ErrPeer + } + if err = c.node.Step(m); err != nil { + return effects, err + } + case Propose: + if !c.leader() { + return effects, ErrLeader + } + if !c.validCommand(in.Command) { + return effects, ErrCommand + } + if err = c.node.Propose(bytes.Clone(in.Command)); err != nil { + return effects, err + } + case Configure: + if !c.leader() { + return effects, ErrLeader + } + cc := &pb.ConfChangeV2{} + if len(in.Change) > MaxMessageBytes || proto.Unmarshal(in.Change, cc) != nil { + return effects, ErrCommand + } + if err = c.node.ProposeConfChange(cc); err != nil { + return effects, err + } + case Read: + if !c.leader() { + return effects, ErrLeader + } + if in.RequestID == "" || len(in.RequestID) > MaxRequestIDBytes || len(c.reads) >= MaxReadRequests { + return effects, ErrRead + } + for _, r := range c.reads { + if r.id == in.RequestID && !r.cancelled { + return effects, ErrRead + } + } + if c.readSerial == ^uint64(0) { + c.stopped = true + return effects, ErrStopped + } + c.readSerial++ + context := make([]byte, 16) + term := c.node.BasicStatus().GetTerm() + binary.BigEndian.PutUint64(context, term) + binary.BigEndian.PutUint64(context[8:], c.readSerial) + c.reads[string(context)] = readRequest{id: in.RequestID, term: term} + case CancelRead: + for key, r := range c.reads { + if r.id == in.RequestID { + if r.submitted { + // RawNode still owns this context until its quorum round + // completes. Keep its slot charged to bound cancelled work. + r.cancelled = true + c.reads[key] = r + } else { + delete(c.reads, key) + } + } + } + case Transfer: + if !c.leader() { + return effects, ErrLeader + } + s := c.node.Status() + progress, ok := s.Progress[in.Target] + // The host's authorized transfer barrier must already have committed. A + // learner, outgoing-only voter or lagging node cannot qualify here. + if !ok || in.Target == c.cfg.NodeID || progress.IsLearner || progress.Match < s.GetCommit() { + return effects, ErrTransfer + } + if _, voter := s.Config.Voters[0][in.Target]; !voter { + return effects, ErrTransfer + } + if len(s.Config.Voters[1]) != 0 { + return effects, ErrTransfer + } + c.node.TransferLeader(in.Target) + case TransportResult: + if in.Peer == 0 || raft.IsLocalMsgTarget(in.Peer) { + return effects, ErrPeer + } + if in.Snapshot { + result := raft.SnapshotFailure + if in.Success { + result = raft.SnapshotFinish + } + c.node.ReportSnapshot(in.Peer, result) + } else if !in.Success { + c.node.ReportUnreachable(in.Peer) + } + default: + return effects, ErrCommand + } + return effects, c.collect(&effects) +} + +func (c *Core) validCommand(data []byte) bool { + if len(data) == 0 || len(data) > MaxMessageBytes { + return false + } + cmd, err := clusterstate.DecodeCommand(data) + return err == nil && cmd.Cluster == c.cfg.ClusterUUID +} + +func (c *Core) validPeer(m *pb.Message) bool { + if m.Type == nil || m.GetTo() != c.cfg.NodeID || m.GetFrom() == 0 || m.GetFrom() == c.cfg.NodeID || raft.IsLocalMsgTarget(m.GetFrom()) || m.GetTerm() == 0 || len(m.GetResponses()) != 0 || len(m.ProtoReflect().GetUnknown()) != 0 { + return false + } + switch m.GetType() { + case pb.MsgApp, pb.MsgAppResp, pb.MsgVote, pb.MsgVoteResp, pb.MsgSnap, pb.MsgHeartbeat, pb.MsgHeartbeatResp, pb.MsgPreVote, pb.MsgPreVoteResp, pb.MsgTimeoutNow: + return true + default: + return false + } +} + +// issueReads keeps pre-commit reads in our bounded map instead of RawNode's +// pending-current-term queue. Cancelled submitted contexts retain admission +// slots until completion, so repeated timeouts cannot grow internal queues. +func (c *Core) issueReads() { + s := c.node.BasicStatus() + keys := make([]string, 0, len(c.reads)) + for key, request := range c.reads { + if !c.leader() || request.term != s.GetTerm() { + delete(c.reads, key) + delete(c.barriers, key) + } else if !request.submitted { + keys = append(keys, key) + } + } + term, err := c.store.Term(s.GetCommit()) + if err != nil || term != s.GetTerm() { + return + } + slices.Sort(keys) + for _, key := range keys { + r := c.reads[key] + r.submitted = true + c.reads[key] = r + c.node.ReadIndex([]byte(key)) + } +} + +func (c *Core) collect(e *Effects) error { + c.issueReads() + if !c.node.HasReady() { + return nil + } + if c.serial == ^uint64(0) { + c.stopped = true + return ErrStopped + } + c.serial++ + rd := c.node.Ready() + c.pending = &pending{id: c.serial, ready: rd} + e.Persist = &PersistBatch{ID: c.serial, HardState: clone(rd.HardState), Entries: copyEntries(rd.Entries), Snapshot: clone(rd.Snapshot)} + return nil +} + +func (c *Core) release() (e Effects, err error) { + p := c.pending + rd := p.ready + // Validate the whole committed batch before changing configuration. Unknown + // schemas stop application and participation, never become successful no-ops. + for _, entry := range rd.CommittedEntries { + switch entry.GetType() { + case pb.EntryNormal: + if len(entry.Data) != 0 && !c.validCommand(entry.Data) { + c.stopped = true + return e, ErrCommand + } + case pb.EntryConfChange, pb.EntryConfChangeV2: + default: + c.stopped = true + return e, ErrCommand + } + } + for _, entry := range rd.CommittedEntries { + switch entry.GetType() { + case pb.EntryConfChange: + cc := &pb.ConfChange{} + if proto.Unmarshal(entry.Data, cc) != nil { + c.stopped = true + return e, ErrCommand + } + c.store.conf = clone(c.node.ApplyConfChange(cc)) + case pb.EntryConfChangeV2: + cc := &pb.ConfChangeV2{} + if proto.Unmarshal(entry.Data, cc) != nil { + c.stopped = true + return e, ErrCommand + } + c.store.conf = clone(c.node.ApplyConfChange(cc)) + } + } + for _, message := range rd.Messages { + encoded, err := wire.Marshal(message) + if err != nil { + c.stopped = true + return Effects{}, ErrStopped + } + e.Send = append(e.Send, Outbound{To: message.GetTo(), Bytes: encoded}) + } + // Preserve RawNode's already stable send order, including ordering to the + // SAME peer. Sorting messages by their bytes could invert dependent appends. + index := rd.Snapshot.GetMetadata().GetIndex() + if n := len(rd.CommittedEntries); n != 0 { + index = rd.CommittedEntries[n-1].GetIndex() + } + if index > c.applied { + p.index = index + e.Apply = &ApplyBatch{ID: p.id, Entries: copyEntries(rd.CommittedEntries), Snapshot: clone(rd.Snapshot), ConfState: clone(c.store.conf), Index: index} + } else { + c.finish(&e) + err = c.collect(&e) + } + return e, err +} + +func (c *Core) finish(e *Effects) { + p := c.pending + s := c.node.BasicStatus() + for _, barrier := range p.ready.ReadStates { + key := string(barrier.RequestCtx) + if _, ok := c.reads[key]; ok { + c.barriers[key] = barrier.Index + } + } + // ReadIndex may run ahead of a size-limited application batch. Retain the + // barrier until its index is durable rather than losing it at Advance. + keys := make([]string, 0, len(c.barriers)) + for key := range c.barriers { + keys = append(keys, key) + } + slices.Sort(keys) + for _, key := range keys { + index := c.barriers[key] + r := c.reads[key] + if r.term == s.GetTerm() && c.leader() && c.applied >= index { + if !r.cancelled { + e.Reads = append(e.Reads, ReadBarrier{RequestID: r.id, Index: index}) + } + delete(c.reads, key) + delete(c.barriers, key) + } + } + c.node.Advance(p.ready) + c.pending = nil +} diff --git a/internal/raftcore/core_test.go b/internal/raftcore/core_test.go new file mode 100644 index 0000000000..87009433e3 --- /dev/null +++ b/internal/raftcore/core_test.go @@ -0,0 +1,630 @@ +package raftcore + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "go.etcd.io/raft/v3" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +type testGroup struct { + t *testing.T + nodes []*Core + messages []Outbound + trace bytes.Buffer + reads []ReadBarrier + applied [][]*pb.Entry + durable []RecoveredState + drop func(from, to uint64) bool +} + +func newGroup(t *testing.T) *testGroup { + t.Helper() + g := &testGroup{t: t, applied: make([][]*pb.Entry, 3), durable: make([]RecoveredState, 3)} + for id := uint64(1); id <= 3; id++ { + c, err := New(Config{ClusterUUID: "test-cluster", NodeID: id, Seed: 100 + id, Protocol: ProtocolVersion, Bootstrap: []uint64{3, 1, 2}}, RecoveredState{}) + if err != nil { + t.Fatal(err) + } + g.nodes = append(g.nodes, c) + } + for id := uint64(1); id <= 3; id++ { + g.call(id, Poll{}) + } + g.pump() + return g +} + +func (g *testGroup) call(id uint64, in Input) { + g.t.Helper() + e, err := g.nodes[id-1].Step(in) + if err != nil { + g.t.Fatalf("node %d input %T: %v", id, in, err) + } + g.accept(id, e, 0) +} +func (g *testGroup) accept(id uint64, e Effects, depth int) { + g.t.Helper() + if depth > 100 { + g.t.Fatal("unbounded Ready completions") + } + b, err := json.Marshal(e) + if err != nil { + g.t.Fatal(err) + } + g.trace.Write(b) + g.trace.WriteByte('\n') + g.messages = append(g.messages, e.Send...) + g.reads = append(g.reads, e.Reads...) + var in Input + if e.Persist != nil { + p := e.Persist + d := &g.durable[id-1] + if !raft.IsEmptyHardState(p.HardState) { + d.HardState = clone(p.HardState) + } + if !raft.IsEmptySnap(p.Snapshot) { + d.Snapshot = clone(p.Snapshot) + d.Entries = nil + } + if len(p.Entries) != 0 { + first := p.Entries[0].GetIndex() + kept := 0 + for _, entry := range d.Entries { + if entry.GetIndex() < first { + kept++ + } + } + d.Entries = append(d.Entries[:kept], copyEntries(p.Entries)...) + } + in = Persisted{p.ID} + } + if e.Apply != nil { + g.durable[id-1].Applied = e.Apply.Index + g.durable[id-1].ConfState = clone(e.Apply.ConfState) + for _, entry := range e.Apply.Entries { + g.applied[id-1] = append(g.applied[id-1], clone(entry)) + } + in = Applied{e.Apply.ID} + } + if in != nil { + next, err := g.nodes[id-1].Step(in) + if err != nil { + g.t.Fatalf("node %d completion %T: %v", id, in, err) + } + g.accept(id, next, depth+1) + } +} +func (g *testGroup) pump() { + g.t.Helper() + for count := 0; len(g.messages) > 0; count++ { + if count > 10000 { + g.t.Fatal("unbounded network effects") + } + out := g.messages[0] + g.messages = g.messages[1:] + m := &pb.Message{} + if err := proto.Unmarshal(out.Bytes, m); err != nil { + g.t.Fatal(err) + } + if g.drop != nil && g.drop(m.GetFrom(), out.To) { + continue + } + g.call(out.To, PeerMessage{out.Bytes}) + } +} +func (g *testGroup) tick(n int) { + for i := 0; i < n; i++ { + for id := uint64(1); id <= 3; id++ { + g.call(id, Tick{}) + } + g.pump() + } +} +func (g *testGroup) leader() uint64 { + g.t.Helper() + var leader uint64 + for _, c := range g.nodes { + if c.status().Role == raft.StateLeader { + if leader != 0 { + g.t.Fatal("multiple leaders") + } + leader = c.cfg.NodeID + } + } + return leader +} +func (g *testGroup) elect() uint64 { + g.t.Helper() + for i := 0; i < 30; i++ { + g.tick(1) + if leader := g.leader(); leader != 0 { + return leader + } + } + g.t.Fatal("no leader within seeded election budget") + return 0 +} +func command(t *testing.T, sequence uint64) []byte { + t.Helper() + b, err := clusterstate.EncodeCommand(clusterstate.Command{Schema: clusterstate.SchemaVersion, Cluster: "test-cluster", Actor: "device", Session: "session", Sequence: sequence, Now: 1234, RequestDigest: "opaque-digest", Change: clusterstate.Mutation{AdvanceTime: &clusterstate.AdvanceTime{}}}) + if err != nil { + t.Fatal(err) + } + return b +} + +// R14 requires elections and emitted bytes to be reproducible, including the +// exact count of PRNG draws. R6's wrapper mechanism must elect the target and +// pass a NEW read barrier before a host can report a completed transfer. +func TestR14SeededReplayAndR6Transfer(t *testing.T) { + run := func() []byte { + g := newGroup(t) + leader := g.elect() + g.call(leader, Propose{command(t, 1)}) + g.pump() + g.call(leader, Read{"before-transfer"}) + g.pump() + if len(g.reads) != 1 { + t.Fatal("no quorum read barrier") + } + target := leader%3 + 1 + g.call(leader, Transfer{target}) + g.pump() + if g.leader() != target { + t.Fatalf("transfer did not elect %d", target) + } + g.call(target, Read{"after-transfer"}) + g.pump() + if len(g.reads) != 2 || g.reads[1].Index < g.reads[0].Index { + t.Fatal("new leader did not pass a barrier") + } + for i := range g.applied { + found := false + for _, e := range g.applied[i] { + if bytes.Equal(e.Data, command(t, 1)) { + found = true + } + } + if !found { + t.Fatalf("member %d did not apply command", i+1) + } + } + return bytes.Clone(g.trace.Bytes()) + } + first, second := run(), run() + if !bytes.Equal(first, second) { + t.Fatal("same seeds/inputs produced different effects") + } + digest := sha256.Sum256(first) + golden, err := os.ReadFile("testdata/replay.sha256") + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(string(golden)) != hex.EncodeToString(digest[:]) { + t.Fatalf("R14 trace changed: got %x (review core/wire/RNG version before changing golden)", digest) + } +} + +// R4 forbids fresh read success and mutation application in a minority, even +// while it still remembers itself as leader. CheckQuorum must then step down. +func TestR4MinorityReadAndCheckQuorum(t *testing.T) { + g := newGroup(t) + leader := g.elect() + g.drop = func(from, to uint64) bool { return from == leader || to == leader } + before := len(g.applied[leader-1]) + g.call(leader, Read{"isolated"}) + g.pump() + g.call(leader, Propose{command(t, 2)}) + g.pump() + if len(g.reads) != 0 || len(g.applied[leader-1]) != before { + t.Fatal("minority exposed success") + } + g.tick(ElectionTicks * 3) + if g.nodes[leader-1].status().Role == raft.StateLeader { + t.Fatal("isolated leader did not step down") + } + if len(g.reads) != 0 || len(g.applied[leader-1]) != before { + t.Fatal("minority applied a proposal or read") + } + if _, err := g.nodes[leader-1].Step(Read{"after-stepdown"}); !errors.Is(err, ErrLeader) { + t.Fatalf("follower read: %v", err) + } +} + +// R5 requires repeated isolated/rejoin attempts not to depose a stable healthy +// majority. This smoke uses the production PreVote path and explicit ticks. +func TestR5PreVoteFlappingMember(t *testing.T) { + g := newGroup(t) + leader := g.elect() + isolated := leader%3 + 1 + term := g.nodes[leader-1].status().Term + for attempt := 0; attempt < 100; attempt++ { + g.drop = func(from, to uint64) bool { return from == isolated || to == isolated } + g.tick(ElectionTicks * 3) + if g.nodes[isolated-1].status().Term != term { + t.Fatal("isolated member raised term without quorum") + } + g.drop = nil + // Deliver a healthy leader heartbeat before any new election deadline. + g.call(leader, Tick{}) + g.pump() + if g.leader() != leader || g.nodes[leader-1].status().Term != term { + t.Fatalf("attempt %d disrupted stable leadership", attempt) + } + } +} + +// R3's persistence protocol must withhold every dependent message, and R4's +// read visibility must additionally wait for the application's durable index. +func TestR3DurableCompletionOrderAndR4AppliedRead(t *testing.T) { + c, err := New(Config{ClusterUUID: "test-cluster", NodeID: 1, Seed: 9, Protocol: 1, Bootstrap: []uint64{1, 2, 3}}, RecoveredState{}) + if err != nil { + t.Fatal(err) + } + e, err := c.Step(Poll{}) + if err != nil || e.Persist == nil || e.Apply != nil || len(e.Send) != 0 { + t.Fatalf("initial Ready: %+v %v", e, err) + } + id := e.Persist.ID + if _, err = c.Step(Applied{id}); !errors.Is(err, ErrCompletion) { + t.Fatal("application accepted before persistence") + } + if _, err = c.Step(Tick{}); !errors.Is(err, ErrBusy) { + t.Fatal("input silently queued during outstanding batch") + } + if _, err = c.Step(Persisted{id + 1}); !errors.Is(err, ErrCompletion) { + t.Fatal("wrong batch completion accepted") + } + // Mutating a host's effect cannot mutate the core's pending Ready. + e.Persist.Entries[0].Data[0] ^= 255 + e, err = c.Step(Persisted{id}) + if err != nil || e.Apply == nil || e.Status.Applied != 0 { + t.Fatalf("persisted: %+v %v", e, err) + } + if _, err = c.Step(Persisted{id}); !errors.Is(err, ErrCompletion) { + t.Fatal("duplicate persistence accepted") + } + e, err = c.Step(Applied{id}) + if err != nil || e.Status.Applied != 3 { + t.Fatalf("applied: %+v %v", e, err) + } + g := newGroup(t) + leader := g.elect() + e, err = g.nodes[leader-1].Step(Read{"delayed"}) + if err != nil { + t.Fatal(err) + } + // Withhold persistence of the leader's ReadIndex heartbeat Ready. + if e.Persist == nil || len(e.Send) != 0 || len(e.Reads) != 0 { + t.Fatal("read escaped persistence boundary") + } + g.accept(leader, e, 0) + g.pump() + if len(g.reads) != 1 { + t.Fatal("durable quorum did not release read") + } +} + +// R14 pins a published SplitMix64 vector independently of a replay generated by +// this wrapper, so accidental algorithm or consumption-order changes are visible. +func TestR14RandomGoldenAndNodeIsolation(t *testing.T) { + r := splitMix64{} + for _, want := range []uint64{0xe220a8397b1dcdaf, 0x6e789e6aa1b965f4, 0x06c45d188009454f, 0xf88bb8a8724c81ec} { + if got := r.next(); got != want { + t.Fatalf("got %x want %x", got, want) + } + } + a, b := splitMix64{state: 7}, splitMix64{state: 7} + other := splitMix64{state: 8} + for i := 0; i < 100; i++ { + if a.bounded(6) != b.bounded(6) { + t.Fatal("interleaved node influenced another seed") + } + other.bounded(7) + } + if a.draws != 100 { + t.Fatal("RNG consumption changed") + } +} + +// R14's deterministic boundary may not acquire hidden time, random, file or +// goroutine inputs. Runtime internals in Go/protobuf are outside the wrapper; +// they must not drive the election PRNG or semantic state. +func TestR14NoAmbientInputs(t *testing.T) { + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, path := range files { + if strings.HasSuffix(path, "_test.go") { + continue + } + f, err := parser.ParseFile(token.NewFileSet(), path, nil, 0) + if err != nil { + t.Fatal(err) + } + for _, im := range f.Imports { + p := strings.Trim(im.Path.Value, "\"") + if p == "time" || p == "os" || p == "syscall" || p == "unsafe" || strings.HasPrefix(p, "net") || strings.Contains(p, "rand") { + t.Fatalf("ambient import in %s: %s", path, p) + } + } + ast.Inspect(f, func(n ast.Node) bool { + if _, ok := n.(*ast.GoStmt); ok { + t.Fatalf("goroutine in %s", path) + } + return true + }) + } +} + +// R3's restart boundary consumes host-recorded durable bytes, not the old +// leader's live memory. R12's applied ConfState must retain a learner on restart. +func TestR3RestartAndR12Configuration(t *testing.T) { + g := newGroup(t) + leader := g.elect() + g.call(leader, Propose{command(t, 3)}) + g.pump() + cc := &pb.ConfChangeV2{Changes: []*pb.ConfChangeSingle{{Type: pb.ConfChangeAddLearnerNode.Enum(), NodeId: new(uint64(4))}}} + data, err := wire.Marshal(cc) + if err != nil { + t.Fatal(err) + } + // Learner four is deliberately absent: it must not block majority progress. + g.drop = func(_, to uint64) bool { return to == 4 } + g.call(leader, Configure{data}) + g.pump() + saved := g.durable[leader-1] + if len(saved.ConfState.GetLearners()) != 1 || saved.ConfState.GetLearners()[0] != 4 { + t.Fatal("learner configuration not durably handed to host") + } + replacement, err := New(Config{ClusterUUID: "test-cluster", NodeID: leader, Seed: 999, Protocol: ProtocolVersion}, saved) + if err != nil { + t.Fatal(err) + } + g.nodes[leader-1] = replacement + if replacement.status().Role != raft.StateFollower || replacement.status().Applied != saved.Applied { + t.Fatal("recovery resumed stale leadership or lost application") + } + g.call(leader, Poll{}) + g.pump() + g.tick(20) + newLeader := g.leader() + if newLeader == 0 { + t.Fatal("recovered group did not elect") + } + g.call(newLeader, Read{"after-restart"}) + g.pump() + if len(g.reads) != 1 || g.reads[0].Index < saved.Applied { + t.Fatal("restart lost committed progress") + } +} + +// R4 forbids peer-injected local triggers, forwarded unsigned proposals, reads, +// transfers and fake storage completions. R14 requires invalid inputs to leave +// the deterministic state untouched. +func TestR4PeerControlRejection(t *testing.T) { + g := newGroup(t) + leader := g.elect() + c := g.nodes[leader-1] + before := c.status() + for _, typ := range []pb.MessageType{pb.MsgHup, pb.MsgBeat, pb.MsgProp, pb.MsgReadIndex, pb.MsgReadIndexResp, pb.MsgTransferLeader, pb.MsgStorageAppend, pb.MsgStorageAppendResp, pb.MsgStorageApply, pb.MsgStorageApplyResp, pb.MessageType(999)} { + data, err := wire.Marshal(&pb.Message{Type: typ.Enum(), From: new(leader%3 + 1), To: new(leader), Term: new(before.Term)}) + if err != nil { + t.Fatal(err) + } + if _, err = c.Step(PeerMessage{data}); !errors.Is(err, ErrPeer) { + t.Fatalf("accepted %s: %v", typ, err) + } + if c.status() != before { + t.Fatalf("rejected %s changed state", typ) + } + } + data, _ := wire.Marshal(&pb.Message{Type: pb.MsgHeartbeat.Enum(), From: new(raft.LocalAppendThread), To: new(leader), Term: new(before.Term)}) + if _, err := c.Step(PeerMessage{data}); !errors.Is(err, ErrPeer) { + t.Fatalf("fake local sender: %v", err) + } + follower := leader%3 + 1 + if _, err := g.nodes[follower-1].Step(Propose{command(t, 1)}); !errors.Is(err, ErrLeader) { + t.Fatal("follower forwarded a proposal") + } +} + +// R3 requires storage errors to stop participation. R14 requires unknown +// committed schemas to stop application, never be mistaken for an empty command. +func TestR3StorageFailureAndR14UnknownSchemaStops(t *testing.T) { + c, err := New(Config{ClusterUUID: "test-cluster", NodeID: 1, Seed: 1, Protocol: 1, Bootstrap: []uint64{1, 2, 3}}, RecoveredState{}) + if err != nil { + t.Fatal(err) + } + e, _ := c.Step(Poll{}) + e, err = c.Step(StorageFailed{e.Persist.ID}) + if !errors.Is(err, ErrStopped) || !e.Status.Stopped || e.NeedNextTick || len(e.Send) != 0 { + t.Fatal("storage failure did not stop member") + } + if _, err = c.Step(Tick{}); !errors.Is(err, ErrStopped) { + t.Fatal("stopped member accepted input") + } + g := newGroup(t) + leader := g.elect() + follower := leader%3 + 1 + status := g.nodes[follower-1].status() + next := status.Commit + 1 + future := bytes.Replace(command(t, 1), []byte(`"schema":1`), []byte(`"schema":999`), 1) + m := &pb.Message{Type: pb.MsgApp.Enum(), From: new(leader), To: new(follower), Term: new(status.Term), Index: new(status.Commit), LogTerm: new(status.Term), Commit: new(next), Entries: []*pb.Entry{{Type: pb.EntryNormal.Enum(), Term: new(status.Term), Index: new(next), Data: future}}} + data, _ := wire.Marshal(m) + e, err = g.nodes[follower-1].Step(PeerMessage{data}) + if err != nil || e.Persist == nil || len(e.Send) != 0 { + t.Fatalf("future entry receipt: %v %+v", err, e) + } + e, err = g.nodes[follower-1].Step(Persisted{e.Persist.ID}) + if !errors.Is(err, ErrCommand) || !e.Status.Stopped || e.Apply != nil || len(e.Send) != 0 { + t.Fatalf("unknown entry became a no-op: %v %+v", err, e) + } +} + +// R4 requires a read to wait for durable application even if its ReadIndex +// result arrives with committed entries. Dropping a size-delayed barrier would +// strand a valid read; exposing it early would authorize stale state. +func TestR4ReadWaitsForApplication(t *testing.T) { + g := newGroup(t) + leader := g.elect() + c := g.nodes[leader-1] + g.call(leader, Propose{command(t, 5)}) + // First deliver the append to one follower and then its durable append ack. + var response Outbound + for _, out := range g.messages { + if out.To == leader%3+1 { + g.messages = nil + g.call(out.To, PeerMessage{out.Bytes}) + for _, reply := range g.messages { + if reply.To == leader { + response = reply + break + } + } + break + } + } + if response.Bytes == nil { + t.Fatal("missing durable append response") + } + g.messages = nil + e, err := c.Step(PeerMessage{response.Bytes}) + if err != nil || e.Persist == nil { + t.Fatalf("commit Ready: %v", err) + } + e, err = c.Step(Persisted{e.Persist.ID}) + if err != nil || e.Apply == nil { + t.Fatal("committed application batch missing") + } + before := e.Status.Applied + if before >= e.Apply.Index || len(e.Reads) != 0 { + t.Fatal("application marked durable early") + } + // Host cannot start another authority read while the transaction is pending. + if _, err = c.Step(Read{"while-applying"}); !errors.Is(err, ErrBusy) { + t.Fatal("read crossed pending transaction") + } + g.accept(leader, e, 0) + g.pump() + g.call(leader, Read{"after-application"}) + g.pump() + if len(g.reads) != 1 || g.reads[0].Index <= before { + t.Fatal("read did not cover durable command") + } +} + +// R3 forbids fabricating a recoverable log when durable metadata contradicts +// it. R14 requires version mismatches to fail before the member participates. +func TestR3RecoveryRefusalAndR14Protocol(t *testing.T) { + cfg := Config{ClusterUUID: "test-cluster", NodeID: 1, Seed: 1, Protocol: ProtocolVersion} + cases := []RecoveredState{ + {HardState: &pb.HardState{Term: new(uint64(1)), Commit: new(uint64(1))}}, + {Applied: 1}, + {HardState: &pb.HardState{Term: new(uint64(1))}, Entries: []*pb.Entry{{Index: new(uint64(2)), Term: new(uint64(1))}}}, + {HardState: &pb.HardState{Term: new(uint64(1))}, Entries: []*pb.Entry{{Index: new(uint64(1)), Term: new(uint64(2))}}}, + {HardState: &pb.HardState{Term: new(uint64(2))}, Entries: []*pb.Entry{{Index: new(uint64(1)), Term: new(uint64(2))}, {Index: new(uint64(2)), Term: new(uint64(1))}}}, + } + for i, state := range cases { + if _, err := New(cfg, state); !errors.Is(err, ErrConfig) { + t.Fatalf("recovery %d: %v", i, err) + } + } + cfg.Protocol++ + if _, err := New(cfg, RecoveredState{}); !errors.Is(err, ErrProtocol) { + t.Fatalf("protocol mismatch: %v", err) + } +} + +// R4 requires a safe current-term quorum even for the first read after an +// election. R14's read backlog is explicitly bounded and cancellable. +func TestR4FirstReadNeedsCurrentTermCommit(t *testing.T) { + g := newGroup(t) + // Keep the future leader's term-start entry uncommitted by withholding both + // followers' append acknowledgments, while still delivering election votes. + g.drop = func(from, to uint64) bool { + for _, c := range g.nodes { + if c.status().Role == raft.StateLeader && to == c.cfg.NodeID && from != to { + return true + } + } + return false + } + var leader uint64 + for i := 0; i < 20 && leader == 0; i++ { + g.tick(1) + leader = g.leader() + } + if leader == 0 { + t.Fatal("election did not finish") + } + c := g.nodes[leader-1] + g.call(leader, Read{"first"}) + g.pump() + if len(g.reads) != 0 { + t.Fatal("first read succeeded without current-term commit") + } + for i := 1; i < MaxReadRequests; i++ { + g.call(leader, Read{string(rune(1000 + i))}) + g.pump() + } + if _, err := c.Step(Read{"excess"}); !errors.Is(err, ErrRead) { + t.Fatal("read backlog unbounded") + } + g.call(leader, CancelRead{"first"}) + g.call(leader, Read{"replacement"}) + g.pump() + g.drop = nil + g.tick(1) + if len(g.reads) == 0 { + t.Fatal("current-term commit did not release queued read") + } + for _, r := range g.reads { + if r.RequestID == "first" { + t.Fatal("cancelled read was released") + } + } +} + +// R4/R27 require cancelled minority reads to retain a bounded admission charge +// while RawNode still holds their contexts. Timeouts cannot grow hidden queues. +func TestR4CancelledReadsStayBoundedR27(t *testing.T) { + g := newGroup(t) + leader := g.elect() + c := g.nodes[leader-1] + g.drop = func(from, to uint64) bool { return from == leader || to == leader } + for i := 0; i < MaxReadRequests; i++ { + id := string(rune(1000 + i)) + g.call(leader, Read{id}) + g.pump() + g.call(leader, CancelRead{id}) + } + if _, err := c.Step(Read{"overflow"}); !errors.Is(err, ErrRead) { + t.Fatalf("cancelled contexts escaped admission: %v", err) + } + g.drop = nil + g.tick(1) + if len(g.reads) != 0 { + t.Fatal("cancelled reads returned success") + } + g.call(leader, Read{"fresh"}) + g.pump() + if len(g.reads) != 1 { + t.Fatal("completed cancellation did not release admission") + } +} diff --git a/internal/raftcore/logger.go b/internal/raftcore/logger.go new file mode 100644 index 0000000000..6d2d134d1f --- /dev/null +++ b/internal/raftcore/logger.go @@ -0,0 +1,19 @@ +package raftcore + +// silentLogger prevents upstream diagnostic formatting from leaking command +// bytes or consulting process time. Invariant failures remain fatal to the +// handle with a constant, payload-free panic; hosts must retain durable stores. +type silentLogger struct{} + +func (silentLogger) Debug(...interface{}) {} +func (silentLogger) Debugf(string, ...interface{}) {} +func (silentLogger) Info(...interface{}) {} +func (silentLogger) Infof(string, ...interface{}) {} +func (silentLogger) Warning(...interface{}) {} +func (silentLogger) Warningf(string, ...interface{}) {} +func (silentLogger) Error(...interface{}) {} +func (silentLogger) Errorf(string, ...interface{}) {} +func (silentLogger) Fatal(...interface{}) { panic("raftcore: consensus invariant failed") } +func (silentLogger) Fatalf(string, ...interface{}) { panic("raftcore: consensus invariant failed") } +func (silentLogger) Panic(...interface{}) { panic("raftcore: consensus invariant failed") } +func (silentLogger) Panicf(string, ...interface{}) { panic("raftcore: consensus invariant failed") } diff --git a/internal/raftcore/random.go b/internal/raftcore/random.go new file mode 100644 index 0000000000..74e3cf679c --- /dev/null +++ b/internal/raftcore/random.go @@ -0,0 +1,30 @@ +package raftcore + +// splitMix64 is specified here rather than delegated to math/rand, whose +// implementation is not our wire-version contract. Addition and products wrap +// modulo 2^64. Rejection sampling avoids biased election choices. This generator +// is NEVER used for keys, certificates, request IDs or other cryptography. +type splitMix64 struct{ state, draws uint64 } + +func (r *splitMix64) next() uint64 { + r.state += 0x9e3779b97f4a7c15 + r.draws++ + z := r.state + z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9 + z = (z ^ (z >> 27)) * 0x94d049bb133111eb + return z ^ (z >> 31) +} + +func (r *splitMix64) bounded(n int) int { + if n <= 0 { + panic("raftcore: nonpositive random bound") + } + b := uint64(n) + threshold := -b % b + for { + v := r.next() + if v >= threshold { + return int(v % b) + } + } +} diff --git a/internal/raftcore/testdata/replay.sha256 b/internal/raftcore/testdata/replay.sha256 new file mode 100644 index 0000000000..bcf51e7cb0 --- /dev/null +++ b/internal/raftcore/testdata/replay.sha256 @@ -0,0 +1 @@ +3657b899e514d48d0eca8b9b9e039d3972542a937e6ac754e64f088eb6c47e2a diff --git a/internal/raftcore/types.go b/internal/raftcore/types.go new file mode 100644 index 0000000000..00733bf695 --- /dev/null +++ b/internal/raftcore/types.go @@ -0,0 +1,159 @@ +// Package raftcore is the single-threaded consensus boundary shared by native +// hosts and the WASI reactor. Hosts own durability, scheduling and transport. +package raftcore + +import ( + "errors" + + "go.etcd.io/raft/v3" + pb "go.etcd.io/raft/v3/raftpb" +) + +const ( + ProtocolVersion uint16 = 1 + ElectionTicks = 6 + HeartbeatTicks = 1 + MaxMessageBytes = 64 << 10 + MaxUncommittedBytes = 1 << 20 + MaxInflightBatches = 8 + MaxReadRequests = 128 + MaxRequestIDBytes = 128 +) + +var ( + ErrConfig = errors.New("raftcore: invalid configuration or recovery state") + ErrProtocol = errors.New("raftcore: unsupported protocol") + ErrBusy = errors.New("raftcore: complete the outstanding batch before retrying input") + ErrCompletion = errors.New("raftcore: unexpected batch completion") + ErrStopped = errors.New("raftcore: member stopped; recover from durable storage") + ErrPeer = errors.New("raftcore: invalid peer message") + ErrLeader = errors.New("raftcore: operation requires the elected leader") + ErrCommand = errors.New("raftcore: invalid command or committed schema") + ErrRead = errors.New("raftcore: invalid, duplicate or excess read request") + ErrTransfer = errors.New("raftcore: transfer target is not a caught-up voter") +) + +// Config contains only explicit inputs. Seed is noncryptographic election +// entropy; hosts must supply a new seed on restart and record it for replay. +// Bootstrap is used once on empty storage, with the SAME voter set on all +// members. Enrollment and topology authorization belong to the host policy. +type Config struct { + ClusterUUID string + NodeID uint64 + Seed uint64 + Protocol uint16 + Bootstrap []uint64 +} + +// RecoveredState is immutable input copied at construction. ConfState is the +// last DURABLY APPLIED configuration, including all joint and learner fields. +// Entries contains the contiguous suffix after Snapshot, not just committed +// entries. Applied belongs to the atomic application transaction. +type RecoveredState struct { + HardState *pb.HardState + ConfState *pb.ConfState + Snapshot *pb.Snapshot + Entries []*pb.Entry + Applied uint64 +} + +// Input is closed so an untrusted peer cannot masquerade as a local completion. +type Input interface{ coreInput() } +type Poll struct{} +type Tick struct{} +type PeerMessage struct{ Bytes []byte } +type Propose struct{ Command []byte } + +// Configure is a mechanism for later authorized membership intents, not a +// public enrollment door. The bytes encode a ConfChangeV2. +type Configure struct{ Change []byte } +type Read struct{ RequestID string } + +// CancelRead marks a host timeout without producing a successful read. A +// submitted context retains its admission slot until Raft finishes that round; +// otherwise repeated cancellation could create unbounded hidden work. +type CancelRead struct{ RequestID string } + +// Transfer starts a request; observing a successful transfer still requires an +// election, a current-term commit and a read barrier on the new leader. +type Transfer struct{ Target uint64 } +type Persisted struct{ BatchID uint64 } +type Applied struct{ BatchID uint64 } + +// StorageFailed poisons the member rather than weakening durability on error. +type StorageFailed struct{ BatchID uint64 } +type TransportResult struct { + Peer uint64 + Snapshot bool + Success bool +} + +func (Poll) coreInput() {} +func (Tick) coreInput() {} +func (PeerMessage) coreInput() {} +func (Propose) coreInput() {} +func (Configure) coreInput() {} +func (Read) coreInput() {} +func (CancelRead) coreInput() {} +func (Transfer) coreInput() {} +func (Persisted) coreInput() {} +func (Applied) coreInput() {} +func (StorageFailed) coreInput() {} +func (TransportResult) coreInput() {} + +// PersistBatch must be written atomically, replacing any conflicting suffix. +// Persisted means the ENTIRE batch is durable, even when Raft's MustSync would +// permit a weaker boundary. An empty batch is a dependency barrier and needs +// no physical write. All exported protobuf values are independent copies. +type PersistBatch struct { + ID uint64 + HardState *pb.HardState + Entries []*pb.Entry + Snapshot *pb.Snapshot +} + +// ApplyBatch is emitted only after persistence. The host atomically applies +// every entry and persists the index, rows, dedup results and ConfState. An +// installed snapshot's application state must be installed in that transaction +// before applying the suffix. Empty normal entries still advance the index. +// Client results may escape only after this transaction is durable; Propose +// itself never returns a successful coordination result. +type ApplyBatch struct { + ID uint64 + Entries []*pb.Entry + Snapshot *pb.Snapshot + ConfState *pb.ConfState + Index uint64 +} + +type Outbound struct { + To uint64 + Bytes []byte +} +type ReadBarrier struct { + RequestID string + Index uint64 +} + +// Status is diagnostic, not evidence of authority. It deliberately avoids the +// upstream status map so serialized effects have a stable order. +type Status struct { + NodeID uint64 + Leader uint64 + Term uint64 + Commit uint64 + Applied uint64 + Role raft.StateType + RNGState uint64 + RNGDraws uint64 + Stopped bool +} + +type Effects struct { + Persist *PersistBatch + Send []Outbound + Apply *ApplyBatch + Reads []ReadBarrier + Status Status + NeedNextTick bool +} From 3c8083322721318e286834e7ce9b6fb180639b19 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Fri, 23 Oct 2026 22:56:33 -0400 Subject: [PATCH 005/103] Define a shared reactor ABI so native and hosted members use the same Go core. Keep consensus synchronous behind owned binary buffers and explicit durability completions. Bound frames and handles, audit the WASI runtime imports, and exercise real lease rules over immutable paged rows without adding a production writer. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/codeaf-raft-wasm/main.go | 49 ++++++ internal/raftabi/README.md | 140 +++++++++++++++ internal/raftabi/abi.go | 319 ++++++++++++++++++++++++++++++++++ internal/raftabi/abi_test.go | 151 ++++++++++++++++ internal/raftabi/probe.go | 79 +++++++++ relay/hosted/src/raft/abi.js | 93 ++++++++++ relay/hosted/src/raft/wasi.js | 61 +++++++ 7 files changed, 892 insertions(+) create mode 100644 cmd/codeaf-raft-wasm/main.go create mode 100644 internal/raftabi/README.md create mode 100644 internal/raftabi/abi.go create mode 100644 internal/raftabi/abi_test.go create mode 100644 internal/raftabi/probe.go create mode 100644 relay/hosted/src/raft/abi.js create mode 100644 relay/hosted/src/raft/wasi.js diff --git a/cmd/codeaf-raft-wasm/main.go b/cmd/codeaf-raft-wasm/main.go new file mode 100644 index 0000000000..48c2122687 --- /dev/null +++ b/cmd/codeaf-raft-wasm/main.go @@ -0,0 +1,49 @@ +//go:build wasip1 && wasm + +// The reactor owns only byte buffers and consensus handles. The host imports +// this precompiled module and resumes calls after its own durability boundary. +package main + +import ( + "unsafe" + + "github.com/Agent-Field/codeaf/internal/raftabi" +) + +var reactor = raftabi.New() +var input, output []byte + +func main() {} + +//go:wasmexport codeaf_abi_version +func abiVersion() uint32 { return raftabi.Version } + +//go:wasmexport codeaf_alloc +func alloc(n uint32) uint32 { + if n == 0 || n > raftabi.MaxBuffer { + return 0 + } + input = make([]byte, n) + return uint32(uintptr(unsafe.Pointer(&input[0]))) +} + +//go:wasmexport codeaf_call +func call(n uint32) uint32 { + if n == 0 || n > uint32(len(input)) { + return 0 + } + output = reactor.Call(input[:n]) + input = nil + return uint32(len(output)) +} + +//go:wasmexport codeaf_result +func result() uint32 { + if len(output) == 0 { + return 0 + } + return uint32(uintptr(unsafe.Pointer(&output[0]))) +} + +//go:wasmexport codeaf_free +func free() { input = nil; output = nil } diff --git a/internal/raftabi/README.md b/internal/raftabi/README.md new file mode 100644 index 0000000000..8055f946ba --- /dev/null +++ b/internal/raftabi/README.md @@ -0,0 +1,140 @@ +# Worker reactor ABI, version 1 + +P03 is an internal feasibility boundary. Neither the local proof Worker nor the +stdio peer is a production directory or a new codeaf command. P08 owns the hosted +actor/queue/storage adapter; P09 owns complete paged application evaluation, +authorization, monotonic time, dedup and conditional transaction policy. No +JavaScript lease or consensus rules are added here. + +Build/run the bounded local proof from the repository root: + +```sh +sh relay/hosted/test/raft/run.sh /tmp/codeaf-p03 +``` + +Install the hosted relay's pinned npm lockfile first if needed. It uses the installed +workerd binary/schema directly, with no retained Miniflare development SDK. The script waits +for I/O pressure before each heavy step, builds serially at nice 10 with two Go +processors and `-p=1`, and runs only the targeted files. It uses local workerd, +loopback inspector, a native stdio peer and disposable durable storage. The +Worker's outbound service throws. The existing local wrapper disables telemetry +and Request.cf downloads. It never deploys, signs in or reads a person's home. +The mixed-consensus and bulk-memory phases run sequentially: Node exits before +the Go streaming driver starts a fresh engine with six handles across two +identities. Both phases assert combined RSS at most 256 MiB. The wrapper limits +local allocator arenas and Node's test heap; it never forces Worker GC. Runtime +reports include sampled V8 heap/backing storage plus wasm linear-memory high +water, separately from process RSS. These local measurements are not billable +CPU or cloud duration. Artifacts and reports stay under the supplied /tmp directory; none is installed +as the person's codeaf binary. + +## Buffers and lifetime + +A precompiled module is imported by the Worker, with exactly the audited WASI +preview1 imports in `relay/hosted/src/raft/wasi.js`. One `Reactor` instance is +shared across all member handles in an isolate. Construction attaches exported +memory before `_initialize`. The exports are `codeaf_abi_version`, +`codeaf_alloc(n)`, `codeaf_call(n)`, `codeaf_result()` and `codeaf_free()`. +Allocation returns a wasm32 pointer; call returns result length; result returns +a pointer. Copy the result before freeing or calling again. No pointer survives +an await or memory growth. Calls are synchronous; the host serializes a member +across its asynchronous persistence and apply transactions. Input and output +buffers are bounded at 2 MiB. Input bytes are decoded completely before stepping +a core. Handles are monotonic uint32 values and are never reused within a +runtime; a new runtime discards all old completions. + +The JS guard stops the whole instance if linear memory exceeds 64 MiB or a call +traps. This is a fail-closed guard, not a substitute for P08's preallocation +admission/cache/eviction policy. `GOMEMLIMIT=48MiB`, `GOGC=50` and +`GOMAXPROCS=1` are the entire runtime environment. Empty args, unsupported fd/ +socket operations, runtime-only clock/entropy and discarded failure output are +the other imports. Poll/yield throw; a build that uses them cannot pass the +compatibility gate. Runtime entropy does not seed elections. + +## Binary grammar + +All lengths/counts, flags, tags and handles are little-endian uint32. Every +counter/node ID/index/seed is little-endian uint64 (BigInt in JS). A blob is +uint32 length followed by exact bytes; a string is a UTF-8 blob. A protobuf +field is an optional deterministic protobuf blob, with zero length for absent. +An entry list is count plus one blob per entry. Bytes cross this boundary +without JSON/base64 wrapping. Application command/row blobs retain P01's +canonical schema inside the binary envelope. + +Every request starts with ABI version and operation. Every response starts +with ABI version and an error string; empty means success and the remaining +bytes are the operation result. Errors are bounded constant/structural messages; +the host must not log private request/row bytes. A Step error is not a successful +coordination response. Oversized effects require stopping/recovering, not retrying +a mutated core blindly. + +| Operation | Request body | Result body | +| --- | --- | --- | +| 1 create/recover | cluster string, node/seed uint64, protocol uint32, bootstrap count/IDs, HardState/ConfState/Snapshot blobs, entries, applied uint64 | handle uint32 | +| 2 drop | handle | none | +| 3 step | handle, input tag, tag-specific data | effects | +| 4 lease feasibility probe | command blob, optional one-row blob | next canonical row blob | +| 5 row representation probe | canonical row blob | same canonical row blob | + +Step tags: 0 Poll, 1 Tick, 2 PeerMessage blob, 3 Propose blob, 4 Read string, +5 Persisted batch ID, 6 Applied batch ID, 7 Transfer target, 8 Configure blob, +9 StorageFailed batch ID, 10 CancelRead string. The core enforces the one-Ready +completion order, leader-only admission and peer-local-control refusal. + +Effects are: optional persist flag and [ID, hard state, entries, snapshot]; +send count and [destination, message blob] tuples; optional apply flag and +[ID, index, entries, snapshot, ConfState]; read count and [request string, +index] tuples; status [node, leader, term, commit, applied uint64, role uint32, +RNG state/draws uint64, stopped flag]; need-tick flag. Preserve send order. + +The build manifest binds the ABI version, core protocol, row schema, source +inventory hash and wasm SHA-256. P05 must add the actual snapshot schema/hash +before snapshot interoperability can be claimed. No snapshot wire compatibility +is claimed by the P03 opaque checkpoint fixture. + +## Oversized legacy representation decision + +Keep each existing canonical row in host storage and pass only one immutable +row (at most the existing 1 MiB legacy envelope) to Go. Inventory/import and +snapshot creation enumerate sorted rows at a fixed applied index, with no +whole-identity listing or JSON string in wasm. Persist the imported byte ceiling +`max(8 MiB, imported application bytes)` outside the reactor alongside the +applied metadata. Preserve every row, sealed title/key and frame plan. Do not +truncate to make import fit. P05 supplies chunk/checksum/install generations; +P09/P11 supply complete read-sets, atomic row/byte accounting and growth policy. + +The proof uses 129 separately persisted rows totaling over 8 MiB, including one +near-limit row and 128 smaller pages. A small native driver compares every row +with wasm and verifies every stored byte on reread, then evaluates a real +release over the large row. Six production handles remain resident while the +engine is sampled through its loopback inspector. This establishes representation/runtime feasibility; it is +not a claim that migration, snapshots, complete policy or production admission +have already shipped. A legacy row over 1 MiB remains a migration refusal until +an owner-approved streaming row extension exists, as the design requires. + + +## P03 stop/go measurement + +The final local proof on Go 1.26.5 and pinned workerd 1.20260926.1 passed. The +full existing relay plus reactor/shim/storage fixture/manifest is 2,248,258 bytes +compressed (3 MiB gate); wasm is 9,285,637 bytes raw. All 26 distinct imports are +covered by the closed shim. The mixed phase compared 743 effect responses +byte-for-byte, including RNG state/draws, committed a real create, passed a +quorum ReadIndex, refused a partitioned read, fell back to native peers and recovered +from an actual workerd process kill with disk-backed DO state. + +The final mixed phase took 5.85 s, observed 230,334,464 bytes combined RSS and +12,949,979 bytes of isolate components, and cold first call was 32.33 ms. +The sequential six-handle/two-identity memory phase took 3.75 s, observed +207,298,560 bytes combined RSS and 39,200,796 bytes of isolate components, +with 17,301,504 bytes wasm linear-memory high water. Its cold call was 28.47 ms; +workerd startup was 145.90 ms. The 129 rows contain 9,205,245 bytes; largest row +is 786,665 bytes. These are observed local samples, not production allocation +admission guarantees or cloud percentiles. + +This is a **go** for the measured shared-Go Worker boundary; no Go-host topology +alternative needs an owner decision at P03. P04/P05 still supply real member +stores and snapshot/compaction, P08 supplies production actors/admission and +serialization, and P09/P11 supply complete policy and migration. P08/P18 must +repeat the size/memory gate with those later components. Application rows are +paged; no production route or feature toggle was added by this proof. diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go new file mode 100644 index 0000000000..f800c6e4b2 --- /dev/null +++ b/internal/raftabi/abi.go @@ -0,0 +1,319 @@ +// Package raftabi defines the bounded binary boundary shared by the Worker +// reactor and its native conformance peer. It owns no storage or transport. +package raftabi + +import ( + "encoding/binary" + "errors" + + "github.com/Agent-Field/codeaf/internal/raftcore" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +const ( + Version uint32 = 1 + MaxBuffer = 2 << 20 + MaxHandles = 96 +) + +var errFrame = errors.New("raftabi: invalid or oversized frame") +var wire = proto.MarshalOptions{Deterministic: true} + +type decoder struct { + b []byte + err error +} + +func (d *decoder) take(n uint32) []byte { + if d.err != nil || uint64(n) > uint64(len(d.b)) { + d.err = errFrame + return nil + } + b := d.b[:n] + d.b = d.b[n:] + return b +} +func (d *decoder) u32() uint32 { + b := d.take(4) + if b == nil { + return 0 + } + return binary.LittleEndian.Uint32(b) +} +func (d *decoder) u64() uint64 { + b := d.take(8) + if b == nil { + return 0 + } + return binary.LittleEndian.Uint64(b) +} +func (d *decoder) blob() []byte { return d.take(d.u32()) } +func (d *decoder) str() string { return string(d.blob()) } +func (d *decoder) message(m proto.Message) bool { + b := d.blob() + if len(b) == 0 { + return false + } + if proto.Unmarshal(b, m) != nil { + d.err = errFrame + } + return true +} +func (d *decoder) entries() []*pb.Entry { + n := d.u32() + if n > MaxBuffer/4 { + d.err = errFrame + return nil + } + out := make([]*pb.Entry, 0) + for i := uint32(0); i < n && d.err == nil; i++ { + e := new(pb.Entry) + if !d.message(e) { + d.err = errFrame + } + out = append(out, e) + } + return out +} +func (d *decoder) end() error { + if d.err != nil || len(d.b) != 0 { + return errFrame + } + return nil +} + +type encoder struct{ b []byte } + +func (e *encoder) u32(v uint32) { e.b = binary.LittleEndian.AppendUint32(e.b, v) } +func (e *encoder) u64(v uint64) { e.b = binary.LittleEndian.AppendUint64(e.b, v) } +func (e *encoder) blob(b []byte) { e.u32(uint32(len(b))); e.b = append(e.b, b...) } +func (e *encoder) str(s string) { e.blob([]byte(s)) } +func (e *encoder) message(m proto.Message) { + if !m.ProtoReflect().IsValid() { + e.blob(nil) + return + } + b, err := wire.Marshal(m) + if err != nil { + panic(errFrame) + } + e.blob(b) +} +func (e *encoder) entries(v []*pb.Entry) { + e.u32(uint32(len(v))) + for _, m := range v { + e.message(m) + } +} +func (e *encoder) flag(v bool) { + if v { + e.u32(1) + } else { + e.u32(0) + } +} + +// Runtime shares one Go runtime across opaque handles. IDs are never reused, +// so a delayed completion cannot accidentally address a replacement member. +// Hosts still serialize each member across asynchronous durability operations. +type Runtime struct { + nodes map[uint32]*raftcore.Core + serial uint32 +} + +func New() *Runtime { return &Runtime{nodes: make(map[uint32]*raftcore.Core)} } + +// Call consumes one complete frame and returns owned bytes. The header is a +// version and operation, followed by length-prefixed binary/protobuf fields. +// A malformed frame is fully decoded before a core can change state. +func (r *Runtime) Call(b []byte) (out []byte) { + defer func() { + if recover() != nil { + e := encoder{} + e.u32(Version) + e.str("raftabi: stopped after invalid call") + out = e.b + } + }() + e := encoder{} + e.u32(Version) + if len(b) > MaxBuffer { + e.str(errFrame.Error()) + return e.b + } + d := decoder{b: b} + if d.u32() != Version { + e.str("raftabi: unsupported version") + return e.b + } + payload, err := r.call(d.u32(), &d) + if err != nil { + e.str(err.Error()) + return e.b + } + e.str("") + e.b = append(e.b, payload...) + if len(e.b) > MaxBuffer { + e = encoder{} + e.u32(Version) + e.str("raftabi: effect exceeds buffer; stop and recover") + } + return e.b +} +func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { + e := encoder{} + switch op { + case 1: + cfg := raftcore.Config{ClusterUUID: d.str(), NodeID: d.u64(), Seed: d.u64()} + protocol := d.u32() + if protocol != uint32(raftcore.ProtocolVersion) { + return nil, raftcore.ErrProtocol + } + cfg.Protocol = uint16(protocol) + n := d.u32() + if n > MaxHandles { + return nil, errFrame + } + for i := uint32(0); i < n; i++ { + cfg.Bootstrap = append(cfg.Bootstrap, d.u64()) + } + h, c, s := new(pb.HardState), new(pb.ConfState), new(pb.Snapshot) + recovered := raftcore.RecoveredState{} + if d.message(h) { + recovered.HardState = h + } + if d.message(c) { + recovered.ConfState = c + } + if d.message(s) { + recovered.Snapshot = s + } + recovered.Entries = d.entries() + recovered.Applied = d.u64() + if err := d.end(); err != nil { + return nil, err + } + if len(r.nodes) >= MaxHandles || r.serial == ^uint32(0) { + return nil, errors.New("raftabi: handle capacity") + } + node, err := raftcore.New(cfg, recovered) + if err != nil { + return nil, err + } + r.serial++ + r.nodes[r.serial] = node + e.u32(r.serial) + case 2: + id := d.u32() + if err := d.end(); err != nil { + return nil, err + } + if r.nodes[id] == nil { + return nil, errFrame + } + delete(r.nodes, id) + case 3: + id, tag := d.u32(), d.u32() + var in raftcore.Input + switch tag { + case 0: + in = raftcore.Poll{} + case 1: + in = raftcore.Tick{} + case 2: + in = raftcore.PeerMessage{Bytes: d.blob()} + case 3: + in = raftcore.Propose{Command: d.blob()} + case 4: + in = raftcore.Read{RequestID: d.str()} + case 5: + in = raftcore.Persisted{BatchID: d.u64()} + case 6: + in = raftcore.Applied{BatchID: d.u64()} + case 7: + in = raftcore.Transfer{Target: d.u64()} + case 8: + in = raftcore.Configure{Change: d.blob()} + case 9: + in = raftcore.StorageFailed{BatchID: d.u64()} + case 10: + in = raftcore.CancelRead{RequestID: d.str()} + default: + return nil, errFrame + } + if err := d.end(); err != nil { + return nil, err + } + node := r.nodes[id] + if node == nil { + return nil, errFrame + } + effects, err := node.Step(in) + if err != nil { + return nil, err + } + encodeEffects(&e, effects) + case 4: + command, row := d.blob(), d.blob() + if err := d.end(); err != nil { + return nil, err + } + next, err := ProbeApply(command, row) + if err != nil { + return nil, err + } + e.blob(next) + case 5: + row := d.blob() + if err := d.end(); err != nil { + return nil, err + } + next, err := RoundTripRow(row) + if err != nil { + return nil, err + } + e.blob(next) + default: + return nil, errFrame + } + return e.b, nil +} +func encodeEffects(e *encoder, v raftcore.Effects) { + e.flag(v.Persist != nil) + if p := v.Persist; p != nil { + e.u64(p.ID) + e.message(p.HardState) + e.entries(p.Entries) + e.message(p.Snapshot) + } + e.u32(uint32(len(v.Send))) + for _, m := range v.Send { + e.u64(m.To) + e.blob(m.Bytes) + } + e.flag(v.Apply != nil) + if a := v.Apply; a != nil { + e.u64(a.ID) + e.u64(a.Index) + e.entries(a.Entries) + e.message(a.Snapshot) + e.message(a.ConfState) + } + e.u32(uint32(len(v.Reads))) + for _, b := range v.Reads { + e.str(b.RequestID) + e.u64(b.Index) + } + s := v.Status + e.u64(s.NodeID) + e.u64(s.Leader) + e.u64(s.Term) + e.u64(s.Commit) + e.u64(s.Applied) + e.u32(uint32(s.Role)) + e.u64(s.RNGState) + e.u64(s.RNGDraws) + e.flag(s.Stopped) + e.flag(v.NeedNextTick) +} diff --git a/internal/raftabi/abi_test.go b/internal/raftabi/abi_test.go new file mode 100644 index 0000000000..6fa241e938 --- /dev/null +++ b/internal/raftabi/abi_test.go @@ -0,0 +1,151 @@ +package raftabi + +import ( + "bytes" + "testing" + + "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +func request(op uint32) *encoder { e := &encoder{}; e.u32(Version); e.u32(op); return e } +func answer(t *testing.T, b []byte) *decoder { + t.Helper() + d := &decoder{b: b} + if d.u32() != Version { + t.Fatal("R14 response version") + } + if err := d.str(); err != "" { + t.Fatal(err) + } + return d +} +func create(t *testing.T, r *Runtime) uint32 { + t.Helper() + e := request(1) + e.str("R14") + e.u64(1) + e.u64(42) + e.u32(1) + e.u32(1) + e.u64(1) + e.blob(nil) + e.blob(nil) + e.blob(nil) + e.u32(0) + e.u64(0) + d := answer(t, r.Call(e.b)) + id := d.u32() + if d.end() != nil { + t.Fatal("trailing response") + } + return id +} + +// R14 requires exact byte replay through the production binary ABI, not a +// second JSON interpretation of diagnostic structs. +func TestR14BinaryReplayAndMalformedInputDoesNotMutate(t *testing.T) { + a, b := New(), New() + id := create(t, a) + other := create(t, b) + for _, tag := range []uint32{0, 1, 1, 1, 1, 1, 1, 1} { + x, y := request(3), request(3) + x.u32(id) + x.u32(tag) + y.u32(other) + y.u32(tag) + bad := append(bytes.Clone(x.b), 1) + d := &decoder{b: a.Call(bad)} + d.u32() + if d.str() == "" { + t.Fatal("R14 trailing input accepted") + } + if !bytes.Equal(a.Call(x.b), b.Call(y.b)) { + t.Fatal("R14 malformed call changed subsequent effect bytes") + } + } +} + +// R3 completion generations cannot cross handle lifetimes after recovery. +func TestR3DroppedHandleNeverTargetsReplacement(t *testing.T) { + r := New() + old := create(t, r) + e := request(2) + e.u32(old) + answer(t, r.Call(e.b)) + next := create(t, r) + if next == old { + t.Fatal("R3 handle ID reused") + } + e = request(3) + e.u32(old) + e.u32(5) + e.u64(1) + d := &decoder{b: r.Call(e.b)} + d.u32() + if d.str() == "" { + t.Fatal("R3 stale completion accepted") + } +} + +// R27 bounds framing before touching the core, including pathological count +// fields that could otherwise request a whole-identity allocation. +func TestR27FrameAndCountAdmission(t *testing.T) { + r := New() + for _, b := range [][]byte{nil, {1}, make([]byte, MaxBuffer+1)} { + d := &decoder{b: r.Call(b)} + d.u32() + if d.str() == "" { + t.Fatal("R27 invalid frame accepted") + } + } + e := request(1) + e.str("R27") + e.u64(1) + e.u64(0) + e.u32(1) + e.u32(^uint32(0)) + d := &decoder{b: r.Call(e.b)} + d.u32() + if d.str() == "" { + t.Fatal("R27 unbounded bootstrap accepted") + } + create(t, r) +} + +// R14/R28 preserve full-width fences and every sealed row byte while paging +// an identity. A round trip cannot truncate an opaque frame plan or title. +func TestR14R28SingleRowKeepsOpaqueDataAndUint64(t *testing.T) { + cell := clusterstate.Created(clusterstate.CellInit{Head: "opaque", Title: string(bytes.Repeat([]byte("A"), 768<<10)), Frames: []string{"f1", "f2"}}, "device", 1000) + cell.Lease.Fence = ^uint64(0) + row, err := clusterstate.EncodeRow(clusterstate.Row{Schema: 1, Key: "cell", Cell: &cell}) + if err != nil { + t.Fatal(err) + } + e := request(5) + e.blob(row) + d := answer(t, New().Call(e.b)) + if !bytes.Equal(d.blob(), row) || d.end() != nil { + t.Fatal("R28 row changed") + } +} + +// R3/R14 prove the feasibility command is the real Go rule and produces the +// same canonical durable row that a native host reads after commitment. +func TestR3R14LeaseProbeUsesSharedRule(t *testing.T) { + c := clusterstate.Command{Schema: 1, Cluster: "R14", Actor: "device", Session: "s", Sequence: 1, RequestDigest: "opaque-digest", Now: 1000, Change: clusterstate.Mutation{Create: &clusterstate.CreateCell{Cell: "cell", Init: clusterstate.CellInit{Head: "opaque-head"}}}} + b, err := clusterstate.EncodeCommand(c) + if err != nil { + t.Fatal(err) + } + e := request(4) + e.blob(b) + e.blob(nil) + d := answer(t, New().Call(e.b)) + row, err := clusterstate.DecodeRow(d.blob()) + if err != nil { + t.Fatal(err) + } + if row.Cell.Head != "opaque-head" || row.Cell.Lease.Fence != 1 || row.Cell.Lease.Expires != 91000 { + t.Fatal("R14 wrong rule result") + } +} diff --git a/internal/raftabi/probe.go b/internal/raftabi/probe.go new file mode 100644 index 0000000000..f011aa3e28 --- /dev/null +++ b/internal/raftabi/probe.go @@ -0,0 +1,79 @@ +package raftabi + +import ( + "errors" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" +) + +// ProbeApply exercises real lease rules over one immutable host-supplied row. +// It is a feasibility seam, NOT an authorized directory writer: P09 must add +// the complete dependency read-set, committed policy, dedup and atomic writes. +// Keeping this probe narrow avoids treating prototype policy as production. +func ProbeApply(command, row []byte) ([]byte, error) { + if len(command) > raftcore.MaxMessageBytes || len(row) > 1<<20 { + return nil, errFrame + } + c, err := clusterstate.DecodeCommand(command) + if err != nil { + return nil, err + } + var old clusterstate.Row + if len(row) > 0 { + old, err = clusterstate.DecodeRow(row) + if err != nil { + return nil, err + } + } + next := old + switch { + case c.Change.Create != nil: + if len(row) != 0 { + return nil, clusterstate.ErrExists + } + v := c.Change.Create + cell := clusterstate.Created(v.Init, c.Actor, c.Now) + next = clusterstate.Row{Schema: clusterstate.SchemaVersion, Key: v.Cell, Cell: &cell} + case c.Change.Acquire != nil: + v := c.Change.Acquire + if old.Cell == nil || old.Key != v.Cell { + return nil, clusterstate.ErrNotFound + } + cell, err := clusterstate.Acquire(*old.Cell, c.Actor, c.Now, v.Force, &v.Expect) + if err != nil { + return nil, err + } + next.Cell = &cell + case c.Change.Release != nil: + v := c.Change.Release + if old.Cell == nil || old.Key != v.Cell { + return nil, clusterstate.ErrNotFound + } + at := c.Now + if v.Quiet { + at = 0 + } + cell, err := clusterstate.ReleaseOf(*old.Cell, c.Actor, v.Fence, at) + if err != nil { + return nil, err + } + next.Cell = &cell + default: + return nil, errors.New("raftabi: unsupported probe command") + } + return clusterstate.EncodeRow(next) +} + +// RoundTripRow proves the one-row representation without accumulating an +// identity in the reactor. The host inventories and streams rows separately. +func RoundTripRow(row []byte) ([]byte, error) { + if len(row) > 1<<20 { + return nil, errFrame + } + v, err := clusterstate.DecodeRow(row) + if err != nil { + return nil, err + } + return clusterstate.EncodeRow(v) +} diff --git a/relay/hosted/src/raft/abi.js b/relay/hosted/src/raft/abi.js new file mode 100644 index 0000000000..993eb12c6c --- /dev/null +++ b/relay/hosted/src/raft/abi.js @@ -0,0 +1,93 @@ +import { wasiPreview1 } from './wasi.js'; + +export const ABI_VERSION = 1; +export const MAX_BUFFER = 2 * 1024 * 1024; +export const CORE_MEMORY_BUDGET = 64 * 1024 * 1024; +const utf8 = new TextEncoder(); +const text = new TextDecoder('utf-8', { fatal: true }); + +export class Writer { + constructor() { this.parts = []; this.length = 0; } + raw(b) { if (!(b instanceof Uint8Array)) throw new TypeError('ABI bytes required'); this.parts.push(b); this.length += b.length; return this; } + u32(n) { if (!Number.isInteger(n) || n < 0 || n > 0xffffffff) throw new RangeError('ABI uint32'); const b = new Uint8Array(4); new DataView(b.buffer).setUint32(0, n, true); return this.raw(b); } + u64(n) { if ((typeof n === 'number' && !Number.isSafeInteger(n)) || BigInt(n) < 0n || BigInt(n) > ((1n << 64n) - 1n)) throw new RangeError('ABI uint64'); const b = new Uint8Array(8); new DataView(b.buffer).setBigUint64(0, BigInt(n), true); return this.raw(b); } + blob(b = new Uint8Array()) { return this.u32(b.length).raw(b); } + str(s) { return this.blob(utf8.encode(s)); } + finish() { const b = new Uint8Array(this.length); let at = 0; for (const p of this.parts) { b.set(p, at); at += p.length; } return b; } +} +export class Reader { + constructor(bytes) { this.bytes = bytes; this.at = 0; this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); } + take(n) { if (!Number.isInteger(n) || n < 0 || this.at + n > this.bytes.length) throw new Error('ABI truncated'); const b = this.bytes.slice(this.at, this.at + n); this.at += n; return b; } + u32() { const at = this.at; this.take(4); return this.view.getUint32(at, true); } + u64() { const at = this.at; this.take(8); return this.view.getBigUint64(at, true); } + blob() { return this.take(this.u32()); } + str() { return text.decode(this.blob()); } + entries() { const n = this.u32(); if (n > MAX_BUFFER / 4) throw new Error('ABI count'); return Array.from({ length: n }, () => this.blob()); } + end() { if (this.at !== this.bytes.length) throw new Error('ABI trailing bytes'); } +} +export const frame = (op) => new Writer().u32(ABI_VERSION).u32(op); +export function response(bytes) { + const r = new Reader(bytes); + if (r.u32() !== ABI_VERSION) throw new Error('ABI version'); + const error = r.str(); if (error) { r.end(); throw new Error(error); } + return r; +} +export function createFrame({ cluster, id, seed, bootstrap = [], hard = new Uint8Array(), conf = new Uint8Array(), snapshot = new Uint8Array(), entries = [], applied = 0n }) { + const w = frame(1).str(cluster).u64(id).u64(seed).u32(1).u32(bootstrap.length); + for (const id of bootstrap) w.u64(id); + w.blob(hard).blob(conf).blob(snapshot).u32(entries.length); + for (const e of entries) w.blob(e); + return w.u64(applied).finish(); +} +export function stepFrame(handle, tag, value) { + const w = frame(3).u32(handle).u32(tag); + if ([2, 3, 8].includes(tag)) w.blob(value); + else if ([4, 10].includes(tag)) w.str(value); + else if ([5, 6, 7, 9].includes(tag)) w.u64(value); + return w.finish(); +} +export function effects(bytes) { + const r = response(bytes), v = {}; + if (r.u32()) v.persist = { id: r.u64(), hard: r.blob(), entries: r.entries(), snapshot: r.blob() }; + v.send = Array.from({ length: r.u32() }, () => ({ to: r.u64(), bytes: r.blob() })); + if (r.u32()) v.apply = { id: r.u64(), index: r.u64(), entries: r.entries(), snapshot: r.blob(), conf: r.blob() }; + v.reads = Array.from({ length: r.u32() }, () => ({ id: r.str(), index: r.u64() })); + v.status = { id: r.u64(), leader: r.u64(), term: r.u64(), commit: r.u64(), applied: r.u64(), role: r.u32(), rngState: r.u64(), rngDraws: r.u64(), stopped: !!r.u32() }; + v.tick = !!r.u32(); r.end(); return v; +} + +// A Worker imports a trusted precompiled module. No runtime compilation or +// await occurs within a call; the shared instance cannot interleave members. +export class Reactor { + constructor(module) { + this.wasi = wasiPreview1(); + this.instance = new WebAssembly.Instance(module, { wasi_snapshot_preview1: this.wasi.imports }); + this.exports = this.instance.exports; + this.wasi.attach(this.exports.memory); + this.poisoned = false; + this.exports._initialize(); + if (this.exports.codeaf_abi_version() !== ABI_VERSION) throw new Error('ABI version'); + this.peakMemory = this.memory; + } + get memory() { return this.exports.memory.buffer.byteLength; } + check() { + this.peakMemory = Math.max(this.peakMemory, this.memory); + if (this.memory > CORE_MEMORY_BUDGET) { this.poisoned = true; throw new Error('core memory capacity; stop and recover'); } + if (this.poisoned) throw new Error('reactor stopped; recover'); + } + call(bytes) { + if (!(bytes instanceof Uint8Array) || bytes.length === 0 || bytes.length > MAX_BUFFER) throw new Error('ABI frame capacity'); + this.check(); + try { + const p = this.exports.codeaf_alloc(bytes.length); + if (!p) throw new Error('ABI allocation'); + this.check(); new Uint8Array(this.exports.memory.buffer, p, bytes.length).set(bytes); + const n = this.exports.codeaf_call(bytes.length); + this.check(); + if (n === 0 || n > MAX_BUFFER) throw new Error('ABI result capacity'); + return new Uint8Array(this.exports.memory.buffer, this.exports.codeaf_result(), n).slice(); + } catch (e) { this.poisoned = true; throw e; } + finally { this.exports.codeaf_free(); } + } + metrics() { return { memory: this.memory, peakMemory: this.peakMemory, wasi: { ...this.wasi.calls } }; } +} diff --git a/relay/hosted/src/raft/wasi.js b/relay/hosted/src/raft/wasi.js new file mode 100644 index 0000000000..43c67c031b --- /dev/null +++ b/relay/hosted/src/raft/wasi.js @@ -0,0 +1,61 @@ +// WASI exists only to initialize Go's runtime. Consensus clocks and election +// entropy enter through raftcore inputs; these imports never supply either. +// There is no filesystem, socket, scheduler loop, environment or payload log. +export function wasiPreview1() { + let memory; + const calls = Object.create(null); + const view = () => new DataView(memory.buffer); + const bytes = (p, n) => { + if (!memory || p < 0 || n < 0 || p + n > memory.buffer.byteLength) throw new Error('WASI bounds'); + return new Uint8Array(memory.buffer, p, n); + }; + const u32 = (p, n) => { bytes(p, 4); view().setUint32(p, n, true); }; + const env = new TextEncoder().encode('GOMEMLIMIT=48MiB\0GOGC=50\0GOMAXPROCS=1\0'); + const offsets = [0]; + for (let i = 0; i < env.length - 1; i++) if (env[i] === 0) offsets.push(i + 1); + const implemented = { + args_sizes_get: (count, size) => { u32(count, 0); u32(size, 0); return 0; }, + args_get: () => 0, + environ_sizes_get: (count, size) => { u32(count, offsets.length); u32(size, env.length); return 0; }, + environ_get: (pointers, data) => { + bytes(data, env.length).set(env); + offsets.forEach((off, i) => u32(pointers + 4 * i, data + off)); + return 0; + }, + clock_time_get: (id, precision, out) => { + if (id !== 0 && id !== 1) return 28; // WASI EINVAL. + bytes(out, 8); + const millis = id === 0 ? Date.now() : performance.now(); + view().setBigUint64(out, BigInt(Math.floor(millis * 1e6)), true); + return 0; + }, + random_get: (p, n) => { + for (let off = 0; off < n; off += 65536) crypto.getRandomValues(bytes(p + off, Math.min(65536, n - off))); + return 0; + }, + // Runtime failure output is discarded rather than risking a command or + // private row in logs. Only byte counters leave this boundary. + fd_write: (fd, iovs, count, out) => { + if (fd !== 1 && fd !== 2) return 8; // WASI EBADF. + if (count > 1024) return 28; + let total = 0; + for (let i = 0; i < count; i++) { bytes(iovs + i * 8, 8); total += view().getUint32(iovs + i * 8 + 4, true); } + u32(out, total); return 0; + }, + proc_exit: () => { throw new Error('Go reactor exited'); }, + // A synchronous reactor must return; silently faking a poll completion + // would conceal an incompatible Go scheduler or blocking operation. + poll_oneoff: () => { throw new Error('Go reactor attempted blocking poll'); }, + sched_yield: () => { throw new Error('Go reactor attempted scheduler yield'); }, + }; + const badFD = ['fd_close', 'fd_read', 'fd_pread', 'fd_pwrite', 'fd_seek', 'fd_fdstat_get', 'fd_fdstat_set_flags', + 'fd_filestat_get', 'fd_filestat_set_size', 'fd_sync', 'fd_datasync', 'fd_prestat_get', 'fd_prestat_dir_name', 'fd_readdir', + 'path_filestat_get', 'path_open', 'sock_accept', 'sock_shutdown']; + for (const name of badFD) implemented[name] = () => 8; + const imports = {}; + for (const [name, fn] of Object.entries(implemented)) imports[name] = (...args) => { + calls[name] = (calls[name] ?? 0) + 1; + return fn(...args); + }; + return { imports, calls, attach: (m) => { memory = m; } }; +} From 723ac56ed12281fd60c5becf7ce5e4d323018b36 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Fri, 23 Oct 2026 22:56:34 -0400 Subject: [PATCH 006/103] Prove the Worker boundary locally before building the hosted cluster on it. Compare native and wasm effects through real commit and ReadIndex exchanges, restart disk-backed workerd storage, and measure the complete package and a grandfathered-size fixture. Run consensus and bulk memory phases serially so the proof fits the shared laptop budget without changing the Worker workload. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/codeaf-raft-probe/main.go | 49 +++ cmd/codeaf-raft-probe/measure.go | 409 +++++++++++++++++++++++ cmd/codeaf-raft-probe/rows.go | 147 ++++++++ relay/hosted/test/raft/abi.test.js | 23 ++ relay/hosted/test/raft/build.mjs | 44 +++ relay/hosted/test/raft/run.sh | 13 + relay/hosted/test/raft/smoke.test.js | 229 +++++++++++++ relay/hosted/test/raft/storage-worker.js | 18 + relay/hosted/test/raft/worker.js | 24 ++ relay/hosted/test/raft/workerd.js | 66 ++++ scripts/raft-proof-build.sh | 8 + scripts/raft-proof-footprint.sh | 12 + 12 files changed, 1042 insertions(+) create mode 100644 cmd/codeaf-raft-probe/main.go create mode 100644 cmd/codeaf-raft-probe/measure.go create mode 100644 cmd/codeaf-raft-probe/rows.go create mode 100644 relay/hosted/test/raft/abi.test.js create mode 100644 relay/hosted/test/raft/build.mjs create mode 100755 relay/hosted/test/raft/run.sh create mode 100644 relay/hosted/test/raft/smoke.test.js create mode 100644 relay/hosted/test/raft/storage-worker.js create mode 100644 relay/hosted/test/raft/worker.js create mode 100644 relay/hosted/test/raft/workerd.js create mode 100755 scripts/raft-proof-build.sh create mode 100755 scripts/raft-proof-footprint.sh diff --git a/cmd/codeaf-raft-probe/main.go b/cmd/codeaf-raft-probe/main.go new file mode 100644 index 0000000000..2ba88dfe42 --- /dev/null +++ b/cmd/codeaf-raft-probe/main.go @@ -0,0 +1,49 @@ +// This local conformance peer speaks the exact reactor ABI over framed stdio. +// It is not a relay service and has no listener, storage or authentication. +package main + +import ( + "encoding/binary" + "io" + "os" + + "github.com/Agent-Field/codeaf/internal/raftabi" +) + +func main() { + if len(os.Args) == 5 && os.Args[1] == "measure" { + if err := measureRows(os.Args[2], os.Args[3], os.Args[4]); err != nil { + os.Stderr.WriteString(err.Error() + "\n") + os.Exit(1) + } + return + } + if len(os.Args) == 3 && os.Args[1] == "rows" { + if err := rowProof(os.Args[2], os.Stdout); err != nil { + os.Stderr.WriteString(err.Error() + "\n") + os.Exit(1) + } + return + } + r := raftabi.New() + for { + var n uint32 + if binary.Read(os.Stdin, binary.LittleEndian, &n) != nil { + return + } + if n == 0 || n > raftabi.MaxBuffer { + os.Exit(2) + } + b := make([]byte, n) + if _, err := io.ReadFull(os.Stdin, b); err != nil { + os.Exit(2) + } + out := r.Call(b) + if binary.Write(os.Stdout, binary.LittleEndian, uint32(len(out))) != nil { + os.Exit(2) + } + if _, err := os.Stdout.Write(out); err != nil { + os.Exit(2) + } + } +} diff --git a/cmd/codeaf-raft-probe/measure.go b/cmd/codeaf-raft-probe/measure.go new file mode 100644 index 0000000000..772f5931fd --- /dev/null +++ b/cmd/codeaf-raft-probe/measure.go @@ -0,0 +1,409 @@ +package main + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/Agent-Field/codeaf/internal/raftabi" + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" +) + +// measureRows runs the R27/R28 memory gate without retaining the Node consensus +// test coordinator. Six production handles and real disk-backed DOs remain in +// workerd; only the bulk driver changes. Inspector sampling never forces GC. +func measureRows(out, binaryPath, schema string) error { + ctx, cancel := context.WithTimeout(context.Background(), 110*time.Second) + defer cancel() + dir, err := os.MkdirTemp("/tmp", "codeaf-p03-rows-") + if err != nil { + return err + } + freePort := func() (string, error) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return "", err + } + addr := l.Addr().String() + err = l.Close() + return addr, err + } + httpAddr, err := freePort() + if err != nil { + return err + } + inspectAddr, err := freePort() + if err != nil { + return err + } + config := fmt.Sprintf(`using Workerd = import %q; +const config :Workerd.Config = ( + services = [ + (name = "raft-proof", worker = ( + compatibilityDate = "2026-09-01", + modules = [(name = "worker.js", esModule = embed %q), (name = "core.wasm", wasm = embed %q)], + bindings = [(name = "PROOF_STORE", durableObjectNamespace = (className = "ProofStore", serviceName = "proof-storage"))], globalOutbound = "blocked" + )), + (name = "proof-storage", worker = ( + compatibilityDate = "2026-09-01", modules = [(name = "storage.js", esModule = embed %q)], + durableObjectNamespaces = [(className = "ProofStore", uniqueKey = "p03-local-proof", enableSql = true)], + durableObjectStorage = (localDisk = "storage"), globalOutbound = "blocked" + )), + (name = "storage", disk = (path = %q, writable = true)), + (name = "blocked", worker = (compatibilityDate = "2026-09-01", modules = [(name = "blocked.js", esModule = "export default {fetch() {throw new Error('proof forbids outbound network')}}")], globalOutbound = "blocked")) + ], sockets = [(name = "http", address = %q, http = (), service = "raft-proof")] +); +`, schema, filepath.Join(out, "worker.js"), filepath.Join(out, "core.wasm"), filepath.Join(out, "storage-worker.js"), dir, httpAddr) + configPath := filepath.Join(dir, "proof.capnp") + if err = os.WriteFile(configPath, []byte(config), 0600); err != nil { + return err + } + cmd := exec.CommandContext(ctx, binaryPath, "serve", "--experimental", "-I/", "--inspector-addr", inspectAddr, configPath) + cmd.Stdout = io.Discard + cmd.Stderr = io.Discard + if err = cmd.Start(); err != nil { + return err + } + stopped := false + defer func() { + if !stopped { + cmd.Process.Kill() + cmd.Wait() + } + }() + client := &http.Client{Transport: &http.Transport{Proxy: nil, MaxIdleConnsPerHost: 2}, Timeout: 5 * time.Second} + defer client.CloseIdleConnections() + origin := "http://" + httpAddr + request := func(method, path string, b []byte) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, method, origin+path, bytes.NewReader(b)) + if err != nil { + return nil, err + } + res, err := client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 && res.StatusCode != 204 { + return nil, errors.New("memory proof HTTP refusal") + } + return io.ReadAll(io.LimitReader(res.Body, raftabi.MaxBuffer+1)) + } + start := time.Now() + for { + if _, err = request("GET", "/ready", nil); err == nil { + break + } + if time.Since(start) > 10*time.Second { + return errors.New("memory proof workerd startup timeout") + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(25 * time.Millisecond): + } + } + readyMs := float64(time.Since(start).Microseconds()) / 1000 + res, err := client.Get("http://" + inspectAddr + "/json/list") + if err != nil { + return err + } + var targets []struct{ Title, WebSocketDebuggerURL string } + err = json.NewDecoder(res.Body).Decode(&targets) + res.Body.Close() + if err != nil { + return err + } + var target string + for _, t := range targets { + if strings.Contains(t.Title, "raft-proof") { + target = t.WebSocketDebuggerURL + break + } + } + if target == "" { + return errors.New("memory proof inspector target absent") + } + conn, _, err := websocket.Dial(ctx, target, nil) + if err != nil { + return err + } + defer conn.CloseNow() + serial := 0 + type heap struct{ UsedSize, TotalSize, EmbedderHeapUsedSize, BackingStorageSize uint64 } + inspect := func() (heap, error) { + serial++ + id := serial + if err := wsjson.Write(ctx, conn, map[string]any{"id": id, "method": "Runtime.getHeapUsage"}); err != nil { + return heap{}, err + } + for { + var reply struct { + ID int + Result heap + Error json.RawMessage + } + if err := wsjson.Read(ctx, conn, &reply); err != nil { + return heap{}, err + } + if reply.ID == id { + if len(reply.Error) > 0 { + return heap{}, errors.New("memory proof inspector refusal") + } + return reply.Result, nil + } + } + } + ticksText, err := exec.Command("getconf", "CLK_TCK").Output() + if err != nil { + return err + } + ticks, err := strconv.ParseFloat(strings.TrimSpace(string(ticksText)), 64) + if err != nil { + return err + } + type proc struct { + PID int + RSSBytes uint64 + CPUSeconds float64 + } + readProc := func(pid int) (proc, error) { + b, err := os.ReadFile(fmt.Sprintf("/proc/%d/status", pid)) + if err != nil { + return proc{}, err + } + p := proc{PID: pid} + for _, line := range strings.Split(string(b), "\n") { + if strings.HasPrefix(line, "VmRSS:") { + f := strings.Fields(line) + n, _ := strconv.ParseUint(f[1], 10, 64) + p.RSSBytes = n * 1024 + } + } + b, err = os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)) + if err != nil { + return proc{}, err + } + f := strings.Fields(string(b)[strings.LastIndex(string(b), ") ")+2:]) + u, _ := strconv.ParseFloat(f[11], 64) + s, _ := strconv.ParseFloat(f[12], 64) + p.CPUSeconds = (u + s) / ticks + return p, nil + } + var peakRSS, peakComponents, peakWasm uint64 + var samples int + var peakProcesses []proc + var sampleErr error + var mu sync.Mutex + sample := func() error { + h, err := inspect() + if err != nil { + return err + } + b, err := request("GET", "/metrics", nil) + if err != nil { + return err + } + var metrics struct{ PeakMemory uint64 } + if err = json.Unmarshal(b, &metrics); err != nil { + return err + } + var sum uint64 + var ps []proc + for _, pid := range []int{os.Getpid(), cmd.Process.Pid} { + p, err := readProc(pid) + if err != nil { + return err + } + sum += p.RSSBytes + ps = append(ps, p) + } + mu.Lock() + defer mu.Unlock() + samples++ + if sum > peakRSS { + peakRSS = sum + peakProcesses = ps + } + peakWasm = max(peakWasm, metrics.PeakMemory) + peakComponents = max(peakComponents, h.TotalSize+h.EmbedderHeapUsedSize+h.BackingStorageSize) + return nil + } + // Complete each bootstrap Ready against real storage before adding load. + // These are dormant followers; the preceding mixed test proves elections, + // leadership, native peers and recovery. No authority read is served here. + header := func(op uint32) *bytes.Buffer { + b := new(bytes.Buffer) + binary.Write(b, binary.LittleEndian, raftabi.Version) + binary.Write(b, binary.LittleEndian, op) + return b + } + put32 := func(b *bytes.Buffer, n uint32) { binary.Write(b, binary.LittleEndian, n) } + put64 := func(b *bytes.Buffer, n uint64) { binary.Write(b, binary.LittleEndian, n) } + putBlob := func(b *bytes.Buffer, v []byte) { binary.Write(b, binary.LittleEndian, uint32(len(v))); b.Write(v) } + payload := func(b []byte) (*bytes.Reader, error) { + r := bytes.NewReader(b) + var version, n uint32 + if binary.Read(r, binary.LittleEndian, &version) != nil || version != raftabi.Version || binary.Read(r, binary.LittleEndian, &n) != nil || n != 0 { + return nil, errors.New("memory proof ABI refusal") + } + return r, nil + } + step := func(handle, tag uint32, batch uint64) ([]byte, error) { + b := header(3) + put32(b, handle) + put32(b, tag) + if tag == 5 || tag == 6 { + put64(b, batch) + } + return request("POST", "/call", b.Bytes()) + } + coldMs := float64(0) + for cluster := 1; cluster <= 2; cluster++ { + for id := uint64(1); id <= 3; id++ { + b := header(1) + putBlob(b, []byte(fmt.Sprintf("bulk-%d", cluster))) + put64(b, id) + put64(b, 100+id) + put32(b, 1) + put32(b, 3) + for _, v := range []uint64{1, 2, 3} { + put64(b, v) + } + for i := 0; i < 3; i++ { + putBlob(b, nil) + } + put32(b, 0) + put64(b, 0) + begin := time.Now() + raw, err := request("POST", "/call", b.Bytes()) + if err != nil { + return err + } + if coldMs == 0 { + coldMs = float64(time.Since(begin).Microseconds()) / 1000 + } + r, err := payload(raw) + if err != nil { + return err + } + var handle uint32 + if binary.Read(r, binary.LittleEndian, &handle) != nil { + return errors.New("memory proof handle missing") + } + raw, err = step(handle, 0, 0) + if err != nil { + return err + } + r, err = payload(raw) + if err != nil { + return err + } + var flag uint32 + var batch uint64 + if binary.Read(r, binary.LittleEndian, &flag) != nil || flag != 1 || binary.Read(r, binary.LittleEndian, &batch) != nil { + return errors.New("memory proof bootstrap persist missing") + } + path := fmt.Sprintf("/store/bulk-%d-%d/", cluster, id) + if _, err = request("PUT", path+"persist", raw); err != nil { + return err + } + raw, err = step(handle, 5, batch) + if err != nil { + return err + } + r, err = payload(raw) + if err != nil { + return err + } + var sends uint32 + if binary.Read(r, binary.LittleEndian, &flag) != nil || flag != 0 || binary.Read(r, binary.LittleEndian, &sends) != nil || sends != 0 || binary.Read(r, binary.LittleEndian, &flag) != nil || flag != 1 || binary.Read(r, binary.LittleEndian, &batch) != nil { + return errors.New("memory proof bootstrap apply missing") + } + if _, err = request("PUT", path+"apply", raw); err != nil { + return err + } + raw, err = step(handle, 6, batch) + if err != nil { + return err + } + if _, err = payload(raw); err != nil { + return err + } + } + } + if err = sample(); err != nil { + return err + } + done := make(chan struct{}) + joined := make(chan struct{}) + go func() { + defer close(joined) + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-done: + return + case <-ticker.C: + if err := sample(); err != nil { + sampleErr = err + return + } + } + } + }() + var legacy bytes.Buffer + rowErr := rowProof(origin, &legacy) + close(done) + <-joined + if sampleErr != nil { + return sampleErr + } + if err = sample(); err != nil { + return err + } + if rowErr != nil { + legacy.Reset() + json.NewEncoder(&legacy).Encode(map[string]string{"error": rowErr.Error()}) + } + report := map[string]any{"readyMs": readyMs, "coldFirstCallMs": coldMs, "residentHandles": 6, "identities": 2, "heapSamples": samples, "observedIsolateBytes": peakComponents + peakWasm, "peakWasmBytes": peakWasm, "observedCombinedRSSBytes": peakRSS, "peakProcesses": peakProcesses, "legacy": json.RawMessage(legacy.Bytes()), "elapsedMs": float64(time.Since(start).Microseconds()) / 1000, "allocatorArenas": os.Getenv("MALLOC_ARENA_MAX")} + encoded, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + if err = os.WriteFile(filepath.Join(out, "rows-runtime.json"), append(encoded, '\n'), 0600); err != nil { + return err + } + cmd.Process.Kill() + cmd.Wait() + stopped = true + if rowErr != nil { + return rowErr + } + if peakRSS > 256<<20 { + return errors.New("R27 combined local process RSS exceeds 256 MiB") + } + if peakComponents+peakWasm > 96<<20 || peakWasm > 64<<20 { + return errors.New("R27 isolate memory exceeds budget") + } + if coldMs >= 1000 { + return errors.New("R27 startup exceeds 1 s envelope") + } + _, err = os.Stdout.Write(append(encoded, '\n')) + return err +} diff --git a/cmd/codeaf-raft-probe/rows.go b/cmd/codeaf-raft-probe/rows.go new file mode 100644 index 0000000000..aad8785c6a --- /dev/null +++ b/cmd/codeaf-raft-probe/rows.go @@ -0,0 +1,147 @@ +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftabi" +) + +// rowProof keeps the oversized R14/R27/R28 fixture in a bounded native driver. +// The Worker still evaluates every row and its actual heap is sampled without +// forcing collection. Keeping HTTP/JSON copies out of the Node coordinator is +// necessary for the shared-laptop budget, not an alternative application rule. +func rowProof(origin string, result io.Writer) error { + u, err := url.Parse(origin) + if err != nil || u.Scheme != "http" || u.Hostname() != "127.0.0.1" || u.Port() == "" || u.Path != "" { + return errors.New("row proof requires the local workerd origin") + } + client := &http.Client{Transport: &http.Transport{MaxIdleConnsPerHost: 1, Proxy: nil}, Timeout: 5 * time.Second} + defer client.CloseIdleConnections() + request := func(method, path string, b []byte) ([]byte, error) { + req, err := http.NewRequest(method, origin+path, bytes.NewReader(b)) + if err != nil { + return nil, err + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return nil, errors.New("row proof HTTP refusal") + } + out, err := io.ReadAll(io.LimitReader(resp.Body, raftabi.MaxBuffer+1)) + if len(out) > raftabi.MaxBuffer { + return nil, errors.New("row proof oversized response") + } + return out, err + } + native := raftabi.New() + check := func(op uint32, blobs ...[]byte) ([]byte, error) { + var b bytes.Buffer + binary.Write(&b, binary.LittleEndian, raftabi.Version) + binary.Write(&b, binary.LittleEndian, op) + for _, blob := range blobs { + binary.Write(&b, binary.LittleEndian, uint32(len(blob))) + b.Write(blob) + } + expected := native.Call(b.Bytes()) + actual, err := request("POST", "/call", b.Bytes()) + if err != nil { + return nil, err + } + if !bytes.Equal(actual, expected) { + return nil, errors.New("R14 row ABI differs between native and wasm") + } + r := bytes.NewReader(actual) + var version, n uint32 + if binary.Read(r, binary.LittleEndian, &version) != nil || version != raftabi.Version || binary.Read(r, binary.LittleEndian, &n) != nil || n != 0 { + return nil, errors.New("R14 row ABI refused") + } + if binary.Read(r, binary.LittleEndian, &n) != nil || uint64(n) != uint64(r.Len()) { + return nil, errors.New("R14 row ABI invalid response") + } + return io.ReadAll(r) + } + const rowCount = 129 + var hashes [rowCount][32]byte + original, restored := sha256.New(), sha256.New() + total, maxRow := 0, 0 + for i := 0; i < rowCount; i++ { + size := 64 << 10 + if i == 0 { + size = 768 << 10 + } + cell := clusterstate.Created(clusterstate.CellInit{Head: "opaque-head", Class: "work", Size: 1, Title: strings.Repeat("A", size), Keys: map[string]map[string]string{"key": {"recipient": "opaque-wrap"}}}, "device-a", 1000) + row, err := clusterstate.EncodeRow(clusterstate.Row{Schema: clusterstate.SchemaVersion, Key: fmt.Sprintf("legacy-%d", i), Cell: &cell}) + if err != nil { + return err + } + if len(row) > 1<<20 { + return errors.New("R27 row exceeds legacy envelope") + } + total += len(row) + maxRow = max(maxRow, len(row)) + hashes[i] = sha256.Sum256(row) + original.Write(row) + got, err := check(5, row) + if err != nil { + return err + } + if !bytes.Equal(got, row) { + return errors.New("R28 opaque row changed") + } + if _, err := request("PUT", fmt.Sprintf("/store/legacy/row-%d", i), got); err != nil { + return err + } + } + if total <= 8<<20 { + return errors.New("R27 fixture must exceed default admission ceiling") + } + for i := 0; i < rowCount; i++ { + row, err := request("GET", fmt.Sprintf("/store/legacy/row-%d", i), nil) + if err != nil { + return err + } + if sha256.Sum256(row) != hashes[i] { + return errors.New("R28 retained row differs") + } + restored.Write(row) + } + digest := original.Sum(nil) + if !bytes.Equal(digest, restored.Sum(nil)) { + return errors.New("R28 logical fixture content differs") + } + row, err := request("GET", "/store/legacy/row-0", nil) + if err != nil { + return err + } + command, err := clusterstate.EncodeCommand(clusterstate.Command{Schema: 1, Cluster: "p03-cluster", Actor: "device-a", Session: "proof", Sequence: 2, RequestDigest: "opaque-digest", Now: 2000, Signature: []byte{1, 2, 3}, Change: clusterstate.Mutation{Release: &clusterstate.ReleaseCell{Cell: "legacy-0", Fence: 1}}}) + if err != nil { + return err + } + next, err := check(4, command, row) + if err != nil { + return err + } + decoded, err := clusterstate.DecodeRow(next) + if err != nil { + return err + } + if decoded.Cell.Lease.Expires != 0 || decoded.Cell.Title != strings.Repeat("A", 768<<10) { + return errors.New("R28 large row release lost state") + } + return json.NewEncoder(result).Encode(map[string]any{"rows": rowCount, "bytes": total, "ceiling": max(8<<20, total), "maxRowBytes": maxRow, "digest": hex.EncodeToString(digest), "representation": "paged immutable single rows; host inventory; grandfathered ceiling; streamed snapshots remain P05"}) +} diff --git a/relay/hosted/test/raft/abi.test.js b/relay/hosted/test/raft/abi.test.js new file mode 100644 index 0000000000..4f83b8eb14 --- /dev/null +++ b/relay/hosted/test/raft/abi.test.js @@ -0,0 +1,23 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { Writer, Reader, frame, response, MAX_BUFFER } from '../../src/raft/abi.js'; +import { wasiPreview1 } from '../../src/raft/wasi.js'; + +test('R14 binary ABI preserves full uint64 and opaque bytes',()=>{ + const bytes=new Writer().u32(1).u64((1n<<64n)-1n).blob(Uint8Array.from({length:256},(_,i)=>i)).str('opaque').finish(); + const r=new Reader(bytes);assert.equal(r.u32(),1);assert.equal(r.u64(),(1n<<64n)-1n);assert.deepEqual(r.blob(),Uint8Array.from({length:256},(_,i)=>i));assert.equal(r.str(),'opaque');r.end(); + assert.throws(()=>r.u32(),/truncated/);assert.throws(()=>response(new Writer().u32(2).str('').finish()),/version/); + assert.throws(()=>new Writer().u32(-1),/uint32/);assert.throws(()=>new Writer().u64(-1n),/uint64/);assert.throws(()=>new Writer().u64(1n<<64n),/uint64/);assert.throws(()=>new Writer().blob('bytes'),/bytes required/); + assert.equal(frame(1).finish().length,8);assert.equal(MAX_BUFFER,2*1024*1024); +}); + +test('R27 WASI shim has fixed environment, no I/O and rejects blocking scheduling',()=>{ + const wasi=wasiPreview1(),memory=new WebAssembly.Memory({initial:1});wasi.attach(memory);const v=new DataView(memory.buffer); + assert.equal(wasi.imports.environ_sizes_get(0,4),0);assert.equal(v.getUint32(0,true),3); + const size=v.getUint32(4,true);assert.equal(wasi.imports.environ_get(8,32),0); + assert.deepEqual([...Array(3)].map((_,i)=>v.getUint32(8+i*4,true)),[32,49,57]); + assert.equal(new TextDecoder().decode(new Uint8Array(memory.buffer,32,size)),'GOMEMLIMIT=48MiB\0GOGC=50\0GOMAXPROCS=1\0'); + assert.throws(()=>wasi.imports.poll_oneoff(),/blocking poll/);assert.throws(()=>wasi.imports.sched_yield(),/scheduler yield/); + assert.equal(wasi.imports.fd_read(),8);assert.equal(wasi.imports.sock_accept(),8);assert.equal(wasi.imports.fd_write(9),8); + assert.throws(()=>wasi.imports.random_get(65536,1),/bounds/); +}); diff --git a/relay/hosted/test/raft/build.mjs b/relay/hosted/test/raft/build.mjs new file mode 100644 index 0000000000..402064b4cb --- /dev/null +++ b/relay/hosted/test/raft/build.mjs @@ -0,0 +1,44 @@ +// Bundling the existing relay together with the reactor host measures the full +// package rather than comparing a naked wasm file against the Worker limit. +import { build } from 'esbuild'; +import { readFile, writeFile, mkdir, readdir } from 'node:fs/promises'; +import { resolve, relative } from 'node:path'; +import { gzipSync } from 'node:zlib'; +import { createHash } from 'node:crypto'; +import { createRequire } from 'node:module'; +import { dirname } from 'node:path'; +import { wasiPreview1 } from '../../src/raft/wasi.js'; +const out = resolve(process.argv[2] ?? '/tmp/codeaf-p03'); +await mkdir(out, { recursive: true }); +const require=createRequire(import.meta.url); +await writeFile(resolve(out,'workerd-path'),require('workerd').default+'\n'); +await writeFile(resolve(out,'workerd-schema-path'),resolve(dirname(require.resolve('workerd/package.json')),'workerd.capnp')+'\n'); +const result = await build({ entryPoints: ['test/raft/worker.js'], bundle: true, format: 'esm', target: 'es2022', external: ['*.wasm', 'cloudflare:*'], write: false }); +const js = result.outputFiles[0].contents; +await writeFile(resolve(out, 'worker.js'), js); +const storage = await readFile('test/raft/storage-worker.js'); +await writeFile(resolve(out,'storage-worker.js'),storage); +const wasm = await readFile(resolve(out, 'core.wasm')); +const imports = [...new Set(WebAssembly.Module.imports(new WebAssembly.Module(wasm)).map(x => { + if (x.module !== 'wasi_snapshot_preview1' || x.kind !== 'function' || typeof wasiPreview1().imports[x.name] !== 'function') throw new Error('R14 unsupported wasm import'); + return x.name; +}))].sort(); +const sha = b => createHash('sha256').update(b).digest('hex'); +const source = []; +async function walk(path) { + for (const entry of (await readdir(path, { withFileTypes: true })).sort((a,b)=>a.name.localeCompare(b.name))) { + const p = resolve(path, entry.name); + if (entry.isDirectory()) await walk(p); + else if ((/\.(go|js|json|mod|sum|patch)$/.test(p) && !p.endsWith('_test.go')) || entry.name === 'LICENSE') source.push([relative(resolve('../..'), p), sha(await readFile(p))]); + } +} +for (const p of ['../../internal/raftcore', '../../internal/raftabi', '../../internal/clusterstate', '../../cmd/codeaf-raft-wasm', '../../third_party/raft', '../../third_party/raft-patch', 'src/raft']) await walk(resolve(p)); +for (const p of ['../../go.mod','../../go.sum']) source.push([relative(resolve('../..'),resolve(p)),sha(await readFile(p))]); +source.sort((a,b)=>a[0].localeCompare(b[0])); +const manifest = { abi: 1, coreProtocol: 1, rowSchema: 1, snapshotSchema: 'P05 pending; no snapshot compatibility claimed', go: 'go1.26.5', sourceHash: sha(JSON.stringify(source)), wasmSHA256: sha(wasm), sources: source }; +await writeFile(resolve(out, 'manifest.json'), JSON.stringify(manifest, null, 2)+'\n'); +const compressed = { wasm: gzipSync(wasm,{level:9}).length, javascript: gzipSync(js,{level:9}).length, storageFixture:gzipSync(storage,{level:9}).length, manifest: gzipSync(JSON.stringify(manifest),{level:9}).length }; +const report = { imports, rawWasm: wasm.length, rawJavascript: js.length, compressed, totalCompressed: Object.values(compressed).reduce((a,b)=>a+b,0), limit: 3*1024*1024, wasmSHA256: manifest.wasmSHA256, sourceHash: manifest.sourceHash }; +await writeFile(resolve(out, 'size.json'), JSON.stringify(report, null, 2)+'\n'); +console.log(JSON.stringify(report)); +if (report.totalCompressed >= report.limit) throw new Error('R27 full package exceeds 3 MiB'); diff --git a/relay/hosted/test/raft/run.sh b/relay/hosted/test/raft/run.sh new file mode 100755 index 0000000000..6dcd0ff8b6 --- /dev/null +++ b/relay/hosted/test/raft/run.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Run from the repository root. No cloud credentials, user home or hosted +# endpoint participates; all storage and build artifacts are disposable. +set -eu +out=${1:-/tmp/codeaf-p03} +sh scripts/raft-proof-build.sh "$out" +sh scripts/raft-proof-footprint.sh go test -p=1 ./internal/raftabi ./internal/raftcore -run '^TestR' -count=1 +sh scripts/raft-proof-footprint.sh sh -c 'cd relay/hosted && sh test/local.sh node test/raft/build.mjs "$1"' proof "$out" +sh scripts/raft-proof-footprint.sh sh -c 'cd relay/hosted && sh test/local.sh timeout --signal=TERM --kill-after=5s 120s node --max-old-space-size=24 test/raft/abi.test.js' +sh scripts/raft-proof-footprint.sh sh -c 'cd relay/hosted && CODEAF_RAFT_PROOF_DIR="$1" sh test/local.sh timeout --signal=TERM --kill-after=5s 120s node --max-old-space-size=24 test/raft/smoke.test.js' proof "$out" +read -r workerd_binary < "$out/workerd-path" +read -r workerd_schema < "$out/workerd-schema-path" +sh scripts/raft-proof-footprint.sh timeout --signal=TERM --kill-after=5s 120s env GOMEMLIMIT=32MiB GOGC=50 "$out/native-peer" measure "$out" "$workerd_binary" "$workerd_schema" diff --git a/relay/hosted/test/raft/smoke.test.js b/relay/hosted/test/raft/smoke.test.js new file mode 100644 index 0000000000..cf95b414e9 --- /dev/null +++ b/relay/hosted/test/raft/smoke.test.js @@ -0,0 +1,229 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn, execFileSync } from 'node:child_process'; +import { readFile, writeFile, mkdtemp, open } from 'node:fs/promises'; +import { resolve, join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { performance } from 'node:perf_hooks'; +import { createHash } from 'node:crypto'; +import { openWorkerd } from './workerd.js'; +import { Writer, Reader, createFrame, stepFrame, frame, response, effects } from '../../src/raft/abi.js'; +import { encodeCanonical, decodeCanonical } from '../../src/raft/codec.js'; + +const clockTicks=Number(execFileSync('getconf',['CLK_TCK'],{encoding:'utf8'}).trim()); +const out = resolve(process.env.CODEAF_RAFT_PROOF_DIR ?? '/tmp/codeaf-p03'); +const text = new TextEncoder(); +const canonical = v => text.encode(encodeCanonical(v)); +const parse = bytes => decodeCanonical(new TextDecoder().decode(bytes)); +const sha = bytes => createHash('sha256').update(bytes).digest('hex'); +async function processSample() { + const ids=[process.pid,...(await readFile(`/proc/${process.pid}/task/${process.pid}/children`,'utf8')).trim().split(/\s+/).filter(Boolean).map(Number)]; + const rows=[]; + for(const pid of ids){try{const status=await readFile(`/proc/${pid}/status`,'utf8');const stat=(await readFile(`/proc/${pid}/stat`,'utf8')).split(') ').slice(1).join(') ').split(' ');rows.push({cpuSeconds:(Number(stat[11])+Number(stat[12]))/clockTicks,pid,name:status.match(/^Name:\s+(.*)$/m)?.[1],rssBytes:Number(status.match(/^VmRSS:\s+(\d+)/m)?.[1]??0)*1024,peakRSSBytes:Number(status.match(/^VmHWM:\s+(\d+)/m)?.[1]??0)*1024});}catch{}} + return {processes:rows,combinedRSSBytes:rows.reduce((sum,r)=>sum+r.rssBytes,0)}; +} +const report = { allocator:{arenas:process.env.MALLOC_ARENA_MAX,trimThreshold:process.env.MALLOC_TRIM_THRESHOLD_}, laptopSamples:[], requirements: ['R3', 'R4', 'R14', 'R27', 'R28'], phases: [], importCalls: {}, measurements: [] }; + +class Native { + constructor(path) { + this.child = spawn(path, [], { stdio: ['pipe', 'pipe', 'inherit'] }); + this.buffer = Buffer.alloc(0); this.waiter = undefined; + this.child.stdout.on('data', b => { this.buffer = Buffer.concat([this.buffer, b]); this.pump(); }); + this.child.on('error', err => this.waiter?.reject(err)); + this.child.on('exit', code => { if (this.waiter) this.waiter.reject(new Error(`native peer exit ${code}`)); }); + } + pump() { + if (!this.waiter || this.buffer.length < 4) return; + const n = this.buffer.readUInt32LE(0); assert(n <= 2*1024*1024); + if (this.buffer.length < n + 4) return; + const b = new Uint8Array(this.buffer.subarray(4,n+4)); this.buffer = this.buffer.subarray(n+4); + const waiter = this.waiter; this.waiter = undefined; clearTimeout(waiter.timer); waiter.resolve(b); + } + call(b) { + assert(!this.waiter, 'one synchronous ABI call per native runtime'); + return new Promise((resolve, reject) => { this.waiter = { resolve, reject, timer:setTimeout(()=>{this.waiter=undefined;reject(new Error('native ABI call exceeded 5 seconds'));},5000) }; const h = Buffer.alloc(4); h.writeUInt32LE(b.length); this.child.stdin.write(h); this.child.stdin.write(b); this.pump(); }); + } + async close() { this.child.stdin.end();if(this.child.exitCode===null)await new Promise(resolve=>this.child.once('exit',resolve)); } +} + +// Protobuf inspection is transport/storage framing only. Application meaning +// is evaluated by Go, and the independent consensus checker remains P06. +function fields(bytes) { + let at = 0; const result = new Map(); + const vint = () => { let n=0n, shift=0n; for(let i=0;i<10;i++){assert(at>3n), kind=Number(key&7n);let value;if(kind===0)value=vint();else if(kind===2){const n=Number(vint());assert(at+n<=bytes.length);value=bytes.slice(at,at+n);at+=n;}else throw new Error('unexpected protobuf wire type');result.set(tag,value);} + return result; +} +const entryIndex = b => fields(b).get(3) ?? 0n; +function checkpoint(d) { + const w = new Writer().blob(d.hard).blob(d.conf).blob(d.snapshot).u32(d.entries.length); + for(const e of d.entries)w.blob(e); + return w.u64(d.applied).blob(d.row).finish(); +} +function recover(bytes) { const r = new Reader(bytes);const d={hard:r.blob(),conf:r.blob(),snapshot:r.blob(),entries:r.entries(),applied:r.u64(),row:r.blob()};r.end();return d; } +const empty = () => ({ hard:new Uint8Array(),conf:new Uint8Array(),snapshot:new Uint8Array(),entries:[],applied:0n,row:new Uint8Array() }); +const decodeHandle = b => {const r=response(b),h=r.u32();r.end();return h;}; +const decodeBlob = b => {const r=response(b),v=r.blob();r.end();return v;}; + +async function startWorker(persist) { + const start = performance.now(); + const mf = await openWorkerd(out,persist); + report.measurements.push({ workerReadyMs:performance.now()-start }); + const call = async b => {const start=performance.now();const res=await mf.dispatchFetch('http://proof/call',{method:'POST',body:b,signal:AbortSignal.timeout(5000)});if(!res.ok)throw new Error(await res.text());const bytes=new Uint8Array(await res.arrayBuffer());report.measurements.push({callMs:performance.now()-start});return bytes;}; + return {mf,call, + save:async(id,b)=>{const res=await mf.dispatchFetch(`http://proof/store/${id}/checkpoint`,{method:'PUT',body:b});assert.equal(res.status,204);}, + load:async id=>{const res=await mf.dispatchFetch(`http://proof/store/${id}/checkpoint`);assert.equal(res.status,200);return new Uint8Array(await res.arrayBuffer());}, + metrics:async()=>{const res=await mf.dispatchFetch('http://proof/metrics');return res.json();}, + }; +} + +async function inspector(mf) { + const base = await mf.getInspectorURL(); + const listing = await (await fetch(new URL('/json/list',base))).json(); + const worker = listing.find(x=>x.title?.includes('raft-proof')) ?? listing[0]; + assert(worker?.webSocketDebuggerUrl,'workerd inspector is required for isolate memory proof'); + const ws = new WebSocket(worker.webSocketDebuggerUrl); + await new Promise((resolve,reject)=>{ws.onopen=resolve;ws.onerror=reject;}); + let serial=0;const waiting=new Map(); + ws.onmessage=e=>{const m=JSON.parse(e.data);if(m.id){const p=waiting.get(m.id);waiting.delete(m.id);if(m.error)p.reject(new Error(JSON.stringify(m.error)));else p.resolve(m.result);}}; + return {call:(method,params={})=>new Promise((resolve,reject)=>{const id=++serial;waiting.set(id,{resolve:v=>{clearTimeout(timer);resolve(v);},reject:e=>{clearTimeout(timer);reject(e);}});const timer=setTimeout(()=>{waiting.delete(id);reject(new Error('inspector call exceeded 5 seconds'));},5000);ws.send(JSON.stringify({id,method,params}));}),close:()=>ws.close()}; +} + +class Group { + constructor(worker,native,dir,wasmIDs=[1],cluster='p03-cluster'){this.worker=worker;this.native=native;this.dir=dir;this.wasmIDs=wasmIDs;this.nodes=new Map();this.queue=[];this.reads=[];this.trace=[];this.block=false;this.cluster=cluster;} + async create(id,recovered) { + const endpoint = this.wasmIDs.includes(id)?this.worker:this.native; + const cfg={cluster:this.cluster,id:BigInt(id),seed:BigInt(100+id),bootstrap:recovered?[]:[1n,2n,3n],...(recovered??{})}; + const handle=decodeHandle(await endpoint.call(createFrame(cfg))); + const shadow=decodeHandle(await this.native.call(createFrame(cfg))); + this.nodes.set(id,{id,endpoint,handle,shadow,d:recovered??empty()}); + } + async call(id,tag,value){ + const n=this.nodes.get(id),bytes=await n.endpoint.call(stepFrame(n.handle,tag,value)); + const expected=await this.native.call(stepFrame(n.shadow,tag,value)); + assert.deepEqual(bytes,expected,`R14 identical effects node ${id} input ${tag}`); + this.trace.push(sha(bytes));report.totalParityCalls=(report.totalParityCalls??0)+1;return effects(bytes); + } + async save(n){ + const bytes=checkpoint(n.d); + if(this.wasmIDs.includes(n.id))await this.worker.save(`${this.cluster}-${n.id}`,bytes); + else {const f=await open(join(this.dir,`native-${this.cluster}-${n.id}`),'w');try{await f.writeFile(bytes);await f.sync();}finally{await f.close();}} + } + async accept(id,e,depth=0){ + assert(depth<100,'bounded Ready completions');const n=this.nodes.get(id);n.status=e.status; + this.queue.push(...e.send.map(m=>({...m,from:id})));this.reads.push(...e.reads.map(r=>({...r,node:id}))); + if(e.persist){ + const p=e.persist;if(p.hard.length)n.d.hard=p.hard; + assert.equal(p.snapshot.length,0,'P05 snapshot lifecycle remains outside P03'); + if(p.entries.length){const first=entryIndex(p.entries[0]);n.d.entries=n.d.entries.filter(b=>entryIndex(b)0){ + const command=f.get(4); + const req=frame(4).blob(command).blob(n.d.row).finish(); + const got=await n.endpoint.call(req),want=await this.native.call(req); + assert.deepEqual(got,want,'R14 same Go lease application');n.d.row=decodeBlob(got); + }} + n.d.conf=a.conf;n.d.applied=a.index;await this.save(n); + await this.accept(id,await this.call(id,6,a.id),depth+1); + } + } + async step(id,tag,value){await this.accept(id,await this.call(id,tag,value));} + async pump(){for(let count=0;this.queue.length;count++){assert(count<2000,'bounded network schedule');const m=this.queue.shift();if(this.block&&(m.from===this.isolated||Number(m.to)===this.isolated))continue;await this.step(Number(m.to),2,m.bytes);}} + async tick(n=1){for(let i=0;in.status?.role===2)?.id;} + async read(id,request){await this.step(id,4,request);await this.pump();return this.reads.find(r=>r.id===request);} +} + +function command(change,sequence=1n,actor='device-a',now=1000){ + return canonical({schema:1,cluster:'p03-cluster',actor,session:'proof',sequence:String(sequence),request_digest:'sealed-command-digest',now,change,signature:'AQID'}); +} + +// R3/R4/R14/R27: actual workerd, patched production cores, real storage, +// identical seeded effects, a current-term quorum read and process restart. +test('R3 R4 R14 R27 mixed Worker/native command, ReadIndex and durable restart', {timeout:120000}, async()=>{ + const dir=await mkdtemp(join(tmpdir(),'codeaf-p03-'));const native=new Native(join(out,'native-peer')); + let worker,inspect,sampling=false;const heapSamples=[]; + const monitor=setInterval(async()=>{if(sampling)return;sampling=true;try{report.laptopSamples.push(await processSample());if(inspect)heapSamples.push(await inspect.call('Runtime.getHeapUsage'));}finally{sampling=false;}},200); + try { + worker=await startWorker(join(dir,'do'));inspect=await inspector(worker.mf); + const samples=heapSamples;const sample=async()=>{const h=await inspect.call('Runtime.getHeapUsage');samples.push(h);return h;}; + await inspect.call('Profiler.enable');await inspect.call('Profiler.start'); + let g=new Group(worker,native,dir); + await g.create(1); + report.coldFirstCallMs=report.measurements.find(x=>x.callMs!==undefined).callMs; + assert(report.coldFirstCallMs<1000,'R27 cold instantiate/initialize/call stays within 1 s startup envelope'); + for(let id=2;id<=3;id++)await g.create(id); + for(let id=1;id<=3;id++)await g.step(id,0); + await g.pump();await g.tick(14);assert(g.leader()); + let leader=g.leader();if(leader!==1){await g.step(leader,7,1n);await g.pump();await g.tick(2);} + assert.equal(g.leader(),1,'wasm is the actual leader'); + const create=command({create:{cell:'cell-proof',init:{head:'sealed-head',class:'work',size:'7',title:'sealed-title',keys:{}},receipt:'durable-receipt'}}); + await g.step(1,3,create);await g.pump(); + const read=await g.read(1,'after-create');assert(read&&read.index>0n,'ReadIndex barrier returned'); + for(const n of g.nodes.values()){const row=parse(n.d.row);assert.equal(row.cell.lease.fence,'1');assert.equal(row.cell.head,'sealed-head');assert(n.d.applied>=read.index);} + await sample(); + // Withhold all native acknowledgments: a remembered Worker leader must + // not satisfy a fresh read before or after CheckQuorum steps it down. + g.block=true;g.isolated=1;assert.equal(await g.read(1,'minority'),undefined);await g.tick(14); + assert.equal(g.reads.find(r=>r.id==='minority'),undefined);assert.notEqual(g.nodes.get(1).status.role,2); + g.block=false;await g.tick(14);leader=g.leader();assert(leader); + assert(await g.read(leader,'healed')); + const before=g.nodes.get(1).d.row;const persisted=recover(await worker.load('p03-cluster-1'));assert.deepEqual(persisted.row,before); + const saved=[];for(let id=1;id<=3;id++)saved.push(id===1?persisted:recover(new Uint8Array(await readFile(join(dir,`native-p03-cluster-${id}`))))); + report.traceDigest=sha(text.encode(g.trace.join('\n')));report.parityCalls=g.trace.length; + report.phases.push('wasm leader command + quorum ReadIndex; minority read refused; native fallback'); + const profile=await inspect.call('Profiler.stop');report.cpuProfileSamples=profile.profile.samples?.length??0; + const functions=new Map(profile.profile.nodes.map(n=>[n.id,n.callFrame.functionName])); + report.activeProfileMicros=(profile.profile.samples??[]).reduce((sum,id,i)=>sum+(functions.get(id)==='(idle)'?0:(profile.profile.timeDeltas?.[i]??0)),0); + report.firstRuntime=await worker.metrics();await sample();inspect.close();inspect=undefined;await worker.mf.dispose();worker=undefined; + // A completely new workerd process recovers only durable storage bytes. + worker=await startWorker(join(dir,'do'));inspect=await inspector(worker.mf); + const reloaded=recover(await worker.load('p03-cluster-1'));assert.deepEqual(checkpoint(reloaded),checkpoint(saved[0])); + g=new Group(worker,native,dir,[1,2]); + for(let id=1;id<=3;id++)await g.create(id,saved[id-1]); + for(let id=1;id<=3;id++)await g.step(id,0);await g.pump();await g.tick(14); + leader=g.leader();assert(leader);assert(await g.read(leader,'restarted')); + for(const n of g.nodes.values())assert.deepEqual(n.d.row,before,'committed row survived process restart'); + report.phases.push('new workerd process storage restart; two wasm + one native parity'); + samples.push(await inspect.call('Runtime.getHeapUsage')); + // Three anchors and a second identity share the same Go runtime. The + // oversized identity remains in the host store and traverses one row at a + // time; no whole-identity JSON/string is passed into the reactor. + const identityCheckpoint=await worker.load('p03-cluster-1'); + await proveCoResident(worker,native,inspect,samples); + assert.deepEqual(await worker.load('p03-cluster-1'),identityCheckpoint,'co-resident identity cannot overwrite the first checkpoint'); + report.finalRuntime=await worker.metrics(); + report.heapSamples=samples; + report.observedIsolateBytes=Math.max(...samples.map(h=>h.totalSize+(h.embedderHeapUsedSize??0)+(h.backingStorageSize??0)))+report.finalRuntime.peakMemory; + assert(report.observedIsolateBytes<=96*1024*1024,'R27 observed isolate envelope ≤96 MiB'); + assert(report.finalRuntime.peakMemory<=64*1024*1024); + assert.equal(report.finalRuntime.wasi.poll_oneoff??0,0,'no blocking runtime poll'); + assert.equal(report.finalRuntime.wasi.sched_yield??0,0,'no scheduler loop'); + report.phases.push('co-resident handles; oversized rows measured by the sequential native driver'); + report.laptopSamples.push(await processSample()); + report.observedCombinedRSSBytes=Math.max(...report.laptopSamples.map(s=>s.combinedRSSBytes)); + assert(report.observedCombinedRSSBytes<=256*1024*1024,'R27 combined local process RSS ≤256 MiB'); + } finally { + clearInterval(monitor);report.laptopSamples.push(await processSample()); + inspect?.close();if(worker)await worker.mf.dispose();await native.close(); + report.observedCombinedRSSBytes=Math.max(...report.laptopSamples.map(s=>s.combinedRSSBytes)); + report.nodePeakRSSBytes=process.resourceUsage().maxRSS*1024; + const cpuByPID=new Map();for(const sample of report.laptopSamples)for(const p of sample.processes)cpuByPID.set(p.pid,Math.max(cpuByPID.get(p.pid)??0,p.cpuSeconds)); + report.sampledTotalCPUSeconds=[...cpuByPID.values()].reduce((a,b)=>a+b,0); + await writeFile(join(out,'runtime.json'),JSON.stringify(report,null,2)+'\n'); + } +}); + +async function proveCoResident(worker,native,inspect,samples){ + // Admission preserves an imported ceiling, never truncating or discarding + // rows. This fixture exercises the representation, not P11 migration policy. + const group=new Group(worker,native,await mkdtemp(join(tmpdir(),'codeaf-p03-extra-')),[1,2,3],'p03-other-identity'); + for(let id=1;id<=3;id++)await group.create(id); + for(let id=1;id<=3;id++)await group.step(id,0); + await group.pump();await group.tick(12);assert(group.leader());assert(await group.read(group.leader(),'other-identity')); + samples.push(await inspect.call('Runtime.getHeapUsage')); +} diff --git a/relay/hosted/test/raft/storage-worker.js b/relay/hosted/test/raft/storage-worker.js new file mode 100644 index 0000000000..20cfcc391e --- /dev/null +++ b/relay/hosted/test/raft/storage-worker.js @@ -0,0 +1,18 @@ +// The proof uses a deliberately tiny opaque checkpoint store, not P08's +// production member schema. Its awaited transaction demonstrates the actual +// workerd durability boundary before the harness supplies completions. +export class ProofStore { + constructor(state) { this.state = state; } + async fetch(request) { + const key = new URL(request.url).pathname.slice(1); + if (request.method === 'PUT') { + const bytes = new Uint8Array(await request.arrayBuffer()); + await this.state.storage.transaction(async tx => { await tx.put(key, bytes); }); + return new Response(null, { status: 204 }); + } + const bytes = await this.state.storage.get(key); + return bytes ? new Response(bytes) : new Response(null, { status: 404 }); + } +} + +export default {fetch:()=>new Response(null,{status:404})}; diff --git a/relay/hosted/test/raft/worker.js b/relay/hosted/test/raft/worker.js new file mode 100644 index 0000000000..346e0049b4 --- /dev/null +++ b/relay/hosted/test/raft/worker.js @@ -0,0 +1,24 @@ +// This local-only door measures the actual Worker runtime. No production route +// imports it; P08 supplies authenticated actors and persistent host adapters. +import core from './core.wasm'; +import { Reactor } from '../../src/raft/abi.js'; +export { IdentityDO, NewcomerGate, LinkGate, LinkRequest, PairGate, Mailbox } from '../../src/worker.js'; +import production from '../../src/worker.js'; +let reactor; +export default { + async fetch(request, env) { + const url = new URL(request.url); + if (url.pathname === '/ready') return new Response(null, {status:204}); + if (url.pathname.startsWith('/store/')) { + const [, , member, ...parts] = url.pathname.split('/'); + const stub = env.PROOF_STORE.get(env.PROOF_STORE.idFromName(member)); + return stub.fetch(new Request('http://proof/' + parts.join('/'), request)); + } + if (url.pathname === '/production') return production.fetch(request, env); + reactor ??= new Reactor(core); + if (url.pathname === '/metrics') return Response.json(reactor.metrics()); + if (url.pathname !== '/call') return new Response('local proof only', { status: 404 }); + return new Response(reactor.call(new Uint8Array(await request.arrayBuffer()))); + }, +}; + diff --git a/relay/hosted/test/raft/workerd.js b/relay/hosted/test/raft/workerd.js new file mode 100644 index 0000000000..ea8d2362df --- /dev/null +++ b/relay/hosted/test/raft/workerd.js @@ -0,0 +1,66 @@ +// The pinned workerd engine is also Miniflare's runtime. Launching it directly +// avoids retaining the entire development SDK in the bounded mixed-peer proof. +import { spawn } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { createServer } from 'node:net'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { join, dirname } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +const require = createRequire(import.meta.url); +const binary = require('workerd').default; +const schema = join(dirname(require.resolve('workerd/package.json')), 'workerd.capnp'); +async function port() { + const s=createServer();await new Promise((resolve,reject)=>{s.on('error',reject);s.listen(0,'127.0.0.1',resolve);}); + const n=s.address().port;await new Promise(resolve=>s.close(resolve));return n; +} +export async function openWorkerd(out,persist) { + await mkdir(persist,{recursive:true}); + const httpPort=await port(),inspectorPort=await port(); + const quote=JSON.stringify; + const config=join(persist,'proof.capnp'); + await writeFile(config,`using Workerd = import ${quote(schema)}; +const config :Workerd.Config = ( + services = [ + (name = "raft-proof", worker = ( + compatibilityDate = "2026-09-01", + modules = [(name = "worker.js", esModule = embed ${quote(join(out,'worker.js'))}), + (name = "core.wasm", wasm = embed ${quote(join(out,'core.wasm'))})], + bindings = [(name = "PROOF_STORE", durableObjectNamespace = (className = "ProofStore", serviceName = "proof-storage"))], + globalOutbound = "blocked" + )), + (name = "proof-storage", worker = ( + compatibilityDate = "2026-09-01", + modules = [(name = "storage.js", esModule = embed ${quote(join(out,'storage-worker.js'))})], + durableObjectNamespaces = [(className = "ProofStore", uniqueKey = "p03-local-proof", enableSql = true)], + durableObjectStorage = (localDisk = "storage"), globalOutbound = "blocked" + )), + (name = "storage", disk = (path = ${quote(persist)}, writable = true)), + (name = "blocked", worker = (compatibilityDate = "2026-09-01", modules = [ + (name = "blocked.js", esModule = "export default {fetch() {throw new Error('proof forbids outbound network')}}") + ], globalOutbound = "blocked")) + ], + sockets = [(name = "http", address = "127.0.0.1:${httpPort}", http = (), service = "raft-proof")] +); +`); + const child=spawn(binary,['serve','--experimental','-I/','--inspector-addr',`127.0.0.1:${inspectorPort}`,config],{stdio:['ignore','pipe','pipe']}); + let errors='',exited=false; + child.stdout.on('data',()=>{});child.stderr.on('data',b=>{if(errors.length<4096)errors+=b.toString();}); + const ended=new Promise(resolve=>child.once('exit',code=>{exited=true;resolve(code);})); + const origin=`http://127.0.0.1:${httpPort}`; + const server={ + origin, + dispatchFetch:(url,options)=>{const u=new URL(url);return fetch(origin+u.pathname+u.search,options);}, + getInspectorURL:async()=>new URL(`http://127.0.0.1:${inspectorPort}`), + dispose:async()=>{if(!exited)child.kill('SIGKILL');await ended;}, + }; + try { + const deadline=Date.now()+10000; + for(;;){ + if(exited)throw new Error(`local workerd exited: ${errors}`); + try{const r=await fetch(origin+'/ready',{signal:AbortSignal.timeout(500)});if(r.status===204)break;}catch{} + if(Date.now()>deadline)throw new Error(`local workerd startup timed out: ${errors}`); + await delay(25); + } + return server; + }catch(err){await server.dispose();throw err;} +} diff --git a/scripts/raft-proof-build.sh b/scripts/raft-proof-build.sh new file mode 100755 index 0000000000..c3a73ec475 --- /dev/null +++ b/scripts/raft-proof-build.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# This feasibility artifact is a WASI reactor and a stdio conformance peer, +# never an alternate installation of the person's codeaf binary. +set -eu +out=${1:?usage: raft-proof-build.sh /tmp/output-directory} +mkdir -p "$out" +sh scripts/raft-proof-footprint.sh env GOMEMLIMIT=192MiB GOOS=wasip1 GOARCH=wasm go build -p=1 -trimpath -buildmode=c-shared -ldflags='-s -w' -o "$out/core.wasm" ./cmd/codeaf-raft-wasm +sh scripts/raft-proof-footprint.sh env GOMEMLIMIT=192MiB go build -p=1 -trimpath -ldflags='-s -w' -o "$out/native-peer" ./cmd/codeaf-raft-probe diff --git a/scripts/raft-proof-footprint.sh b/scripts/raft-proof-footprint.sh new file mode 100755 index 0000000000..3f21cadf20 --- /dev/null +++ b/scripts/raft-proof-footprint.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# The shared laptop is in use; pressure admission precedes every heavy step. +set -eu +while ! awk '/^some / {split($2,a,"="); exit !(a[2]<=40)}' /proc/pressure/io; do + cat /proc/pressure/io + sleep 60 +done +cat /proc/pressure/io +export GOFLAGS=-buildvcs=false GOMAXPROCS=2 +# Bound allocator arenas in the local Node/workerd hosts on many-core WSL. +export MALLOC_ARENA_MAX=1 MALLOC_TRIM_THRESHOLD_=65536 +exec nice -n 10 "$@" From 26b6717a39b5522539694c098b4885d6386f006e Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Fri, 23 Oct 2026 23:11:06 -0400 Subject: [PATCH 007/103] Persist native Raft members before releasing dependent effects. Keep each member in its own FULL-synchronous SQLite WAL store so votes, conflicting suffixes and application results survive restart. Refuse corrupt authoritative state and commit rows, dedup results, configuration and the applied index together. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/raftstore/README.md | 106 ++++++ internal/raftstore/recover.go | 178 ++++++++++ internal/raftstore/sqlite.go | 607 ++++++++++++++++++++++++++++++++++ 3 files changed, 891 insertions(+) create mode 100644 internal/raftstore/README.md create mode 100644 internal/raftstore/recover.go create mode 100644 internal/raftstore/sqlite.go diff --git a/internal/raftstore/README.md b/internal/raftstore/README.md new file mode 100644 index 0000000000..ee642d02c5 --- /dev/null +++ b/internal/raftstore/README.md @@ -0,0 +1,106 @@ +# Native member durability + +This is P04's internal host adapter, with no production directory routing or new +person-facing door. Each member has its own SQLite file, permanently bound to +`Identity{ClusterUUID, NodeID, Protocol}` and `FormatVersion`. The same identity's +three anchors must use three distinct files. The caller creates the parent +directory; `Open` creates only an absent file with mode 0600 and syncs the parent +directory after initialization. Existing empty, corrupt, mismatched or incomplete +files are refused and retained. They are never repaired into new empty members. + +SQLite uses WAL, `synchronous=FULL`, immediate transactions and one connection. +The connection pragmas are checked at startup and applied to any replacement +connection by the driver DSN. There is no weaker durability option. A transaction +error stops the handle; close it, retain the file and recover before participating +again. A validation refusal (`ErrBatch`, `ErrConflict`, `ErrSnapshot`) rolls back +without poisoning otherwise valid storage. A COMMIT error is an unknown outcome, +never permission to acknowledge a client or send a Raft append response. + +## Driving the core + +1. `Open(path, identity)`, then `Recover(ctx)`. Supply its `Core` to + `raftcore.New` with explicit seed/protocol/member inputs. Supply a shared + bootstrap voter set only when `Recovery.Fresh` is true. Persisted but unapplied + bootstrap entries are recovered without bootstrapping a second time. +2. On `Effects.Persist`, call `Persist(ctx, *batch)`. Only after success submit + `raftcore.Persisted{BatchID: batch.ID}`. This flushes the entire Ready, including + commit-only hard state, before the core releases any messages. An empty Ready + is a no-write barrier. Batch IDs belong to one core lifetime and are not durable + sequence numbers; discard old-handle completions after restart. +3. Evaluate `Effects.Apply` against immutable rows returned by `Read` at the + current applied index. Pass its complete ConfState, entries and index unchanged + to `Apply`, with a host-evaluated `Update`. Apply compares `ExpectedApplied`, + checks contiguous entries against the durable committed log, and commits the + write-set with visible version, monotonic logical time, complete ConfState and + applied index. Empty normal and configuration entries also advance that index. +4. Only after Apply succeeds may client results escape and the host submit + `raftcore.Applied{BatchID: batch.ID}`. Recovery skips already-applied entries and + replays committed entries above Applied. Results in `Sessions` share the same + transaction, so a dropped reply does not lose its stored result. +5. A local `Read` is a storage dependency read, not an authority response. Serve a + public read only after a fresh core quorum ReadIndex barrier and durable + Applied reaching that barrier. The host serializes all core work and storage + completions; the database mutex alone does not serialize a whole actor. + +P09 owns signatures, authorization, declared read dependencies, deterministic +evaluation, response encoding, dedup sequencing and admission policy. This adapter +does not infer these from stored bytes. It accepts opaque records with a 1 MiB +per-record and aggregate read-set envelope. A missing key is absent from the map; +an existing empty record remains present. `Update.Delete` is for an explicitly +authorized application change; no consensus maintenance path deletes records. + +## Format and corruption checks + +`meta` holds checksummed manifest, hard state, log tip, application header and +complete protobuf ConfState. `raft_log` holds protobuf entries by eight-byte +big-endian index, covering the full uint64 range with correct SQLite ordering. +`records` is keyed by namespace and opaque record key. Namespaces separate state, +sessions/dedup results, membership intents, enrollments, tombstones and durable +outboxes while keeping them in one transaction. + +SHA-256 checksums bind payloads to their table and key. The application header +also holds record count and an XOR accumulator of the keyed row hashes to detect +missing rows without loading every row on each write. The log tip detects a +missing uncommitted tail as well as holes. Recovery streams all application rows +through checksum validation, checks SQLite integrity and index/term/commit/applied +invariants, then validates the supplied recovery with the production core. These +checks detect accidental corruption, not a malicious storage operator or rollback +of an entire consistent database to an old backup. + +`ErrCorrupt` stops that member and retains evidence. Recovery from a healthy group +uses a newly enrolled replacement ID, not deletion of the file. There is no path +here that deletes chats, frames, branches, saved heads or legacy original stores. + +## P05 boundary + +Snapshot input explicitly returns `ErrSnapshot`; no snapshot acknowledgment may +escape. P05 must add streamed manifest/chunk generations, checksums, two retained +complete generations, atomic installation and compaction. Its recovered snapshot +must already be installed in the application before constructing the core. The +current schema retains the whole log and recovery materializes that suffix; this +is not the completed bounded log/cache policy. P05 must update the format and +recovery checks together, preserving all application namespaces and exact results. + +## Focused proof + +Run through the wave's pressure/CPU wrapper: + +```sh +sh scripts/raft-proof-footprint.sh go test -p=1 ./internal/raftstore ./internal/raftcore -run '^TestR' -count=1 -timeout=120s -v +``` + +The R3/R12/R17 transaction tests terminate subprocesses without Close/rollback at +every Ready/application statement and both sides of COMMIT. They recover copies +of only the database and WAL, excluding process memory and the shared-memory index. +The Linux `strace` probe asserts successful WAL fsync/fdatasync between invocation +and return for term/vote, commit-only and application commits. This observes the +production SQLite VFS rather than substituting a fake persistence completion. +Physical power cuts or hardware that lies about flushes are not simulated. + +The bounded three-native-node fixture runs real cores/stores and real create/ +publish lease rules, then loses a leader and recovers another member from durable +bytes before a quorum read. It also recovers and leaves real joint membership. +Its one-cell evaluator is test-only, not P09's authorized directory writer or the +independent P06 history oracle. Full fault schedules, snapshots, transport, +membership policy, cross-runtime storage and real-client journeys remain later +packets; these mechanism tests do not claim complete R1–R30 release proof. diff --git a/internal/raftstore/recover.go b/internal/raftstore/recover.go new file mode 100644 index 0000000000..f1ebedac2c --- /dev/null +++ b/internal/raftstore/recover.go @@ -0,0 +1,178 @@ +package raftstore + +import ( + "bytes" + "context" + "database/sql" + "encoding/binary" + "encoding/json" + "errors" + + "github.com/Agent-Field/codeaf/internal/raftcore" + "go.etcd.io/raft/v3" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// Recovery has no leadership, timers or batch IDs to restore. Fresh means the +// store has never persisted Raft work; only then may the host supply Bootstrap. +// Entries after Applied, including committed ones, are replayed by the core. +type Recovery struct { + Core raftcore.RecoveredState + Application Application + Fresh bool +} + +func decodeEntry(key, data, hash []byte) (*pb.Entry, error) { + if len(key) != 8 || !bytes.Equal(hash, checksum([]byte("raft_log"), key, data)) { + return nil, ErrCorrupt + } + e := &pb.Entry{} + if proto.Unmarshal(data, e) != nil || !validEntry(e) || e.GetIndex() == 0 || e.GetIndex() != binary.BigEndian.Uint64(key) || e.GetTerm() == 0 { + return nil, ErrCorrupt + } + return e, nil +} + +func getEntry(ctx context.Context, tx *sql.Tx, index uint64) (*pb.Entry, error) { + var data, hash []byte + key := indexKey(index) + if err := tx.QueryRowContext(ctx, `SELECT data,checksum FROM raft_log WHERE idx=?`, key).Scan(&data, &hash); err != nil { + return nil, ErrCorrupt + } + return decodeEntry(key, data, hash) +} + +func lastEntry(ctx context.Context, tx *sql.Tx) (*pb.Entry, error) { + var key, data, hash []byte + err := tx.QueryRowContext(ctx, `SELECT idx,data,checksum FROM raft_log ORDER BY idx DESC LIMIT 1`).Scan(&key, &data, &hash) + if errors.Is(err, sql.ErrNoRows) { + return &pb.Entry{}, nil + } + if err != nil { + return nil, err + } + return decodeEntry(key, data, hash) +} + +// Recover checks physical integrity, each record's checksum, member identity +// and cross-record invariants in one consistent transaction. Corrupt evidence +// stays in place; a missing log entry is never synthesized from an index. +// P05 will replace the un-compacted suffix read with snapshot-backed recovery. +func (s *SQLite) Recover(ctx context.Context) (Recovery, error) { + s.mu.Lock() + defer s.mu.Unlock() + var out Recovery + err := s.transaction(ctx, func(tx *sql.Tx) error { + var result string + if err := tx.QueryRowContext(ctx, `PRAGMA quick_check`).Scan(&result); err != nil || result != "ok" { + return ErrCorrupt + } + data, err := getMeta(ctx, tx, "manifest") + if err != nil { + return err + } + var m manifest + if json.Unmarshal(data, &m) != nil { + return ErrCorrupt + } + if m.Format != FormatVersion || m.Identity != s.identity { + return ErrIdentity + } + var count int + if err := tx.QueryRowContext(ctx, `SELECT count(*) FROM meta`).Scan(&count); err != nil || count != 5 { + return ErrCorrupt + } + h := &pb.HardState{} + if err := getProto(ctx, tx, "hardstate", h); err != nil { + return err + } + a, err := getApplication(ctx, tx) + if err != nil { + return err + } + rows, err := tx.QueryContext(ctx, `SELECT idx,data,checksum FROM raft_log ORDER BY idx`) + if err != nil { + return ErrCorrupt + } + defer rows.Close() + var entries []*pb.Entry + var last, term uint64 + for rows.Next() { + var key, data, hash []byte + if err := rows.Scan(&key, &data, &hash); err != nil { + return ErrCorrupt + } + e, err := decodeEntry(key, data, hash) + if err != nil { + return err + } + if last == ^uint64(0) || e.GetIndex() != last+1 || e.GetTerm() < term { + return ErrCorrupt + } + last, term = e.GetIndex(), e.GetTerm() + entries = append(entries, e) + } + if err := rows.Err(); err != nil { + return ErrCorrupt + } + if err := rows.Close(); err != nil { + return err + } + tip, err := getMeta(ctx, tx, "log_tip") + if err != nil || !bytes.Equal(tip, indexKey(last)) { + return ErrCorrupt + } + if a.Applied > h.GetCommit() || h.GetCommit() > last || term > h.GetTerm() || (h.GetTerm() == 0 && h.GetVote() != 0) || (a.Applied == 0 && !proto.Equal(a.ConfState, &pb.ConfState{})) { + return ErrCorrupt + } + // Stream application rows through checksum validation. Recovery does not + // materialize an identity or silently discard unknown namespaces. + r, err := tx.QueryContext(ctx, `SELECT namespace,key,data,checksum FROM records ORDER BY namespace,key`) + if err != nil { + return ErrCorrupt + } + defer r.Close() + var appRows uint64 + var recordsHash [32]byte + for r.Next() { + var namespace Namespace + var key string + var data, hash []byte + if err := r.Scan(&namespace, &key, &data, &hash); err != nil { + return ErrCorrupt + } + if !validKey(Key{namespace, key}) || len(data) > MaxRecordBytes || !bytes.Equal(hash, checksum([]byte("records"), []byte(namespace), []byte(key), data)) { + return ErrCorrupt + } + appRows++ + xorHash(&recordsHash, hash) + } + if err := r.Err(); err != nil { + return ErrCorrupt + } + if err := r.Close(); err != nil { + return err + } + if a.RecordCount != appRows || a.RecordsHash != recordsHash || (a.Applied == 0 && appRows != 0) { + return ErrCorrupt + } + out.Core = raftcore.RecoveredState{HardState: h, ConfState: a.ConfState, Entries: entries, Applied: a.Applied} + out.Application = a + out.Fresh = last == 0 && raft.IsEmptyHardState(h) && a.Applied == 0 + if out.Fresh { + // A nonnil recovered ConfState forbids core bootstrap. Empty + // initialized storage has no configuration to recover yet. + out.Core.ConfState = nil + } + cfg := raftcore.Config{ClusterUUID: s.identity.ClusterUUID, NodeID: s.identity.NodeID, Protocol: s.identity.Protocol} + if _, err := raftcore.New(cfg, out.Core); err != nil { + return ErrCorrupt + } + return nil + }) + if err != nil { + return Recovery{}, err + } + return out, nil +} diff --git a/internal/raftstore/sqlite.go b/internal/raftstore/sqlite.go new file mode 100644 index 0000000000..21ca24c7d2 --- /dev/null +++ b/internal/raftstore/sqlite.go @@ -0,0 +1,607 @@ +// Package raftstore owns native per-member durability. It is a host adapter; +// neither SQLite nor a filesystem callback enters the deterministic core. +package raftstore + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/binary" + "encoding/json" + "errors" + "net/url" + "os" + "path/filepath" + "sync" + + "github.com/Agent-Field/codeaf/internal/raftcore" + "go.etcd.io/raft/v3" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" + _ "modernc.org/sqlite" // This is the repository's existing native SQLite driver. +) + +const ( + FormatVersion = 1 + MaxRecordBytes = 1 << 20 +) + +var ( + ErrCorrupt = errors.New("raftstore: invalid authoritative store; retain it and recover a replacement from the group") + ErrIdentity = errors.New("raftstore: member identity or format mismatch") + ErrBatch = errors.New("raftstore: invalid persistence or application batch") + ErrConflict = errors.New("raftstore: application read index changed") + ErrStopped = errors.New("raftstore: storage failed; close and recover before participating") + ErrSnapshot = errors.New("raftstore: snapshot installation requires the snapshot lifecycle adapter") +) + +// Identity permanently binds a file to one member, not just to one identity's +// directory. A different member must have its own path and freshly enrolled ID. +type Identity struct { + ClusterUUID string `json:"cluster"` + NodeID uint64 `json:"node"` + Protocol uint16 `json:"protocol"` +} + +type manifest struct { + Format int `json:"format"` + Identity +} + +// Application is committed with every application write-set. ConfState is the +// complete DURABLY APPLIED configuration, including joint and learner fields. +// VisibleVersion deliberately does not move for ordinary consensus entries. +type Application struct { + Applied uint64 `json:"applied"` + VisibleVersion uint64 `json:"version"` + LogicalTime int64 `json:"now"` + ConfState *pb.ConfState `json:"-"` + RecordCount uint64 `json:"record_count"` + RecordsHash [32]byte `json:"records_hash"` +} + +// Namespace keeps application dependencies and durable effect results in the +// same transaction without teaching this storage layer directory policy. +type Namespace string + +const ( + State Namespace = "state" + Sessions Namespace = "sessions" + Membership Namespace = "membership" + Enrollment Namespace = "enrollment" + Tombstones Namespace = "tombstones" + Outbox Namespace = "outbox" +) + +type Key struct { + Namespace Namespace + Key string +} + +// Write is a host-evaluated write-set, not permission to mutate a directory. +// Delete is reserved for explicitly authorized application changes. Consensus +// compaction never deletes these records or any sealed chat data. +type Write struct { + Key + Value []byte + Delete bool +} + +type Update struct { + ExpectedApplied uint64 + VisibleVersion uint64 + LogicalTime int64 + Writes []Write +} + +// SQLite serializes all use of its sole connection. Each transaction takes +// SQLite's writer lock immediately; a second opener cannot race a read-set CAS. +// The host must also serialize its core and discard callbacks from old handles. +type SQLite struct { + mu sync.Mutex + db *sql.DB + identity Identity + stopped bool + // cut lets package tests kill the PROCESS at actual transaction boundaries. + // Production never installs a callback or receives a weaker durability mode. + cut func(string) +} + +// Indexes are fixed-width big-endian BLOBs, so SQLite sorts the full uint64 +// range without signed INTEGER overflow or decimal-string collation mistakes. +const schema = ` +CREATE TABLE meta (key TEXT PRIMARY KEY, data BLOB NOT NULL, checksum BLOB NOT NULL) WITHOUT ROWID; +CREATE TABLE raft_log (idx BLOB PRIMARY KEY CHECK(length(idx)=8), data BLOB NOT NULL, checksum BLOB NOT NULL) WITHOUT ROWID; +CREATE TABLE records (namespace TEXT NOT NULL, key TEXT NOT NULL, data BLOB NOT NULL, checksum BLOB NOT NULL, PRIMARY KEY(namespace,key)) WITHOUT ROWID; +` + +var wire = proto.MarshalOptions{Deterministic: true} + +// Open creates an absent file and otherwise validates the existing store before +// returning it. It NEVER creates missing tables in an existing authoritative +// file, repairs corrupt rows, truncates it or falls back to an empty database. +func Open(path string, identity Identity) (*SQLite, error) { + if path == "" || path == ":memory:" || identity.ClusterUUID == "" || identity.NodeID == 0 || raft.IsLocalMsgTarget(identity.NodeID) || identity.Protocol != raftcore.ProtocolVersion { + return nil, ErrIdentity + } + abs, err := filepath.Abs(path) + if err != nil { + return nil, err + } + created := false + f, err := os.OpenFile(abs, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err == nil { + created = true + err = f.Close() + } else if errors.Is(err, os.ErrExist) { + var info os.FileInfo + info, err = os.Lstat(abs) + if err == nil && (!info.Mode().IsRegular() || info.Size() == 0) { + err = ErrCorrupt + } + } + if err != nil { + return nil, err + } + u := url.URL{Scheme: "file", Path: abs} + q := url.Values{"_txlock": {"immediate"}, "_pragma": {"busy_timeout(10000)", "journal_mode(WAL)", "synchronous(FULL)"}, "mode": {"rw"}} + u.RawQuery = q.Encode() + db, err := sql.Open("sqlite", u.String()) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + s := &SQLite{db: db, identity: identity} + ok := false + defer func() { + if !ok { + db.Close() + } + }() + ctx := context.Background() + var mode string + var synchronous int + if err = db.QueryRowContext(ctx, `PRAGMA journal_mode`).Scan(&mode); err != nil { + return nil, ErrCorrupt + } + if err = db.QueryRowContext(ctx, `PRAGMA synchronous`).Scan(&synchronous); err != nil || mode != "wal" || synchronous != 2 { + return nil, ErrCorrupt + } + if created { + err = s.transaction(ctx, func(tx *sql.Tx) error { + if _, err := tx.ExecContext(ctx, schema); err != nil { + return err + } + m, _ := json.Marshal(manifest{FormatVersion, identity}) + if err := putMeta(ctx, tx, "manifest", m); err != nil { + return err + } + if err := putProto(ctx, tx, "hardstate", &pb.HardState{}); err != nil { + return err + } + if err := putMeta(ctx, tx, "log_tip", indexKey(0)); err != nil { + return err + } + return putApplication(ctx, tx, Application{ConfState: &pb.ConfState{}}) + }) + if err != nil { + return nil, err + } + // SQLite syncs the WAL before COMMIT returns. Sync the containing + // directory too so first creation survives losing power immediately. + dir, err := os.Open(filepath.Dir(abs)) + if err != nil { + return nil, err + } + err = dir.Sync() + closeErr := dir.Close() + if err != nil { + return nil, err + } + if closeErr != nil { + return nil, closeErr + } + } + if _, err = s.Recover(ctx); err != nil { + return nil, err + } + ok = true + return s, nil +} + +func (s *SQLite) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + s.stopped = true + return s.db.Close() +} + +func indexKey(index uint64) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, index) + return b +} + +// Checksums bind each payload to its table and key, detecting both bit damage +// and misplaced rows. They detect corruption, not a malicious storage operator. +func checksum(parts ...[]byte) []byte { + h := sha256.New() + for _, p := range parts { + h.Write(indexKey(uint64(len(p)))) + h.Write(p) + } + return h.Sum(nil) +} + +func putMeta(ctx context.Context, tx *sql.Tx, key string, data []byte) error { + _, err := tx.ExecContext(ctx, `INSERT INTO meta VALUES (?,?,?) ON CONFLICT(key) DO UPDATE SET data=excluded.data,checksum=excluded.checksum`, key, data, checksum([]byte("meta"), []byte(key), data)) + return err +} + +func getMeta(ctx context.Context, tx *sql.Tx, key string) ([]byte, error) { + var data, hash []byte + if err := tx.QueryRowContext(ctx, `SELECT data,checksum FROM meta WHERE key=?`, key).Scan(&data, &hash); err != nil { + return nil, ErrCorrupt + } + if !bytes.Equal(hash, checksum([]byte("meta"), []byte(key), data)) { + return nil, ErrCorrupt + } + return data, nil +} + +func putProto(ctx context.Context, tx *sql.Tx, key string, v proto.Message) error { + data, err := wire.Marshal(v) + if err != nil { + return ErrBatch + } + return putMeta(ctx, tx, key, data) +} + +func getProto(ctx context.Context, tx *sql.Tx, key string, v proto.Message) error { + data, err := getMeta(ctx, tx, key) + if err != nil { + return err + } + if proto.Unmarshal(data, v) != nil || len(v.ProtoReflect().GetUnknown()) != 0 { + return ErrCorrupt + } + return nil +} + +func putApplication(ctx context.Context, tx *sql.Tx, a Application) error { + data, err := json.Marshal(a) + if err != nil { + return err + } + if err := putMeta(ctx, tx, "application", data); err != nil { + return err + } + return putProto(ctx, tx, "confstate", a.ConfState) +} + +func getApplication(ctx context.Context, tx *sql.Tx) (Application, error) { + var a Application + data, err := getMeta(ctx, tx, "application") + if err != nil { + return a, err + } + if json.Unmarshal(data, &a) != nil { + return a, ErrCorrupt + } + a.ConfState = &pb.ConfState{} + err = getProto(ctx, tx, "confstate", a.ConfState) + return a, err +} + +func (s *SQLite) boundary(name string) { + if s.cut != nil { + s.cut(name) + } +} + +// A failed storage operation poisons the handle. Even a COMMIT error can have +// an unknown durable outcome; only recovery may decide which bytes survived. +func (s *SQLite) transaction(ctx context.Context, fn func(*sql.Tx) error) error { + if s.stopped { + return ErrStopped + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + s.stopped = true + return err + } + defer tx.Rollback() + s.boundary("after-begin") + if err = fn(tx); err != nil { + if !errors.Is(err, ErrBatch) && !errors.Is(err, ErrConflict) && !errors.Is(err, ErrSnapshot) { + s.stopped = true + } + return err + } + s.boundary("before-commit") + if err = tx.Commit(); err != nil { + s.stopped = true + return err + } + s.boundary("after-commit") + return nil +} + +// Persist returns only after FULL-synchronous COMMIT. Submit Persisted to the +// core only on success; a queued write or transport receipt is insufficient. +// Snapshot input is explicitly refused until P05 supplies atomic installation. +func (s *SQLite) Persist(ctx context.Context, batch raftcore.PersistBatch) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.stopped { + return ErrStopped + } + if !raft.IsEmptySnap(batch.Snapshot) { + return ErrSnapshot + } + if len(batch.Entries) == 0 && raft.IsEmptyHardState(batch.HardState) { + return nil + } + return s.transaction(ctx, func(tx *sql.Tx) error { + old := &pb.HardState{} + if err := getProto(ctx, tx, "hardstate", old); err != nil { + return err + } + h := batch.HardState + if raft.IsEmptyHardState(h) { + h = old + } + if len(h.ProtoReflect().GetUnknown()) != 0 || h.GetTerm() < old.GetTerm() || h.GetCommit() < old.GetCommit() || (h.GetTerm() == old.GetTerm() && old.GetVote() != 0 && h.GetVote() != old.GetVote()) { + return ErrBatch + } + last, err := lastEntry(ctx, tx) + if err != nil { + return err + } + tip, err := getMeta(ctx, tx, "log_tip") + if err != nil || !bytes.Equal(tip, indexKey(last.GetIndex())) { + return ErrCorrupt + } + if len(batch.Entries) != 0 { + first := batch.Entries[0].GetIndex() + if first == 0 || (last.GetIndex() != ^uint64(0) && first > last.GetIndex()+1) { + return ErrBatch + } + previous := &pb.Entry{} + if first > 1 { + previous, err = getEntry(ctx, tx, first-1) + if err != nil { + return err + } + } + for i, entry := range batch.Entries { + if entry == nil || previous.GetIndex() == ^uint64(0) || entry.GetIndex() != previous.GetIndex()+1 || entry.GetTerm() == 0 || entry.GetTerm() < previous.GetTerm() || !validEntry(entry) { + return ErrBatch + } + if entry.GetIndex() <= old.GetCommit() { + stored, err := getEntry(ctx, tx, entry.GetIndex()) + if err != nil { + return err + } + if !proto.Equal(stored, entry) { + return ErrBatch + } + } + previous = entry + if i == len(batch.Entries)-1 && entry.GetIndex() < old.GetCommit() { + return ErrBatch + } + } + last = previous + if h.GetCommit() > last.GetIndex() || h.GetTerm() < last.GetTerm() { + return ErrBatch + } + if _, err := tx.ExecContext(ctx, `DELETE FROM raft_log WHERE idx>=?`, indexKey(first)); err != nil { + return err + } + s.boundary("after-truncate") + for _, entry := range batch.Entries { + data, err := wire.Marshal(entry) + if err != nil { + return ErrBatch + } + key := indexKey(entry.GetIndex()) + if _, err := tx.ExecContext(ctx, `INSERT INTO raft_log VALUES (?,?,?)`, key, data, checksum([]byte("raft_log"), key, data)); err != nil { + return err + } + s.boundary("after-entry") + } + } + if h.GetCommit() > last.GetIndex() || h.GetTerm() < last.GetTerm() || (h.GetTerm() == 0 && h.GetVote() != 0) { + return ErrBatch + } + if !raft.IsEmptyHardState(batch.HardState) { + if err := putProto(ctx, tx, "hardstate", batch.HardState); err != nil { + return err + } + s.boundary("after-hardstate") + } + if err := putMeta(ctx, tx, "log_tip", indexKey(last.GetIndex())); err != nil { + return err + } + s.boundary("after-log-tip") + return nil + }) +} + +func validEntry(e *pb.Entry) bool { + if e == nil || len(e.ProtoReflect().GetUnknown()) != 0 { + return false + } + switch e.GetType() { + case pb.EntryNormal, pb.EntryConfChange, pb.EntryConfChangeV2: + return true + default: + return false + } +} + +func validKey(k Key) bool { + if k.Key == "" { + return false + } + switch k.Namespace { + case State, Sessions, Membership, Enrollment, Tombstones, Outbox: + return true + default: + return false + } +} + +// Apply compares the read-set's applied index, checks every input against the +// durable committed log, and commits rows, dedup results and metadata together. +// The caller may acknowledge clients and submit Applied ONLY after success. +func (s *SQLite) Apply(ctx context.Context, batch raftcore.ApplyBatch, update Update) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.stopped { + return ErrStopped + } + if !raft.IsEmptySnap(batch.Snapshot) { + return ErrSnapshot + } + return s.transaction(ctx, func(tx *sql.Tx) error { + a, err := getApplication(ctx, tx) + if err != nil { + return err + } + if update.ExpectedApplied != a.Applied { + return ErrConflict + } + h := &pb.HardState{} + if err := getProto(ctx, tx, "hardstate", h); err != nil { + return err + } + if len(batch.Entries) == 0 || batch.Index <= a.Applied || batch.Index > h.GetCommit() || batch.ConfState == nil || len(batch.ConfState.ProtoReflect().GetUnknown()) != 0 || update.LogicalTime < a.LogicalTime || update.VisibleVersion < a.VisibleVersion { + return ErrBatch + } + previous := a.Applied + for _, entry := range batch.Entries { + if entry == nil || previous == ^uint64(0) || entry.GetIndex() != previous+1 { + return ErrBatch + } + stored, err := getEntry(ctx, tx, entry.GetIndex()) + if err != nil { + return err + } + if !proto.Equal(stored, entry) { + return ErrBatch + } + previous = entry.GetIndex() + } + if previous != batch.Index { + return ErrBatch + } + seen := make(map[Key]bool, len(update.Writes)) + for _, w := range update.Writes { + if !validKey(w.Key) || seen[w.Key] || len(w.Value) > MaxRecordBytes || (w.Delete && len(w.Value) != 0) { + return ErrBatch + } + seen[w.Key] = true + var oldData, oldHash []byte + err = tx.QueryRowContext(ctx, `SELECT data,checksum FROM records WHERE namespace=? AND key=?`, w.Namespace, w.Key.Key).Scan(&oldData, &oldHash) + if err == nil { + if a.RecordCount == 0 || !bytes.Equal(oldHash, checksum([]byte("records"), []byte(w.Namespace), []byte(w.Key.Key), oldData)) { + return ErrCorrupt + } + a.RecordCount-- + xorHash(&a.RecordsHash, oldHash) + } else if !errors.Is(err, sql.ErrNoRows) { + return err + } + if w.Delete { + _, err = tx.ExecContext(ctx, `DELETE FROM records WHERE namespace=? AND key=?`, w.Namespace, w.Key.Key) + } else { + data := w.Value + if data == nil { + data = []byte{} + } + hash := checksum([]byte("records"), []byte(w.Namespace), []byte(w.Key.Key), data) + _, err = tx.ExecContext(ctx, `INSERT INTO records VALUES (?,?,?,?) ON CONFLICT(namespace,key) DO UPDATE SET data=excluded.data,checksum=excluded.checksum`, w.Namespace, w.Key.Key, data, hash) + if a.RecordCount == ^uint64(0) { + return ErrCorrupt + } + a.RecordCount++ + xorHash(&a.RecordsHash, hash) + } + if err != nil { + return err + } + s.boundary("after-row") + } + a.Applied, a.VisibleVersion, a.LogicalTime, a.ConfState = batch.Index, update.VisibleVersion, update.LogicalTime, batch.ConfState + data, err := json.Marshal(a) + if err != nil { + return err + } + if err := putMeta(ctx, tx, "application", data); err != nil { + return err + } + s.boundary("after-application-header") + if err := putProto(ctx, tx, "confstate", a.ConfState); err != nil { + return err + } + s.boundary("after-configuration") + return nil + }) +} + +func xorHash(target *[32]byte, hash []byte) { + for i := range target { + target[i] ^= hash[i] + } +} + +// Read supplies immutable copies at exactly expectedApplied. A missing key is +// absent from the map; it is not confused with an existing empty opaque value. +// This is a LOCAL storage read, never a linearizable directory door by itself. +func (s *SQLite) Read(ctx context.Context, expectedApplied uint64, keys []Key) (map[Key][]byte, Application, error) { + s.mu.Lock() + defer s.mu.Unlock() + out := make(map[Key][]byte, len(keys)) + var a Application + err := s.transaction(ctx, func(tx *sql.Tx) error { + var err error + a, err = getApplication(ctx, tx) + if err != nil { + return err + } + if a.Applied != expectedApplied { + return ErrConflict + } + readBytes := 0 + for _, k := range keys { + if !validKey(k) { + return ErrBatch + } + var data, hash []byte + err := tx.QueryRowContext(ctx, `SELECT data,checksum FROM records WHERE namespace=? AND key=?`, k.Namespace, k.Key).Scan(&data, &hash) + if errors.Is(err, sql.ErrNoRows) { + continue + } + if err != nil { + return err + } + if !bytes.Equal(hash, checksum([]byte("records"), []byte(k.Namespace), []byte(k.Key), data)) { + return ErrCorrupt + } + readBytes += len(data) + if readBytes > MaxRecordBytes { + return ErrBatch + } + out[k] = data + } + return nil + }) + if err != nil { + return nil, Application{}, err + } + return out, a, nil +} From 23745767197f114bd465fb6e939f8bed1242177c Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Fri, 23 Oct 2026 23:11:06 -0400 Subject: [PATCH 008/103] Prove native recovery across persistence and application failures. Cut subprocesses at every transaction boundary and observe actual WAL flushes before acknowledgment. Exercise production cores with three real stores so leader loss, joint configuration and dropped application completions recover without losing committed results. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/raftstore/native_test.go | 356 ++++++++++++++++++ internal/raftstore/sqlite_test.go | 607 ++++++++++++++++++++++++++++++ 2 files changed, 963 insertions(+) create mode 100644 internal/raftstore/native_test.go create mode 100644 internal/raftstore/sqlite_test.go diff --git a/internal/raftstore/native_test.go b/internal/raftstore/native_test.go new file mode 100644 index 0000000000..135aace3dd --- /dev/null +++ b/internal/raftstore/native_test.go @@ -0,0 +1,356 @@ +package raftstore + +import ( + "cmp" + "fmt" + "path/filepath" + "slices" + "testing" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + "go.etcd.io/raft/v3" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// nativeGroup is a bounded, socket-free host fixture using REAL per-member +// SQLite databases and production cores. Its one-cell rule evaluator is not +// P09's authorized directory writer or P06's independent history oracle. +type nativeGroup struct { + t *testing.T + nodes [3]*raftcore.Core + stores [3]*SQLite + paths [3]string + apps [3]Application + alive [3]bool + status [3]raftcore.Status + queue []raftcore.Outbound + reads []raftcore.ReadBarrier + events int + cutApply func(int, string, *raftcore.ApplyBatch) bool +} + +func newNativeGroup(t *testing.T) *nativeGroup { + t.Helper() + g := &nativeGroup{t: t} + for i := range g.nodes { + g.paths[i] = filepath.Join(t.TempDir(), "member.sqlite") + g.start(i) + } + g.pump() + return g +} + +func (g *nativeGroup) start(i int) { + g.t.Helper() + id := Identity{testIdentity.ClusterUUID, uint64(i + 1), raftcore.ProtocolVersion} + s, err := Open(g.paths[i], id) + must(g.t, err) + g.t.Cleanup(func() { s.Close() }) + r := recovered(g.t, s) + cfg := raftcore.Config{ClusterUUID: id.ClusterUUID, NodeID: id.NodeID, Protocol: id.Protocol, Seed: uint64(100 + i + g.events)} + if r.Fresh { + cfg.Bootstrap = []uint64{1, 2, 3} + } + c, err := raftcore.New(cfg, r.Core) + must(g.t, err) + g.nodes[i], g.stores[i], g.apps[i], g.alive[i] = c, s, r.Application, true + g.call(i, raftcore.Poll{}) +} + +func (g *nativeGroup) call(i int, input raftcore.Input) { + g.t.Helper() + if !g.alive[i] { + return + } + g.events++ + if g.events > 2000 { + g.t.Fatal("native fixture exceeded its fixed event budget") + } + e, err := g.nodes[i].Step(input) + must(g.t, err) + for depth := 0; ; depth++ { + if depth > 100 { + g.t.Fatal("unbounded Ready completion chain") + } + g.status[i] = e.Status + g.queue = append(g.queue, e.Send...) + for _, barrier := range e.Reads { + if g.apps[i].Applied < barrier.Index { + g.t.Fatal("read escaped before durable application") + } + g.reads = append(g.reads, barrier) + } + if e.Persist != nil { + if len(e.Send) != 0 || e.Apply != nil || len(e.Reads) != 0 { + g.t.Fatal("dependent effect escaped before Ready durability") + } + must(g.t, g.stores[i].Persist(testContext, *e.Persist)) + e, err = g.nodes[i].Step(raftcore.Persisted{BatchID: e.Persist.ID}) + } else if e.Apply != nil { + if g.cutApply != nil && g.cutApply(i, "before-apply", e.Apply) { + g.alive[i] = false + return + } + u := g.evaluate(i, *e.Apply) + must(g.t, g.stores[i].Apply(testContext, *e.Apply, u)) + if g.cutApply != nil && g.cutApply(i, "after-apply", e.Apply) { + g.alive[i] = false + return + } + g.apps[i] = recovered(g.t, g.stores[i]).Application + e, err = g.nodes[i].Step(raftcore.Applied{BatchID: e.Apply.ID}) + } else { + return + } + must(g.t, err) + } +} + +func (g *nativeGroup) evaluate(i int, batch raftcore.ApplyBatch) Update { + g.t.Helper() + a := g.apps[i] + u := Update{ExpectedApplied: a.Applied, VisibleVersion: a.VisibleVersion, LogicalTime: a.LogicalTime} + working := make(map[Key][]byte) + for _, entry := range batch.Entries { + if entry.GetType() != pb.EntryNormal || len(entry.Data) == 0 { + continue + } + cmd, err := clusterstate.DecodeCommand(entry.Data) + must(g.t, err) + key := Key{State, "cell"} + data, ok := working[key] + if !ok { + rows, _, err := g.stores[i].Read(testContext, a.Applied, []Key{key}) + must(g.t, err) + data = rows[key] + } + var cell clusterstate.Cell + if len(data) != 0 { + row, err := clusterstate.DecodeRow(data) + must(g.t, err) + cell = *row.Cell + } + u.LogicalTime = max(u.LogicalTime, cmd.Now) + switch { + case cmd.Change.Create != nil: + if len(data) != 0 { + g.t.Fatal("fixture create duplicated") + } + cell = clusterstate.Created(cmd.Change.Create.Init, cmd.Actor, u.LogicalTime) + case cmd.Change.Publish != nil: + cell, err = clusterstate.PublishTo(cell, cmd.Actor, cmd.Change.Publish.Publish, u.LogicalTime) + must(g.t, err) + default: + g.t.Fatal("unsupported native fixture command") + } + u.VisibleVersion++ + working[key], err = clusterstate.EncodeRow(clusterstate.Row{Schema: clusterstate.SchemaVersion, Key: "cell", Cell: &cell}) + must(g.t, err) + result, err := clusterstate.EncodeResult(clusterstate.Result{Schema: clusterstate.SchemaVersion, Code: clusterstate.OK, Now: u.LogicalTime, Version: u.VisibleVersion}) + must(g.t, err) + working[Key{Sessions, cmd.Session}] = result + } + for key, data := range working { + u.Writes = append(u.Writes, Write{Key: key, Value: data}) + } + slices.SortFunc(u.Writes, func(a, b Write) int { + if c := cmp.Compare(a.Namespace, b.Namespace); c != 0 { + return c + } + return cmp.Compare(a.Key.Key, b.Key.Key) + }) + return u +} + +func (g *nativeGroup) pump() { + g.t.Helper() + for len(g.queue) != 0 { + out := g.queue[0] + g.queue = g.queue[1:] + g.call(int(out.To)-1, raftcore.PeerMessage{Bytes: out.Bytes}) + } +} + +func (g *nativeGroup) tick() { + for i := range g.nodes { + g.call(i, raftcore.Tick{}) + } + g.pump() +} + +func (g *nativeGroup) elect() int { + g.t.Helper() + for ticks := 0; ticks < 30; ticks++ { + g.tick() + for i, status := range g.status { + if g.alive[i] && status.Role == raft.StateLeader { + return i + } + } + } + g.t.Fatal("no leader within bounded native election") + return -1 +} + +func (g *nativeGroup) crash(i int) { + g.t.Helper() + // Copy only the database and WAL while the completed store is still open. + // No Close/checkpoint can make an unflushed application transaction durable. + g.paths[i] = copyDurable(g.t, g.paths[i]) + must(g.t, g.stores[i].Close()) + g.nodes[i], g.alive[i] = nil, false +} + +func (g *nativeGroup) propose(i int, seq uint64, mutation clusterstate.Mutation) { + g.t.Helper() + b, err := clusterstate.EncodeCommand(clusterstate.Command{Schema: clusterstate.SchemaVersion, Cluster: testIdentity.ClusterUUID, Actor: "device", Session: "session", Sequence: seq, RequestDigest: fmt.Sprintf("digest-%d", seq), Now: int64(1000 + seq), Change: mutation}) + must(g.t, err) + g.call(i, raftcore.Propose{Command: b}) + g.pump() +} + +func (g *nativeGroup) cell(i int) clusterstate.Cell { + g.t.Helper() + key := Key{State, "cell"} + rows, _, err := g.stores[i].Read(testContext, g.apps[i].Applied, []Key{key}) + must(g.t, err) + row, err := clusterstate.DecodeRow(rows[key]) + must(g.t, err) + return *row.Cell +} + +// R3's native mechanism uses three actual stores: a completed create/publish +// survives losing the leader and restoring a second member from durable bytes. +// A fresh quorum ReadIndex then observes the same head, fence and exact result. +func TestR3ThreeNativeStoresCrashRecoverAndRead(t *testing.T) { + g := newNativeGroup(t) + leader := g.elect() + g.propose(leader, 1, clusterstate.Mutation{Create: &clusterstate.CreateCell{Cell: "cell", Init: clusterstate.CellInit{Head: "opaque-head-1"}, Receipt: "uploaded-1"}}) + g.propose(leader, 2, clusterstate.Mutation{Publish: &clusterstate.PublishCell{Cell: "cell", Publish: clusterstate.Publish{OldHead: "opaque-head-1", Head: "opaque-head-2", Fence: 1}, Receipt: "uploaded-2"}}) + key := Key{Sessions, "session"} + before, _, err := g.stores[leader].Read(testContext, g.apps[leader].Applied, []Key{key}) + must(t, err) + for i := range g.nodes { + if c := g.cell(i); c.Head != "opaque-head-2" || c.Lease.Fence != 1 { + t.Fatal("publish acknowledged before all connected replicas applied") + } + } + other := (leader + 1) % 3 + g.crash(leader) + g.crash(other) + g.start(other) + g.pump() + fallback := g.elect() + g.call(fallback, raftcore.Read{RequestID: "after-power-recovery"}) + g.pump() + if len(g.reads) != 1 || g.cell(fallback).Head != "opaque-head-2" { + t.Fatal("quorum read lost acknowledged publication after restart") + } + after, _, err := g.stores[fallback].Read(testContext, g.apps[fallback].Applied, []Key{key}) + must(t, err) + if string(before[key]) != string(after[key]) { + t.Fatal("recovery lost exact retained application result") + } + g.start(leader) + g.pump() + for range 2 { + g.tick() + } + if g.cell(leader).Head != "opaque-head-2" || g.status[leader].Role == raft.StateLeader { + t.Fatal("old leader did not recover as a caught-up follower") + } +} + +// R12 preserves the library's entire joint ConfState across process recovery; +// a demoted member cannot regain an incoming vote from an older bootstrap list. +func TestR12NativeJointConfigurationRecovery(t *testing.T) { + g := newNativeGroup(t) + leader := g.elect() + if leader == 2 { + g.call(leader, raftcore.Transfer{Target: 1}) + g.pump() + leader = g.elect() + } + change, err := wire.Marshal(&pb.ConfChangeV2{Transition: pb.ConfChangeTransitionJointExplicit.Enum(), Changes: []*pb.ConfChangeSingle{{Type: pb.ConfChangeAddLearnerNode.Enum(), NodeId: proto.Uint64(3)}}}) + must(t, err) + g.call(leader, raftcore.Configure{Change: change}) + g.pump() + for i := range g.nodes { + conf := recovered(t, g.stores[i]).Core.ConfState + if len(conf.VotersOutgoing) != 3 || !slices.Contains(conf.LearnersNext, 3) { + t.Fatal("joint/learner-next configuration was not durably applied") + } + g.crash(i) + } + for i := range g.nodes { + g.start(i) + } + g.pump() + leader = g.elect() + change, err = wire.Marshal(&pb.ConfChangeV2{}) + must(t, err) + g.call(leader, raftcore.Configure{Change: change}) + g.pump() + for i := range g.nodes { + conf := recovered(t, g.stores[i]).Core.ConfState + if len(conf.VotersOutgoing) != 0 || len(conf.LearnersNext) != 0 || !slices.Equal(conf.Voters, []uint64{1, 2}) || !slices.Equal(conf.Learners, []uint64{3}) { + t.Fatal("leaving joint state lost its committed learner/voter configuration") + } + } + // An outgoing voter may have won the joint-state election. Leaving joint + // consensus then correctly steps that leader down, so wait for an incoming + // voter to win before testing the demoted member's restart. + leader = g.elect() + if leader == 2 { + t.Fatal("demoted learner won an election after leaving joint state") + } + g.crash(2) + g.start(2) + for range 12 { + g.tick() + } + if g.status[2].Role != raft.StateFollower || g.status[2].Leader != uint64(leader+1) { + t.Fatalf("recovered learner regained a vote or displaced leader %d: %+v", leader+1, g.status[2]) + } +} + +// R3 and R17 require replay after durable consensus but before application, and +// skipping already-applied work after application COMMIT but before completion. +// Both paths preserve one create and the same persisted result without a reply. +func TestR3R17NativeCommittedApplyRecovery(t *testing.T) { + for _, phase := range []string{"before-apply", "after-apply"} { + t.Run(phase, func(t *testing.T) { + g := newNativeGroup(t) + leader := g.elect() + target := g.status[leader].Commit + 1 + g.cutApply = func(i int, current string, batch *raftcore.ApplyBatch) bool { + return i == leader && phase == current && batch.Index == target + } + g.propose(leader, 1, clusterstate.Mutation{Create: &clusterstate.CreateCell{Cell: "cell", Init: clusterstate.CellInit{Head: "committed-without-reply"}, Receipt: "uploaded"}}) + if g.alive[leader] { + t.Fatal("requested host completion boundary was not reached") + } + r := recovered(t, g.stores[leader]) + if r.Core.HardState.GetCommit() < target || (r.Core.Applied >= target) != (phase == "after-apply") { + t.Fatal("cut did not straddle the durable application boundary") + } + g.crash(leader) + g.cutApply = nil + g.start(leader) + g.pump() + if g.cell(leader).Head != "committed-without-reply" || g.apps[leader].VisibleVersion != 1 || g.apps[leader].RecordCount != 2 { + t.Fatal("recovery lost or applied the command twice") + } + key := Key{Sessions, "session"} + rows, _, err := g.stores[leader].Read(testContext, g.apps[leader].Applied, []Key{key}) + must(t, err) + result, err := clusterstate.DecodeResult(rows[key]) + must(t, err) + if result.Code != clusterstate.OK || result.Version != 1 { + t.Fatal("reply-drop recovery lost the exact application result") + } + }) + } +} diff --git a/internal/raftstore/sqlite_test.go b/internal/raftstore/sqlite_test.go new file mode 100644 index 0000000000..a81a43b5ca --- /dev/null +++ b/internal/raftstore/sqlite_test.go @@ -0,0 +1,607 @@ +package raftstore + +import ( + "bytes" + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/raftcore" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +var testIdentity = Identity{"store-test", 1, raftcore.ProtocolVersion} +var testContext = context.Background() + +func entry(index, term uint64, data string) *pb.Entry { + return &pb.Entry{Index: proto.Uint64(index), Term: proto.Uint64(term), Type: pb.EntryNormal.Enum(), Data: []byte(data)} +} + +func hard(term, vote, commit uint64) *pb.HardState { + return &pb.HardState{Term: proto.Uint64(term), Vote: proto.Uint64(vote), Commit: proto.Uint64(commit)} +} + +func openTest(t *testing.T, path string) *SQLite { + t.Helper() + s, err := Open(path, testIdentity) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + return s +} + +func must(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} + +func recovered(t *testing.T, s *SQLite) Recovery { + t.Helper() + r, err := s.Recover(testContext) + must(t, err) + return r +} + +func seed(t *testing.T, s *SQLite) { + t.Helper() + must(t, s.Persist(testContext, raftcore.PersistBatch{ID: 1, HardState: hard(2, 2, 1), Entries: []*pb.Entry{entry(1, 1, ""), entry(2, 2, "old-two"), entry(3, 2, "old-three")}})) + must(t, s.Apply(testContext, raftcore.ApplyBatch{ID: 1, Index: 1, Entries: []*pb.Entry{entry(1, 1, "")}, ConfState: &pb.ConfState{Voters: []uint64{1, 2, 3}}}, Update{ExpectedApplied: 0, VisibleVersion: 7, LogicalTime: 1000, Writes: allRows("old")})) +} + +func replacement() raftcore.PersistBatch { + return raftcore.PersistBatch{ID: 2, HardState: hard(3, 3, 3), Entries: []*pb.Entry{entry(2, 3, "new-two"), entry(3, 3, "new-three")}} +} + +func allRows(value string) []Write { + var out []Write + for _, namespace := range []Namespace{State, Sessions, Membership, Enrollment, Tombstones, Outbox} { + out = append(out, Write{Key: Key{namespace, "retained"}, Value: []byte(value)}) + } + return out +} + +func nextApply() (raftcore.ApplyBatch, Update) { + p := replacement() + return raftcore.ApplyBatch{ID: 2, Index: 3, Entries: p.Entries, ConfState: &pb.ConfState{Voters: []uint64{1, 2, 4}, VotersOutgoing: []uint64{1, 2, 3}, Learners: []uint64{5}, LearnersNext: []uint64{3}, AutoLeave: proto.Bool(true)}}, Update{ExpectedApplied: 1, VisibleVersion: 8, LogicalTime: 1100, Writes: allRows("new")} +} + +// R3 requires the whole Ready, including a replaced suffix and commit-only +// hard state, to survive restart before the host acknowledges persistence. +func TestR3ReadyDurabilityAndConflictingSuffix(t *testing.T) { + path := filepath.Join(t.TempDir(), "member.sqlite") + s := openTest(t, path) + if !recovered(t, s).Fresh { + t.Fatal("new member is not empty") + } + seed(t, s) + must(t, s.Persist(testContext, replacement())) + r := recovered(t, s) + if r.Core.Applied != 1 || r.Core.HardState.GetCommit() != 3 || len(r.Core.Entries) != 3 || string(r.Core.Entries[1].Data) != "new-two" { + t.Fatal("Ready was not stored together with the entire new suffix") + } + // Reject committed-prefix replacement, holes and double votes rather than + // allowing a malformed host batch to rewrite already acknowledged history. + bad := []raftcore.PersistBatch{ + {HardState: hard(2, 2, 3)}, + {HardState: hard(3, 2, 3)}, + {HardState: hard(3, 0, 3)}, + {HardState: hard(3, 3, 2)}, + {HardState: hard(3, 3, 4)}, + {Entries: []*pb.Entry{entry(5, 3, "hole")}}, + {Entries: []*pb.Entry{entry(2, 3, "changed-commit")}}, + {Entries: []*pb.Entry{entry(1, 1, "")}}, + {Entries: []*pb.Entry{nil}}, + } + for i, batch := range bad { + if err := s.Persist(testContext, batch); !errors.Is(err, ErrBatch) { + t.Fatalf("invalid batch %d: %v", i, err) + } + } + must(t, s.Close()) + s = openTest(t, path) + r = recovered(t, s) + if !proto.Equal(r.Core.HardState, hard(3, 3, 3)) || string(r.Core.Entries[2].Data) != "new-three" { + t.Fatal("restart changed the durable term/vote/suffix") + } +} + +// R12 and R17 require complete configuration and exact dedup bytes to share +// the applied-index transaction. R28 keeps every application namespace intact. +func TestR12R17AppliedReadSetCASAndRecords(t *testing.T) { + s := openTest(t, filepath.Join(t.TempDir(), "member.sqlite")) + seed(t, s) + must(t, s.Persist(testContext, replacement())) + a, u := nextApply() + must(t, s.Apply(testContext, a, u)) + r := recovered(t, s) + if r.Application.VisibleVersion != 8 || r.Application.LogicalTime != 1100 || !proto.Equal(r.Core.ConfState, a.ConfState) { + t.Fatal("applied metadata/configuration lost fields") + } + var keys []Key + for _, w := range u.Writes { + keys = append(keys, w.Key) + } + keys = append(keys, Key{State, "missing"}) + rows, app, err := s.Read(testContext, 3, keys) + must(t, err) + if len(rows) != 6 || app.RecordCount != 6 || app.Applied != 3 { + t.Fatal("lost retained state or invented a missing record") + } + for _, data := range rows { + if string(data) != "new" { + t.Fatal("dedup/application result does not match applied index") + } + } + rows[keys[0]][0] = 'X' + again, _, err := s.Read(testContext, 3, keys[:1]) + must(t, err) + if string(again[keys[0]]) != "new" { + t.Fatal("caller mutated retained storage") + } + if err := s.Apply(testContext, a, u); !errors.Is(err, ErrConflict) { + t.Fatalf("stale apply read-set: %v", err) + } + if _, _, err := s.Read(testContext, 1, keys); !errors.Is(err, ErrConflict) { + t.Fatalf("stale storage read-set: %v", err) + } + // A second connection observes the same CAS, rather than overwriting rows + // evaluated before the first connection committed its newer index. + other, err := Open(databasePath(t, s), testIdentity) + must(t, err) + defer other.Close() + if err := other.Apply(testContext, a, u); !errors.Is(err, ErrConflict) { + t.Fatalf("cross-connection applied CAS: %v", err) + } +} + +func databasePath(t *testing.T, s *SQLite) string { + t.Helper() + var seq int + var name, path string + must(t, s.db.QueryRow(`PRAGMA database_list`).Scan(&seq, &name, &path)) + return path +} + +// R3's recovery boundary is tested by exiting WITHOUT Close or deferred +// rollback. Parent tests copy only durable database/WAL files, never memory or +// the shared-memory index, into a distinct member directory before recovery. +func TestR3CrashHelper(t *testing.T) { + mode := os.Getenv("P04_HELPER") + if mode == "" { + return + } + s, err := Open(os.Getenv("P04_PATH"), testIdentity) + if err != nil { + t.Fatal(err) + } + if mode == "flush" { + for _, step := range []struct { + name string + batch raftcore.PersistBatch + }{ + {"term-vote", raftcore.PersistBatch{HardState: hard(4, 2, 0), Entries: []*pb.Entry{entry(1, 4, "")}}}, + {"commit-only", raftcore.PersistBatch{HardState: hard(4, 2, 1)}}, + } { + os.Stderr.WriteString("R3-BEGIN-" + step.name + "\n") + if err := s.Persist(testContext, step.batch); err != nil { + t.Fatal(err) + } + os.Stderr.WriteString("R3-END-" + step.name + "\n") + } + os.Stderr.WriteString("R3-BEGIN-apply\n") + must(t, s.Apply(testContext, raftcore.ApplyBatch{Index: 1, Entries: []*pb.Entry{entry(1, 4, "")}, ConfState: &pb.ConfState{Voters: []uint64{1, 2, 3}}}, Update{Writes: allRows("durable-result")})) + os.Stderr.WriteString("R3-END-apply\n") + os.Exit(0) + } + if mode == "apply" { + must(t, s.Persist(testContext, replacement())) + } + cutNumber, err := strconv.Atoi(os.Getenv("P04_NUMBER")) + must(t, err) + count := 0 + s.cut = func(phase string) { + if phase == os.Getenv("P04_PHASE") { + count++ + if count == cutNumber { + os.Exit(86) + } + } + } + if mode == "apply" { + a, u := nextApply() + must(t, s.Apply(testContext, a, u)) + } else { + must(t, s.Persist(testContext, replacement())) + } + t.Fatal("requested crash boundary was not reached") +} + +func copyDurable(t *testing.T, source string) string { + t.Helper() + target := filepath.Join(t.TempDir(), "member.sqlite") + for _, suffix := range []string{"", "-wal"} { + data, err := os.ReadFile(source + suffix) + if errors.Is(err, os.ErrNotExist) && suffix != "" { + continue + } + must(t, err) + must(t, os.WriteFile(target+suffix, data, 0600)) + } + return target +} + +// R3 cuts each Ready persistence statement, then both sides of COMMIT. R12 +// and R17 cut every application row, metadata and configuration statement. +func TestR3R12R17CrashEveryTransactionBoundary(t *testing.T) { + for _, mode := range []string{"persist", "apply"} { + cuts := map[string]int{"after-begin": 1, "before-commit": 1, "after-commit": 1} + if mode == "persist" { + cuts["after-truncate"], cuts["after-entry"], cuts["after-hardstate"], cuts["after-log-tip"] = 1, 2, 1, 1 + } else { + cuts["after-row"], cuts["after-application-header"], cuts["after-configuration"] = 6, 1, 1 + } + var phases []string + for phase := range cuts { + phases = append(phases, phase) + } + slices.Sort(phases) + for _, phase := range phases { + for number := 1; number <= cuts[phase]; number++ { + t.Run(mode+"/"+phase+"/"+strconv.Itoa(number), func(t *testing.T) { + path := filepath.Join(t.TempDir(), "member.sqlite") + s := openTest(t, path) + seed(t, s) + must(t, s.Close()) + cmd := exec.Command(os.Args[0], "-test.run=^TestR3CrashHelper$", "-test.count=1") + cmd.Env = append(os.Environ(), "P04_HELPER="+mode, "P04_PATH="+path, "P04_PHASE="+phase, "P04_NUMBER="+strconv.Itoa(number)) + output, err := cmd.CombinedOutput() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 86 { + t.Fatalf("helper did not crash at %s: %v %s", phase, err, output) + } + s = openTest(t, copyDurable(t, path)) + r := recovered(t, s) + committed := phase == "after-commit" + wantNewLog := mode == "apply" || committed + want := hard(2, 2, 1) + data := "old-two" + if wantNewLog { + want, data = hard(3, 3, 3), "new-two" + } + if !proto.Equal(r.Core.HardState, want) || len(r.Core.Entries) != 3 || string(r.Core.Entries[1].Data) != data { + t.Fatal("recovered a partial Ready or conflicting suffix") + } + wantApplied, wantRow, wantVersion, wantNow := uint64(1), "old", uint64(7), int64(1000) + conf := &pb.ConfState{Voters: []uint64{1, 2, 3}} + if mode == "apply" && committed { + wantApplied, wantRow, wantVersion, wantNow = 3, "new", 8, 1100 + a, _ := nextApply() + conf = a.ConfState + } + if r.Core.Applied != wantApplied || r.Application.VisibleVersion != wantVersion || r.Application.LogicalTime != wantNow || !proto.Equal(conf, r.Core.ConfState) { + t.Fatal("applied metadata or joint configuration was partially committed") + } + var keys []Key + for _, w := range allRows("") { + keys = append(keys, w.Key) + } + rows, _, err := s.Read(testContext, wantApplied, keys) + must(t, err) + if len(rows) != len(keys) { + t.Fatal("lost a retained application record") + } + for _, value := range rows { + if string(value) != wantRow { + t.Fatal("rows/dedup outcome is ahead of or behind the applied index") + } + } + }) + } + } + } +} + +// R3 requires real fsync/fdatasync on the WAL, including a commit-only Ready. +// This Linux proof observes the production SQLite VFS; PRAGMA alone would not +// show whether acknowledgments actually follow successful flush syscalls. +func TestR3TermVoteCommitAndApplicationFsync(t *testing.T) { + strace, err := exec.LookPath("strace") + if err != nil { + t.Skip("the syscall durability probe requires Linux strace") + } + path := filepath.Join(t.TempDir(), "member.sqlite") + s := openTest(t, path) + must(t, s.Close()) + trace := filepath.Join(t.TempDir(), "flush.trace") + cmd := exec.Command(strace, "-f", "-yy", "-s", "200", "-e", "trace=fsync,fdatasync,write,pwrite64", "-o", trace, os.Args[0], "-test.run=^TestR3CrashHelper$", "-test.count=1") + cmd.Env = append(os.Environ(), "P04_HELPER=flush", "P04_PATH="+path) + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("syscall durability probe: %v %s", err, output) + } + data, err := os.ReadFile(trace) + must(t, err) + for _, phase := range []string{"term-vote", "commit-only", "apply"} { + _, tail, found := strings.Cut(string(data), "R3-BEGIN-"+phase) + if !found { + t.Fatalf("missing before marker for %s", phase) + } + segment, _, found := strings.Cut(tail, "R3-END-"+phase) + if !found { + t.Fatalf("missing durable return marker for %s", phase) + } + flushed := false + for _, line := range strings.Split(segment, "\n") { + if strings.Contains(line, path+"-wal") && (strings.Contains(line, "fsync(") || strings.Contains(line, "fdatasync(")) && strings.HasSuffix(line, "= 0") { + flushed = true + } + } + if !flushed { + t.Fatalf("no successful WAL flush before %s return: %s", phase, segment) + } + } + s = openTest(t, copyDurable(t, path)) + r := recovered(t, s) + if !proto.Equal(r.Core.HardState, hard(4, 2, 1)) || r.Core.Applied != 1 { + t.Fatal("term, vote, committed application did not survive abrupt process exit") + } +} + +// R13 requires retaining corrupt stores rather than recreating them. Missing +// rows and log tails are corruption too, even when remaining row hashes match. +func TestR13CorruptStoreRefusalAndEvidence(t *testing.T) { + for name, damage := range map[string]string{ + "log-bytes": `UPDATE raft_log SET data=x'01' WHERE idx=x'0000000000000002'`, + "log-hole": `DELETE FROM raft_log WHERE idx=x'0000000000000002'`, + "log-tail": `DELETE FROM raft_log WHERE idx=x'0000000000000003'`, + "hardstate": `UPDATE meta SET data=x'0102' WHERE key='hardstate'`, + "confstate": `UPDATE meta SET checksum=x'00' WHERE key='confstate'`, + "applied": `UPDATE meta SET data=x'00' WHERE key='application'`, + "state-bytes": `UPDATE records SET data=x'00' WHERE namespace='state'`, + "dedup-absent": `DELETE FROM records WHERE namespace='sessions'`, + "missing-meta": `DELETE FROM meta WHERE key='manifest'`, + "missing-table": `DROP TABLE records`, + } { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "member.sqlite") + s := openTest(t, path) + seed(t, s) + _, err := s.db.Exec(damage) + must(t, err) + must(t, s.Close()) + before, err := os.ReadFile(path) + must(t, err) + _, err = Open(path, testIdentity) + if !errors.Is(err, ErrCorrupt) { + t.Fatalf("damaged store opened: %v", err) + } + after, err := os.ReadFile(path) + must(t, err) + if !bytes.Equal(before, after) { + t.Fatal("refusal rewrote authoritative evidence") + } + }) + } + for _, data := range [][]byte{nil, []byte("not a SQLite database")} { + path := filepath.Join(t.TempDir(), "member.sqlite") + must(t, os.WriteFile(path, data, 0600)) + _, err := Open(path, testIdentity) + if !errors.Is(err, ErrCorrupt) { + t.Fatalf("invalid file silently initialized: %v", err) + } + got, err := os.ReadFile(path) + must(t, err) + if !bytes.Equal(got, data) { + t.Fatal("invalid file changed") + } + } +} + +// R3 and R12 prohibit restoring another node's authority from the same file. +func TestR3MemberIdentityAndFullUint64IndexKeys(t *testing.T) { + path := filepath.Join(t.TempDir(), "member ?#%.sqlite") + s := openTest(t, path) + seed(t, s) + must(t, s.Close()) + for _, identity := range []Identity{{"other-cluster", 1, raftcore.ProtocolVersion}, {testIdentity.ClusterUUID, 2, raftcore.ProtocolVersion}, {testIdentity.ClusterUUID, 1, raftcore.ProtocolVersion + 1}} { + if _, err := Open(path, identity); !errors.Is(err, ErrIdentity) { + t.Fatalf("member/cluster/protocol mismatch: %v", err) + } + } + indexes := []uint64{0, 1, 1<<53 - 1, 1 << 53, 1<<63 - 1, 1 << 63, ^uint64(0)} + for i := 1; i < len(indexes); i++ { + if bytes.Compare(indexKey(indexes[i-1]), indexKey(indexes[i])) >= 0 { + t.Fatal("uint64 index collation loses order") + } + } +} + +// R3 forbids progress after a disk error. A real SQLite page quota exercises +// the production error path; the core must receive no fabricated completion. +func TestR3StorageFailureStopsParticipation(t *testing.T) { + path := filepath.Join(t.TempDir(), "member.sqlite") + s := openTest(t, path) + seed(t, s) + var pages int + must(t, s.db.QueryRow(`PRAGMA page_count`).Scan(&pages)) + var limit int + must(t, s.db.QueryRow(`PRAGMA max_page_count=`+strconv.Itoa(pages)).Scan(&limit)) + p := replacement() + p.Entries[0].Data = bytes.Repeat([]byte{0x80}, 2<<20) + if err := s.Persist(testContext, p); err == nil || errors.Is(err, ErrBatch) { + t.Fatalf("SQLite disk quota did not produce a storage failure: %v", err) + } + if err := s.Persist(testContext, raftcore.PersistBatch{}); !errors.Is(err, ErrStopped) { + t.Fatalf("failed store still permits acknowledgment: %v", err) + } + if _, err := s.Recover(testContext); !errors.Is(err, ErrStopped) { + t.Fatalf("failed handle recovered without reopening: %v", err) + } + must(t, s.Close()) + s = openTest(t, path) + r := recovered(t, s) + if !proto.Equal(r.Core.HardState, hard(2, 2, 1)) || string(r.Core.Entries[1].Data) != "old-two" { + t.Fatal("disk failure committed half of the conflicting suffix") + } + c, err := raftcore.New(raftcore.Config{ClusterUUID: testIdentity.ClusterUUID, NodeID: 1, Protocol: raftcore.ProtocolVersion, Seed: 9, Bootstrap: []uint64{1, 2, 3}}, raftcore.RecoveredState{}) + must(t, err) + e, err := c.Step(raftcore.Poll{}) + must(t, err) + if e.Persist == nil { + t.Fatal("no persistence dependency to fail") + } + e, err = c.Step(raftcore.StorageFailed{BatchID: e.Persist.ID}) + if !errors.Is(err, raftcore.ErrStopped) || len(e.Send) != 0 || e.Apply != nil || len(e.Reads) != 0 { + t.Fatal("storage failure released a dependent acknowledgment") + } +} + +// R3 requires refusing incomplete or invented application work. R17 requires +// a failed row write to roll back earlier rows and the dedup result together. +func TestR3R17InvalidApplyRollsBackAndEmptyReadyDoesNotWrite(t *testing.T) { + s := openTest(t, filepath.Join(t.TempDir(), "member.sqlite")) + seed(t, s) + // A dependency-only Ready must not open a write transaction or flush a WAL. + s.cut = func(string) { t.Fatal("empty Ready started a transaction") } + must(t, s.Persist(testContext, raftcore.PersistBatch{ID: 2})) + s.cut = nil + must(t, s.Persist(testContext, replacement())) + for _, name := range []string{"uncommitted", "hole", "wrong-entry", "wrong-end", "clock", "version", "duplicate-key", "unknown-namespace", "large-row", "nil-configuration"} { + t.Run(name, func(t *testing.T) { + a, u := nextApply() + switch name { + case "uncommitted": + a.Index = 4 + case "hole": + a.Entries = a.Entries[1:] + case "wrong-entry": + a.Entries[0] = entry(2, 3, "invented") + case "wrong-end": + a.Index = 2 + case "clock": + u.LogicalTime = 999 + case "version": + u.VisibleVersion = 6 + case "duplicate-key": + u.Writes = append(u.Writes, u.Writes[0]) + case "unknown-namespace": + u.Writes = append(u.Writes, Write{Key: Key{"unknown", "no"}, Value: []byte("refused")}) + case "large-row": + u.Writes = append(u.Writes, Write{Key: Key{State, "large"}, Value: make([]byte, MaxRecordBytes+1)}) + case "nil-configuration": + a.ConfState = nil + } + if err := s.Apply(testContext, a, u); !errors.Is(err, ErrBatch) { + t.Fatalf("invalid apply succeeded: %v", err) + } + r := recovered(t, s) + if r.Core.Applied != 1 || r.Application.VisibleVersion != 7 { + t.Fatal("refused apply advanced durable state") + } + rows, _, err := s.Read(testContext, 1, []Key{{State, "retained"}, {Sessions, "retained"}}) + must(t, err) + for _, value := range rows { + if string(value) != "old" { + t.Fatal("refused late write leaked an earlier row or dedup result") + } + } + }) + } + // An application storage error, unlike a validation refusal, stops the + // handle. The earlier row write still rolls back with the rest of the set. + _, err := s.db.Exec(`CREATE TRIGGER fail_session BEFORE UPDATE ON records WHEN NEW.namespace='sessions' BEGIN SELECT RAISE(ABORT,'injected storage failure'); END`) + must(t, err) + a, u := nextApply() + if err := s.Apply(testContext, a, u); err == nil || errors.Is(err, ErrBatch) { + t.Fatalf("application storage failure: %v", err) + } + if err := s.Apply(testContext, a, u); !errors.Is(err, ErrStopped) { + t.Fatalf("failed application handle kept participating: %v", err) + } + path := databasePath(t, s) + must(t, s.Close()) + s = openTest(t, path) + if recovered(t, s).Core.Applied != 1 { + t.Fatal("application storage failure moved the applied index") + } +} + +// R3 requires replacing the conflicting suffix, including deleting a longer +// uncommitted tail, rather than retaining entries from the displaced leader. +func TestR3ShorterConflictingSuffixSurvivesRestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "member.sqlite") + s := openTest(t, path) + seed(t, s) + p := replacement() + p.Entries = p.Entries[:1] + p.HardState = hard(3, 3, 2) + must(t, s.Persist(testContext, p)) + must(t, s.Close()) + s = openTest(t, path) + r := recovered(t, s) + if len(r.Core.Entries) != 2 || r.Core.Entries[1].GetTerm() != 3 || r.Core.HardState.GetCommit() != 2 { + t.Fatal("old uncommitted tail survived replacement") + } +} + +// R13's snapshot lifecycle belongs to P05. Until then a snapshot cannot receive +// a false success that would permit dependent replication acknowledgments. +func TestR13SnapshotInputFailsClosed(t *testing.T) { + s := openTest(t, filepath.Join(t.TempDir(), "member.sqlite")) + seed(t, s) + snap := &pb.Snapshot{Data: []byte("not-installed"), Metadata: &pb.SnapshotMetadata{Index: proto.Uint64(2), Term: proto.Uint64(2)}} + if err := s.Persist(testContext, raftcore.PersistBatch{Snapshot: snap}); !errors.Is(err, ErrSnapshot) { + t.Fatalf("snapshot persistence falsely succeeded: %v", err) + } + if err := s.Apply(testContext, raftcore.ApplyBatch{Snapshot: snap, Index: 2}, Update{ExpectedApplied: 1}); !errors.Is(err, ErrSnapshot) { + t.Fatalf("snapshot application falsely succeeded: %v", err) + } + if recovered(t, s).Core.Applied != 1 { + t.Fatal("snapshot refusal modified authoritative state") + } +} + +// R28 preserves records outside an explicitly authorized application write-set; +// the store never interprets compaction or pressure as permission to erase them. +func TestR28ExplicitWriteSetAndReadEnvelope(t *testing.T) { + s := openTest(t, filepath.Join(t.TempDir(), "member.sqlite")) + seed(t, s) + must(t, s.Persist(testContext, replacement())) + a, u := nextApply() + u.Writes = []Write{ + {Key: Key{State, "retained"}, Delete: true}, + {Key: Key{State, "empty"}}, + {Key: Key{State, "large-a"}, Value: make([]byte, MaxRecordBytes/2+1)}, + {Key: Key{State, "large-b"}, Value: make([]byte, MaxRecordBytes/2+1)}, + } + must(t, s.Apply(testContext, a, u)) + r := recovered(t, s) + if r.Application.RecordCount != 8 { + t.Fatal("explicit writes changed an unmentioned namespace") + } + keys := []Key{{State, "retained"}, {State, "empty"}, {Tombstones, "retained"}, {Sessions, "retained"}} + rows, _, err := s.Read(testContext, 3, keys) + must(t, err) + if _, ok := rows[keys[0]]; ok { + t.Fatal("explicitly deleted row survived") + } + if empty, ok := rows[keys[1]]; !ok || len(empty) != 0 { + t.Fatal("empty opaque value was confused with an absent row") + } + if string(rows[keys[2]]) != "old" || string(rows[keys[3]]) != "old" { + t.Fatal("unmentioned tombstone or dedup record was discarded") + } + if rows, _, err := s.Read(testContext, 3, []Key{{State, "large-a"}, {State, "large-b"}}); !errors.Is(err, ErrBatch) || rows != nil { + t.Fatalf("oversized evaluation read-set escaped: %v", err) + } +} From 149161f88cc3d83f1e4f58f86af08664d9210aba Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Fri, 23 Oct 2026 23:26:11 -0400 Subject: [PATCH 009/103] Keep durable coordination checkpoints without retaining learner history. Stream complete application snapshots, retain two checked generations and atomically install their state before acknowledging Raft persistence. Compact only covered applied history so a disconnected learner catches up without pinning the log. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/raftcore/README.md | 27 +- internal/raftcore/core.go | 66 ++- internal/raftcore/snapshot_test.go | 79 ++++ internal/raftcore/types.go | 29 +- internal/raftsnapshot/README.md | 69 +++ internal/raftsnapshot/format.go | 152 +++++++ internal/raftsnapshot/format_test.go | 69 +++ internal/raftstore/README.md | 73 ++- internal/raftstore/compact.go | 252 +++++++++++ internal/raftstore/native_test.go | 54 ++- internal/raftstore/recover.go | 79 +++- internal/raftstore/snapshot.go | 429 ++++++++++++++++++ internal/raftstore/snapshot_test.go | 648 +++++++++++++++++++++++++++ internal/raftstore/sqlite.go | 51 ++- 14 files changed, 2027 insertions(+), 50 deletions(-) create mode 100644 internal/raftcore/snapshot_test.go create mode 100644 internal/raftsnapshot/README.md create mode 100644 internal/raftsnapshot/format.go create mode 100644 internal/raftsnapshot/format_test.go create mode 100644 internal/raftstore/compact.go create mode 100644 internal/raftstore/snapshot.go create mode 100644 internal/raftstore/snapshot_test.go diff --git a/internal/raftcore/README.md b/internal/raftcore/README.md index c5a7c3c947..73459c71be 100644 --- a/internal/raftcore/README.md +++ b/internal/raftcore/README.md @@ -62,7 +62,7 @@ cluster/protocol, enrollment and revocation generations before this call. Local signature or apply policy. P09 must recheck committed authorization and produce application results. Normal commands are bounded by the shared 64 KiB batch budget; uncommitted payload is bounded at 1 MiB and inflight append batches at 8. -Transport chunking/snapshot admission remain P05/P07. +Authenticated transport and per-identity snapshot byte admission remain P07/P08/P09. `Configure` and `Transfer` are mechanisms for authorized host intents, not public APIs. Transfer requires a caught-up incoming voter outside joint state. @@ -110,3 +110,28 @@ not the later simulator, persistent power-loss, joint-swap, cross-runtime or real-client release proofs. The initial trace SHA-256 is a regression fixture; SplitMix64 vectors are independently published values. Full fault checking is P06 and the actual Worker stop/go measurement is P03. + +## Applied checkpoints and bounded history + +At 1,024 retained applied entries or 4 MiB of applied protobuf log, effects on an +idle completion boundary carry `SnapshotNeeded{Index, Term}`. This is a pure +maintenance hint; the host must service it before admitting further growth. It +does not wait for learners or change membership. Quotas/backpressure and durable +snapshot storage belong to the host. The core's separate uncommitted limit still +applies. See `internal/raftsnapshot/README.md` for portable format 1. + +Serialize checkpoint creation with this actor. Create a durable checkpoint at the +current applied index, compact its covered durable prefix, and only then supply +`CompactSnapshot{Snapshot}`. The input refuses a pending Ready, wrong index/term, +cluster/protocol, noncanonical manifest or incomplete configuration. It copies +the manifest into MemoryStorage and compacts the in-memory prefix. A lagging +member then receives a manifest-only MsgSnap; the host streams chunks separately. +Never report successful snapshot transport until the destination has installed +its application and durable pointer. Chunk upload/completion alone is insufficient. + +On incoming snapshots, install the application with the durable Raft batch before +`Persisted`; native raftstore does this atomically. This permits the core's +conservative persist-before-send protocol without sending an acknowledgment for +an uninstalled snapshot. The later Apply effect confirms that installed index +and applies any committed suffix before `Applied`. Recover the application before +constructing a core; no application rows or chunks belong in its log cache. diff --git a/internal/raftcore/core.go b/internal/raftcore/core.go index 79fef56913..c18acc19e9 100644 --- a/internal/raftcore/core.go +++ b/internal/raftcore/core.go @@ -6,6 +6,7 @@ import ( "slices" "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" "go.etcd.io/raft/v3" pb "go.etcd.io/raft/v3/raftpb" "google.golang.org/protobuf/proto" @@ -15,7 +16,7 @@ var wire = proto.MarshalOptions{Deterministic: true} // storage is an explicit in-memory view of recovered and completed durable // bytes. No host callback or filesystem read occurs while stepping consensus. -// P05 will bound retained history by snapshots; this is not a durable store. +// Durable checkpoints bound this cache; this is not a durable store. type storage struct { *raft.MemoryStorage conf *pb.ConfState @@ -177,6 +178,9 @@ func (c *Core) Step(input Input) (effects Effects, err error) { } effects.Status = c.status() effects.NeedNextTick = !c.stopped + if !c.stopped && c.pending == nil { + effects.SnapshotNeeded = c.SnapshotNeeded() + } }() if c.stopped { return effects, ErrStopped @@ -226,6 +230,37 @@ func (c *Core) Step(input Input) (effects Effects, err error) { return effects, ErrBusy } switch in := input.(type) { + case CompactSnapshot: + p := in.Snapshot + m, e := raftsnapshot.Decode(p.GetData()) + index := p.GetMetadata().GetIndex() + term, e2 := c.store.Term(index) + if e != nil || e2 != nil || index != c.applied || m.Index != index || m.Term != term || m.Cluster != c.cfg.ClusterUUID || m.Protocol != uint64(c.cfg.Protocol) || p.GetMetadata().GetTerm() != term || !proto.Equal(p.GetMetadata().GetConfState(), c.store.conf) { + return effects, ErrSnapshot + } + conf, e := m.Configuration() + if e != nil || !proto.Equal(conf, c.store.conf) { + return effects, ErrSnapshot + } + existing, e := c.store.Snapshot() + if e != nil { + return effects, ErrSnapshot + } + if existing.GetMetadata().GetIndex() == index { + if !proto.Equal(existing, p) { + return effects, ErrSnapshot + } + } else { + if _, e = c.store.CreateSnapshot(index, clone(c.store.conf), bytes.Clone(p.Data)); e != nil { + return effects, ErrSnapshot + } + } + first, _ := c.store.FirstIndex() + if first <= index { + if e = c.store.Compact(index); e != nil { + return effects, ErrSnapshot + } + } case Poll: case Tick: c.node.Tick() @@ -483,3 +518,32 @@ func (c *Core) finish(e *Effects) { c.node.Advance(p.ready) c.pending = nil } + +// SnapshotNeeded is a pure maintenance hint, never a client receipt. Hosts +// service it before admitting more growth. Its bounds exclude uncommitted +// entries, whose separate Raft admission budget remains unchanged. +func (c *Core) SnapshotNeeded() *SnapshotPoint { + if c.stopped || c.pending != nil { + return nil + } + first, e := c.store.FirstIndex() + if e != nil || first > c.applied { + return nil + } + entries, e := c.store.Entries(first, c.applied+1, ^uint64(0)) + if e != nil { + return nil + } + var size uint64 + for _, entry := range entries { + size += uint64(proto.Size(entry)) + } + if len(entries) < SnapshotEntries && size < SnapshotLogBytes { + return nil + } + term, e := c.store.Term(c.applied) + if e != nil { + return nil + } + return &SnapshotPoint{Index: c.applied, Term: term} +} diff --git a/internal/raftcore/snapshot_test.go b/internal/raftcore/snapshot_test.go new file mode 100644 index 0000000000..2ac639eb44 --- /dev/null +++ b/internal/raftcore/snapshot_test.go @@ -0,0 +1,79 @@ +package raftcore + +import ( + "bytes" + "errors" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// R11/R13 require deterministic maintenance at the entry threshold and cache +// compaction only after a host checkpoint of the exact applied configuration. +func TestR11R12R13SnapshotHintAndDurableCompactInput(t *testing.T) { + conf := &pb.ConfState{Voters: []uint64{1, 2, 4}, VotersOutgoing: []uint64{1, 2, 3}, Learners: []uint64{5}, LearnersNext: []uint64{3}, AutoLeave: proto.Bool(true)} + entries := make([]*pb.Entry, SnapshotEntries) + for i := range entries { + entries[i] = &pb.Entry{Index: proto.Uint64(uint64(i + 1)), Term: proto.Uint64(1), Type: pb.EntryNormal.Enum()} + } + c, e := New(Config{ClusterUUID: "snapshot", NodeID: 1, Seed: 1, Protocol: 1}, RecoveredState{HardState: &pb.HardState{Term: proto.Uint64(1), Commit: proto.Uint64(SnapshotEntries)}, ConfState: conf, Entries: entries, Applied: SnapshotEntries}) + if e != nil { + t.Fatal(e) + } + effects, e := c.Step(Poll{}) + if e != nil { + t.Fatal(e) + } + if effects.SnapshotNeeded == nil || effects.SnapshotNeeded.Index != SnapshotEntries || effects.SnapshotNeeded.Term != 1 { + t.Fatal("R11 entry threshold did not request an applied checkpoint") + } + encodedConf, e := raftsnapshot.Configuration(conf) + if e != nil { + t.Fatal(e) + } + m := raftsnapshot.Manifest{Version: 1, Cluster: "snapshot", Protocol: 1, Index: SnapshotEntries, Term: 1, Conf: encodedConf, RecordsHash: strings.Repeat("0", 64), Digest: raftsnapshot.ID(nil)} + data, e := m.Encode() + if e != nil { + t.Fatal(e) + } + p := &pb.Snapshot{Data: data, Metadata: &pb.SnapshotMetadata{Index: proto.Uint64(SnapshotEntries), Term: proto.Uint64(1), ConfState: conf}} + bad := clone(p) + bad.Metadata.ConfState.LearnersNext = nil + if _, e = c.Step(CompactSnapshot{Snapshot: bad}); !errors.Is(e, ErrSnapshot) { + t.Fatal("R12 discarded learner-next during checkpoint") + } + bad = clone(p) + bad.Data = append(bad.Data, ' ') + if _, e = c.Step(CompactSnapshot{Snapshot: bad}); !errors.Is(e, ErrSnapshot) { + t.Fatal("R13 accepted noncanonical snapshot") + } + effects, e = c.Step(CompactSnapshot{Snapshot: p}) + if e != nil { + t.Fatal(e) + } + first, e := c.store.FirstIndex() + if e != nil { + t.Fatal(e) + } + retained, e := c.store.Snapshot() + if e != nil { + t.Fatal(e) + } + if first != SnapshotEntries+1 || !proto.Equal(retained, p) || effects.SnapshotNeeded != nil { + t.Fatal("R11 cache kept the covered applied prefix") + } + // Effects and completion input cannot mutate the checkpoint retained for a + // lagging learner; those bytes must remain identical across retries. + original := bytes.Clone(data) + p.Data[0] ^= 1 + retained, e = c.store.Snapshot() + if e != nil { + t.Fatal(e) + } + if !bytes.Equal(retained.Data, original) { + t.Fatal("R13 snapshot retained caller aliases") + } +} diff --git a/internal/raftcore/types.go b/internal/raftcore/types.go index 00733bf695..981bbb4be2 100644 --- a/internal/raftcore/types.go +++ b/internal/raftcore/types.go @@ -18,6 +18,8 @@ const ( MaxInflightBatches = 8 MaxReadRequests = 128 MaxRequestIDBytes = 128 + SnapshotEntries = 1024 + SnapshotLogBytes = 4 << 20 ) var ( @@ -30,6 +32,7 @@ var ( ErrLeader = errors.New("raftcore: operation requires the elected leader") ErrCommand = errors.New("raftcore: invalid command or committed schema") ErrRead = errors.New("raftcore: invalid, duplicate or excess read request") + ErrSnapshot = errors.New("raftcore: invalid durable snapshot checkpoint") ErrTransfer = errors.New("raftcore: transfer target is not a caught-up voter") ) @@ -59,6 +62,19 @@ type RecoveredState struct { // Input is closed so an untrusted peer cannot masquerade as a local completion. type Input interface{ coreInput() } + +// CompactSnapshot is supplied ONLY after the host has durably created and +// compacted this exact applied checkpoint. It carries the portable manifest, +// never application chunks. No outstanding Ready may overlap the operation. +type CompactSnapshot struct{ Snapshot *pb.Snapshot } + +func (CompactSnapshot) coreInput() {} + +type SnapshotPoint struct { + Index uint64 + Term uint64 +} + type Poll struct{} type Tick struct{} type PeerMessage struct{ Bytes []byte } @@ -150,10 +166,11 @@ type Status struct { } type Effects struct { - Persist *PersistBatch - Send []Outbound - Apply *ApplyBatch - Reads []ReadBarrier - Status Status - NeedNextTick bool + Persist *PersistBatch + Send []Outbound + Apply *ApplyBatch + Reads []ReadBarrier + Status Status + NeedNextTick bool + SnapshotNeeded *SnapshotPoint `json:",omitempty"` } diff --git a/internal/raftsnapshot/README.md b/internal/raftsnapshot/README.md new file mode 100644 index 0000000000..f3155edcdd --- /dev/null +++ b/internal/raftsnapshot/README.md @@ -0,0 +1,69 @@ +# Portable coordination snapshot, format 1 + +This pure package is shared by native Go and the WASI core. SQLite and DO hosts +own chunk staging, durable generation publication and installation. The format +contains coordination only, with no plaintext chat content or private keys. It +copies every opaque application row without interpreting its policy. P09 owns +complete authorization/evaluation; P08 owns the hosted storage implementation. + +`raftpb.Snapshot.Data` is a canonical manifest, at most 16 KiB. Its metadata index, +term and complete ConfState must match that manifest. It never embeds chunks or +a whole identity. Its generation ID is lowercase SHA-256 hex of those exact +manifest bytes. The manifest is compact UTF-8 JSON, fields in the following order, +with no extra fields, whitespace, duplicate keys or alternate encodings: + +| Field | Encoding and meaning | +| --- | --- | +| format | decimal string `"1"` | +| cluster | cluster UUID string | +| protocol | decimal string of the core protocol | +| index, term | nonzero uint64 decimal strings | +| conf | standard padded base64 of deterministic raftpb.ConfState protobuf | +| version | uint64 decimal string of the visible application version | +| now | int64 decimal string of committed logical milliseconds | +| records | uint64 decimal string, complete application record count | +| records_hash | lowercase SHA-256-sized hex, XOR of keyed application row hashes | +| bytes, chunks | uint64 decimal strings, total stream length and ceiling(bytes / 262144) | +| digest | lowercase SHA-256 hex of the concatenated stream bytes | + +Unsigned strings have no leading zero except `"0"`. Signed time uses ordinary +base-ten with no plus or negative zero. JSON string escapes follow Go's compact +encoding/json spelling (including HTML escapes), and the ConfState wire bytes +use deterministic protobuf. Decode re-encodes and compares exact bytes. This +makes noncanonical or unknown schemas fail closed. Every ConfState field, +including outgoing voters, learners-next and AutoLeave presence, is preserved. +The Worker build manifest pins format version and the format source hash beside +ABI/core/source/wasm hashes. The native format tests supply independent framing +and greater-than-2^53 integer vectors. + +The stream contains rows sorted ascending by namespace bytes, then key bytes. +The six namespaces are state, sessions, membership, enrollment, tombstones and +outbox. Every record is three unsigned big-endian uint32 lengths (namespace, +key, value), followed by those exact byte strings. Namespace is ASCII. Key is +nonempty and at most 1 MiB; value is opaque, may be empty, and is at most 1 MiB. +No record may be filtered. Framing may cross any chunk boundary, including within +a length field or a near-limit row. A reader retains at most one row and one +chunk. Duplicate/out-of-order records, bad lengths, incomplete headers/values, +wrong count, wrong keyed-row aggregate and extra trailing bytes are refused. + +Chunks are numbered from zero. Every chunk except the last is exactly 256 KiB; +the last has bytes - number * 262144 bytes. Empty application state has zero +chunks and the SHA-256 digest of an empty stream. Each chunk has its own SHA-256 +for transport; native storage additionally binds its checksum to generation and +chunk number. The final manifest digest detects reordered or substituted chunks. +The native keyed checksum is SHA-256 of a sequence of byte parts, each prefixed +by its eight-byte unsigned big-endian length. An application row's parts are +`records`, namespace, key, value. The count plus XOR of those hashes is the +native application's missing-row detector; the ordered stream digest separately +binds the exact complete portable representation. + +Hosts inventory and admit a transfer against their identity's application byte +ceiling before staging; the format does not silently impose an 8 MiB ceiling on +grandfathered legacy data. Only one incomplete generation may stage. Verify a +new complete generation before retiring redundant consensus bytes. Keep two +complete generations, including the installed one while another awaits install. +A completed upload is not a successful Raft snapshot delivery. Atomically install +application rows, application metadata, complete configuration, included index / +term and durable Raft pointer before releasing the dependent acknowledgment. +An interrupted installation keeps its prior complete generation. Never erase +corrupt authoritative evidence to manufacture a fresh member. diff --git a/internal/raftsnapshot/format.go b/internal/raftsnapshot/format.go new file mode 100644 index 0000000000..45043eab4d --- /dev/null +++ b/internal/raftsnapshot/format.go @@ -0,0 +1,152 @@ +// Package raftsnapshot defines the portable, streamed coordination snapshot. +// Hosts persist chunks; consensus retains only the small manifest in Snapshot.Data. +package raftsnapshot + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "io" + "unicode/utf8" + + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +const ( + Version = 1 + ChunkBytes = 256 << 10 + RecordBytes = 1 << 20 + ManifestBytes = 16 << 10 + RetainedGenerations = 2 +) + +var ErrFormat = errors.New("raftsnapshot: invalid snapshot format or checksum") + +// Manifest uses decimal strings for every integer crossing JavaScript. Conf is +// deterministic protobuf, preserving joint voters, learners-next and AutoLeave. +// Digest covers the concatenated fixed-size chunks, independent of transport. +type Manifest struct { + Version uint64 `json:"format,string"` + Cluster string `json:"cluster"` + Protocol uint64 `json:"protocol,string"` + Index uint64 `json:"index,string"` + Term uint64 `json:"term,string"` + Conf []byte `json:"conf"` + VisibleVersion uint64 `json:"version,string"` + LogicalTime int64 `json:"now,string"` + RecordCount uint64 `json:"records,string"` + RecordsHash string `json:"records_hash"` + Bytes uint64 `json:"bytes,string"` + Chunks uint64 `json:"chunks,string"` + Digest string `json:"digest"` +} + +var wire = proto.MarshalOptions{Deterministic: true} + +func (m Manifest) Configuration() (*pb.ConfState, error) { + c := &pb.ConfState{} + if proto.Unmarshal(m.Conf, c) != nil || len(c.ProtoReflect().GetUnknown()) != 0 { + return nil, ErrFormat + } + b, err := wire.Marshal(c) + if err != nil || !bytes.Equal(b, m.Conf) { + return nil, ErrFormat + } + return c, nil +} + +func Configuration(c *pb.ConfState) ([]byte, error) { return wire.Marshal(c) } + +func validHash(s string) bool { + b, e := hex.DecodeString(s) + return e == nil && len(b) == sha256.Size && hex.EncodeToString(b) == s +} + +func (m Manifest) Encode() ([]byte, error) { + if m.Version != Version || m.Cluster == "" || !utf8.ValidString(m.Cluster) || m.Protocol == 0 || m.Index == 0 || m.Term == 0 || !validHash(m.Digest) || !validHash(m.RecordsHash) || m.Chunks != m.Bytes/ChunkBytes+boolUint(m.Bytes%ChunkBytes != 0) { + return nil, ErrFormat + } + if _, e := m.Configuration(); e != nil { + return nil, e + } + b, e := json.Marshal(m) + if e != nil || len(b) > ManifestBytes { + return nil, ErrFormat + } + return b, nil +} +func boolUint(b bool) uint64 { + if b { + return 1 + } + return 0 +} + +func Decode(b []byte) (Manifest, error) { + var m Manifest + if len(b) > ManifestBytes || json.Unmarshal(b, &m) != nil { + return m, ErrFormat + } + canonical, e := m.Encode() + if e != nil || !bytes.Equal(canonical, b) { + return Manifest{}, ErrFormat + } + return m, nil +} + +func ID(data []byte) string { sum := sha256.Sum256(data); return hex.EncodeToString(sum[:]) } + +// Row framing has three big-endian uint32 lengths followed by namespace, key +// and opaque value. A record may cross chunk boundaries; it is never truncated. +type Row struct { + Namespace string + Key string + Value []byte +} + +func validRow(r Row) bool { + switch r.Namespace { + case "state", "sessions", "membership", "enrollment", "tombstones", "outbox": + default: + return false + } + return len(r.Key) > 0 && len(r.Key) <= RecordBytes && len(r.Value) <= RecordBytes +} +func WriteRow(w io.Writer, r Row) error { + if !validRow(r) { + return ErrFormat + } + var header [12]byte + binary.BigEndian.PutUint32(header[:4], uint32(len(r.Namespace))) + binary.BigEndian.PutUint32(header[4:8], uint32(len(r.Key))) + binary.BigEndian.PutUint32(header[8:], uint32(len(r.Value))) + for _, b := range [][]byte{header[:], []byte(r.Namespace), []byte(r.Key), r.Value} { + if _, e := w.Write(b); e != nil { + return e + } + } + return nil +} +func ReadRow(r io.Reader) (Row, error) { + var header [12]byte + if _, e := io.ReadFull(r, header[:]); e != nil { + return Row{}, e + } + n, k, v := binary.BigEndian.Uint32(header[:4]), binary.BigEndian.Uint32(header[4:8]), binary.BigEndian.Uint32(header[8:]) + if n > 16 || k == 0 || k > RecordBytes || v > RecordBytes { + return Row{}, ErrFormat + } + b := make([]byte, int(n)+int(k)+int(v)) + if _, e := io.ReadFull(r, b); e != nil { + return Row{}, e + } + row := Row{string(b[:n]), string(b[n : n+k]), b[n+k:]} + if !validRow(row) { + return Row{}, ErrFormat + } + return row, nil +} diff --git a/internal/raftsnapshot/format_test.go b/internal/raftsnapshot/format_test.go new file mode 100644 index 0000000000..b3802c0216 --- /dev/null +++ b/internal/raftsnapshot/format_test.go @@ -0,0 +1,69 @@ +package raftsnapshot + +import ( + "bytes" + "encoding/hex" + "io" + "strings" + "testing" + + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// R13/R14 require a runtime-independent framing contract, including integers +// beyond JavaScript's exact-number range and opaque bytes across chunk cuts. +func TestR13R14PortableManifestAndRowFraming(t *testing.T) { + conf := &pb.ConfState{Voters: []uint64{1, 2, 9007199254740993}, VotersOutgoing: []uint64{1, 2, 3}, Learners: []uint64{5}, LearnersNext: []uint64{3}, AutoLeave: proto.Bool(true)} + c, e := Configuration(conf) + if e != nil { + t.Fatal(e) + } + m := Manifest{Version: 1, Cluster: "portable", Protocol: 1, Index: ^uint64(0), Term: 9007199254740993, Conf: c, VisibleVersion: 9007199254740994, LogicalTime: -1, RecordsHash: strings.Repeat("0", 64), Digest: ID(nil)} + b, e := m.Encode() + if e != nil { + t.Fatal(e) + } + if !bytes.Contains(b, []byte(`"index":"18446744073709551615"`)) || !bytes.Contains(b, []byte(`"term":"9007199254740993"`)) { + t.Fatal("R14 uint64 lost precision") + } + decoded, e := Decode(b) + if e != nil { + t.Fatal(e) + } + got, e := decoded.Configuration() + if e != nil || !proto.Equal(got, conf) { + t.Fatal("R12 joint configuration changed") + } + for _, bad := range [][]byte{append(append([]byte{}, b...), 10), bytes.Replace(b, []byte(`"format":"1"`), []byte(`"format":1`), 1), bytes.Replace(b, []byte(`"index":"18446744073709551615"`), []byte(`"index":"018446744073709551615"`), 1), bytes.Replace(b, []byte(`"cluster":"portable"`), []byte(`"cluster":"portable","unknown":1`), 1)} { + if _, e := Decode(bad); e == nil { + t.Fatal("R14 noncanonical manifest accepted") + } + } + var stream bytes.Buffer + row := Row{"state", "k", []byte{0, 255}} + if e = WriteRow(&stream, row); e != nil { + t.Fatal(e) + } + // This vector is specified independently from WriteRow's implementation. + if hex.EncodeToString(stream.Bytes()) != "00000005000000010000000273746174656b00ff" { + t.Fatal("R14 framing differs from the portable vector") + } + round, e := ReadRow(bytes.NewReader(stream.Bytes())) + if e != nil || round.Namespace != row.Namespace || round.Key != row.Key || !bytes.Equal(round.Value, row.Value) { + t.Fatal("R13 row changed") + } + for n := 1; n < len(stream.Bytes()); n++ { + if _, e = ReadRow(bytes.NewReader(stream.Bytes()[:n])); e == nil { + t.Fatal("R13 truncated record accepted") + } + } + if _, e = ReadRow(bytes.NewReader(nil)); e != io.EOF { + t.Fatal("R13 empty stream is not EOF") + } + for _, bad := range []Row{{"other", "k", nil}, {"state", "", nil}, {"state", "k", make([]byte, RecordBytes+1)}} { + if WriteRow(io.Discard, bad) == nil { + t.Fatal("R13 unsupported row accepted") + } + } +} diff --git a/internal/raftstore/README.md b/internal/raftstore/README.md index ee642d02c5..68786b729a 100644 --- a/internal/raftstore/README.md +++ b/internal/raftstore/README.md @@ -1,6 +1,6 @@ # Native member durability -This is P04's internal host adapter, with no production directory routing or new +This is the internal native host adapter, with no production directory routing or new person-facing door. Each member has its own SQLite file, permanently bound to `Identity{ClusterUUID, NodeID, Protocol}` and `FormatVersion`. The same identity's three anchors must use three distinct files. The caller creates the parent @@ -71,15 +71,56 @@ of an entire consistent database to an old backup. uses a newly enrolled replacement ID, not deletion of the file. There is no path here that deletes chats, frames, branches, saved heads or legacy original stores. -## P05 boundary - -Snapshot input explicitly returns `ErrSnapshot`; no snapshot acknowledgment may -escape. P05 must add streamed manifest/chunk generations, checksums, two retained -complete generations, atomic installation and compaction. Its recovered snapshot -must already be installed in the application before constructing the core. The -current schema retains the whole log and recovery materializes that suffix; this -is not the completed bounded log/cache policy. P05 must update the format and -recovery checks together, preserving all application namespaces and exact results. +## Snapshot lifecycle and compaction + +Storage format 2 adds a checksummed `meta.snapshot` pointer, `snapshots` manifests +and `snapshot_chunks`, with full-width big-endian chunk numbers. Format-1 stores +from the earlier development packet are refused, retained and never upgraded in +place; this is an internal pre-cutover format, not a legacy directory migration. +The cross-runtime stream is format 1 in `internal/raftsnapshot/README.md`. + +For a local checkpoint, serialize the actor, call `CreateSnapshot`, then +`Compact(snapshot index)`, then `core.Step(CompactSnapshot{Snapshot})`. Creation +streams sorted rows from one fixed applied SQLite transaction, with at most one +record and one 256 KiB chunk in memory. It retains all namespaces and application +metadata. Compact deletes only the applied log prefix covered by the complete +durable pointer. `SnapshotDue` and core `SnapshotNeeded` share the 1,024-entry / +4 MiB thresholds. Hosts must service maintenance before admitting more growth; +learners do not pin it. Recovery loads only the suffix after the active snapshot. + +For incoming transfer, call `BeginSnapshot(manifest bytes)`, then +`PutSnapshotChunk(id, chunk number, data, SHA-256 hex)`. A duplicate chunk is +idempotent; conflicting bytes, wrong digest, offset or length are refused. One +incomplete generation may stage; starting a new one abandons only incomplete +consensus bytes. A completed transfer awaiting installation keeps that slot. +`CompleteSnapshot(id)` validates all chunks, ordering, stream digest, count and +application hash and returns the portable protobuf pointer. This publishes a +complete transfer, not application authority. The active pointer remains old. +The host must enforce the imported/application byte ceiling and transport quota +before staging; this adapter does not choose P09/P11's admission policy. + +Only `Persist` of the real incoming Ready atomically replaces application rows, +header/ConfState, active snapshot pointer, log prefix and hard state. Successful +return means installation is durable, so the core may now receive `Persisted`. +Re-read application metadata after that return before evaluating a suffix. A +snapshot-only Apply is an idempotent confirmation with unchanged metadata and no +writes. It can confirm the install even if the host's previous expected index +was older; suffix writes still require the installed index through normal CAS. + +Two complete generations are retained. Completion of a new transfer can retire +the oldest redundant generation while retaining the active one; publication of a +local checkpoint or installation retains its predecessor. Incomplete generations +never replace an installed pointer. On restart, validate both complete snapshots +and restore the active pointer; an uninstalled complete transfer cannot supersede +the Raft decision. Corruption of authoritative complete bytes refuses that node +and retains evidence. An incomplete transfer is ignored for authority and may be +retried. Recovery never silently falls back from corrupt committed state. + +Creation, installation, compaction and generation retirement never delete an +application record, saved chat, frame, branch, vault or legacy original. Snapshot +installation replaces the local application with the complete authoritative +replicated view, not a filtered subset. Explicit application deletes retain their +existing policy boundary. No frame garbage collection is introduced here. ## Focused proof @@ -101,6 +142,16 @@ The bounded three-native-node fixture runs real cores/stores and real create/ publish lease rules, then loses a leader and recovers another member from durable bytes before a quorum read. It also recovers and leaves real joint membership. Its one-cell evaluator is test-only, not P09's authorized directory writer or the -independent P06 history oracle. Full fault schedules, snapshots, transport, +independent P06 history oracle. Full simulator fault schedules, hosted snapshot interoperability, transport, membership policy, cross-runtime storage and real-client journeys remain later packets; these mechanism tests do not claim complete R1–R30 release proof. + +The P05 R13 tests cut real subprocesses at staging/chunk/completion, each +source/install row and created chunk, application/pointer/log publication, +compaction and both sides of COMMIT. They retain database/WAL only. Missing and +corrupt chunk tests cover every chunk in a near-limit record, plus canonical +manifest refusals. The R11 native fixture demotes a member to learner, disconnects +it, commits more than the 4 MiB trigger through the majority, compacts both stores +and caches, and then installs/replays/restarts the learner. The pure format tests +pin full-width integer and binary framing vectors independently of the encoder. +These are the P05 mechanism gates; P06/P08 own simulator/cross-runtime campaigns. diff --git a/internal/raftstore/compact.go b/internal/raftstore/compact.go new file mode 100644 index 0000000000..a3a8075c88 --- /dev/null +++ b/internal/raftstore/compact.go @@ -0,0 +1,252 @@ +package raftstore + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "hash" + + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + pb "go.etcd.io/raft/v3/raftpb" +) + +type chunkWriter struct { + ctx context.Context + tx *sql.Tx + buf []byte + offset, bytes uint64 + digest hash.Hash +} + +func (w *chunkWriter) flush() error { + if len(w.buf) == 0 { + return nil + } + if e := putChunk(w.ctx, w.tx, "creating", w.offset, w.buf); e != nil { + return e + } + w.offset++ + w.buf = w.buf[:0] + return nil +} +func (w *chunkWriter) Write(p []byte) (int, error) { + length := len(p) + for len(p) > 0 { + n := min(len(p), raftsnapshot.ChunkBytes-len(w.buf)) + w.buf = append(w.buf, p[:n]...) + w.digest.Write(p[:n]) + w.bytes += uint64(n) + p = p[n:] + if len(w.buf) == raftsnapshot.ChunkBytes { + if e := w.flush(); e != nil { + return 0, e + } + } + } + return length, nil +} + +// CreateSnapshot streams a fixed applied view inside one transaction. It +// publishes a complete local checkpoint but leaves physical log deletion to +// Compact. No application data is removed, even under storage pressure. +func (s *SQLite) CreateSnapshot(ctx context.Context) (*pb.Snapshot, error) { + s.mu.Lock() + defer s.mu.Unlock() + var p *pb.Snapshot + e := s.transaction(ctx, func(tx *sql.Tx) error { + a, e := getApplication(ctx, tx) + if e != nil { + return e + } + if a.Applied == 0 { + return ErrSnapshot + } + old, e := getSnapshot(ctx, tx) + if e != nil { + return e + } + if old.GetMetadata().GetIndex() == a.Applied { + p = old + return nil + } + entry, e := getEntry(ctx, tx, a.Applied) + if e != nil { + return e + } + conf, e := raftsnapshot.Configuration(a.ConfState) + if e != nil { + return e + } + m := raftsnapshot.Manifest{Version: raftsnapshot.Version, Cluster: s.identity.ClusterUUID, Protocol: uint64(s.identity.Protocol), Index: a.Applied, Term: entry.GetTerm(), Conf: conf, VisibleVersion: a.VisibleVersion, LogicalTime: a.LogicalTime, RecordCount: a.RecordCount, RecordsHash: hex.EncodeToString(a.RecordsHash[:])} + w := &chunkWriter{ctx: ctx, tx: tx, buf: make([]byte, 0, raftsnapshot.ChunkBytes), digest: sha256.New()} + rows, e := tx.QueryContext(ctx, `SELECT namespace,key,data,checksum FROM records ORDER BY namespace,key`) + if e != nil { + return e + } + var count uint64 + var rowsHash [32]byte + for rows.Next() { + var row raftsnapshot.Row + var sum []byte + if e = rows.Scan(&row.Namespace, &row.Key, &row.Value, &sum); e != nil { + rows.Close() + return e + } + if !bytes.Equal(sum, checksum([]byte("records"), []byte(row.Namespace), []byte(row.Key), row.Value)) { + rows.Close() + return ErrCorrupt + } + count++ + xorHash(&rowsHash, sum) + if e = raftsnapshot.WriteRow(w, row); e != nil { + rows.Close() + if errors.Is(e, raftsnapshot.ErrFormat) { + return ErrBatch + } + return e + } + s.boundary("after-snapshot-source-row") + } + e = rows.Err() + rows.Close() + if e != nil { + return e + } + if count != a.RecordCount || rowsHash != a.RecordsHash { + return ErrCorrupt + } + if e = w.flush(); e != nil { + return e + } + m.Bytes, m.Chunks, m.Digest = w.bytes, w.offset, hex.EncodeToString(w.digest.Sum(nil)) + data, e := m.Encode() + if e != nil { + return ErrSnapshot + } + id := raftsnapshot.ID(data) + existing, existingData, complete, existingErr := snapshotManifest(ctx, tx, id) + if existingErr == nil { + // A completed transfer may coincide with local log catch-up after + // restart. Reuse its identity without treating a duplicate as I/O + // failure, and never overwrite damaged complete evidence. + if !bytes.Equal(existingData, data) { + return ErrCorrupt + } + if complete { + if e = walkSnapshot(ctx, tx, id, existing, nil); e != nil { + return ErrCorrupt + } + } + if _, e = tx.ExecContext(ctx, `DELETE FROM snapshot_chunks WHERE generation=?`, id); e != nil { + return e + } + if _, e = tx.ExecContext(ctx, `UPDATE snapshots SET complete=1 WHERE id=?`, id); e != nil { + return e + } + } else if errors.Is(existingErr, ErrSnapshot) { + if _, e = tx.ExecContext(ctx, `INSERT INTO snapshots VALUES (?,?,?,1)`, id, data, checksum([]byte("snapshots"), []byte(id), data)); e != nil { + return e + } + } else { + return existingErr + } + if _, e = tx.ExecContext(ctx, `UPDATE snapshot_chunks SET generation=? WHERE generation='creating'`, id); e != nil { + return e + } + // The temporary key never escapes the transaction. Rebind each checksum + // without loading more than one chunk after its final content ID is known. + for offset := uint64(0); offset < m.Chunks; offset++ { + var chunk []byte + if e = tx.QueryRowContext(ctx, `SELECT data FROM snapshot_chunks WHERE generation=? AND offset=?`, id, indexKey(offset)).Scan(&chunk); e != nil { + return e + } + if e = putChunk(ctx, tx, id, offset, chunk); e != nil { + return e + } + s.boundary("after-snapshot-created-chunk") + } + p, e = snapshotPointer(m, data) + if e != nil { + return e + } + if e = putProto(ctx, tx, "snapshot", p); e != nil { + return e + } + s.boundary("after-snapshot-pointer") + return retainSnapshots(ctx, tx, id) + }) + return p, e +} + +// Compact removes only the applied consensus prefix covered by the current +// complete durable checkpoint. A dead learner must install that checkpoint; +// its progress never participates in this decision. +func (s *SQLite) Compact(ctx context.Context, index uint64) error { + s.mu.Lock() + defer s.mu.Unlock() + return s.transaction(ctx, func(tx *sql.Tx) error { + p, e := getSnapshot(ctx, tx) + if e != nil { + return e + } + if index == 0 || index != p.GetMetadata().GetIndex() { + return ErrSnapshot + } + a, e := getApplication(ctx, tx) + if e != nil { + return e + } + if index > a.Applied { + return ErrSnapshot + } + m, e := decodeSnapshot(p, s.identity) + if e != nil { + return e + } + id := raftsnapshot.ID(p.Data) + _, _, complete, e := snapshotManifest(ctx, tx, id) + if e != nil { + return e + } + if !complete { + return ErrSnapshot + } + if e = walkSnapshot(ctx, tx, id, m, nil); e != nil { + return e + } + if _, e = tx.ExecContext(ctx, `DELETE FROM raft_log WHERE idx<=?`, indexKey(index)); e != nil { + return e + } + s.boundary("after-compaction") + return nil + }) +} + +// SnapshotDue measures only applied history after the last complete snapshot. +// The host services this before admitting more growth; learners cannot pin it. +func (s *SQLite) SnapshotDue(ctx context.Context) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + var due bool + e := s.transaction(ctx, func(tx *sql.Tx) error { + a, e := getApplication(ctx, tx) + if e != nil { + return e + } + p, e := getSnapshot(ctx, tx) + if e != nil { + return e + } + var count, size uint64 + if e = tx.QueryRowContext(ctx, `SELECT count(*),coalesce(sum(length(data)),0) FROM raft_log WHERE idx>? AND idx<=?`, indexKey(p.GetMetadata().GetIndex()), indexKey(a.Applied)).Scan(&count, &size); e != nil { + return e + } + due = count >= raftcore.SnapshotEntries || size >= raftcore.SnapshotLogBytes + return nil + }) + return due, e +} diff --git a/internal/raftstore/native_test.go b/internal/raftstore/native_test.go index 135aace3dd..7781e762f1 100644 --- a/internal/raftstore/native_test.go +++ b/internal/raftstore/native_test.go @@ -9,6 +9,7 @@ import ( "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" "go.etcd.io/raft/v3" pb "go.etcd.io/raft/v3/raftpb" "google.golang.org/protobuf/proto" @@ -18,17 +19,19 @@ import ( // SQLite databases and production cores. Its one-cell rule evaluator is not // P09's authorized directory writer or P06's independent history oracle. type nativeGroup struct { - t *testing.T - nodes [3]*raftcore.Core - stores [3]*SQLite - paths [3]string - apps [3]Application - alive [3]bool - status [3]raftcore.Status - queue []raftcore.Outbound - reads []raftcore.ReadBarrier - events int - cutApply func(int, string, *raftcore.ApplyBatch) bool + t *testing.T + nodes [3]*raftcore.Core + stores [3]*SQLite + paths [3]string + apps [3]Application + alive [3]bool + status [3]raftcore.Status + queue []raftcore.Outbound + reads []raftcore.ReadBarrier + events int + cutApply func(int, string, *raftcore.ApplyBatch) bool + maintain bool + snapshotsSent int } func newNativeGroup(t *testing.T) *nativeGroup { @@ -87,6 +90,9 @@ func (g *nativeGroup) call(i int, input raftcore.Input) { g.t.Fatal("dependent effect escaped before Ready durability") } must(g.t, g.stores[i].Persist(testContext, *e.Persist)) + if !raft.IsEmptySnap(e.Persist.Snapshot) { + g.apps[i] = recovered(g.t, g.stores[i]).Application + } e, err = g.nodes[i].Step(raftcore.Persisted{BatchID: e.Persist.ID}) } else if e.Apply != nil { if g.cutApply != nil && g.cutApply(i, "before-apply", e.Apply) { @@ -101,6 +107,11 @@ func (g *nativeGroup) call(i int, input raftcore.Input) { } g.apps[i] = recovered(g.t, g.stores[i]).Application e, err = g.nodes[i].Step(raftcore.Applied{BatchID: e.Apply.ID}) + } else if g.maintain && e.SnapshotNeeded != nil { + p, createErr := g.stores[i].CreateSnapshot(testContext) + must(g.t, createErr) + must(g.t, g.stores[i].Compact(testContext, p.GetMetadata().GetIndex())) + e, err = g.nodes[i].Step(raftcore.CompactSnapshot{Snapshot: p}) } else { return } @@ -169,7 +180,26 @@ func (g *nativeGroup) pump() { for len(g.queue) != 0 { out := g.queue[0] g.queue = g.queue[1:] - g.call(int(out.To)-1, raftcore.PeerMessage{Bytes: out.Bytes}) + target := int(out.To) - 1 + message := &pb.Message{} + must(g.t, proto.Unmarshal(out.Bytes, message)) + if g.alive[target] && message.GetType() == pb.MsgSnap { + source := int(message.GetFrom()) - 1 + pointer := message.GetSnapshot() + id, err := g.stores[target].BeginSnapshot(testContext, pointer.Data) + must(g.t, err) + m, err := raftsnapshot.Decode(pointer.Data) + must(g.t, err) + for offset := uint64(0); offset < m.Chunks; offset++ { + data, err := g.stores[source].ReadSnapshotChunk(testContext, id, offset) + must(g.t, err) + must(g.t, g.stores[target].PutSnapshotChunk(testContext, id, offset, data, raftsnapshot.ID(data))) + } + _, err = g.stores[target].CompleteSnapshot(testContext, id) + must(g.t, err) + g.snapshotsSent++ + } + g.call(target, raftcore.PeerMessage{Bytes: out.Bytes}) } } diff --git a/internal/raftstore/recover.go b/internal/raftstore/recover.go index f1ebedac2c..33fbf58e06 100644 --- a/internal/raftstore/recover.go +++ b/internal/raftstore/recover.go @@ -9,6 +9,7 @@ import ( "errors" "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" "go.etcd.io/raft/v3" pb "go.etcd.io/raft/v3/raftpb" "google.golang.org/protobuf/proto" @@ -35,6 +36,13 @@ func decodeEntry(key, data, hash []byte) (*pb.Entry, error) { } func getEntry(ctx context.Context, tx *sql.Tx, index uint64) (*pb.Entry, error) { + p, err := getSnapshot(ctx, tx) + if err != nil { + return nil, err + } + if index > 0 && p.GetMetadata().GetIndex() == index { + return &pb.Entry{Index: proto.Uint64(index), Term: proto.Uint64(p.GetMetadata().GetTerm())}, nil + } var data, hash []byte key := indexKey(index) if err := tx.QueryRowContext(ctx, `SELECT data,checksum FROM raft_log WHERE idx=?`, key).Scan(&data, &hash); err != nil { @@ -47,7 +55,11 @@ func lastEntry(ctx context.Context, tx *sql.Tx) (*pb.Entry, error) { var key, data, hash []byte err := tx.QueryRowContext(ctx, `SELECT idx,data,checksum FROM raft_log ORDER BY idx DESC LIMIT 1`).Scan(&key, &data, &hash) if errors.Is(err, sql.ErrNoRows) { - return &pb.Entry{}, nil + p, e := getSnapshot(ctx, tx) + if e != nil { + return nil, e + } + return &pb.Entry{Index: proto.Uint64(p.GetMetadata().GetIndex()), Term: proto.Uint64(p.GetMetadata().GetTerm())}, nil } if err != nil { return nil, err @@ -58,7 +70,7 @@ func lastEntry(ctx context.Context, tx *sql.Tx) (*pb.Entry, error) { // Recover checks physical integrity, each record's checksum, member identity // and cross-record invariants in one consistent transaction. Corrupt evidence // stays in place; a missing log entry is never synthesized from an index. -// P05 will replace the un-compacted suffix read with snapshot-backed recovery. +// Only the suffix after the complete active checkpoint enters the core cache. func (s *SQLite) Recover(ctx context.Context) (Recovery, error) { s.mu.Lock() defer s.mu.Unlock() @@ -80,7 +92,7 @@ func (s *SQLite) Recover(ctx context.Context) (Recovery, error) { return ErrIdentity } var count int - if err := tx.QueryRowContext(ctx, `SELECT count(*) FROM meta`).Scan(&count); err != nil || count != 5 { + if err := tx.QueryRowContext(ctx, `SELECT count(*) FROM meta`).Scan(&count); err != nil || count != 6 { return ErrCorrupt } h := &pb.HardState{} @@ -91,13 +103,65 @@ func (s *SQLite) Recover(ctx context.Context) (Recovery, error) { if err != nil { return err } - rows, err := tx.QueryContext(ctx, `SELECT idx,data,checksum FROM raft_log ORDER BY idx`) + snapshot, err := getSnapshot(ctx, tx) + if err != nil { + return err + } + snapIndex := snapshot.GetMetadata().GetIndex() + if snapIndex > 0 { + if _, err = decodeSnapshot(snapshot, s.identity); err != nil { + return ErrCorrupt + } + id := raftsnapshot.ID(snapshot.Data) + _, data, complete, err := snapshotManifest(ctx, tx, id) + if err != nil || !complete || !bytes.Equal(data, snapshot.Data) { + return ErrCorrupt + } + } else if !proto.Equal(snapshot, &pb.Snapshot{}) { + return ErrCorrupt + } + // Incomplete staged transfers are ignored. Every retained complete + // generation is checked so damaged authoritative evidence is refused. + generations, err := tx.QueryContext(ctx, `SELECT id FROM snapshots WHERE complete=1`) + if err != nil { + return ErrCorrupt + } + var ids []string + for generations.Next() { + var id string + if err = generations.Scan(&id); err != nil { + generations.Close() + return err + } + ids = append(ids, id) + } + err = generations.Err() + generations.Close() + if err != nil { + return err + } + if len(ids) > raftsnapshot.RetainedGenerations { + return ErrCorrupt + } + for _, id := range ids { + m, _, _, err := snapshotManifest(ctx, tx, id) + if err != nil { + return ErrCorrupt + } + if err = walkSnapshot(ctx, tx, id, m, nil); err != nil { + return ErrCorrupt + } + } + if a.Applied < snapIndex { + return ErrCorrupt + } + rows, err := tx.QueryContext(ctx, `SELECT idx,data,checksum FROM raft_log WHERE idx>? ORDER BY idx`, indexKey(snapIndex)) if err != nil { return ErrCorrupt } defer rows.Close() var entries []*pb.Entry - var last, term uint64 + last, term := snapIndex, snapshot.GetMetadata().GetTerm() for rows.Next() { var key, data, hash []byte if err := rows.Scan(&key, &data, &hash); err != nil { @@ -157,7 +221,10 @@ func (s *SQLite) Recover(ctx context.Context) (Recovery, error) { if a.RecordCount != appRows || a.RecordsHash != recordsHash || (a.Applied == 0 && appRows != 0) { return ErrCorrupt } - out.Core = raftcore.RecoveredState{HardState: h, ConfState: a.ConfState, Entries: entries, Applied: a.Applied} + out.Core = raftcore.RecoveredState{HardState: h, ConfState: a.ConfState, Snapshot: snapshot, Entries: entries, Applied: a.Applied} + if snapIndex == 0 { + out.Core.Snapshot = nil + } out.Application = a out.Fresh = last == 0 && raft.IsEmptyHardState(h) && a.Applied == 0 if out.Fresh { diff --git a/internal/raftstore/snapshot.go b/internal/raftstore/snapshot.go new file mode 100644 index 0000000000..8d333b81fb --- /dev/null +++ b/internal/raftstore/snapshot.go @@ -0,0 +1,429 @@ +package raftstore + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "hash" + "io" + + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// Snapshot.Data is the canonical manifest. Chunks travel on a separate bounded +// host channel and must be complete before Persist accepts the Raft snapshot. +func snapshotPointer(m raftsnapshot.Manifest, data []byte) (*pb.Snapshot, error) { + c, e := m.Configuration() + if e != nil { + return nil, ErrSnapshot + } + return &pb.Snapshot{Data: bytes.Clone(data), Metadata: &pb.SnapshotMetadata{Index: proto.Uint64(m.Index), Term: proto.Uint64(m.Term), ConfState: c}}, nil +} +func getSnapshot(ctx context.Context, tx *sql.Tx) (*pb.Snapshot, error) { + p := &pb.Snapshot{} + e := getProto(ctx, tx, "snapshot", p) + return p, e +} +func decodeSnapshot(p *pb.Snapshot, identity Identity) (raftsnapshot.Manifest, error) { + m, e := raftsnapshot.Decode(p.GetData()) + if e != nil || m.Cluster != identity.ClusterUUID || m.Protocol != uint64(identity.Protocol) || m.Index != p.GetMetadata().GetIndex() || m.Term != p.GetMetadata().GetTerm() { + return m, ErrSnapshot + } + c, e := m.Configuration() + if e != nil || !proto.Equal(c, p.GetMetadata().GetConfState()) || len(p.ProtoReflect().GetUnknown()) != 0 || len(p.GetMetadata().ProtoReflect().GetUnknown()) != 0 { + return m, ErrSnapshot + } + return m, nil +} + +func stage(ctx context.Context, tx *sql.Tx, data []byte) (string, error) { + id := raftsnapshot.ID(data) + var existing []byte + e := tx.QueryRowContext(ctx, `SELECT manifest FROM snapshots WHERE id=?`, id).Scan(&existing) + if e == nil { + if !bytes.Equal(data, existing) { + return "", ErrCorrupt + } + return id, nil + } + if !errors.Is(e, sql.ErrNoRows) { + return "", e + } + // Only one incoming generation may stage at once. Abandoning incomplete + // consensus bytes never removes a complete snapshot or application record. + if _, e = tx.ExecContext(ctx, `DELETE FROM snapshot_chunks WHERE generation IN (SELECT id FROM snapshots WHERE complete=0)`); e != nil { + return "", e + } + if _, e = tx.ExecContext(ctx, `DELETE FROM snapshots WHERE complete=0`); e != nil { + return "", e + } + _, e = tx.ExecContext(ctx, `INSERT INTO snapshots VALUES (?,?,?,0)`, id, data, checksum([]byte("snapshots"), []byte(id), data)) + return id, e +} + +// BeginSnapshot admits a canonical manifest without exposing it to recovery. +// Retrying the same generation preserves already durable chunks. +func (s *SQLite) BeginSnapshot(ctx context.Context, data []byte) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + m, e := raftsnapshot.Decode(data) + if e != nil || m.Cluster != s.identity.ClusterUUID || m.Protocol != uint64(s.identity.Protocol) { + return "", ErrSnapshot + } + var id string + e = s.transaction(ctx, func(tx *sql.Tx) error { + active, err := getSnapshot(ctx, tx) + if err != nil { + return err + } + if m.Index <= active.GetMetadata().GetIndex() && !bytes.Equal(active.Data, data) { + return ErrSnapshot + } + rows, err := tx.QueryContext(ctx, `SELECT id,manifest FROM snapshots WHERE complete=1`) + if err != nil { + return err + } + blocked := false + for rows.Next() { + var id string + var b []byte + if err = rows.Scan(&id, &b); err != nil { + rows.Close() + return err + } + existing, err := raftsnapshot.Decode(b) + if err != nil { + rows.Close() + return ErrCorrupt + } + if existing.Index > active.GetMetadata().GetIndex() && id != raftsnapshot.ID(data) { + blocked = true + } + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + if blocked { + return ErrSnapshot + } + + var e error + id, e = stage(ctx, tx, data) + s.boundary("after-snapshot-stage") + return e + }) + return id, e +} +func snapshotManifest(ctx context.Context, tx *sql.Tx, id string) (raftsnapshot.Manifest, []byte, bool, error) { + var data, sum []byte + var complete int + e := tx.QueryRowContext(ctx, `SELECT manifest,checksum,complete FROM snapshots WHERE id=?`, id).Scan(&data, &sum, &complete) + if errors.Is(e, sql.ErrNoRows) { + return raftsnapshot.Manifest{}, nil, false, ErrSnapshot + } + if e != nil { + return raftsnapshot.Manifest{}, nil, false, e + } + m, e := raftsnapshot.Decode(data) + if e != nil || raftsnapshot.ID(data) != id || !bytes.Equal(sum, checksum([]byte("snapshots"), []byte(id), data)) || (complete != 0 && complete != 1) { + return m, nil, false, ErrCorrupt + } + return m, data, complete == 1, nil +} +func chunkLength(m raftsnapshot.Manifest, offset uint64) uint64 { + if offset >= m.Chunks { + return 0 + } + if offset == m.Chunks-1 { + return m.Bytes - offset*raftsnapshot.ChunkBytes + } + return raftsnapshot.ChunkBytes +} +func putChunk(ctx context.Context, tx *sql.Tx, id string, offset uint64, data []byte) error { + key := indexKey(offset) + _, e := tx.ExecContext(ctx, `INSERT INTO snapshot_chunks VALUES (?,?,?,?) ON CONFLICT(generation,offset) DO UPDATE SET data=excluded.data,checksum=excluded.checksum`, id, key, data, checksum([]byte("snapshot_chunks"), []byte(id), key, data)) + return e +} + +// PutSnapshotChunk checks size and the transport's SHA-256 before any write. +// A conflicting retry cannot overwrite a previously accepted chunk. +func (s *SQLite) PutSnapshotChunk(ctx context.Context, id string, offset uint64, data []byte, digest string) error { + s.mu.Lock() + defer s.mu.Unlock() + if len(data) > raftsnapshot.ChunkBytes || raftsnapshot.ID(data) != digest { + return ErrSnapshot + } + return s.transaction(ctx, func(tx *sql.Tx) error { + m, _, complete, e := snapshotManifest(ctx, tx, id) + if e != nil { + return e + } + if uint64(len(data)) != chunkLength(m, offset) || len(data) == 0 { + return ErrSnapshot + } + var old []byte + e = tx.QueryRowContext(ctx, `SELECT data FROM snapshot_chunks WHERE generation=? AND offset=?`, id, indexKey(offset)).Scan(&old) + if e == nil { + if !bytes.Equal(old, data) { + return ErrSnapshot + } + return nil + } + if !errors.Is(e, sql.ErrNoRows) { + return e + } + if complete { + return ErrSnapshot + } + if e = putChunk(ctx, tx, id, offset, data); e != nil { + return e + } + s.boundary("after-snapshot-chunk") + return nil + }) +} + +type chunkReader struct { + ctx context.Context + tx *sql.Tx + id string + m raftsnapshot.Manifest + offset uint64 + buf []byte + digest hash.Hash + read uint64 +} + +func (r *chunkReader) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + if len(r.buf) == 0 { + if r.offset == r.m.Chunks { + return 0, io.EOF + } + var sum []byte + key := indexKey(r.offset) + if e := r.tx.QueryRowContext(r.ctx, `SELECT data,checksum FROM snapshot_chunks WHERE generation=? AND offset=?`, r.id, key).Scan(&r.buf, &sum); e != nil { + return 0, ErrSnapshot + } + if uint64(len(r.buf)) != chunkLength(r.m, r.offset) || !bytes.Equal(sum, checksum([]byte("snapshot_chunks"), []byte(r.id), key, r.buf)) { + return 0, ErrCorrupt + } + r.digest.Write(r.buf) + r.offset++ + } + n := copy(p, r.buf) + r.buf = r.buf[n:] + r.read += uint64(n) + return n, nil +} + +// walkSnapshot never loads the whole identity. At most one record and one +// chunk are resident; canonical row order detects duplicate or reordered keys. +func walkSnapshot(ctx context.Context, tx *sql.Tx, id string, m raftsnapshot.Manifest, visit func(raftsnapshot.Row) error) error { + r := &chunkReader{ctx: ctx, tx: tx, id: id, m: m, digest: sha256.New()} + var count uint64 + var rowsHash [32]byte + var previousNamespace, previousKey string + for { + row, e := raftsnapshot.ReadRow(r) + if e == io.EOF { + break + } + if e != nil { + return ErrSnapshot + } + if count > 0 && (row.Namespace < previousNamespace || (row.Namespace == previousNamespace && row.Key <= previousKey)) { + return ErrSnapshot + } + previousNamespace, previousKey = row.Namespace, row.Key + count++ + xorHash(&rowsHash, checksum([]byte("records"), []byte(row.Namespace), []byte(row.Key), row.Value)) + if visit != nil { + if e = visit(row); e != nil { + return e + } + } + } + var chunks uint64 + if e := tx.QueryRowContext(ctx, `SELECT count(*) FROM snapshot_chunks WHERE generation=?`, id).Scan(&chunks); e != nil { + return e + } + if count != m.RecordCount || hex.EncodeToString(rowsHash[:]) != m.RecordsHash || r.read != m.Bytes || chunks != m.Chunks || hex.EncodeToString(r.digest.Sum(nil)) != m.Digest { + return ErrSnapshot + } + return nil +} + +// CompleteSnapshot publishes only the immutable verified generation, not the +// active application pointer. Persist performs that second atomic transition. +func (s *SQLite) CompleteSnapshot(ctx context.Context, id string) (*pb.Snapshot, error) { + s.mu.Lock() + defer s.mu.Unlock() + var p *pb.Snapshot + e := s.transaction(ctx, func(tx *sql.Tx) error { + m, data, _, e := snapshotManifest(ctx, tx, id) + if e != nil { + return e + } + if e = walkSnapshot(ctx, tx, id, m, nil); e != nil { + return e + } + active, e := getSnapshot(ctx, tx) + if e != nil { + return e + } + if m.Index <= active.GetMetadata().GetIndex() && !bytes.Equal(active.Data, data) { + return ErrSnapshot + } + if _, e = tx.ExecContext(ctx, `UPDATE snapshots SET complete=1 WHERE id=?`, id); e != nil { + return e + } + if e = retainSnapshots(ctx, tx, id); e != nil { + return e + } + s.boundary("after-snapshot-complete") + p, e = snapshotPointer(m, data) + return e + }) + return p, e +} + +// ReadSnapshotChunk returns a bounded immutable transport buffer. A successful +// upload is not a Raft snapshot delivery acknowledgment; installation is. +func (s *SQLite) ReadSnapshotChunk(ctx context.Context, id string, offset uint64) ([]byte, error) { + s.mu.Lock() + defer s.mu.Unlock() + var out []byte + e := s.transaction(ctx, func(tx *sql.Tx) error { + m, _, complete, e := snapshotManifest(ctx, tx, id) + if e != nil { + return e + } + if !complete || offset >= m.Chunks { + return ErrSnapshot + } + var sum []byte + key := indexKey(offset) + if e = tx.QueryRowContext(ctx, `SELECT data,checksum FROM snapshot_chunks WHERE generation=? AND offset=?`, id, key).Scan(&out, &sum); e != nil { + return e + } + if uint64(len(out)) != chunkLength(m, offset) || !bytes.Equal(sum, checksum([]byte("snapshot_chunks"), []byte(id), key, out)) { + return ErrCorrupt + } + return nil + }) + return out, e +} + +func retainSnapshots(ctx context.Context, tx *sql.Tx, active string) error { + rows, e := tx.QueryContext(ctx, `SELECT id,manifest FROM snapshots WHERE complete=1`) + if e != nil { + return e + } + type generation struct { + id string + index uint64 + } + var keep generation + var remove []string + for rows.Next() { + var id string + var data []byte + if e = rows.Scan(&id, &data); e != nil { + rows.Close() + return e + } + m, e := raftsnapshot.Decode(data) + if e != nil { + rows.Close() + return ErrCorrupt + } + if id != active { + if keep.id == "" || m.Index > keep.index { + if keep.id != "" { + remove = append(remove, keep.id) + } + keep = generation{id, m.Index} + } else { + remove = append(remove, id) + } + } + } + e = rows.Err() + rows.Close() + if e != nil { + return e + } + for _, id := range remove { + if _, e = tx.ExecContext(ctx, `DELETE FROM snapshot_chunks WHERE generation=?`, id); e != nil { + return e + } + if _, e = tx.ExecContext(ctx, `DELETE FROM snapshots WHERE id=?`, id); e != nil { + return e + } + } + return nil +} + +func (s *SQLite) installSnapshot(ctx context.Context, tx *sql.Tx, p *pb.Snapshot) error { + m, e := decodeSnapshot(p, s.identity) + if e != nil { + return e + } + id := raftsnapshot.ID(p.Data) + _, data, complete, e := snapshotManifest(ctx, tx, id) + if e != nil { + return e + } + if !complete || !bytes.Equal(data, p.Data) { + return ErrSnapshot + } + a, e := getApplication(ctx, tx) + if e != nil { + return e + } + if m.Index <= a.Applied || m.LogicalTime < a.LogicalTime || m.VisibleVersion < a.VisibleVersion { + return ErrSnapshot + } + if _, e = tx.ExecContext(ctx, `DELETE FROM records`); e != nil { + return e + } + s.boundary("after-snapshot-clear") + e = walkSnapshot(ctx, tx, id, m, func(r raftsnapshot.Row) error { + _, e := tx.ExecContext(ctx, `INSERT INTO records VALUES (?,?,?,?)`, r.Namespace, r.Key, r.Value, checksum([]byte("records"), []byte(r.Namespace), []byte(r.Key), r.Value)) + s.boundary("after-snapshot-row") + return e + }) + if e != nil { + return e + } + var recordsHash [32]byte + h, _ := hex.DecodeString(m.RecordsHash) + copy(recordsHash[:], h) + c, _ := m.Configuration() + if e = putApplication(ctx, tx, Application{Applied: m.Index, VisibleVersion: m.VisibleVersion, LogicalTime: m.LogicalTime, RecordCount: m.RecordCount, RecordsHash: recordsHash, ConfState: c}); e != nil { + return e + } + s.boundary("after-snapshot-application") + if e = putProto(ctx, tx, "snapshot", p); e != nil { + return e + } + s.boundary("after-snapshot-pointer") + if _, e = tx.ExecContext(ctx, `DELETE FROM raft_log`); e != nil { + return e + } + if e = putMeta(ctx, tx, "log_tip", indexKey(m.Index)); e != nil { + return e + } + s.boundary("after-snapshot-log") + return retainSnapshots(ctx, tx, id) +} diff --git a/internal/raftstore/snapshot_test.go b/internal/raftstore/snapshot_test.go new file mode 100644 index 0000000000..f3c408c48f --- /dev/null +++ b/internal/raftstore/snapshot_test.go @@ -0,0 +1,648 @@ +package raftstore + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + "go.etcd.io/raft/v3" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +func snapshotSource(t *testing.T) (*SQLite, *pb.Snapshot) { + t.Helper() + s := openTest(t, filepath.Join(t.TempDir(), "member.sqlite")) + seed(t, s) + // Keep a previous complete generation before a joint configuration and a + // near-limit opaque value force the next snapshot across several chunks. + _, e := s.CreateSnapshot(testContext) + must(t, e) + must(t, s.Persist(testContext, replacement())) + a, u := nextApply() + u.Writes = append(u.Writes, Write{Key: Key{State, "large"}, Value: bytes.Repeat([]byte{0, 255, 13, 17}, MaxRecordBytes/4)}) + must(t, s.Apply(testContext, a, u)) + p, e := s.CreateSnapshot(testContext) + must(t, e) + return s, p +} +func stageCopy(t *testing.T, source, target *SQLite, p *pb.Snapshot, omit int) string { + t.Helper() + id, e := target.BeginSnapshot(testContext, p.Data) + must(t, e) + m, e := raftsnapshot.Decode(p.Data) + must(t, e) + for offset := uint64(0); offset < m.Chunks; offset++ { + if int(offset) == omit { + continue + } + b, e := source.ReadSnapshotChunk(testContext, id, offset) + must(t, e) + must(t, target.PutSnapshotChunk(testContext, id, offset, b, raftsnapshot.ID(b))) + } + return id +} +func installBatch(p *pb.Snapshot) raftcore.PersistBatch { + return raftcore.PersistBatch{Snapshot: p, HardState: hard(max(4, p.GetMetadata().GetTerm()), 2, p.GetMetadata().GetIndex())} +} +func rowInventory(t *testing.T, s *SQLite) map[Key]string { + t.Helper() + out := map[Key]string{} + rows, e := s.db.Query(`SELECT namespace,key,data FROM records ORDER BY namespace,key`) + must(t, e) + defer rows.Close() + for rows.Next() { + var k Key + var b []byte + must(t, rows.Scan(&k.Namespace, &k.Key, &b)) + out[k] = string(b) + } + must(t, rows.Err()) + return out +} +func assertInventory(t *testing.T, a, b map[Key]string) { + t.Helper() + if len(a) != len(b) { + t.Fatal("R28 record count changed") + } + for k, v := range a { + if b[k] != v { + t.Fatalf("R13/R17/R28 record changed in %s", k.Namespace) + } + } +} + +// R13 and R28 require complete logical-content parity across a streamed +// snapshot; R12 includes every joint configuration field, and R17 exact results. +func TestR12R13R17R28SnapshotReplayAndRetainedGenerations(t *testing.T) { + source, p := snapshotSource(t) + before := rowInventory(t, source) + m, e := raftsnapshot.Decode(p.Data) + must(t, e) + if m.Chunks < 4 { + t.Fatal("R13 fixture did not cross chunk boundaries") + } + target := openTest(t, filepath.Join(t.TempDir(), "member.sqlite")) + id := stageCopy(t, source, target, p, -1) + delivered, e := target.CompleteSnapshot(testContext, id) + must(t, e) + if !proto.Equal(delivered, p) { + t.Fatal("R14 manifest pointer changed") + } + must(t, target.Persist(testContext, installBatch(p))) + r := recovered(t, target) + if r.Core.Applied != m.Index || !proto.Equal(r.Core.ConfState, p.Metadata.ConfState) || !proto.Equal(r.Core.Snapshot, p) || len(r.Core.Entries) != 0 { + t.Fatal("R12/R13 snapshot recovery changed metadata") + } + assertInventory(t, before, rowInventory(t, target)) + must(t, target.Apply(testContext, raftcore.ApplyBatch{Snapshot: p, Index: m.Index, ConfState: r.Application.ConfState}, Update{ExpectedApplied: m.Index, VisibleVersion: r.Application.VisibleVersion, LogicalTime: r.Application.LogicalTime})) + // Preserve a committed suffix for native replay after the snapshot boundary. + suffix := entry(4, 4, "") + must(t, target.Persist(testContext, raftcore.PersistBatch{Entries: []*pb.Entry{suffix}, HardState: hard(4, 2, 4)})) + path := copyDurable(t, targetPath(t, target)) + reopened := openTest(t, path) + r = recovered(t, reopened) + if len(r.Core.Entries) != 1 || r.Core.Entries[0].GetIndex() != 4 || r.Core.Applied != 3 { + t.Fatal("R13 committed suffix was not retained for replay") + } + core, e := raftcore.New(raftcore.Config{ClusterUUID: testIdentity.ClusterUUID, NodeID: 1, Protocol: 1, Seed: 99}, r.Core) + must(t, e) + effect, e := core.Step(raftcore.Poll{}) + must(t, e) + if effect.Persist == nil { + t.Fatal("R13 committed suffix did not replay") + } + must(t, reopened.Persist(testContext, *effect.Persist)) + effect, e = core.Step(raftcore.Persisted{BatchID: effect.Persist.ID}) + must(t, e) + if effect.Apply == nil || effect.Apply.Index != 4 { + t.Fatal("R13 replay skipped committed suffix") + } + must(t, reopened.Apply(testContext, *effect.Apply, Update{ExpectedApplied: 3, VisibleVersion: r.Application.VisibleVersion, LogicalTime: r.Application.LogicalTime})) + _, e = core.Step(raftcore.Applied{BatchID: effect.Apply.ID}) + must(t, e) + newer, e := reopened.CreateSnapshot(testContext) + must(t, e) + must(t, reopened.Compact(testContext, 4)) + _, e = core.Step(raftcore.CompactSnapshot{Snapshot: newer}) + must(t, e) + assertInventory(t, before, rowInventory(t, reopened)) + // A third checkpoint retires only consensus chunks from the oldest complete + // generation. Application rows and the second generation stay byte exact. + must(t, reopened.Persist(testContext, raftcore.PersistBatch{Entries: []*pb.Entry{entry(5, 4, "")}, HardState: hard(4, 2, 5)})) + must(t, reopened.Apply(testContext, raftcore.ApplyBatch{Index: 5, Entries: []*pb.Entry{entry(5, 4, "")}, ConfState: r.Application.ConfState}, Update{ExpectedApplied: 4, VisibleVersion: r.Application.VisibleVersion, LogicalTime: r.Application.LogicalTime})) + _, e = reopened.CreateSnapshot(testContext) + must(t, e) + var count int + must(t, reopened.db.QueryRow(`SELECT count(*) FROM snapshots WHERE complete=1`).Scan(&count)) + if count != 2 { + t.Fatal("R13 did not retain exactly two complete generations") + } + for _, pointer := range []*pb.Snapshot{newer, recovered(t, reopened).Core.Snapshot} { + b, e := reopened.ReadSnapshotChunk(testContext, raftsnapshot.ID(pointer.Data), 0) + must(t, e) + if len(b) == 0 { + t.Fatal("R13 retained generation has no bytes") + } + restored := openTest(t, filepath.Join(t.TempDir(), "member.sqlite")) + id := stageCopy(t, reopened, restored, pointer, -1) + _, e = restored.CompleteSnapshot(testContext, id) + must(t, e) + must(t, restored.Persist(testContext, installBatch(pointer))) + if !proto.Equal(recovered(t, restored).Core.Snapshot, pointer) { + t.Fatal("R13 retained generation could not restore a fresh member") + } + assertInventory(t, before, rowInventory(t, restored)) + } + assertInventory(t, before, rowInventory(t, reopened)) +} + +func targetPath(t *testing.T, s *SQLite) string { + t.Helper() + var seq int + var name, path string + must(t, s.db.QueryRow(`PRAGMA database_list`).Scan(&seq, &name, &path)) + return path +} + +// R13 requires every missing chunk and corrupt chunk to leave the previously +// installed generation authoritative. A transfer cannot acknowledge upload as +// installation, and a retry cannot overwrite accepted bytes with other bytes. +func TestR13PartialCorruptAndConflictingSnapshots(t *testing.T) { + source, p := snapshotSource(t) + m, e := raftsnapshot.Decode(p.Data) + must(t, e) + for offset := 0; offset < int(m.Chunks); offset++ { + t.Run(fmt.Sprintf("missing-%d", offset), func(t *testing.T) { + target := openTest(t, filepath.Join(t.TempDir(), "member.sqlite")) + seed(t, target) + old, e := target.CreateSnapshot(testContext) + must(t, e) + id := stageCopy(t, source, target, p, offset) + if _, e = target.CompleteSnapshot(testContext, id); !errors.Is(e, ErrSnapshot) { + t.Fatalf("R13 incomplete snapshot completed: %v", e) + } + if e = target.Persist(testContext, installBatch(p)); !errors.Is(e, ErrSnapshot) { + t.Fatal("R13 incomplete snapshot installed") + } + restart := openTest(t, copyDurable(t, targetPath(t, target))) + if !proto.Equal(recovered(t, restart).Core.Snapshot, old) { + t.Fatal("R13 incomplete staging replaced the last complete generation") + } + }) + t.Run(fmt.Sprintf("corrupt-%d", offset), func(t *testing.T) { + target := openTest(t, filepath.Join(t.TempDir(), "member.sqlite")) + id := stageCopy(t, source, target, p, -1) + b, e := source.ReadSnapshotChunk(testContext, id, uint64(offset)) + must(t, e) + b[0] ^= 1 + if e = target.PutSnapshotChunk(testContext, id, uint64(offset), b, raftsnapshot.ID(b)); !errors.Is(e, ErrSnapshot) { + t.Fatal("R13 conflicting chunk replaced durable bytes") + } + _, e = target.db.Exec(`UPDATE snapshot_chunks SET data=? WHERE generation=? AND offset=?`, b, id, indexKey(uint64(offset))) + must(t, e) + if _, e = target.CompleteSnapshot(testContext, id); e == nil { + t.Fatal("R13 corrupt chunk completed") + } + }) + } + target := openTest(t, filepath.Join(t.TempDir(), "member.sqlite")) + id := stageCopy(t, source, target, p, -1) + _, e = target.CompleteSnapshot(testContext, id) + must(t, e) + must(t, target.Persist(testContext, installBatch(p))) + _, e = target.db.Exec(`DELETE FROM snapshot_chunks WHERE generation=? AND offset=?`, id, indexKey(0)) + must(t, e) + if _, e = target.Recover(testContext); !errors.Is(e, ErrCorrupt) { + t.Fatal("R13 corrupt authoritative snapshot did not refuse recovery") + } +} + +// R13's process helper exits at actual SQLite boundaries without Close or a +// deferred rollback, preserving only durable database/WAL bytes for the parent. +func TestR13SnapshotCrashHelper(t *testing.T) { + mode := os.Getenv("P05_MODE") + if mode == "" { + return + } + s, e := Open(os.Getenv("P05_PATH"), testIdentity) + must(t, e) + number, e := strconv.Atoi(os.Getenv("P05_NUMBER")) + must(t, e) + count := 0 + s.cut = func(phase string) { + if phase == os.Getenv("P05_PHASE") { + count++ + if count == number { + os.Exit(86) + } + } + } + switch mode { + case "begin": + data, e2 := os.ReadFile(os.Getenv("P05_POINTER")) + must(t, e2) + p := &pb.Snapshot{} + must(t, proto.Unmarshal(data, p)) + _, e = s.BeginSnapshot(testContext, p.Data) + case "create": + _, e = s.CreateSnapshot(testContext) + case "compact": + e = s.Compact(testContext, 3) + case "install": + data, e2 := os.ReadFile(os.Getenv("P05_POINTER")) + must(t, e2) + p := &pb.Snapshot{} + must(t, proto.Unmarshal(data, p)) + e = s.Persist(testContext, installBatch(p)) + case "complete": + _, e = s.CompleteSnapshot(testContext, os.Getenv("P05_ID")) + case "chunk": + data, e2 := os.ReadFile(os.Getenv("P05_CHUNK")) + must(t, e2) + e = s.PutSnapshotChunk(testContext, os.Getenv("P05_ID"), 0, data, raftsnapshot.ID(data)) + } + must(t, e) + t.Fatal("R13 requested snapshot crash boundary not reached") +} + +// R3/R13 cover every chunk, record and pointer switch plus both sides of the +// install COMMIT. Recovery observes one complete application generation. +func TestR3R13SnapshotCrashBoundaries(t *testing.T) { + source, p := snapshotSource(t) + m, e := raftsnapshot.Decode(p.Data) + must(t, e) + for _, mode := range []string{"begin", "create", "install", "compact", "complete", "chunk"} { + phases := []struct { + name string + count int + }{{"after-begin", 1}, {"before-commit", 1}, {"after-commit", 1}} + switch mode { + case "begin": + phases = append(phases, struct { + name string + count int + }{"after-snapshot-stage", 1}) + case "create": + phases = append(phases, struct { + name string + count int + }{"after-snapshot-source-row", 7}, struct { + name string + count int + }{"after-snapshot-created-chunk", int(m.Chunks)}, struct { + name string + count int + }{"after-snapshot-pointer", 1}) + case "install": + for _, phase := range []string{"after-snapshot-clear", "after-snapshot-application", "after-snapshot-pointer", "after-snapshot-log", "after-hardstate", "after-log-tip"} { + phases = append(phases, struct { + name string + count int + }{phase, 1}) + } + phases = append(phases, struct { + name string + count int + }{"after-snapshot-row", 7}) + case "compact": + phases = append(phases, struct { + name string + count int + }{"after-compaction", 1}) + case "complete": + phases = append(phases, struct { + name string + count int + }{"after-snapshot-complete", 1}) + case "chunk": + phases = append(phases, struct { + name string + count int + }{"after-snapshot-chunk", 1}) + } + for _, phase := range phases { + for number := 1; number <= phase.count; number++ { + t.Run(fmt.Sprintf("%s/%s/%d", mode, phase.name, number), func(t *testing.T) { + path := filepath.Join(t.TempDir(), "member.sqlite") + s := openTest(t, path) + seed(t, s) + old, e := s.CreateSnapshot(testContext) + must(t, e) + var id string + if mode == "create" || mode == "compact" { + must(t, s.Persist(testContext, replacement())) + a, u := nextApply() + u.Writes = append(u.Writes, Write{Key: Key{State, "large"}, Value: bytes.Repeat([]byte{0, 255, 13, 17}, MaxRecordBytes/4)}) + must(t, s.Apply(testContext, a, u)) + if mode == "compact" { + _, e = s.CreateSnapshot(testContext) + must(t, e) + } + } else if mode != "begin" { + omit := -1 + if mode == "chunk" { + omit = 0 + } + id = stageCopy(t, source, s, p, omit) + if mode == "install" { + _, e = s.CompleteSnapshot(testContext, id) + must(t, e) + } + } + pointer := filepath.Join(t.TempDir(), "pointer") + b, e := wire.Marshal(p) + must(t, e) + must(t, os.WriteFile(pointer, b, 0600)) + chunk := filepath.Join(t.TempDir(), "chunk") + b, e = source.ReadSnapshotChunk(testContext, raftsnapshot.ID(p.Data), 0) + must(t, e) + must(t, os.WriteFile(chunk, b, 0600)) + must(t, s.Close()) + cmd := exec.Command(os.Args[0], "-test.run=^TestR13SnapshotCrashHelper$", "-test.count=1") + cmd.Env = append(os.Environ(), "P05_MODE="+mode, "P05_PATH="+path, "P05_PHASE="+phase.name, "P05_NUMBER="+strconv.Itoa(number), "P05_ID="+id, "P05_POINTER="+pointer, "P05_CHUNK="+chunk) + output, e := cmd.CombinedOutput() + var exit *exec.ExitError + if !errors.As(e, &exit) || exit.ExitCode() != 86 { + t.Fatalf("R13 process did not cut: %v %s", e, output) + } + restored := openTest(t, copyDurable(t, path)) + r := recovered(t, restored) + committed := phase.name == "after-commit" + if mode == "install" { + if committed { + if !proto.Equal(r.Core.Snapshot, p) || r.Application.Applied != 3 { + t.Fatal("R13 completed install not durable") + } + assertInventory(t, rowInventory(t, source), rowInventory(t, restored)) + } else { + if !proto.Equal(r.Core.Snapshot, old) || r.Application.Applied != 1 { + t.Fatal("R13 partial install escaped") + } + for _, v := range rowInventory(t, restored) { + if v != "old" { + t.Fatal("R13 old application partially replaced") + } + } + } + } + if mode == "create" { + want := old + if committed { + want = p + } + if !proto.Equal(r.Core.Snapshot, want) { + t.Fatal("R13 snapshot publication was not atomic") + } + assertInventory(t, rowInventory(t, source), rowInventory(t, restored)) + } + if mode == "compact" { + if !proto.Equal(r.Core.Snapshot, p) { + t.Fatal("R13 compaction changed checkpoint") + } + var count int + must(t, restored.db.QueryRow(`SELECT count(*) FROM raft_log`).Scan(&count)) + if (count == 0) != committed { + t.Fatal("R13 compaction cut changed retained suffix") + } + } + if mode == "begin" { + if !proto.Equal(r.Core.Snapshot, old) { + t.Fatal("R13 staging changed installed snapshot") + } + var count int + must(t, restored.db.QueryRow(`SELECT count(*) FROM snapshots WHERE id=?`, raftsnapshot.ID(p.Data)).Scan(&count)) + if (count == 1) != committed { + t.Fatal("R13 stage publication was not atomic") + } + } + if mode == "complete" || mode == "chunk" { + if !proto.Equal(r.Core.Snapshot, old) { + t.Fatal("R13 upload changed installed snapshot") + } + var complete int + must(t, restored.db.QueryRow(`SELECT complete FROM snapshots WHERE id=?`, id).Scan(&complete)) + if mode == "complete" && (complete == 1) != committed { + t.Fatal("R13 completion was not atomic") + } + if mode == "chunk" { + var count int + must(t, restored.db.QueryRow(`SELECT count(*) FROM snapshot_chunks WHERE generation=? AND offset=?`, id, indexKey(0)).Scan(&count)) + if (count == 1) != committed { + t.Fatal("R13 chunk was not atomic") + } + } + } + }) + } + } + } +} + +// R11/R13 require a learner that missed the compacted prefix to install the +// snapshot while the healthy voter majority keeps committing ordinary writes. +func TestR11R13NativeLaggingLearnerDoesNotPinLog(t *testing.T) { + g := newNativeGroup(t) + leader := g.elect() + if leader == 2 { + g.call(leader, raftcore.Transfer{Target: 1}) + g.pump() + leader = g.elect() + } + change, e := wire.Marshal(&pb.ConfChangeV2{Changes: []*pb.ConfChangeSingle{{Type: pb.ConfChangeAddLearnerNode.Enum(), NodeId: proto.Uint64(3)}}}) + must(t, e) + g.call(leader, raftcore.Configure{Change: change}) + g.pump() + conf := recovered(t, g.stores[0]).Core.ConfState + if len(conf.Voters) != 2 || conf.Voters[0] != 1 || conf.Voters[1] != 2 || len(conf.Learners) != 1 || conf.Learners[0] != 3 || len(conf.VotersOutgoing) != 0 { + t.Fatalf("R11 fixture did not finish learner demotion: %v", conf) + } + g.propose(leader, 1, clusterstate.Mutation{Create: &clusterstate.CreateCell{Cell: "cell", Init: clusterstate.CellInit{Head: "head-1"}, Receipt: "uploaded"}}) + g.crash(2) + g.maintain = true + for seq := uint64(2); seq <= 88; seq++ { + g.propose(leader, seq, clusterstate.Mutation{Publish: &clusterstate.PublishCell{Cell: "cell", Publish: clusterstate.Publish{OldHead: fmt.Sprintf("head-%d", seq-1), Head: fmt.Sprintf("head-%d", seq), Fence: 1, Title: strings.Repeat("x", 53<<10)}, Receipt: "uploaded"}}) + } + for _, i := range []int{0, 1} { + r := recovered(t, g.stores[i]) + if r.Core.Snapshot.GetMetadata().GetIndex() == 0 { + t.Fatal("R11 applied byte threshold did not checkpoint") + } + var size uint64 + must(t, g.stores[i].db.QueryRow(`SELECT coalesce(sum(length(data)),0) FROM raft_log`).Scan(&size)) + if size > raftcore.SnapshotLogBytes+raftcore.MaxMessageBytes { + t.Fatal("R11 learner pinned the durable log") + } + if len(r.Core.Entries) > 20 { + t.Fatal("R11 recovery still materialized compacted history") + } + } + g.start(2) + g.pump() + for range 3 { + g.tick() + } + if g.snapshotsSent == 0 || g.cell(2).Head != "head-88" || g.cell(2).Lease.Fence != 1 { + t.Fatal("R11/R13 learner did not install and replay snapshot") + } + if g.status[2].Role == raft.StateLeader { + t.Fatal("R12 learner regained a vote") + } + g.crash(2) + g.start(2) + g.pump() + if g.cell(2).Head != "head-88" { + t.Fatal("R13 installed learner snapshot did not survive native restart") + } +} + +// R13/R28 forbid compaction without an applied durable checkpoint, and reject +// unexpected schema/identity or damaged manifests without deleting evidence. +func TestR13R28SnapshotAdmissionAndCompactionRefusals(t *testing.T) { + s, p := snapshotSource(t) + if e := s.Compact(testContext, 2); !errors.Is(e, ErrSnapshot) { + t.Fatal("R13 compacted an uncovered index") + } + before := rowInventory(t, s) + must(t, s.Compact(testContext, 3)) + assertInventory(t, before, rowInventory(t, s)) + for _, bad := range [][]byte{[]byte(`{}`), append(bytes.Clone(p.Data), ' '), bytes.Replace(p.Data, []byte(`"cluster":"store-test"`), []byte(`"cluster":"other"`), 1)} { + if _, e := s.BeginSnapshot(testContext, bad); !errors.Is(e, ErrSnapshot) { + t.Fatal("R13 invalid manifest admitted") + } + } +} + +// R11 requires both entry-count and byte triggers to measure applied history, +// and never compact uncommitted work or retained application records. +func TestR11R13AppliedEntryTriggerAndUncommittedSuffix(t *testing.T) { + s := openTest(t, filepath.Join(t.TempDir(), "member.sqlite")) + entries := make([]*pb.Entry, raftcore.SnapshotEntries+1) + for i := range entries { + entries[i] = entry(uint64(i+1), 1, "") + } + must(t, s.Persist(testContext, raftcore.PersistBatch{Entries: entries, HardState: hard(1, 1, raftcore.SnapshotEntries)})) + due, e := s.SnapshotDue(testContext) + must(t, e) + if due { + t.Fatal("R11 uncommitted/unapplied entries triggered a checkpoint") + } + must(t, s.Apply(testContext, raftcore.ApplyBatch{Index: raftcore.SnapshotEntries, Entries: entries[:raftcore.SnapshotEntries], ConfState: &pb.ConfState{Voters: []uint64{1, 2, 3}}}, Update{Writes: allRows("retained")})) + due, e = s.SnapshotDue(testContext) + must(t, e) + if !due { + t.Fatal("R11 applied entry threshold did not trigger") + } + p, e := s.CreateSnapshot(testContext) + must(t, e) + must(t, s.Compact(testContext, p.GetMetadata().GetIndex())) + r := recovered(t, s) + if len(r.Core.Entries) != 1 || r.Core.Entries[0].GetIndex() != raftcore.SnapshotEntries+1 { + t.Fatal("R13 compaction removed uncommitted suffix") + } + due, e = s.SnapshotDue(testContext) + must(t, e) + if due { + t.Fatal("R11 compacted history counted toward next trigger") + } + if len(rowInventory(t, s)) != 6 { + t.Fatal("R28 compaction removed retained records") + } +} + +// R3/R13 require a chunk write failure to stop participation, even when it +// occurs inside the streaming row encoder before manifest publication. +func TestR3R13SnapshotWriteFailureStopsMember(t *testing.T) { + s, p := snapshotSource(t) + must(t, s.Persist(testContext, raftcore.PersistBatch{Entries: []*pb.Entry{entry(4, 4, "")}, HardState: hard(4, 2, 4)})) + a := recovered(t, s).Application + must(t, s.Apply(testContext, raftcore.ApplyBatch{Index: 4, Entries: []*pb.Entry{entry(4, 4, "")}, ConfState: a.ConfState}, Update{ExpectedApplied: 3, VisibleVersion: a.VisibleVersion, LogicalTime: a.LogicalTime})) + _, e := s.db.Exec(`CREATE TRIGGER fail_chunk BEFORE INSERT ON snapshot_chunks BEGIN SELECT RAISE(ABORT,'fixture storage failure'); END`) + must(t, e) + if _, e = s.CreateSnapshot(testContext); e == nil { + t.Fatal("R3 failed chunk write reported successful snapshot") + } + if e = s.Persist(testContext, raftcore.PersistBatch{}); !errors.Is(e, ErrStopped) { + t.Fatal("R3 failed chunk writer left member participating") + } + restored := openTest(t, copyDurable(t, targetPath(t, s))) + r := recovered(t, restored) + if !proto.Equal(r.Core.Snapshot, p) || r.Core.Applied != 4 { + t.Fatal("R13 failed checkpoint changed authoritative bytes") + } +} + +// R13 binds chunks to the manifest's whole-stream digest even when each +// individual transport hash is valid. R11 bounds completed staging to one slot. +func TestR11R13StreamDigestAndPendingGenerationAdmission(t *testing.T) { + source, p := snapshotSource(t) + original, e := raftsnapshot.Decode(p.Data) + must(t, e) + target := openTest(t, filepath.Join(t.TempDir(), "member.sqlite")) + bad := original + bad.Digest = strings.Repeat("0", 64) + data, e := bad.Encode() + must(t, e) + id, e := target.BeginSnapshot(testContext, data) + must(t, e) + for offset := uint64(0); offset < original.Chunks; offset++ { + b, e := source.ReadSnapshotChunk(testContext, raftsnapshot.ID(p.Data), offset) + must(t, e) + must(t, target.PutSnapshotChunk(testContext, id, offset, b, raftsnapshot.ID(b))) + } + if _, e = target.CompleteSnapshot(testContext, id); !errors.Is(e, ErrSnapshot) { + t.Fatal("R13 individually valid chunks bypassed the stream digest") + } + if !recovered(t, target).Fresh { + t.Fatal("R13 rejected stream became authoritative") + } + id = stageCopy(t, source, target, p, -1) + _, e = target.CompleteSnapshot(testContext, id) + must(t, e) + newer := original + newer.Index++ + data, e = newer.Encode() + must(t, e) + if _, e = target.BeginSnapshot(testContext, data); !errors.Is(e, ErrSnapshot) { + t.Fatal("R11 a second completed transfer displaced the staging slot") + } + must(t, target.Persist(testContext, installBatch(p))) + // Installing the pending generation releases the slot without deleting its + // complete checkpoint or any application records. + _, e = target.BeginSnapshot(testContext, data) + must(t, e) + if !proto.Equal(recovered(t, target).Core.Snapshot, p) { + t.Fatal("R13 new staging changed installed generation") + } +} + +// R13 permits log catch-up to reach a completed but uninstalled transfer's +// exact checkpoint after restart. Local snapshot creation must remain resumable. +func TestR13LocalCheckpointReusesCompletedTransfer(t *testing.T) { + source, p := snapshotSource(t) + target := openTest(t, filepath.Join(t.TempDir(), "member.sqlite")) + seed(t, target) + id := stageCopy(t, source, target, p, -1) + _, e := target.CompleteSnapshot(testContext, id) + must(t, e) + must(t, target.Persist(testContext, replacement())) + a, u := nextApply() + u.Writes = append(u.Writes, Write{Key: Key{State, "large"}, Value: bytes.Repeat([]byte{0, 255, 13, 17}, MaxRecordBytes/4)}) + must(t, target.Apply(testContext, a, u)) + local, e := target.CreateSnapshot(testContext) + must(t, e) + if !proto.Equal(local, p) { + t.Fatal("R13 local checkpoint differs from the identical completed transfer") + } + must(t, target.Compact(testContext, 3)) + assertInventory(t, rowInventory(t, source), rowInventory(t, target)) +} diff --git a/internal/raftstore/sqlite.go b/internal/raftstore/sqlite.go index 21ca24c7d2..7ef5a145e4 100644 --- a/internal/raftstore/sqlite.go +++ b/internal/raftstore/sqlite.go @@ -16,6 +16,7 @@ import ( "sync" "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" "go.etcd.io/raft/v3" pb "go.etcd.io/raft/v3/raftpb" "google.golang.org/protobuf/proto" @@ -23,8 +24,8 @@ import ( ) const ( - FormatVersion = 1 - MaxRecordBytes = 1 << 20 + FormatVersion = 2 + MaxRecordBytes = raftsnapshot.RecordBytes ) var ( @@ -33,7 +34,7 @@ var ( ErrBatch = errors.New("raftstore: invalid persistence or application batch") ErrConflict = errors.New("raftstore: application read index changed") ErrStopped = errors.New("raftstore: storage failed; close and recover before participating") - ErrSnapshot = errors.New("raftstore: snapshot installation requires the snapshot lifecycle adapter") + ErrSnapshot = errors.New("raftstore: snapshot is incomplete, inconsistent or not installable") ) // Identity permanently binds a file to one member, not just to one identity's @@ -113,6 +114,8 @@ type SQLite struct { const schema = ` CREATE TABLE meta (key TEXT PRIMARY KEY, data BLOB NOT NULL, checksum BLOB NOT NULL) WITHOUT ROWID; CREATE TABLE raft_log (idx BLOB PRIMARY KEY CHECK(length(idx)=8), data BLOB NOT NULL, checksum BLOB NOT NULL) WITHOUT ROWID; +CREATE TABLE snapshots (id TEXT PRIMARY KEY, manifest BLOB NOT NULL, checksum BLOB NOT NULL, complete INTEGER NOT NULL) WITHOUT ROWID; +CREATE TABLE snapshot_chunks (generation TEXT NOT NULL, offset BLOB NOT NULL CHECK(length(offset)=8), data BLOB NOT NULL, checksum BLOB NOT NULL, PRIMARY KEY(generation,offset)) WITHOUT ROWID; CREATE TABLE records (namespace TEXT NOT NULL, key TEXT NOT NULL, data BLOB NOT NULL, checksum BLOB NOT NULL, PRIMARY KEY(namespace,key)) WITHOUT ROWID; ` @@ -184,6 +187,9 @@ func Open(path string, identity Identity) (*SQLite, error) { if err := putMeta(ctx, tx, "log_tip", indexKey(0)); err != nil { return err } + if err := putProto(ctx, tx, "snapshot", &pb.Snapshot{}); err != nil { + return err + } return putApplication(ctx, tx, Application{ConfState: &pb.ConfState{}}) }) if err != nil { @@ -331,17 +337,14 @@ func (s *SQLite) transaction(ctx context.Context, fn func(*sql.Tx) error) error // Persist returns only after FULL-synchronous COMMIT. Submit Persisted to the // core only on success; a queued write or transport receipt is insufficient. -// Snapshot input is explicitly refused until P05 supplies atomic installation. +// Snapshot input is accepted only after the host has durably staged all chunks. func (s *SQLite) Persist(ctx context.Context, batch raftcore.PersistBatch) error { s.mu.Lock() defer s.mu.Unlock() if s.stopped { return ErrStopped } - if !raft.IsEmptySnap(batch.Snapshot) { - return ErrSnapshot - } - if len(batch.Entries) == 0 && raft.IsEmptyHardState(batch.HardState) { + if len(batch.Entries) == 0 && raft.IsEmptyHardState(batch.HardState) && raft.IsEmptySnap(batch.Snapshot) { return nil } return s.transaction(ctx, func(tx *sql.Tx) error { @@ -356,6 +359,14 @@ func (s *SQLite) Persist(ctx context.Context, batch raftcore.PersistBatch) error if len(h.ProtoReflect().GetUnknown()) != 0 || h.GetTerm() < old.GetTerm() || h.GetCommit() < old.GetCommit() || (h.GetTerm() == old.GetTerm() && old.GetVote() != 0 && h.GetVote() != old.GetVote()) { return ErrBatch } + if !raft.IsEmptySnap(batch.Snapshot) { + if batch.Snapshot.GetMetadata().GetIndex() < old.GetCommit() || h.GetCommit() < batch.Snapshot.GetMetadata().GetIndex() { + return ErrSnapshot + } + if err := s.installSnapshot(ctx, tx, batch.Snapshot); err != nil { + return err + } + } last, err := lastEntry(ctx, tx) if err != nil { return err @@ -364,9 +375,13 @@ func (s *SQLite) Persist(ctx context.Context, batch raftcore.PersistBatch) error if err != nil || !bytes.Equal(tip, indexKey(last.GetIndex())) { return ErrCorrupt } + base, err := getSnapshot(ctx, tx) + if err != nil { + return err + } if len(batch.Entries) != 0 { first := batch.Entries[0].GetIndex() - if first == 0 || (last.GetIndex() != ^uint64(0) && first > last.GetIndex()+1) { + if first <= base.GetMetadata().GetIndex() || first == 0 || (last.GetIndex() != ^uint64(0) && first > last.GetIndex()+1) { return ErrBatch } previous := &pb.Entry{} @@ -444,7 +459,7 @@ func validEntry(e *pb.Entry) bool { } func validKey(k Key) bool { - if k.Key == "" { + if k.Key == "" || len(k.Key) > MaxRecordBytes { return false } switch k.Namespace { @@ -464,14 +479,23 @@ func (s *SQLite) Apply(ctx context.Context, batch raftcore.ApplyBatch, update Up if s.stopped { return ErrStopped } - if !raft.IsEmptySnap(batch.Snapshot) { - return ErrSnapshot - } return s.transaction(ctx, func(tx *sql.Tx) error { a, err := getApplication(ctx, tx) if err != nil { return err } + if !raft.IsEmptySnap(batch.Snapshot) { + p, err := getSnapshot(ctx, tx) + if err != nil { + return err + } + if !proto.Equal(p, batch.Snapshot) || a.Applied < p.GetMetadata().GetIndex() { + return ErrSnapshot + } + if len(batch.Entries) == 0 && batch.Index == a.Applied && proto.Equal(batch.ConfState, a.ConfState) && len(update.Writes) == 0 && update.VisibleVersion == a.VisibleVersion && update.LogicalTime == a.LogicalTime { + return nil + } + } if update.ExpectedApplied != a.Applied { return ErrConflict } @@ -479,6 +503,7 @@ func (s *SQLite) Apply(ctx context.Context, batch raftcore.ApplyBatch, update Up if err := getProto(ctx, tx, "hardstate", h); err != nil { return err } + if len(batch.Entries) == 0 || batch.Index <= a.Applied || batch.Index > h.GetCommit() || batch.ConfState == nil || len(batch.ConfState.ProtoReflect().GetUnknown()) != 0 || update.LogicalTime < a.LogicalTime || update.VisibleVersion < a.VisibleVersion { return ErrBatch } From 1f01cc563d2cec4b617ff157a2f066279ad54092 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Fri, 23 Oct 2026 23:26:11 -0400 Subject: [PATCH 010/103] Carry portable snapshot maintenance across the reactor boundary. Preserve ABI-1 effects while adding checkpoint completion and maintenance queries. Pin the snapshot schema and source hash in the Worker bundle so native and hosted adapters share the same format. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/raftabi/README.md | 12 ++++-- internal/raftabi/abi.go | 19 ++++++++++ internal/raftabi/snapshot_test.go | 60 ++++++++++++++++++++++++++++++ relay/hosted/src/raft/abi.js | 10 ++++- relay/hosted/test/raft/abi.test.js | 12 +++++- relay/hosted/test/raft/build.mjs | 4 +- 6 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 internal/raftabi/snapshot_test.go diff --git a/internal/raftabi/README.md b/internal/raftabi/README.md index 8055f946ba..ae0b524bbb 100644 --- a/internal/raftabi/README.md +++ b/internal/raftabi/README.md @@ -75,10 +75,11 @@ a mutated core blindly. | 3 step | handle, input tag, tag-specific data | effects | | 4 lease feasibility probe | command blob, optional one-row blob | next canonical row blob | | 5 row representation probe | canonical row blob | same canonical row blob | +| 6 snapshot maintenance query | handle | present flag, then index/term uint64 when present | Step tags: 0 Poll, 1 Tick, 2 PeerMessage blob, 3 Propose blob, 4 Read string, 5 Persisted batch ID, 6 Applied batch ID, 7 Transfer target, 8 Configure blob, -9 StorageFailed batch ID, 10 CancelRead string. The core enforces the one-Ready +9 StorageFailed batch ID, 10 CancelRead string, 11 CompactSnapshot protobuf blob. The core enforces the one-Ready completion order, leader-only admission and peer-local-control refusal. Effects are: optional persist flag and [ID, hard state, entries, snapshot]; @@ -88,9 +89,12 @@ index] tuples; status [node, leader, term, commit, applied uint64, role uint32, RNG state/draws uint64, stopped flag]; need-tick flag. Preserve send order. The build manifest binds the ABI version, core protocol, row schema, source -inventory hash and wasm SHA-256. P05 must add the actual snapshot schema/hash -before snapshot interoperability can be claimed. No snapshot wire compatibility -is claimed by the P03 opaque checkpoint fixture. +inventory hash and wasm SHA-256. It also pins portable snapshot schema 1 and the +SHA-256 of `internal/raftsnapshot/format.go`. Operation 6 exposes the core's +SnapshotNeeded maintenance hint without changing ABI-1 effect framing; tag 11 +completes a host's durable local checkpoint/compaction. Actual hosted chunk +storage and bidirectional native/wasm installation are P08's gate; the P03 opaque +checkpoint fixture alone does not establish that interoperability. ## Oversized legacy representation decision diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index f800c6e4b2..4c165201ac 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -239,6 +239,10 @@ func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { in = raftcore.StorageFailed{BatchID: d.u64()} case 10: in = raftcore.CancelRead{RequestID: d.str()} + case 11: + p := &pb.Snapshot{} + d.message(p) + in = raftcore.CompactSnapshot{Snapshot: p} default: return nil, errFrame } @@ -274,6 +278,21 @@ func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { return nil, err } e.blob(next) + case 6: + id := d.u32() + if err := d.end(); err != nil { + return nil, err + } + node := r.nodes[id] + if node == nil { + return nil, errFrame + } + point := node.SnapshotNeeded() + e.flag(point != nil) + if point != nil { + e.u64(point.Index) + e.u64(point.Term) + } default: return nil, errFrame } diff --git a/internal/raftabi/snapshot_test.go b/internal/raftabi/snapshot_test.go new file mode 100644 index 0000000000..9ce95442ee --- /dev/null +++ b/internal/raftabi/snapshot_test.go @@ -0,0 +1,60 @@ +package raftabi + +import ( + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// R13/R14 require host checkpoint inputs and maintenance counters to preserve +// the same core semantics across the binary native/WASI boundary. +func TestR13R14SnapshotMaintenanceABI(t *testing.T) { + r := New() + id := create(t, r) + conf := &pb.ConfState{Voters: []uint64{1}} + entries := make([]*pb.Entry, raftcore.SnapshotEntries) + for i := range entries { + entries[i] = &pb.Entry{Index: proto.Uint64(uint64(i + 1)), Term: proto.Uint64(1), Type: pb.EntryNormal.Enum()} + } + c, e := raftcore.New(raftcore.Config{ClusterUUID: "R14", NodeID: 1, Protocol: 1, Seed: 42}, raftcore.RecoveredState{HardState: &pb.HardState{Term: proto.Uint64(1), Commit: proto.Uint64(raftcore.SnapshotEntries)}, ConfState: conf, Entries: entries, Applied: raftcore.SnapshotEntries}) + if e != nil { + t.Fatal(e) + } + r.nodes[id] = c + query := request(6) + query.u32(id) + d := answer(t, r.Call(query.b)) + if d.u32() != 1 || d.u64() != raftcore.SnapshotEntries || d.u64() != 1 || d.end() != nil { + t.Fatal("R14 maintenance counters changed") + } + config, e := raftsnapshot.Configuration(conf) + if e != nil { + t.Fatal(e) + } + manifest := raftsnapshot.Manifest{Version: 1, Cluster: "R14", Protocol: 1, Index: raftcore.SnapshotEntries, Term: 1, Conf: config, RecordsHash: strings.Repeat("0", 64), Digest: raftsnapshot.ID(nil)} + data, e := manifest.Encode() + if e != nil { + t.Fatal(e) + } + p := &pb.Snapshot{Data: data, Metadata: &pb.SnapshotMetadata{Index: proto.Uint64(raftcore.SnapshotEntries), Term: proto.Uint64(1), ConfState: conf}} + frame := request(3) + frame.u32(id) + frame.u32(11) + frame.message(p) + // Validate framing before stepping the core, including checkpoint inputs. + bad := append(append([]byte{}, frame.b...), 1) + invalid := &decoder{b: r.Call(bad)} + invalid.u32() + if invalid.str() == "" { + t.Fatal("R14 malformed checkpoint frame accepted") + } + answer(t, r.Call(frame.b)) + d = answer(t, r.Call(query.b)) + if d.u32() != 0 || d.end() != nil { + t.Fatal("R13 checkpoint did not compact ABI core") + } +} diff --git a/relay/hosted/src/raft/abi.js b/relay/hosted/src/raft/abi.js index 993eb12c6c..80f6953c06 100644 --- a/relay/hosted/src/raft/abi.js +++ b/relay/hosted/src/raft/abi.js @@ -41,7 +41,7 @@ export function createFrame({ cluster, id, seed, bootstrap = [], hard = new Uint } export function stepFrame(handle, tag, value) { const w = frame(3).u32(handle).u32(tag); - if ([2, 3, 8].includes(tag)) w.blob(value); + if ([2, 3, 8, 11].includes(tag)) w.blob(value); else if ([4, 10].includes(tag)) w.str(value); else if ([5, 6, 7, 9].includes(tag)) w.u64(value); return w.finish(); @@ -91,3 +91,11 @@ export class Reactor { } metrics() { return { memory: this.memory, peakMemory: this.peakMemory, wasi: { ...this.wasi.calls } }; } } + +// Maintenance is queried separately so ABI-1 effect frames stay byte compatible. +// Call tag 11 only after the host has durably checkpointed and compacted storage. +export function snapshotNeeded(bytes) { + const r = response(bytes); + const point = r.u32() ? { index: r.u64(), term: r.u64() } : null; + r.end(); return point; +} diff --git a/relay/hosted/test/raft/abi.test.js b/relay/hosted/test/raft/abi.test.js index 4f83b8eb14..2d9be3bf4b 100644 --- a/relay/hosted/test/raft/abi.test.js +++ b/relay/hosted/test/raft/abi.test.js @@ -1,6 +1,6 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { Writer, Reader, frame, response, MAX_BUFFER } from '../../src/raft/abi.js'; +import { Writer, Reader, frame, response, MAX_BUFFER, stepFrame, snapshotNeeded } from '../../src/raft/abi.js'; import { wasiPreview1 } from '../../src/raft/wasi.js'; test('R14 binary ABI preserves full uint64 and opaque bytes',()=>{ @@ -21,3 +21,13 @@ test('R27 WASI shim has fixed environment, no I/O and rejects blocking schedulin assert.equal(wasi.imports.fd_read(),8);assert.equal(wasi.imports.sock_accept(),8);assert.equal(wasi.imports.fd_write(9),8); assert.throws(()=>wasi.imports.random_get(65536,1),/bounds/); }); + +// R13/R14 pin the additive ABI-1 maintenance grammar used by a streamed host. +test('R13 R14 snapshot maintenance framing preserves exact indexes and bytes',()=>{ + const p=Uint8Array.of(0,255,1),r=new Reader(stepFrame(7,11,p)); + assert.equal(r.u32(),1);assert.equal(r.u32(),3);assert.equal(r.u32(),7);assert.equal(r.u32(),11);assert.deepEqual(r.blob(),p);r.end(); + const index=(1n<<64n)-1n; + assert.deepEqual(snapshotNeeded(new Writer().u32(1).str('').u32(1).u64(index).u64(9007199254740993n).finish()),{index,term:9007199254740993n}); + assert.equal(snapshotNeeded(new Writer().u32(1).str('').u32(0).finish()),null); + assert.throws(()=>snapshotNeeded(new Writer().u32(1).str('').u32(1).finish()),/truncated/); +}); diff --git a/relay/hosted/test/raft/build.mjs b/relay/hosted/test/raft/build.mjs index 402064b4cb..da795bd607 100644 --- a/relay/hosted/test/raft/build.mjs +++ b/relay/hosted/test/raft/build.mjs @@ -32,10 +32,10 @@ async function walk(path) { else if ((/\.(go|js|json|mod|sum|patch)$/.test(p) && !p.endsWith('_test.go')) || entry.name === 'LICENSE') source.push([relative(resolve('../..'), p), sha(await readFile(p))]); } } -for (const p of ['../../internal/raftcore', '../../internal/raftabi', '../../internal/clusterstate', '../../cmd/codeaf-raft-wasm', '../../third_party/raft', '../../third_party/raft-patch', 'src/raft']) await walk(resolve(p)); +for (const p of ['../../internal/raftsnapshot', '../../internal/raftcore', '../../internal/raftabi', '../../internal/clusterstate', '../../cmd/codeaf-raft-wasm', '../../third_party/raft', '../../third_party/raft-patch', 'src/raft']) await walk(resolve(p)); for (const p of ['../../go.mod','../../go.sum']) source.push([relative(resolve('../..'),resolve(p)),sha(await readFile(p))]); source.sort((a,b)=>a[0].localeCompare(b[0])); -const manifest = { abi: 1, coreProtocol: 1, rowSchema: 1, snapshotSchema: 'P05 pending; no snapshot compatibility claimed', go: 'go1.26.5', sourceHash: sha(JSON.stringify(source)), wasmSHA256: sha(wasm), sources: source }; +const manifest = { abi: 1, coreProtocol: 1, rowSchema: 1, snapshotSchema: 1, snapshotFormatSHA256: sha(await readFile('../../internal/raftsnapshot/format.go')), go: 'go1.26.5', sourceHash: sha(JSON.stringify(source)), wasmSHA256: sha(wasm), sources: source }; await writeFile(resolve(out, 'manifest.json'), JSON.stringify(manifest, null, 2)+'\n'); const compressed = { wasm: gzipSync(wasm,{level:9}).length, javascript: gzipSync(js,{level:9}).length, storageFixture:gzipSync(storage,{level:9}).length, manifest: gzipSync(JSON.stringify(manifest),{level:9}).length }; const report = { imports, rawWasm: wasm.length, rawJavascript: js.length, compressed, totalCompressed: Object.values(compressed).reduce((a,b)=>a+b,0), limit: 3*1024*1024, wasmSHA256: manifest.wasmSHA256, sourceHash: manifest.sourceHash }; From 4ae978c0f7ff857e27e8c73bb7ec08b6e90991d2 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Fri, 23 Oct 2026 23:43:46 -0400 Subject: [PATCH 011/103] Add an independent bounded checker for coordination histories. Keep timed-out operations as unknown outcomes and retain session results and tombstones so the oracle can detect stale reads, double takes and duplicate execution without sharing production rules. Co-Authored-By: Claude Opus 5.5 (1M context) --- go.mod | 1 + go.sum | 2 + internal/raftsim/model/check.go | 90 +++++++++++++++++++++ internal/raftsim/model/check_test.go | 116 +++++++++++++++++++++++++++ internal/raftsim/model/model.go | 69 ++++++++++++++-- 5 files changed, 271 insertions(+), 7 deletions(-) create mode 100644 internal/raftsim/model/check.go create mode 100644 internal/raftsim/model/check_test.go diff --git a/go.mod b/go.mod index 64a13f7fc1..0a6265394d 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/Agent-Field/agentfield/sdk/go v0.0.0-20260801225427-e6587ade0886 github.com/alecthomas/chroma/v2 v2.27.0 github.com/amp-labs/connectors v0.0.0-20260814231226-e12d95c297eb + github.com/anishathalye/porcupine v1.1.0 github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/lipgloss v1.1.0 github.com/charmbracelet/ultraviolet v0.0.0-20260703014108-f5a850f9c2b7 diff --git a/go.sum b/go.sum index a7ea105399..bdfa3e9bdc 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/amp-labs/amp-common v0.0.0-20260814191756-775714ec2997 h1:tSy0lHivSjB github.com/amp-labs/amp-common v0.0.0-20260814191756-775714ec2997/go.mod h1:jbTw6fXlE4qWGsAjxrwnRz8lA7UoX3JkLtXHd17hLSM= github.com/amp-labs/connectors v0.0.0-20260814231226-e12d95c297eb h1:Irpqzr0Mg+YPs+yLOFTTUef6walS1HbZqdNwTMC4rfU= github.com/amp-labs/connectors v0.0.0-20260814231226-e12d95c297eb/go.mod h1:4gpg6Xqe/zgmFGyZryhpUBN+9dxAq6CVZqckKY6RUxE= +github.com/anishathalye/porcupine v1.1.0 h1:jkMLqDejaWqvhvjxYKyqwQO3d1Jw+/08wHiIw0O4wcU= +github.com/anishathalye/porcupine v1.1.0/go.mod h1:WM0SsFjWNl2Y4BqHr/E/ll2yY1GY1jqn+W7Z/84Zoog= github.com/antchfx/xmlquery v1.5.1 h1:T9I4Ns1EXiWHy0IqKupGhnfTQtJwlGrpXtauYOoNv78= github.com/antchfx/xmlquery v1.5.1/go.mod h1:bVqnl7TaDXSReKINrhZz+2E/PbCu2tUahb+wZ7WZNT8= github.com/antchfx/xpath v1.3.6 h1:s0y+ElRRtTQdfHP609qFu0+c6bglDv20pqOViQjjdPI= diff --git a/internal/raftsim/model/check.go b/internal/raftsim/model/check.go new file mode 100644 index 0000000000..d64cafb997 --- /dev/null +++ b/internal/raftsim/model/check.go @@ -0,0 +1,90 @@ +package model + +import ( + "encoding/json" + "reflect" + "time" + + "github.com/anishathalye/porcupine" +) + +// Operation keeps the real invocation and response ordering. Unknown means a +// timeout, not a failed mutation: it may take effect anywhere before history end. +type Operation struct { + ID int + Call int64 + Return int64 + Input Call + Output Outcome + Unknown bool +} + +// Check never partitions an identity: vault, revocation and session state can +// couple cells. A timeout is inconclusive and callers must treat it as failure. +func Check(history []Operation) porcupine.CheckResult { + return CheckWithin(history, time.Second) +} + +func CheckWithin(history []Operation, budget time.Duration) porcupine.CheckResult { + if budget <= 0 || budget > time.Second || len(history) > 100 { + return porcupine.Unknown + } + end := int64(1) + for _, op := range history { + end = max(end, op.Call+1, op.Return+1) + } + operations := make([]porcupine.Operation, 0, len(history)) + for _, op := range history { + if op.Call < 0 || !op.Unknown && op.Return <= op.Call { + return porcupine.Illegal + } + until := op.Return + if op.Unknown { + until = end + } + operations = append(operations, porcupine.Operation{ClientId: op.ID, Input: op.Input, Output: op, Call: op.Call, Return: until}) + } + nondeterministic := porcupine.NondeterministicModel{ + Init: func() []any { return []any{State{}} }, + Equal: func(a, b any) bool { return reflect.DeepEqual(a, b) }, + Step: func(state, input, output any) []any { + s := state.(State) + call := input.(Call) + observed := output.(Operation) + next, result := Step(s, call) + if observed.Unknown { + return []any{s, next} + } + if result == observed.Output { + return []any{next} + } + return nil + }, + } + return porcupine.CheckOperationsTimeout(nondeterministic.ToModel(), operations, budget) +} + +// Minimize preserves invocation order and only removes a group when the +// remaining history is still definitely illegal. Inconclusive probes keep it. +func Minimize(history []Operation) []Operation { + out := append([]Operation(nil), history...) + if Check(out) != porcupine.Illegal { + return out + } + for width := len(out) / 2; width > 0; width /= 2 { + // Keep the state-establishing first operation; dropping it can turn + // a race into an irrelevant standalone read of an absent cell. + for start := 1; start+width <= len(out); { + candidate := append(append([]Operation(nil), out[:start]...), out[start+width:]...) + if Check(candidate) == porcupine.Illegal { + out = candidate + } else { + start++ + } + } + } + return out +} + +// Digest is a diagnostic state encoding, independent of production row bytes. +func Digest(s State) string { b, _ := json.Marshal(s); return string(b) } diff --git a/internal/raftsim/model/check_test.go b/internal/raftsim/model/check_test.go new file mode 100644 index 0000000000..3f22456590 --- /dev/null +++ b/internal/raftsim/model/check_test.go @@ -0,0 +1,116 @@ +package model + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/anishathalye/porcupine" +) + +func TestR3R4UnknownTimeoutCanCommitAfterCallerStoppedWaiting(t *testing.T) { + create := Call{Kind: "create", Actor: "a", Cell: "c", Head: "h"} + won := Outcome{Code: "ok", Cell: Cell{Holder: "a", Fence: 1, Until: 90000, Head: "h"}} + for _, known := range []bool{false, true} { + history := []Operation{{ID: 0, Call: 1, Return: 2, Input: create, Unknown: true}, {ID: 1, Call: 3, Return: 4, Input: Call{Kind: "read", Actor: "a", Cell: "c"}, Output: Outcome{Code: "not_found"}}} + if known { + history[1].Output = won + } + if Check(history) != porcupine.Ok { + t.Fatal("timeout was treated as a failed or completed-before-timeout operation", known) + } + } + // Deleting the unknown operation would make this legitimate history illegal. + if Check([]Operation{{Call: 1, Return: 2, Input: Call{Kind: "read", Cell: "c"}, Output: won}}) != porcupine.Illegal { + t.Fatal("unexplained read was accepted") + } +} +func TestR1R4WholeIdentityOrderingAndCheckerDeadline(t *testing.T) { + s, create := Step(State{}, Call{Kind: "create", Actor: "a", Cell: "c", Head: "h"}) + _, take := Step(s, Call{Kind: "acquire", Actor: "b", Cell: "c", Expected: create.Cell, Force: true}) + history := []Operation{{Call: 1, Return: 2, Input: Call{Kind: "create", Actor: "a", Cell: "c", Head: "h"}, Output: create}, {Call: 3, Return: 4, Input: Call{Kind: "acquire", Actor: "b", Cell: "c", Expected: create.Cell, Force: true}, Output: take}, {Call: 5, Return: 6, Input: Call{Kind: "read", Actor: "a", Cell: "c"}, Output: create}} + if Check(history) != porcupine.Illegal || len(Minimize(history)) == 0 { + t.Fatal("stale completed read escaped") + } + for _, budget := range []time.Duration{0, -1, time.Second + 1} { + if CheckWithin(history, budget) != porcupine.Unknown { + t.Fatal("invalid/unbounded checker budget accepted") + } + } + if Check(make([]Operation, 101)) != porcupine.Unknown { + t.Fatal("unbounded history admitted") + } +} +func TestR17SessionsKeepExactResultAndClosedTombstone(t *testing.T) { + s := State{} + for i := 0; i < 128; i++ { + call := Call{Kind: "advance_time", Actor: "a", Session: strconv.Itoa(i), Sequence: 1, Digest: "d", At: int64(i)} + var out Outcome + s, out = Step(s, call) + if out.Code != "ok" { + t.Fatal(i, out) + } + } + full := Call{Kind: "advance_time", Actor: "a", Session: "full", Sequence: 1, Digest: "d"} + if _, out := Step(s, full); out.Code != "capacity" { + t.Fatal(out) + } + retry := Call{Kind: "advance_time", Actor: "a", Session: "0", Sequence: 1, Digest: "d", At: 9000} + original := s.Sessions["a/0"].Result + next, out := Step(s, retry) + if out != original || next.Now != s.Now { + t.Fatal("retry was re-executed") + } + for _, call := range []Call{{Kind: "advance_time", Actor: "a", Session: "0", Sequence: 1, Digest: "changed"}, {Kind: "advance_time", Actor: "a", Session: "0", Sequence: 2, Digest: "new"}, {Kind: "advance_time", Actor: "a", Session: "0", Sequence: 3, Digest: "new", Ack: 1}} { + if _, out := Step(s, call); out.Code != "conflict" { + t.Fatal(out) + } + } + s, out = Step(s, Call{Kind: "session_close", Actor: "a", Session: "0", Sequence: 2, Ack: 1, Digest: "close"}) + if out.Code != "ok" { + t.Fatal(out) + } + if _, out := Step(s, retry); out.Code != "conflict" { + t.Fatal("closed session ID reused") + } + if _, out := Step(s, full); out.Code != "ok" { + t.Fatal("closing did not free a live-session slot") + } +} + +// R1/R4/R17 require an independent oracle. AST inspection keeps a future +// refactor from silently sharing the production rules with the checker. +func TestR1R4R17OracleImportsNoProductionRules(t *testing.T) { + entries, err := os.ReadDir(".") + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".go") || strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + file, err := parser.ParseFile(token.NewFileSet(), entry.Name(), nil, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + ast.Inspect(file, func(node ast.Node) bool { + spec, ok := node.(*ast.ImportSpec) + if !ok { + return true + } + path, err := strconv.Unquote(spec.Path.Value) + if err != nil { + t.Fatal(err) + } + if strings.Contains(path, "codeaf/internal/") || strings.Contains(path, "etcd.io/") { + t.Fatal("oracle imports production state", path) + } + return true + }) + } +} diff --git a/internal/raftsim/model/model.go b/internal/raftsim/model/model.go index 743989798e..b93d4e0357 100644 --- a/internal/raftsim/model/model.go +++ b/internal/raftsim/model/model.go @@ -1,15 +1,17 @@ // Package model is the independent sequential ownership/vault oracle for the // fault simulator. It deliberately imports no production state types or rules. -// P06 supplies history checking, unknown timeout outcomes and assignment cases. +// Histories include timed-out requests as unknown outcomes; assignments extend +// this domain when their production adapter lands. package model // State describes observable coordination, independent of Raft internals. // Calls advance committed time explicitly; the model never reads a host clock. type State struct { - Now int64 - Vault string - Cells map[string]Cell - Revoked map[string]bool + Now int64 + Vault string + Cells map[string]Cell + Revoked map[string]bool + Sessions map[string]Session } type Cell struct { @@ -24,6 +26,10 @@ type Cell struct { // Call contains only the small ownership/vault domain. Expected is an observed // holder/fence, not a fresh read substituted by the checker behind the caller. type Call struct { + Session string + Sequence uint64 + Digest string + Ack uint64 Kind string Actor string Cell string @@ -49,7 +55,7 @@ type Outcome struct { // Step returns a new state so alternative history orders cannot mutate one // another. The independent 90-second lease specification is intentional: using // the production constant or apply function would hide a shared defect. -func Step(s State, op Call) (State, Outcome) { +func domain(s State, op Call) (State, Outcome) { next := State{Now: s.Now, Vault: s.Vault, Cells: map[string]Cell{}, Revoked: map[string]bool{}} for k, v := range s.Cells { next.Cells[k] = v @@ -57,7 +63,8 @@ func Step(s State, op Call) (State, Outcome) { for k, v := range s.Revoked { next.Revoked[k] = v } - if op.At > next.Now { + next.Sessions = s.Sessions + if op.Kind != "read" && op.Kind != "read_vault" && op.At > next.Now { next.Now = op.At } if next.Revoked[op.Actor] { @@ -132,3 +139,51 @@ func Step(s State, op Call) (State, Outcome) { out.Cell = c return next, out } + +// Session retains one result until an acknowledged advance, and closed IDs stay +// tombstoned. The oracle intentionally spells the contract limits independently. +type Session struct { + Actor string + Sequence uint64 + Digest string + Result Outcome + Closed bool +} + +func Step(s State, op Call) (State, Outcome) { + if op.Session == "" || op.Kind == "read" || op.Kind == "read_vault" { + return domain(s, op) + } + key := op.Actor + "/" + op.Session + old, exists := s.Sessions[key] + if exists && (old.Closed || op.Sequence < old.Sequence || op.Sequence == old.Sequence && op.Digest != old.Digest) { + return s, Outcome{Code: "conflict"} + } + if exists && op.Sequence == old.Sequence { + return s, old.Result + } + if op.Sequence == 0 || !exists && op.Sequence != 1 || exists && (op.Sequence != old.Sequence+1 || op.Ack != old.Sequence) { + return s, Outcome{Code: "conflict"} + } + if !exists { + live := 0 + for _, v := range s.Sessions { + if !v.Closed { + live++ + } + } + if live >= 128 { + return s, Outcome{Code: "capacity"} + } + } + next, out := s, Outcome{Code: "ok"} + if op.Kind != "session_close" { + next, out = domain(s, op) + } + next.Sessions = map[string]Session{} + for k, v := range s.Sessions { + next.Sessions[k] = v + } + next.Sessions[key] = Session{Actor: op.Actor, Sequence: op.Sequence, Digest: op.Digest, Result: out, Closed: op.Kind == "session_close"} + return next, out +} From 9bb7712bbd9cee3ed84244b7291be416f10b3240 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Fri, 23 Oct 2026 23:43:46 -0400 Subject: [PATCH 012/103] Exercise the production Raft core with deterministic durability and network faults. Drive explicit write, flush, application and reply boundaries through seeded virtual-time schedules, preserve portable snapshots across recovery and compare the disk model with native SQLite. Keep replayable counterexamples and require broken fence, read and flush controls to fail. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/raftsim/README.md | 105 ++ internal/raftsim/application.go | 222 +++++ internal/raftsim/disk.go | 208 ++++ internal/raftsim/native_test.go | 124 +++ internal/raftsim/replay.go | 70 ++ internal/raftsim/simulator.go | 753 ++++++++++++++ internal/raftsim/simulator_test.go | 693 +++++++++++++ internal/raftsim/snapshot.go | 110 ++ internal/raftsim/snapshot_test.go | 130 +++ internal/raftsim/testdata/r1-double-take.json | 77 ++ .../testdata/r3-timeout-later-commit.json | 37 + internal/raftsim/testdata/r4-stale-read.json | 66 ++ internal/raftsim/testdata/replay/fence.json | 499 ++++++++++ internal/raftsim/testdata/replay/flush.json | 57 ++ internal/raftsim/testdata/replay/read.json | 941 ++++++++++++++++++ 15 files changed, 4092 insertions(+) create mode 100644 internal/raftsim/README.md create mode 100644 internal/raftsim/application.go create mode 100644 internal/raftsim/disk.go create mode 100644 internal/raftsim/native_test.go create mode 100644 internal/raftsim/replay.go create mode 100644 internal/raftsim/simulator.go create mode 100644 internal/raftsim/simulator_test.go create mode 100644 internal/raftsim/snapshot.go create mode 100644 internal/raftsim/snapshot_test.go create mode 100644 internal/raftsim/testdata/r1-double-take.json create mode 100644 internal/raftsim/testdata/r3-timeout-later-commit.json create mode 100644 internal/raftsim/testdata/r4-stale-read.json create mode 100644 internal/raftsim/testdata/replay/fence.json create mode 100644 internal/raftsim/testdata/replay/flush.json create mode 100644 internal/raftsim/testdata/replay/read.json diff --git a/internal/raftsim/README.md b/internal/raftsim/README.md new file mode 100644 index 0000000000..f880d1f2f5 --- /dev/null +++ b/internal/raftsim/README.md @@ -0,0 +1,105 @@ +# Deterministic fault gate + +This development package runs 3–5 **production** `raftcore` instances on one +virtual-time event queue. It adds no product route, environment switch, socket, +clock read or background consensus loop. P09 owns the authorized production +application adapter; this packet's `ProbeApplication` exercises the real pure +lease rules and a small session-policy fixture. The independent oracle in +`model` uses neither those rules nor production state types. Assignment policy +extends that oracle in P16; it is not advertised as implemented here. + +## Run the bounded gate + +From the repository root, with no other heavy command running: + +```sh +sh scripts/raft-proof-footprint.sh nice -n 5 env GOMEMLIMIT=128MiB \ + /usr/bin/time -v go test -p=1 ./internal/raftsim/... \ + -run '^TestR' -count=1 -timeout=120s -v +``` + +The wrapper admits work only at I/O `some avg10 <= 40`, uses GOMAXPROCS=2, +GOFLAGS=-buildvcs=false, one compiler worker and effective nice 15. The campaign +uses seeds 0–99 serially, at most 2,000 processed events per seed, 2 cells and +fewer than 100 operations. One whole-identity Porcupine check gets at most one +second; timeout/Unknown fails the proof. No checker partition hides cross-cell +vault/revocation/session dependencies. No HTML or large fixture is generated. +Supplementary bounded cases cover all 16 surviving joint-majority subsets, +every enter/leave Ready boundary, 100 minority flaps, every snapshot chunk and +install boundary, both retained generations and native SQLite parity. + +To keep exact negative counterexamples, pass the simulator package alone: + +```sh +sh scripts/raft-proof-footprint.sh nice -n 5 env GOMEMLIMIT=128MiB \ + go test -p=1 ./internal/raftsim -run '^TestR1R3R4Broken' -count=1 \ + -raftsim-evidence-dir=/tmp/codeaf-raftsim-evidence +``` + +The evidence flag belongs only to that test binary. Do not pass it to the +separate `model` package. `testdata` contains independently specified good and +bad histories; broken fence/read/flush variants must be detected by the same +checker/ordering monitor used for healthy runs. + +## Drive and replay + +`New(Config)` creates three identically bootstrapped voters and optional empty +members 4–5. `Do(Action)` records every external command/fault. `Drain` processes +bounded queued work; `Ticks` advances explicit election ticks. A proposal's +admission is never a successful operation. `History` records invocation and +response order; a timeout stays as an unknown outcome and may commit later. +Cancelled reads use the actual core cancellation input. Reads complete only +from quorum ReadIndex and durable application state. + +`Evidence()` includes seed/config, ordered actions, event trace, separate flush +trace, whole history, checker result and minimized illegal history. `Run(replay)` +reconstructs all RNG seeds, deliveries, effects and replies; `Digest` compares +the exact trace and observations. `WriteEvidence(dir)` is explicit host I/O for +synthetic fixtures, outside the scheduler. Tests export this bundle on failure; +negative controls export it when the evidence flag is supplied. + +Network faults include omission, duplicates, seeded delay/reorder, directed +partitions, held read acknowledgments and temporary stalls. Recovery gets a new +recorded seed and handle generation. Old delivery/completion callbacks cannot +complete work on that handle. Persist-write, flush, completion, apply-write, +application flush, completion and reply are distinct events. Power loss keeps +only the flushed image and discards staged/torn bytes; process death may retain +a complete unflushed transaction. Empty Ready barriers have no physical write. +Disk errors stop participation. Every dependent send/application/reply is checked +against its flush, and every state transition checks leader-per-term, committed +log agreement, monotonic indexes/fences and application equivalence. + +The disk model is a fault model, not another durable storage implementation. +P04/P05 prove SQLite process cuts and actual WAL flushes. The native parity test +replays the simulator's exact production batches into real SQLite stores and +compares recovery bytes, application state and portable snapshots; simulated +snapshot chunks are also installed through the real store's API. The simulated +one-row application stays below the portable 1 MiB row envelope; it is not the +paged production directory evaluator or a migration representation. + +## Constraints subsequent packets must keep + +Create a new checkpoint **after committed learner admission** when catch-up +requires a compacted prefix. etcd refuses an older snapshot whose configuration +omits the recipient. Wait until the learner applies the advertised barrier +before proposing its promotion. This test host supplies the mechanism; P12 owns +the durable enrollment/intent and policy workflow. + +Snapshot transfer completion is separate from Raft installation. Chunks stage +without changing application authority, complete streams are checksummed, +incomplete generations are ignored on recovery, and corrupt complete bytes +refuse recovery. Retain two complete generations. Report transport success only +after the real snapshot Ready is installed durably; report interruption/timeout +so RawNode can retry. A transport receipt alone is never an append acknowledgment. + +The fixture session contract keeps one exact result per live session, requires +an acknowledgment to advance, retains closed-ID tombstones, refuses conflicting +digests and caps live sessions at 128. Acks/close are replay-catalog inputs until +P09 supplies their real schema; no public protocol for them is invented here. +Authentication, durable upload receipts, complete directory verbs, assignments, +learner quotas/transport queues, hosted actors and client journeys remain the +packets that own them. R1's displayed sentence, unsent local branches and the +other X/J requirements are not claimed by a metadata simulator. + +Porcupine v1.1.0 is MIT licensed and only imported by the simulator checker. +Neither this dependency nor the simulator is linked into `bin/codeaf`. diff --git a/internal/raftsim/application.go b/internal/raftsim/application.go new file mode 100644 index 0000000000..351db93e49 --- /dev/null +++ b/internal/raftsim/application.go @@ -0,0 +1,222 @@ +// Package raftsim drives production consensus cores with explicit virtual time, +// durability and faults. It is a development gate, never a directory writer. +package raftsim + +import ( + "bytes" + "errors" + "fmt" + "slices" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftsim/model" +) + +// ProbeApplication exercises real pure lease rules while P09 supplies the full +// authorized writer. Session policy here is a contract fixture, not a second +// production implementation. The checker uses its own rules and state types. +type ProbeApplication struct { + Now int64 + Vault string + Cells map[string]clusterstate.Cell + Revoked map[string]bool + Sessions map[string]probeSession + Padding []byte +} +type probeSession struct { + Sequence uint64 + Digest string + Closed bool + Result model.Outcome +} + +func freshApp() ProbeApplication { + return ProbeApplication{Cells: map[string]clusterstate.Cell{}, Revoked: map[string]bool{}, Sessions: map[string]probeSession{}} +} +func (a ProbeApplication) copy() ProbeApplication { + out := ProbeApplication{Now: a.Now, Vault: a.Vault, Cells: map[string]clusterstate.Cell{}, Revoked: map[string]bool{}, Sessions: map[string]probeSession{}, Padding: bytes.Clone(a.Padding)} + for key, cell := range a.Cells { + cell.Frames = slices.Clone(cell.Frames) + if cell.Keys != nil { + keys := map[string]map[string]string{} + for id, recipients := range cell.Keys { + keys[id] = map[string]string{} + for who, wrapped := range recipients { + keys[id][who] = wrapped + } + } + cell.Keys = keys + } + out.Cells[key] = cell + } + for key, value := range a.Revoked { + out.Revoked[key] = value + } + for key, value := range a.Sessions { + out.Sessions[key] = value + } + return out +} +func observed(c clusterstate.Cell) model.Cell { + return model.Cell{Holder: c.Lease.Device, Fence: c.Lease.Fence, Until: c.Lease.Expires, Head: c.Head, Pending: c.Lease.Pending, ReleasedAt: c.Lease.ReleasedAt} +} +func (a ProbeApplication) read(op model.Call) model.Outcome { + if a.Revoked[op.Actor] { + return model.Outcome{Code: "revoked"} + } + if op.Kind == "read_vault" { + return model.Outcome{Code: "ok", Vault: a.Vault} + } + c, ok := a.Cells[op.Cell] + if !ok { + return model.Outcome{Code: "not_found"} + } + return model.Outcome{Code: "ok", Cell: observed(c)} +} +func (a *ProbeApplication) evaluate(op model.Call, brokenFence bool) model.Outcome { + key := op.Actor + "/" + op.Session + last, known := a.Sessions[key] + if op.Session != "" { + if known { + if last.Closed || op.Sequence < last.Sequence || op.Sequence == last.Sequence && op.Digest != last.Digest { + return model.Outcome{Code: "conflict"} + } + if op.Sequence == last.Sequence { + return last.Result + } + if op.Sequence != last.Sequence+1 || op.Ack != last.Sequence { + return model.Outcome{Code: "conflict"} + } + } else { + if op.Sequence != 1 { + return model.Outcome{Code: "conflict"} + } + live := 0 + for _, s := range a.Sessions { + if !s.Closed { + live++ + } + } + if live == 128 { + return model.Outcome{Code: "capacity"} + } + } + } + out := model.Outcome{Code: "ok"} + if op.Kind != "session_close" { + a.Now = max(a.Now, op.At) + out = a.mutate(op, brokenFence) + } + if op.Session != "" { + a.Sessions[key] = probeSession{Sequence: op.Sequence, Digest: op.Digest, Result: out, Closed: op.Kind == "session_close"} + } + return out +} +func (a *ProbeApplication) mutate(op model.Call, brokenFence bool) model.Outcome { + if a.Revoked[op.Actor] { + return model.Outcome{Code: "revoked"} + } + switch op.Kind { + case "advance_time": + return model.Outcome{Code: "ok"} + case "set_vault": + if a.Vault != op.OldVault { + return model.Outcome{Code: "cas"} + } + a.Vault = op.NewVault + return model.Outcome{Code: "ok"} + case "revoke": + _, err := clusterstate.RevokeOf(clusterstate.Device{}, op.Actor, op.Target) + if err != nil { + return model.Outcome{Code: "self_revoke"} + } + a.Revoked[op.Target] = true + return model.Outcome{Code: "ok"} + } + c, exists := a.Cells[op.Cell] + var err error + if op.Kind == "create" { + if exists { + return model.Outcome{Code: "exists"} + } + c = clusterstate.Created(clusterstate.CellInit{Head: op.Head}, op.Actor, a.Now) + } else { + if !exists { + return model.Outcome{Code: "not_found"} + } + switch op.Kind { + case "acquire": + if c.Lease.Fence == ^uint64(0) { + return model.Outcome{Code: "capacity"} + } + expect := &clusterstate.LeaseExpectation{Device: op.Expected.Holder, Fence: op.Expected.Fence} + if brokenFence { + expect = nil + } + c, err = clusterstate.Acquire(c, op.Actor, a.Now, op.Force, expect) + case "publish": + c, err = clusterstate.PublishTo(c, op.Actor, clusterstate.Publish{Fence: op.Fence, OldHead: op.OldHead, Head: op.Head, Pending: op.Pending}, a.Now) + case "beat": + c, err = clusterstate.Heartbeat(c, op.Actor, clusterstate.Beat{Fence: op.Fence, Pending: op.Pending}, a.Now) + case "release": + at := a.Now + if op.Quiet { + at = 0 + } + c, err = clusterstate.ReleaseOf(c, op.Actor, op.Fence, at) + default: + return model.Outcome{Code: "unsupported"} + } + } + if err != nil { + code := "" + for _, pair := range []struct { + err error + code string + }{{clusterstate.ErrLeaseHeld, "lease_held"}, {clusterstate.ErrFenceStale, "fence_stale"}, {clusterstate.ErrHeadMoved, "head_moved"}} { + if errors.Is(err, pair.err) { + code = pair.code + break + } + } + if code == "" { + panic(err) + } + return model.Outcome{Code: code} + } + a.Cells[op.Cell] = c + return model.Outcome{Code: "ok", Cell: observed(c)} +} + +// command keeps real canonical admission in the loop. Extra fixture policy +// inputs (session acknowledgments) live in the replay catalog, not a wire fork. +func command(op model.Call, id int) ([]byte, error) { + change := clusterstate.Mutation{} + switch op.Kind { + case "create": + change.Create = &clusterstate.CreateCell{Cell: op.Cell, Init: clusterstate.CellInit{Head: op.Head}, Receipt: "durable-probe-receipt"} + case "acquire": + change.Acquire = &clusterstate.AcquireCell{Cell: op.Cell, Force: op.Force, Expect: clusterstate.LeaseExpectation{Device: op.Expected.Holder, Fence: op.Expected.Fence}} + case "publish": + change.Publish = &clusterstate.PublishCell{Cell: op.Cell, Publish: clusterstate.Publish{Fence: op.Fence, OldHead: op.OldHead, Head: op.Head, Pending: op.Pending}, Receipt: "durable-probe-receipt"} + case "beat": + change.Beat = &clusterstate.BeatCell{Cell: op.Cell, Beat: clusterstate.Beat{Fence: op.Fence, Pending: op.Pending}} + case "release": + change.Release = &clusterstate.ReleaseCell{Cell: op.Cell, Fence: op.Fence, Quiet: op.Quiet} + case "revoke": + change.Revoke = &clusterstate.DeviceID{Device: op.Target} + case "set_vault": + change.SetVault = &clusterstate.VaultCAS{Old: op.OldVault, New: op.NewVault} + case "advance_time", "session_close": + change.AdvanceTime = &clusterstate.AdvanceTime{} + default: + return nil, fmt.Errorf("unsupported probe operation %s", op.Kind) + } + session, digest, sequence := op.Session, op.Digest, op.Sequence + if session == "" { + session = fmt.Sprintf("unsequenced-%d", id) + digest = session + sequence = 1 + } + return clusterstate.EncodeCommand(clusterstate.Command{Schema: clusterstate.SchemaVersion, Cluster: clusterUUID, Actor: op.Actor, Session: session, Sequence: sequence, RequestDigest: digest, Now: op.At, Change: change}) +} diff --git a/internal/raftsim/disk.go b/internal/raftsim/disk.go new file mode 100644 index 0000000000..dab70cd945 --- /dev/null +++ b/internal/raftsim/disk.go @@ -0,0 +1,208 @@ +package raftsim + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "slices" + + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +var wire = proto.MarshalOptions{Deterministic: true} +var errCorrupt = errors.New("raftsim: corrupt complete snapshot") + +type generation struct { + Pointer *pb.Snapshot + Chunks [][]byte + Complete bool +} +type image struct { + Hard *pb.HardState + Conf *pb.ConfState + Entries []*pb.Entry + Snapshot *pb.Snapshot + Applied uint64 + App ProbeApplication + Generations []generation +} + +func freshImage() image { return image{App: freshApp()} } +func clonePB[T proto.Message](p T) T { + if !p.ProtoReflect().IsValid() { + return p + } + return proto.Clone(p).(T) +} +func (d image) copy() image { + out := d + out.Hard = clonePB(d.Hard) + out.Conf = clonePB(d.Conf) + out.Snapshot = clonePB(d.Snapshot) + out.App = d.App.copy() + out.Entries = nil + for _, e := range d.Entries { + out.Entries = append(out.Entries, clonePB(e)) + } + out.Generations = nil + for _, g := range d.Generations { + out.Generations = append(out.Generations, cloneGeneration(g)) + } + return out +} +func cloneGeneration(g generation) generation { + out := generation{Pointer: clonePB(g.Pointer), Complete: g.Complete} + for _, b := range g.Chunks { + out.Chunks = append(out.Chunks, bytes.Clone(b)) + } + return out +} +func (d image) recovered() (raftcore.RecoveredState, error) { + for _, g := range d.Generations { + if g.Complete { + if _, e := decodeGeneration(g); e != nil { + return raftcore.RecoveredState{}, e + } + } + } + return raftcore.RecoveredState{HardState: clonePB(d.Hard), ConfState: clonePB(d.Conf), Snapshot: clonePB(d.Snapshot), Entries: d.copy().Entries, Applied: d.Applied}, nil +} +func (d *image) persist(p *raftcore.PersistBatch) error { + if p.Snapshot.GetMetadata().GetIndex() > 0 { + found := false + for _, g := range d.Generations { + if proto.Equal(g.Pointer, p.Snapshot) && g.Complete { + app, err := decodeGeneration(g) + if err != nil { + return err + } + d.App = app + d.Applied = p.Snapshot.Metadata.GetIndex() + d.Conf = clonePB(p.Snapshot.Metadata.ConfState) + d.Snapshot = clonePB(p.Snapshot) + found = true + break + } + } + if !found { + return fmt.Errorf("snapshot persist before complete transfer") + } + d.Entries = nil + } + if len(p.Entries) > 0 { + first := p.Entries[0].GetIndex() + d.Entries = slices.DeleteFunc(d.Entries, func(e *pb.Entry) bool { return e.GetIndex() >= first }) + for _, e := range p.Entries { + d.Entries = append(d.Entries, clonePB(e)) + } + } + if p.HardState != nil { + d.Hard = clonePB(p.HardState) + } + return nil +} +func (d image) term(index uint64) uint64 { + if d.Snapshot.GetMetadata().GetIndex() == index { + return d.Snapshot.GetMetadata().GetTerm() + } + for _, e := range d.Entries { + if e.GetIndex() == index { + return e.GetTerm() + } + } + return 0 +} +func rowHash(row raftsnapshot.Row) []byte { + h := sha256.New() + for _, b := range [][]byte{[]byte("records"), []byte(row.Namespace), []byte(row.Key), row.Value} { + var n [8]byte + binary.BigEndian.PutUint64(n[:], uint64(len(b))) + h.Write(n[:]) + h.Write(b) + } + return h.Sum(nil) +} +func (d image) checkpoint() (generation, error) { + data, err := json.Marshal(d.App) + if err != nil { + return generation{}, err + } + row := raftsnapshot.Row{Namespace: "state", Key: "probe", Value: data} + var stream bytes.Buffer + if err = raftsnapshot.WriteRow(&stream, row); err != nil { + return generation{}, err + } + payload := stream.Bytes() + conf, err := raftsnapshot.Configuration(d.Conf) + if err != nil { + return generation{}, err + } + manifest := raftsnapshot.Manifest{Version: raftsnapshot.Version, Cluster: clusterUUID, Protocol: uint64(raftcore.ProtocolVersion), Index: d.Applied, Term: d.term(d.Applied), Conf: conf, LogicalTime: d.App.Now, RecordCount: 1, RecordsHash: hex.EncodeToString(rowHash(row)), Bytes: uint64(len(payload)), Chunks: uint64((len(payload) + raftsnapshot.ChunkBytes - 1) / raftsnapshot.ChunkBytes), Digest: raftsnapshot.ID(payload)} + encoded, err := manifest.Encode() + if err != nil { + return generation{}, err + } + g := generation{Pointer: &pb.Snapshot{Data: encoded, Metadata: &pb.SnapshotMetadata{Index: proto.Uint64(manifest.Index), Term: proto.Uint64(manifest.Term), ConfState: clonePB(d.Conf)}}, Complete: true} + for len(payload) > 0 { + n := min(len(payload), raftsnapshot.ChunkBytes) + g.Chunks = append(g.Chunks, bytes.Clone(payload[:n])) + payload = payload[n:] + } + return g, nil +} +func decodeGeneration(g generation) (ProbeApplication, error) { + m, err := raftsnapshot.Decode(g.Pointer.GetData()) + if err != nil || !g.Complete || len(g.Chunks) != int(m.Chunks) { + return ProbeApplication{}, errCorrupt + } + conf, confErr := m.Configuration() + if confErr != nil || m.Cluster != clusterUUID || m.Protocol != uint64(raftcore.ProtocolVersion) || m.Index != g.Pointer.GetMetadata().GetIndex() || m.Term != g.Pointer.GetMetadata().GetTerm() || !proto.Equal(conf, g.Pointer.GetMetadata().GetConfState()) { + return ProbeApplication{}, errCorrupt + } + var payload bytes.Buffer + for i, b := range g.Chunks { + if len(b) > raftsnapshot.ChunkBytes || i < len(g.Chunks)-1 && len(b) != raftsnapshot.ChunkBytes { + return ProbeApplication{}, errCorrupt + } + payload.Write(b) + } + if uint64(payload.Len()) != m.Bytes || raftsnapshot.ID(payload.Bytes()) != m.Digest { + return ProbeApplication{}, errCorrupt + } + row, err := raftsnapshot.ReadRow(&payload) + if err != nil || row.Namespace != "state" || row.Key != "probe" || hex.EncodeToString(rowHash(row)) != m.RecordsHash || m.RecordCount != 1 { + return ProbeApplication{}, errCorrupt + } + if _, err = raftsnapshot.ReadRow(&payload); err != io.EOF { + return ProbeApplication{}, errCorrupt + } + var a ProbeApplication + if json.Unmarshal(row.Value, &a) != nil || a.Now != m.LogicalTime { + return ProbeApplication{}, errCorrupt + } + return a, nil +} +func (d *image) retain(g generation) { + for i, old := range d.Generations { + if bytes.Equal(old.Pointer.Data, g.Pointer.Data) { + d.Generations[i] = cloneGeneration(g) + return + } + } + d.Generations = append(d.Generations, cloneGeneration(g)) + for len(d.Generations) > 2 { + remove := 0 + if proto.Equal(d.Generations[0].Pointer, d.Snapshot) { + remove = 1 + } + d.Generations = slices.Delete(d.Generations, remove, remove+1) + } +} diff --git a/internal/raftsim/native_test.go b/internal/raftsim/native_test.go new file mode 100644 index 0000000000..5c90cafeed --- /dev/null +++ b/internal/raftsim/native_test.go @@ -0,0 +1,124 @@ +package raftsim + +import ( + "bytes" + "context" + "path/filepath" + "testing" + + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsim/model" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + "github.com/Agent-Field/codeaf/internal/raftstore" + "google.golang.org/protobuf/proto" +) + +// R3/R12/R13 compare the simulated durability protocol with the real native +// adapter. SQLite and the pure snapshot codec receive the exact production +// batches observed by the simulator, with no alternative consensus algorithm. +func TestR3R12R13SimulatedDiskMatchesNativeStoreAndSnapshot(t *testing.T) { + s := group(t, Config{Seed: 311, RecordDisk: true}) + create(t, s, "a") + leader := s.Leader() + submit(t, s, leader, model.Call{Kind: "acquire", Actor: "b", Cell: "a", Expected: observed(s.Nodes[leader].Stable.App.Cells["a"]), Force: true, Session: "exact", Sequence: 1, Digest: "opaque"}) + submit(t, s, leader, model.Call{Kind: "revoke", Actor: "owner", Target: "removed"}) + submit(t, s, leader, model.Call{Kind: "set_vault", Actor: "owner", NewVault: "opaque-vault"}) + ctx := context.Background() + stores := make([]*raftstore.SQLite, len(s.Nodes)) + applied := make([]uint64, len(s.Nodes)) + for node := range stores { + store, err := raftstore.Open(filepath.Join(t.TempDir(), "member.sqlite"), raftstore.Identity{ClusterUUID: clusterUUID, NodeID: uint64(node + 1), Protocol: raftcore.ProtocolVersion}) + if err != nil { + t.Fatal(err) + } + stores[node] = store + t.Cleanup(func() { store.Close() }) + } + key := raftstore.Key{Namespace: raftstore.State, Key: "probe"} + for _, tx := range s.journal { + store := stores[tx.Node] + if tx.Persist != nil { + if err := store.Persist(ctx, *tx.Persist); err != nil { + t.Fatal(err) + } + } + if tx.Apply != nil { + update := raftstore.Update{ExpectedApplied: applied[tx.Node], LogicalTime: tx.App.Now, Writes: []raftstore.Write{{Key: key, Value: encodedApp(tx.App)}}} + if err := store.Apply(ctx, *tx.Apply, update); err != nil { + t.Fatal(err) + } + applied[tx.Node] = tx.Apply.Index + } + } + for node, store := range stores { + recovered, err := store.Recover(ctx) + if err != nil { + t.Fatal(err) + } + d := s.Nodes[node].Stable + if recovered.Core.Applied != d.Applied || !proto.Equal(recovered.Core.HardState, d.Hard) || !proto.Equal(recovered.Core.ConfState, d.Conf) || len(recovered.Core.Entries) != len(d.Entries) { + t.Fatal("simulated durable metadata differs from native store") + } + for i, e := range d.Entries { + if !proto.Equal(e, recovered.Core.Entries[i]) { + t.Fatal("simulated log differs from native store") + } + } + rows, _, err := store.Read(ctx, d.Applied, []raftstore.Key{key}) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(rows[key], encodedApp(d.App)) { + t.Fatal("simulated application differs from native") + } + simulated, err := d.checkpoint() + if err != nil { + t.Fatal(err) + } + native, err := store.CreateSnapshot(ctx) + if err != nil { + t.Fatal(err) + } + if !proto.Equal(native, simulated.Pointer) { + t.Fatal("portable manifest differs from real native snapshot") + } + if err = store.Compact(ctx, d.Applied); err != nil { + t.Fatal(err) + } + // The portable simulated chunks must also be installable by the production + // store, including the exact retained result and revocation/vault state. + target, err := raftstore.Open(filepath.Join(t.TempDir(), "restored.sqlite"), raftstore.Identity{ClusterUUID: clusterUUID, NodeID: uint64(node + 1), Protocol: raftcore.ProtocolVersion}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { target.Close() }) + id, err := target.BeginSnapshot(ctx, simulated.Pointer.Data) + if err != nil { + t.Fatal(err) + } + for i, chunk := range simulated.Chunks { + if err = target.PutSnapshotChunk(ctx, id, uint64(i), chunk, raftsnapshot.ID(chunk)); err != nil { + t.Fatal(err) + } + } + pointer, err := target.CompleteSnapshot(ctx, id) + if err != nil { + t.Fatal(err) + } + if err = target.Persist(ctx, raftcore.PersistBatch{ID: 1, HardState: d.Hard, Snapshot: pointer}); err != nil { + t.Fatal(err) + } + restored, err := target.Recover(ctx) + if err != nil { + t.Fatal(err) + } + if restored.Core.Applied != d.Applied || !proto.Equal(restored.Core.ConfState, d.Conf) { + t.Fatal("native snapshot lost application/configuration metadata") + } + rows, _, err = target.Read(ctx, d.Applied, []raftstore.Key{key}) + if err != nil || !bytes.Equal(rows[key], encodedApp(d.App)) { + t.Fatal("native snapshot lost observable state", err) + } + } + finish(t, s) +} diff --git a/internal/raftsim/replay.go b/internal/raftsim/replay.go new file mode 100644 index 0000000000..b06684f048 --- /dev/null +++ b/internal/raftsim/replay.go @@ -0,0 +1,70 @@ +package raftsim + +import ( + "encoding/json" + "os" + "path/filepath" + + "github.com/Agent-Field/codeaf/internal/raftsim/model" + "github.com/anishathalye/porcupine" +) + +// Replay contains only explicit inputs. Generation seeds, flush order and +// message digests can be compared against Evidence after exact reproduction. +type Replay struct { + Config Config + Actions []Action +} +type Evidence struct { + Replay Replay + Trace []Trace + FlushTrace []Trace + History []model.Operation + Minimized []model.Operation + Check porcupine.CheckResult + Error string + Digest string + Events int +} + +func Run(replay Replay) (*Simulator, error) { + s, err := New(replay.Config) + if err != nil { + return s, err + } + for _, a := range replay.Actions { + s.Do(a) + } + return s, s.Err +} +func (s *Simulator) Evidence() Evidence { + history := s.History() + check := model.Check(history) + e := Evidence{Replay: Replay{Config: s.initial, Actions: s.Actions}, Trace: s.Trace, History: history, Check: check, Digest: s.Digest(), Events: s.Events} + if s.Err != nil { + e.Error = s.Err.Error() + } + for _, entry := range s.Trace { + switch entry.Phase { + case "persist-write", "persist-flush", "persist-ack", "apply-write", "apply-flush", "apply-ack", "reply": + e.FlushTrace = append(e.FlushTrace, entry) + } + } + if check == porcupine.Illegal { + e.Minimized = model.Minimize(history) + } + return e +} + +// WriteEvidence is opt-in host I/O, outside the deterministic scheduler. Only +// synthetic histories belong here; callers choose a disposable output folder. +func (s *Simulator) WriteEvidence(dir string) error { + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + b, err := json.MarshalIndent(s.Evidence(), "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, "evidence.json"), append(b, '\n'), 0600) +} diff --git a/internal/raftsim/simulator.go b/internal/raftsim/simulator.go new file mode 100644 index 0000000000..3410fbd4ae --- /dev/null +++ b/internal/raftsim/simulator.go @@ -0,0 +1,753 @@ +package raftsim + +import ( + "container/heap" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "slices" + + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsim/model" + "go.etcd.io/raft/v3" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +const clusterUUID = "raftsim-contract" +const MaxEvents = 2000 + +type Config struct { + PaddingBytes int + CutSnapshot bool + RecordDisk bool + Seed uint64 + Nodes int + MaxEvents int + // Fault rates are per thousand and apply only to network messages. + Drop int + Duplicate int + Delay int64 + CutNode int + CutPhase string + CutOccurrence int + Broken string +} +type Action struct { + Kind string + Node int + Other int + Count int + Value int64 + Request model.Call +} +type Trace struct { + Order int64 + Time int64 + Node int + Generation uint64 + Phase string + Batch uint64 + Digest string +} +type client struct { + History model.Operation + Node int + Bytes string + Done bool + Lost bool +} +type member struct { + SnapshotSource *origin + + Core *raftcore.Core + Stable image + Written *image + Torn []byte + Status raftcore.Status + Generation uint64 + Pending uint64 + Persisted uint64 + ApplyFlushed uint64 + Alive bool +} +type origin struct { + Node int + Generation uint64 +} +type event struct { + Target int + + Source origin + + At int64 + Serial uint64 + Node int + Generation uint64 + Kind string + Input raftcore.Input + Persist *raftcore.PersistBatch + Apply *raftcore.ApplyBatch + Results map[string]model.Outcome + Client int + Result model.Outcome + Snapshot *generation + Chunk int +} +type queue []event + +func (q queue) Len() int { return len(q) } +func (q queue) Less(i, j int) bool { + if q[i].At == q[j].At { + return q[i].Serial < q[j].Serial + } + return q[i].At < q[j].At +} +func (q queue) Swap(i, j int) { q[i], q[j] = q[j], q[i] } +func (q *queue) Push(x any) { *q = append(*q, x.(event)) } +func (q *queue) Pop() any { old := *q; e := old[len(old)-1]; *q = old[:len(old)-1]; return e } + +// Simulator owns one serial event queue. No wall time, sleep or background +// goroutine changes simulated state. Checker deadlines are outside this loop. +type diskRecord struct { + Node int + Persist *raftcore.PersistBatch + Apply *raftcore.ApplyBatch + App ProbeApplication +} +type Simulator struct { + ReadAckDelay map[int]int64 + + journal []diskRecord + + initial Config + Config Config + Nodes []*member + Time int64 + Events int + Trace []Trace + Actions []Action + Clients []client + Err error + CutHit bool + FailNext map[int]bool + Blocked map[[2]int]bool + Stalled map[int]int64 + catalog map[string]model.Call + queue queue + serial uint64 + order int64 + rng uint64 + leaders map[uint64]int + commits map[uint64]string + applications map[uint64]string + maxCommit []uint64 + maxApplied []uint64 + appGeneration []uint64 + maxFence map[string]uint64 + cutCount int +} + +func New(config Config) (*Simulator, error) { + if config.Nodes == 0 { + config.Nodes = 3 + } + if config.MaxEvents == 0 { + config.MaxEvents = MaxEvents + } + if config.PaddingBytes < 0 || config.PaddingBytes > 640<<10 || config.Nodes < 3 || config.Nodes > 5 || config.MaxEvents > MaxEvents || config.MaxEvents < 1 || config.Delay < 0 || config.Delay > 20 || config.Drop < 0 || config.Drop > 1000 || config.Duplicate < 0 || config.Duplicate > 1000 { + return nil, errors.New("raftsim: invalid bounded configuration") + } + s := &Simulator{initial: config, Config: config, ReadAckDelay: map[int]int64{}, rng: config.Seed + 1, FailNext: map[int]bool{}, Blocked: map[[2]int]bool{}, Stalled: map[int]int64{}, catalog: map[string]model.Call{}, leaders: map[uint64]int{}, commits: map[uint64]string{}, applications: map[uint64]string{}, maxFence: map[string]uint64{}, maxCommit: make([]uint64, config.Nodes), maxApplied: make([]uint64, config.Nodes), appGeneration: make([]uint64, config.Nodes)} + for i := 0; i < config.Nodes; i++ { + d := freshImage() + d.App.Padding = make([]byte, config.PaddingBytes) + s.Nodes = append(s.Nodes, &member{Stable: d}) + } + for i := range s.Nodes { + s.start(i) + } + return s, s.Err +} +func (s *Simulator) random(n uint64) uint64 { + s.rng = s.rng*6364136223846793005 + 1442695040888963407 + return s.rng % n +} +func (s *Simulator) stamp() int64 { s.order++; return s.order } +func (s *Simulator) fail(format string, args ...any) { + if s.Err == nil { + s.Err = fmt.Errorf(format, args...) + } +} +func (s *Simulator) schedule(e event) { s.serial++; e.Serial = s.serial; heap.Push(&s.queue, e) } +func (s *Simulator) later(node int, kind string) eBuilder { + return eBuilder{s: s, e: event{Node: node, Kind: kind, Generation: s.Nodes[node].Generation, At: s.Time + 1}} +} + +type eBuilder struct { + s *Simulator + e event +} + +func (b eBuilder) push() { b.s.schedule(b.e) } +func (s *Simulator) record(node int, phase string, batch uint64, digest string) { + s.Trace = append(s.Trace, Trace{s.stamp(), s.Time, node, s.Nodes[node].Generation, phase, batch, digest}) +} +func (s *Simulator) start(node int) { + n := s.Nodes[node] + recovered, err := n.Stable.recovered() + if err != nil { + s.fail("recovery: %v", err) + return + } + cfg := raftcore.Config{ClusterUUID: clusterUUID, NodeID: uint64(node + 1), Protocol: raftcore.ProtocolVersion, Seed: s.random(^uint64(0))} + if n.Stable.Hard == nil && len(n.Stable.Entries) == 0 && n.Stable.Snapshot == nil && node < 3 { + cfg.Bootstrap = []uint64{1, 2, 3} + } + n.Core, err = raftcore.New(cfg, recovered) + if err != nil { + s.fail("node %d recovery: %v", node, err) + return + } + n.Generation++ + n.Alive = true + n.Pending = 0 + n.Persisted = 0 + n.ApplyFlushed = 0 + n.Written = nil + n.Torn = nil + s.record(node, "restart", 0, fmt.Sprint(cfg.Seed)) + s.step(node, raftcore.Poll{}) +} +func (s *Simulator) crash(node int, power bool) { + n := s.Nodes[node] + sources := map[int]uint64{} + if n.SnapshotSource != nil { + sources[n.SnapshotSource.Node] = n.SnapshotSource.Generation + n.SnapshotSource = nil + } + for _, event := range s.queue { + if event.Node == node && event.Snapshot != nil && event.Kind != "snapshot-timeout" { + sources[event.Source.Node] = event.Source.Generation + } + } + ids := make([]int, 0, len(sources)) + for id := range sources { + ids = append(ids, id) + } + slices.Sort(ids) + for _, id := range ids { + s.reportSnapshot(origin{id, sources[id]}, node, false) + } + if !power && n.Written != nil { + n.Stable = n.Written.copy() + } + n.Alive = false + n.Core = nil + n.Written = nil + n.Torn = nil + s.record(node, "crash", n.Pending, fmt.Sprint(power)) +} +func (s *Simulator) step(node int, input raftcore.Input) { + if s.Err != nil || !s.Nodes[node].Alive { + return + } + n := s.Nodes[node] + e, err := n.Core.Step(input) + if errors.Is(err, raftcore.ErrBusy) { + b := s.later(node, "input") + b.e.Input = input + b.push() + return + } + if errors.Is(err, raftcore.ErrLeader) || errors.Is(err, raftcore.ErrRead) || errors.Is(err, raftcore.ErrTransfer) { + return + } + if err != nil { + s.fail("node %d %T: %v", node, input, err) + return + } + n.Status = e.Status + stateBytes, _ := json.Marshal(e.Status) + s.record(node, "step-"+fmt.Sprintf("%T", input), n.Pending, raftMessageDigest(stateBytes)) + if n.Status.Role == raft.StateLeader { + if old, ok := s.leaders[n.Status.Term]; ok && old != node { + s.fail("R4: two leaders in term %d", n.Status.Term) + } + s.leaders[n.Status.Term] = node + } + s.effects(node, e) +} +func (s *Simulator) effects(node int, e raftcore.Effects) { + n := s.Nodes[node] + if e.Persist != nil { + if len(e.Send) > 0 || e.Apply != nil || len(e.Reads) > 0 { + s.fail("R3: effects before flush") + } + n.Pending = e.Persist.ID + n.Persisted = 0 + n.ApplyFlushed = 0 + b := s.later(node, "persist-write") + if e.Persist.HardState == nil && len(e.Persist.Entries) == 0 && e.Persist.Snapshot.GetMetadata().GetIndex() == 0 { + // Empty Ready barriers have no bytes to flush. Their explicit + // completion still releases messages on the next scheduled event. + n.Persisted = e.Persist.ID + b.e.Kind = "persist-ack" + s.record(node, "persist-empty", e.Persist.ID, "") + } + b.e.Persist = e.Persist + b.push() + } + for _, out := range e.Send { + if n.Pending != 0 && n.Persisted != n.Pending { + s.fail("R3: send before flush") + } + m := &pb.Message{} + if err := proto.Unmarshal(out.Bytes, m); err != nil { + s.fail("message decode: %v", err) + return + } + s.record(node, "send-"+m.GetType().String(), n.Pending, raftMessageDigest(out.Bytes)) + target := int(out.To) - 1 + if target < 0 || target >= len(s.Nodes) { + s.fail("unknown destination") + return + } + if m.GetType() == pb.MsgSnap { + s.schedule(event{At: s.Time + 32, Node: node, Generation: n.Generation, Kind: "snapshot-timeout", Target: target, Snapshot: &generation{Pointer: clonePB(m.Snapshot)}}) + } + if s.Blocked[[2]int{node, target}] || int(s.random(1000)) < s.Config.Drop { + continue + } + copies := 1 + if int(s.random(1000)) < s.Config.Duplicate { + copies++ + } + for j := 0; j < copies; j++ { + b := s.later(target, "input") + b.e.At += int64(s.random(uint64(s.Config.Delay + 1))) + if m.GetType() == pb.MsgHeartbeatResp && len(m.Context) > 0 { + b.e.At += s.ReadAckDelay[target] + } + b.e.Input = raftcore.PeerMessage{Bytes: slices.Clone(out.Bytes)} + if m.GetType() == pb.MsgSnap { + g, ok := s.snapshotFor(node, m.Snapshot) + if !ok { + s.fail("missing durable snapshot source=%d pointer=%v generations=%d", node, m.Snapshot, len(n.Stable.Generations)) + continue + } + b.e.Snapshot = &g + b.e.Source = origin{node, n.Generation} + } + b.push() + } + } + if e.Apply != nil { + if n.Persisted != e.Apply.ID { + s.fail("R3: apply before flush") + } + b := s.later(node, "apply-write") + b.e.Apply = e.Apply + b.push() + } + for _, barrier := range e.Reads { + id := -1 + if _, err := fmt.Sscanf(barrier.RequestID, "read-%d", &id); err != nil || id < 0 || id >= len(s.Clients) { + s.fail("unknown read context") + return + } + if n.Stable.Applied < barrier.Index { + s.fail("R4: read before applied barrier") + } + s.reply(node, id, n.Stable.App.read(s.Clients[id].History.Input)) + } + if e.Persist == nil && e.Apply == nil { + n.Pending = 0 + } +} +func raftMessageDigest(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } +func (s *Simulator) reply(node, id int, out model.Outcome) { + b := s.later(node, "reply") + b.e.Client = id + b.e.Result = out + b.push() +} +func (s *Simulator) apply(node int, batch *raftcore.ApplyBatch) (image, map[string]model.Outcome) { + n := s.Nodes[node] + d := n.Stable.copy() + results := map[string]model.Outcome{} + for _, entry := range batch.Entries { + if entry.GetType() != pb.EntryNormal || len(entry.Data) == 0 { + continue + } + call, ok := s.catalog[string(entry.Data)] + if !ok { + s.fail("uncatalogued committed command") + return d, results + } + results[string(entry.Data)] = d.App.evaluate(call, s.Config.Broken == "fence") + } + d.Applied = batch.Index + d.Conf = clonePB(batch.ConfState) + return d, results +} +func (s *Simulator) process(e event) { + n := s.Nodes[e.Node] + if !n.Alive || e.Generation != n.Generation { + return + } + if until := s.Stalled[e.Node]; until > s.Time { + e.At = until + s.schedule(e) + return + } + batch := uint64(0) + if e.Persist != nil { + batch = e.Persist.ID + } + if e.Apply != nil { + batch = e.Apply.ID + } + s.record(e.Node, e.Kind, batch, "") + if e.Snapshot != nil && e.Kind != "snapshot-timeout" { + source := e.Source + n.SnapshotSource = &source + } + isSnapshot := e.Snapshot != nil || e.Persist != nil && e.Persist.Snapshot.GetMetadata().GetIndex() > 0 || e.Apply != nil && e.Apply.Snapshot.GetMetadata().GetIndex() > 0 + if !s.CutHit && e.Node == s.Config.CutNode && e.Kind == s.Config.CutPhase && (!s.Config.CutSnapshot || isSnapshot) { + s.cutCount++ + if s.cutCount >= max(1, s.Config.CutOccurrence) { + s.CutHit = true + s.crash(e.Node, true) + return + } + } + switch e.Kind { + case "snapshot-timeout": + if s.Nodes[e.Target].Stable.Applied < e.Snapshot.Pointer.GetMetadata().GetIndex() { + s.step(e.Node, raftcore.TransportResult{Peer: uint64(e.Target + 1), Snapshot: true, Success: false}) + } + case "input": + if e.Snapshot != nil && !s.hasComplete(e.Node, e.Snapshot.Pointer) { + if n.Written != nil { + e.At = s.Time + 1 + s.schedule(e) + return + } + e.Kind = "snapshot-begin" + e.At = s.Time + 1 + s.schedule(e) + return + } + if e.Snapshot != nil { + source := e.Source + n.SnapshotSource = &source + } + s.step(e.Node, e.Input) + case "snapshot-begin", "snapshot-chunk", "snapshot-complete": + s.snapshotEvent(e) + case "persist-write": + if s.FailNext[e.Node] { + delete(s.FailNext, e.Node) + _, err := n.Core.Step(raftcore.StorageFailed{BatchID: e.Persist.ID}) + if !errors.Is(err, raftcore.ErrStopped) { + s.fail("disk error did not stop core") + } + s.crash(e.Node, true) + return + } + d := n.Stable.copy() + if err := d.persist(e.Persist); err != nil { + s.fail("persist: %v", err) + return + } + n.Written = &d + if len(e.Persist.Entries) > 0 { + b, _ := wire.Marshal(e.Persist.Entries[0]) + n.Torn = slices.Clone(b[:len(b)/2]) + } + e.Kind = "persist-flush" + e.At = s.Time + 1 + s.schedule(e) + if s.Config.Broken == "flush" { + s.step(e.Node, raftcore.Persisted{BatchID: e.Persist.ID}) + } + case "persist-flush": + if n.Written == nil { + s.fail("flush without write") + return + } + n.Stable = n.Written.copy() + n.Written = nil + n.Torn = nil + n.Persisted = e.Persist.ID + if s.Config.RecordDisk { + s.journal = append(s.journal, diskRecord{Node: e.Node, Persist: e.Persist}) + } + if e.Persist.Snapshot.GetMetadata().GetIndex() > 0 && n.SnapshotSource != nil { + s.reportSnapshot(*n.SnapshotSource, e.Node, true) + n.SnapshotSource = nil + } + e.Kind = "persist-ack" + e.At = s.Time + 1 + s.schedule(e) + case "persist-ack": + s.step(e.Node, raftcore.Persisted{BatchID: e.Persist.ID}) + case "apply-write": + d, results := s.apply(e.Node, e.Apply) + n.Written = &d + e.Results = results + e.Kind = "apply-flush" + e.At = s.Time + 1 + s.schedule(e) + case "apply-flush": + if n.Written == nil { + s.fail("application flush without write") + return + } + n.Stable = n.Written.copy() + n.Written = nil + n.ApplyFlushed = e.Apply.ID + if s.Config.RecordDisk { + s.journal = append(s.journal, diskRecord{Node: e.Node, Apply: e.Apply, App: n.Stable.App.copy()}) + } + e.Kind = "apply-ack" + e.At = s.Time + 1 + s.schedule(e) + case "apply-ack": + if n.ApplyFlushed != e.Apply.ID { + s.fail("R3: application completion before flush") + } + s.step(e.Node, raftcore.Applied{BatchID: e.Apply.ID}) + for id := range s.Clients { + c := &s.Clients[id] + if c.Node != e.Node || c.Done || c.Lost { + continue + } + if out, ok := e.Results[c.Bytes]; ok { + s.reply(e.Node, id, out) + } + } + case "reply": + c := &s.Clients[e.Client] + if !c.Lost && !c.Done { + c.History.Output = e.Result + c.History.Return = s.stamp() + c.Done = true + } + default: + s.fail("unknown event %s", e.Kind) + } + s.invariants() +} +func (s *Simulator) invariants() { + for i, n := range s.Nodes { + d := n.Stable + commit := d.Hard.GetCommit() + if commit < s.maxCommit[i] || d.Applied < s.maxApplied[i] { + s.fail("R3: durable index regressed on node %d", i) + } + applicationChanged := s.maxApplied[i] != d.Applied || s.appGeneration[i] != n.Generation + s.maxCommit[i] = commit + s.maxApplied[i] = d.Applied + s.appGeneration[i] = n.Generation + for _, e := range d.Entries { + if e.GetIndex() > commit { + break + } + data, _ := wire.Marshal(e) + digest := raftMessageDigest(data) + if old, ok := s.commits[e.GetIndex()]; ok && old != digest { + s.fail("R3: committed log disagreement at %d", e.GetIndex()) + } + s.commits[e.GetIndex()] = digest + } + if d.Applied > 0 && applicationChanged { + b, _ := json.Marshal(d.App) + conf, _ := wire.Marshal(d.Conf) + digest := raftMessageDigest(append(b, conf...)) + if old, ok := s.applications[d.Applied]; ok && old != digest { + s.fail("R13: application disagreement at %d", d.Applied) + } + s.applications[d.Applied] = digest + } + for key, c := range d.App.Cells { + k := fmt.Sprintf("%d/%s", i, key) + if c.Lease.Fence < s.maxFence[k] { + s.fail("R2: fence regression") + } + s.maxFence[k] = c.Lease.Fence + } + } +} +func (s *Simulator) run(count int, until int64) { + for len(s.queue) > 0 && s.Err == nil && (count < 0 || count > 0) { + if s.queue[0].At > until { + return + } + if s.Events == s.Config.MaxEvents { + s.fail("event budget exhausted (%d)", s.Events) + return + } + e := heap.Pop(&s.queue).(event) + s.Time = max(s.Time, e.At) + s.Events++ + s.process(e) + if count > 0 { + count-- + } + } +} + +// Do records every external input for exact replay. Network deliveries and disk +// completions are derived from the recorded seed and these ordered actions. +func (s *Simulator) Do(a Action) int { + s.Actions = append(s.Actions, a) + s.stamp() + if s.Err != nil { + return -1 + } + switch a.Kind { + case "delay-read-acks": + s.ReadAckDelay[a.Node] = a.Value + case "arm-cut": + s.Config.CutSnapshot = a.Count != 0 + s.Config.CutNode = a.Node + s.Config.CutPhase = a.Request.Kind + s.Config.CutOccurrence = a.Other + s.cutCount = 0 + s.CutHit = false + case "drain": + s.run(-1, s.Time+max(1, a.Value)) + case "events": + s.run(a.Count, 1<<62) + case "node-ticks": + for j := 0; j < a.Count; j++ { + b := s.later(a.Node, "input") + b.e.Input = raftcore.Tick{} + b.push() + s.run(-1, s.Time+100) + } + case "ticks": + for j := 0; j < a.Count; j++ { + for i := range s.Nodes { + if s.Nodes[i].Alive { + b := s.later(i, "input") + b.e.Input = raftcore.Tick{} + b.push() + } + } + s.run(-1, s.Time+100) + } + case "crash": + s.crash(a.Node, a.Value != 0) + case "restart": + s.start(a.Node) + case "block": + s.Blocked[[2]int{a.Node, a.Other}] = a.Value != 0 + case "stall": + s.Stalled[a.Node] = s.Time + a.Value + case "disk-fail": + s.FailNext[a.Node] = true + case "network": + s.Config.Drop = a.Count + s.Config.Duplicate = a.Other + s.Config.Delay = a.Value + case "snapshot": + s.checkpoint(a.Node) + case "configure": + var change *pb.ConfChangeV2 + switch a.Count { + case 1: + change = &pb.ConfChangeV2{Changes: []*pb.ConfChangeSingle{{Type: pb.ConfChangeAddLearnerNode.Enum(), NodeId: proto.Uint64(uint64(a.Other + 1))}}} + case 2: + change = &pb.ConfChangeV2{Transition: pb.ConfChangeTransitionJointExplicit.Enum(), Changes: []*pb.ConfChangeSingle{{Type: pb.ConfChangeAddLearnerNode.Enum(), NodeId: proto.Uint64(3)}, {Type: pb.ConfChangeAddNode.Enum(), NodeId: proto.Uint64(4)}}} + case 3: + change = &pb.ConfChangeV2{} + default: + s.fail("unknown configuration action") + return -1 + } + data, _ := wire.Marshal(change) + s.step(a.Node, raftcore.Configure{Change: data}) + case "timeout": + c := &s.Clients[a.Node] + if !c.Done { + c.Lost = true + if c.History.Input.Kind == "read" || c.History.Input.Kind == "read_vault" { + s.step(c.Node, raftcore.CancelRead{RequestID: fmt.Sprintf("read-%d", a.Node)}) + } + c.History.Unknown = true + c.History.Return = s.order + } + case "submit": + id := len(s.Clients) + if id == 100 { + s.fail("history operation budget exceeded") + return -1 + } + call := a.Request + c := client{Node: a.Node, History: model.Operation{ID: id, Call: s.order, Input: call}} + if call.Kind == "read" || call.Kind == "read_vault" { + s.Clients = append(s.Clients, c) + if s.Config.Broken == "read" { + s.reply(a.Node, id, s.Nodes[a.Node].Stable.App.read(call)) + } else { + s.step(a.Node, raftcore.Read{RequestID: fmt.Sprintf("read-%d", id)}) + } + } else { + data, err := command(call, id) + if err != nil { + s.fail("command: %v", err) + return -1 + } + c.Bytes = string(data) + if previous, exists := s.catalog[c.Bytes]; exists && previous != call { + s.fail("ambiguous probe catalog command") + return -1 + } + s.catalog[c.Bytes] = call + s.Clients = append(s.Clients, c) + s.step(a.Node, raftcore.Propose{Command: data}) + } + return id + default: + s.fail("unknown action %s", a.Kind) + } + return -1 +} +func (s *Simulator) Drain() { s.Do(Action{Kind: "drain", Value: 1000}) } +func (s *Simulator) Ticks(count int) { s.Do(Action{Kind: "ticks", Count: count}) } +func (s *Simulator) Submit(node int, op model.Call) int { + return s.Do(Action{Kind: "submit", Node: node, Request: op}) +} +func (s *Simulator) Leader() int { + winner := -1 + var term uint64 + for i, n := range s.Nodes { + if n.Alive && n.Status.Role == raft.StateLeader && n.Status.Term >= term { + winner = i + term = n.Status.Term + } + } + return winner +} +func (s *Simulator) History() []model.Operation { + out := make([]model.Operation, len(s.Clients)) + for i, c := range s.Clients { + out[i] = c.History + if !c.Done { + out[i].Unknown = true + } + } + return out +} +func (s *Simulator) Digest() string { + b, _ := json.Marshal(struct { + Trace []Trace + History []model.Operation + }{s.Trace, s.History()}) + return raftMessageDigest(b) +} diff --git a/internal/raftsim/simulator_test.go b/internal/raftsim/simulator_test.go new file mode 100644 index 0000000000..ad248b8a02 --- /dev/null +++ b/internal/raftsim/simulator_test.go @@ -0,0 +1,693 @@ +package raftsim + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/raftsim/model" + "github.com/anishathalye/porcupine" + "go.etcd.io/raft/v3" +) + +var evidenceDir = flag.String("raftsim-evidence-dir", "", "write synthetic replay evidence outside the repository") + +func group(t *testing.T, c Config) *Simulator { + t.Helper() + s, err := New(c) + if err != nil { + t.Fatal(err) + } + failureDir := filepath.Join(t.TempDir(), "failure") + if *evidenceDir != "" { + failureDir = filepath.Join(*evidenceDir, t.Name()) + } + t.Cleanup(func() { + if t.Failed() { + if err := s.WriteEvidence(failureDir); err != nil { + t.Log(err) + } + t.Logf("exact seed/action/flush/history replay: %s", failureDir) + } + }) + s.Drain() + elect(t, s) + return s +} +func elect(t *testing.T, s *Simulator) int { + t.Helper() + for range 25 { + if s.Err != nil { + finish(t, s) + } + if l := s.Leader(); l >= 0 { + return l + } + s.Ticks(1) + } + t.Fatal("no leader within bounded healthy ticks") + return -1 +} +func finish(t *testing.T, s *Simulator) { + t.Helper() + result := model.Check(s.History()) + if s.Err != nil || result != porcupine.Ok { + dir := filepath.Join(t.TempDir(), "failure") + if *evidenceDir != "" { + dir = filepath.Join(*evidenceDir, t.Name()) + } + if err := s.WriteEvidence(dir); err != nil { + t.Log(err) + } + t.Fatalf("seed=%d events=%d core=%v history=%s evidence=%s", s.initial.Seed, s.Events, s.Err, result, dir) + } +} +func done(t *testing.T, s *Simulator, id int) model.Outcome { + t.Helper() + s.Drain() + for tries := 0; tries < 12 && !s.Clients[id].Done && s.Err == nil; tries++ { + s.Ticks(1) + } + if s.Err != nil { + finish(t, s) + } + if !s.Clients[id].Done { + t.Fatalf("request %d did not finish at seed %d", id, s.initial.Seed) + } + return s.Clients[id].History.Output +} +func submit(t *testing.T, s *Simulator, node int, op model.Call) model.Outcome { + t.Helper() + return done(t, s, s.Submit(node, op)) +} +func create(t *testing.T, s *Simulator, cell string) model.Cell { + t.Helper() + out := submit(t, s, s.Leader(), model.Call{Kind: "create", Actor: "owner", Cell: cell, Head: "head-0", At: 100}) + if out.Code != "ok" { + t.Fatal(out) + } + return out.Cell +} +func isolate(s *Simulator, node int, blocked bool) { + value := int64(0) + if blocked { + value = 1 + } + for i := range s.Nodes { + if i != node { + s.Do(Action{Kind: "block", Node: node, Other: i, Value: value}) + s.Do(Action{Kind: "block", Node: i, Other: node, Value: value}) + } + } +} +func freshLeader(t *testing.T, s *Simulator, old int) int { + t.Helper() + for range 25 { + s.Ticks(1) + if l := s.Leader(); l >= 0 && l != old { + return l + } + if s.Err != nil { + finish(t, s) + } + } + t.Fatal("healthy majority failed to elect") + return -1 +} + +// R1–R5/R13/R17 share one whole-identity history per seed, including unknown +// outcomes. Each run stays below 2,000 effect events and 100 operations. +func TestR1R2R3R4R5R13R17SeededFaultCampaign(t *testing.T) { + total, maxEvents := 0, 0 + for seed := uint64(0); seed < 100; seed++ { + t.Run(fmt.Sprintf("seed-%03d", seed), func(t *testing.T) { + s := group(t, Config{Seed: seed, Nodes: 4, Drop: 10, Duplicate: 100, Delay: 3}) + a := create(t, s, "a") + create(t, s, "b") + leader := s.Leader() + one := s.Submit(leader, model.Call{Kind: "acquire", Actor: "take-a", Cell: "a", Expected: a, Force: true, At: 110}) + two := s.Submit(leader, model.Call{Kind: "acquire", Actor: "take-b", Cell: "a", Expected: a, Force: seed%2 == 0, At: 90}) + s.Drain() + won, lost := done(t, s, one), done(t, s, two) + if won.Code == "lease_held" { + won, lost = lost, won + } + if won.Code != "ok" || lost.Code != "lease_held" || won.Cell.Fence != 2 { + t.Fatalf("R1: %+v %+v", won, lost) + } + for _, kind := range []string{"publish", "beat", "release"} { + out := submit(t, s, leader, model.Call{Kind: kind, Actor: "owner", Cell: "a", Fence: 1, OldHead: "wrong", Head: "displaced", At: 120}) + if out.Code != "fence_stale" { + t.Fatalf("R2 %s: %+v", kind, out) + } + } + // The explicit clock log includes backward samples and long sleep; it + // changes expiry semantics, never ticks or ownership fencing. + submit(t, s, leader, model.Call{Kind: "advance_time", Actor: "owner", At: 3_600_000}) + submit(t, s, leader, model.Call{Kind: "advance_time", Actor: "owner", At: -400}) + take := model.Call{Kind: "acquire", Actor: "retry", Cell: "b", Expected: observed(s.Nodes[leader].Stable.App.Cells["b"]), Force: true, At: 3_600_001, Session: "durable", Sequence: 1, Digest: "opaque-1"} + dropped := s.Submit(leader, take) + s.Do(Action{Kind: "timeout", Node: dropped}) + s.Drain() + first := s.Nodes[leader].Stable.App.Sessions["retry/durable"].Result + again := submit(t, s, leader, take) + if again != first || again.Code != "ok" || again.Cell.Fence != 2 { + t.Fatalf("R17: retry changed result %+v %+v", first, again) + } + conflict := take + conflict.Digest = "other-digest" + if submit(t, s, leader, conflict).Code != "conflict" { + t.Fatal("R17: conflicting digest succeeded") + } + // A minority leader may accept admission but can produce neither a fresh + // authority read nor a successful mutation after the majority moves on. + isolate(s, leader, true) + next := freshLeader(t, s, leader) + before := observed(s.Nodes[next].Stable.App.Cells["a"]) + moved := submit(t, s, next, model.Call{Kind: "acquire", Actor: "majority", Cell: "a", Expected: before, Force: true, At: 3_600_010}) + if moved.Code != "ok" { + t.Fatal(moved) + } + read := s.Submit(leader, model.Call{Kind: "read", Actor: "owner", Cell: "a"}) + write := s.Submit(leader, model.Call{Kind: "publish", Actor: won.Cell.Holder, Cell: "a", Fence: won.Cell.Fence, OldHead: "head-0", Head: "isolated", At: 3_600_020}) + s.Drain() + if s.Clients[read].Done || s.Clients[write].Done { + t.Fatal("R4: minority success") + } + s.Do(Action{Kind: "timeout", Node: read}) + s.Do(Action{Kind: "timeout", Node: write}) + s.Do(Action{Kind: "crash", Node: leader, Value: 1}) + s.Do(Action{Kind: "restart", Node: leader}) + isolate(s, leader, false) + s.Ticks(2) + s.Drain() + leader = s.Leader() + if out := submit(t, s, leader, model.Call{Kind: "read", Actor: "owner", Cell: "a"}); out.Cell != moved.Cell { + t.Fatal("R3: recovery lost winner") + } + // Snapshot/compaction keeps exact session results through a second crash. + s.Do(Action{Kind: "snapshot", Node: leader}) + s.Do(Action{Kind: "crash", Node: leader, Value: 1}) + s.Do(Action{Kind: "restart", Node: leader}) + s.Drain() + s.Ticks(2) + leader = s.Leader() + if leader < 0 { + leader = elect(t, s) + } + if out := submit(t, s, leader, take); out != first { + t.Fatal("R13/R17: compact/restart changed retry") + } + // Every seed also crosses the old/new-majority intersection. + // Node 4 starts empty, catches up (including the compacted prefix), + // then participates in the explicit joint swap. + s.Do(Action{Kind: "network"}) + s.Do(Action{Kind: "configure", Node: leader, Other: 3, Count: 1}) + s.Drain() + // A joining learner must be in the snapshot's ConfState. A + // pre-admission checkpoint cannot bootstrap that new member. + s.Do(Action{Kind: "snapshot", Node: leader}) + for attempt := 0; attempt < 20 && s.Nodes[3].Stable.Applied < s.Nodes[leader].Stable.Applied; attempt++ { + s.Ticks(1) + } + if s.Nodes[3].Stable.Applied < s.Nodes[leader].Stable.Applied { + t.Fatalf("R12: learner did not reach the promotion barrier: %v", s.Err) + } + s.Ticks(1) + s.Do(Action{Kind: "configure", Node: leader, Count: 2}) + s.Drain() + s.Do(Action{Kind: "configure", Node: leader, Count: 3}) + s.Drain() + s.Ticks(1) + if s.Leader() < 0 { + elect(t, s) + } + for _, n := range s.Nodes { + if !slices.Equal(n.Stable.Conf.GetVoters(), []uint64{1, 2, 4}) || len(n.Stable.Conf.GetVotersOutgoing()) > 0 { + t.Fatalf("R12: seeded joint swap did not finish: conf=%v status=%+v err=%v events=%d", n.Stable.Conf, n.Status, s.Err, s.Events) + } + } + finish(t, s) + total += s.Events + maxEvents = max(maxEvents, s.Events) + if seed == 42 { + replayed, replayErr := Run(s.Evidence().Replay) + if replayErr != nil || replayed.Digest() != s.Digest() { + t.Fatal("seed replay diverged") + } + } + }) + } + t.Logf("100 fixed seeds, %d events, maximum %d/2000 per seed", total, maxEvents) +} + +func TestR5HundredMinorityFlapsKeepHealthyTerm(t *testing.T) { + s := group(t, Config{Seed: 501}) + leader := s.Leader() + term := s.Nodes[leader].Status.Term + minority := (leader + 1) % 3 + for range 100 { + isolate(s, minority, true) + s.Do(Action{Kind: "node-ticks", Node: minority, Count: 7}) + isolate(s, minority, false) + s.Ticks(1) + if s.Nodes[leader].Status.Term != term || s.Nodes[leader].Status.Role != raft.StateLeader { + t.Fatal("R5: flapping minority displaced healthy leader") + } + } + finish(t, s) + t.Logf("100 flaps, %d events, stable term %d", s.Events, term) +} + +// These deliberate faults live only in the development harness. The same +// checker/ordering monitor must reject each, otherwise the gate proves little. +func TestR1R3R4BrokenFenceReadAndFlushAreDetected(t *testing.T) { + for _, variant := range []string{"fence", "read", "flush"} { + t.Run(variant, func(t *testing.T) { + s, err := New(Config{Seed: 77, Broken: variant}) + if err != nil { + t.Fatal(err) + } + s.Drain() + if variant != "flush" { + elect(t, s) + original := create(t, s, "a") + leader := s.Leader() + if variant == "fence" { + s.Submit(leader, model.Call{Kind: "acquire", Actor: "b", Cell: "a", Expected: original, Force: true}) + s.Submit(leader, model.Call{Kind: "acquire", Actor: "c", Cell: "a", Expected: original, Force: true}) + s.Drain() + } else { + isolate(s, leader, true) + next := freshLeader(t, s, leader) + submit(t, s, next, model.Call{Kind: "publish", Actor: "owner", Cell: "a", Fence: 1, OldHead: "head-0", Head: "new"}) + s.Submit(leader, model.Call{Kind: "read", Actor: "owner", Cell: "a"}) + s.Drain() + } + if model.Check(s.History()) != porcupine.Illegal { + t.Fatal("broken variant escaped independent checker") + } + if len(model.Minimize(s.History())) == 0 { + t.Fatal("missing minimized counterexample") + } + } else if s.Err == nil || !strings.Contains(s.Err.Error(), "before flush") { + t.Fatal("early acknowledgment escaped flush monitor", s.Err) + } + if *evidenceDir != "" { + if err := s.WriteEvidence(filepath.Join(*evidenceDir, "broken-"+variant)); err != nil { + t.Fatal(err) + } + } + replayed, replayErr := Run(s.Evidence().Replay) + if replayErr == nil && s.Err != nil || replayed.Digest() != s.Digest() || fmt.Sprint(replayed.Err) != fmt.Sprint(s.Err) { + t.Fatal("negative replay diverged") + } + }) + } +} + +func TestR3R12CrashEveryReadyBoundary(t *testing.T) { + phases := []string{"persist-write", "persist-flush", "persist-ack", "apply-write", "apply-flush", "apply-ack", "reply"} + for _, phase := range phases { + for node := 0; node < 3; node++ { + if phase == "reply" && node > 0 { + continue + } + t.Run(fmt.Sprintf("%s/%d", phase, node), func(t *testing.T) { + s := group(t, Config{Seed: 1001}) + create(t, s, "a") + cutNode := node + if phase == "reply" { + cutNode = s.Leader() + } + // Replay must start from the same initial config, with fault arming recorded. + s.Do(Action{Kind: "arm-cut", Node: cutNode, Other: 1, Request: model.Call{Kind: phase}}) + leader := s.Leader() + request := s.Submit(leader, model.Call{Kind: "publish", Actor: "owner", Cell: "a", Fence: 1, OldHead: "head-0", Head: "durable"}) + s.Drain() + if !s.CutHit { + t.Fatal("required cut was not reached") + } + s.Do(Action{Kind: "restart", Node: cutNode}) + s.Drain() + if !s.Clients[request].Done { + s.Do(Action{Kind: "timeout", Node: request}) + } + s.Ticks(2) + leader = s.Leader() + if leader < 0 { + leader = elect(t, s) + } + // Unknown mutation may be absent or present, and a read decides which. No + // invocation is removed just because its successful reply was lost. + submit(t, s, leader, model.Call{Kind: "read", Actor: "owner", Cell: "a"}) + finish(t, s) + }) + } + } +} + +func TestR3DiskFailuresStallsTornWritesAndOldGeneration(t *testing.T) { + s := group(t, Config{Seed: 801, Duplicate: 200, Delay: 4}) + create(t, s, "a") + leader := s.Leader() + follower := (leader + 1) % 3 + s.Do(Action{Kind: "disk-fail", Node: follower}) + s.Do(Action{Kind: "stall", Node: (leader + 2) % 3, Value: 20}) + out := submit(t, s, leader, model.Call{Kind: "publish", Actor: "owner", Cell: "a", Fence: 1, OldHead: "head-0", Head: "after-error"}) + if out.Code != "ok" || s.Nodes[follower].Alive { + t.Fatal("disk failure did not stop member") + } + s.Do(Action{Kind: "restart", Node: follower}) + s.Ticks(2) + s.Drain() + if s.Nodes[follower].Stable.App.Cells["a"].Head != "after-error" { + t.Fatal("failed follower did not recover") + } + finish(t, s) +} + +func TestR1R4R17ReplayFixtures(t *testing.T) { + files, err := filepath.Glob("testdata/*.json") + if err != nil || len(files) == 0 { + t.Fatal("missing replay fixtures", err) + } + for _, file := range files { + t.Run(filepath.Base(file), func(t *testing.T) { + data, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + var fixture struct { + History []model.Operation + Expected porcupine.CheckResult + } + if err = json.Unmarshal(data, &fixture); err != nil { + t.Fatal(err) + } + if got := model.Check(fixture.History); got != fixture.Expected { + t.Fatalf("fixture %s: %s", file, got) + } + }) + } +} + +// All joint fields must survive recovery; an outgoing member may lead while +// joint but must never regain a vote after the durable leave configuration. +func TestR12JointSwapCrashIntersections(t *testing.T) { + for _, stage := range []string{"enter", "leave"} { + for node := 0; node < 4; node++ { + for _, phase := range []string{"persist-write", "persist-flush", "persist-ack", "apply-write", "apply-flush", "apply-ack"} { + t.Run(fmt.Sprintf("%s/%d/%s", stage, node, phase), func(t *testing.T) { + s := group(t, Config{Seed: 1201, Nodes: 4}) + leader := s.Leader() + s.Do(Action{Kind: "configure", Node: leader, Other: 3, Count: 1}) + s.Drain() + s.Ticks(1) + if stage == "leave" { + s.Do(Action{Kind: "configure", Node: leader, Count: 2}) + s.Drain() + } + s.Do(Action{Kind: "arm-cut", Node: node, Other: 1, Request: model.Call{Kind: phase}}) + kind := 2 + if stage == "leave" { + kind = 3 + } + s.Do(Action{Kind: "configure", Node: leader, Count: kind}) + s.Drain() + if !s.CutHit { + t.Fatal("joint boundary not reached") + } + s.Do(Action{Kind: "restart", Node: node}) + s.Drain() + s.Ticks(2) + leader = s.Leader() + if leader < 0 { + leader = elect(t, s) + } + // A pre-commit crash leaves an unknown configuration + // result. Reconcile the durable state and resubmit the + // missing stage instead of treating admission as success. + for attempt := 0; attempt < 25; attempt++ { + conf := s.Nodes[leader].Stable.Conf + if slices.Equal(conf.GetVoters(), []uint64{1, 2, 4}) && len(conf.GetVotersOutgoing()) == 0 { + break + } + change := 2 + if len(conf.GetVotersOutgoing()) > 0 { + change = 3 + } + s.Do(Action{Kind: "configure", Node: leader, Count: change}) + s.Drain() + s.Ticks(1) + leader = s.Leader() + if leader < 0 { + leader = elect(t, s) + } + } + s.Ticks(1) + for i, n := range s.Nodes { + c := n.Stable.Conf + if !slices.Equal(c.GetVoters(), []uint64{1, 2, 4}) || len(c.GetVotersOutgoing()) != 0 || !slices.Equal(c.GetLearners(), []uint64{3}) { + t.Fatalf("node %d lost final configuration: %v", i, c) + } + } + s.Do(Action{Kind: "crash", Node: 2, Value: 1}) + s.Do(Action{Kind: "restart", Node: 2}) + s.Ticks(12) + if s.Nodes[2].Status.Role == raft.StateLeader { + t.Fatal("removed voter regained leadership") + } + finish(t, s) + }) + } + } + } +} + +func TestR12EveryOldNewMajorityIntersection(t *testing.T) { + for live := 0; live < 16; live++ { + t.Run(fmt.Sprintf("live-%04b", live), func(t *testing.T) { + s := group(t, Config{Seed: 1202, Nodes: 4}) + create(t, s, "a") + leader := s.Leader() + s.Do(Action{Kind: "configure", Node: leader, Other: 3, Count: 1}) + s.Drain() + s.Do(Action{Kind: "configure", Node: leader, Count: 2}) + s.Drain() + old, newCount := 0, 0 + for i := 0; i < 4; i++ { + if live&(1<= 2 && newCount >= 2 + s.Ticks(15) + if quorum && s.Leader() < 0 { + elect(t, s) + } + node := s.Leader() + if node < 0 { + for i := range s.Nodes { + if live&(1<= 0 { + request = s.Submit(node, model.Call{Kind: "publish", Actor: "owner", Cell: "a", Fence: 1, OldHead: "head-0", Head: "joint-result"}) + s.Drain() + s.Ticks(2) + } + if quorum && (request < 0 || !s.Clients[request].Done || s.Clients[request].History.Output.Code != "ok") { + t.Fatal("joint quorum could not commit") + } + if !quorum && request >= 0 && s.Clients[request].Done { + t.Fatal("joint change accepted only one side's majority") + } + if request >= 0 && !s.Clients[request].Done { + s.Do(Action{Kind: "timeout", Node: request}) + } + for i, n := range s.Nodes { + if !n.Alive { + s.Do(Action{Kind: "restart", Node: i}) + } + } + s.Ticks(12) + s.Drain() + for _, n := range s.Nodes { + if len(n.Stable.Conf.GetVotersOutgoing()) != 3 { + t.Fatal("recovery silently left joint state") + } + } + finish(t, s) + }) + } +} + +func TestR3AcknowledgedPublishSurvivesLeaderAndSecondMemberPowerLoss(t *testing.T) { + s := group(t, Config{Seed: 303}) + create(t, s, "a") + leader := s.Leader() + published := submit(t, s, leader, model.Call{Kind: "publish", Actor: "owner", Cell: "a", Fence: 1, OldHead: "head-0", Head: "retained"}) + other := (leader + 1) % 3 + s.Do(Action{Kind: "crash", Node: leader, Value: 1}) + s.Do(Action{Kind: "crash", Node: other, Value: 1}) + s.Do(Action{Kind: "restart", Node: other}) + s.Drain() + fallback := elect(t, s) + if out := submit(t, s, fallback, model.Call{Kind: "read", Actor: "owner", Cell: "a"}); out != published { + t.Fatal("acknowledged head/fence lost after second-member power recovery") + } + s.Do(Action{Kind: "restart", Node: leader}) + s.Ticks(2) + if s.Nodes[leader].Stable.App.Cells["a"].Head != "retained" { + t.Fatal("old leader lost acknowledged head") + } + finish(t, s) +} + +func TestR4DelayedOldReadAcksCannotAuthorizeAFreshMinorityRead(t *testing.T) { + s := group(t, Config{Seed: 404}) + create(t, s, "a") + old := s.Leader() + s.Do(Action{Kind: "delay-read-acks", Node: old, Value: 1000}) + before := s.Submit(old, model.Call{Kind: "read", Actor: "owner", Cell: "a"}) + s.Do(Action{Kind: "drain", Value: 20}) + isolate(s, old, true) + next := -1 + // Only the reachable majority's clocks advance. The isolated old leader + // retains its volatile role while old barrier acknowledgments are delayed. + for attempt := 0; attempt < 25 && next < 0; attempt++ { + for node := range s.Nodes { + if node != old { + s.Do(Action{Kind: "node-ticks", Node: node, Count: 1}) + } + } + for node, n := range s.Nodes { + if node != old && n.Status.Role == raft.StateLeader { + next = node + } + } + } + if next < 0 { + t.Fatal("reachable majority failed to elect") + } + changed := s.Submit(next, model.Call{Kind: "publish", Actor: "owner", Cell: "a", Fence: 1, OldHead: "head-0", Head: "new"}) + for attempts := 0; attempts < 150 && !s.Clients[changed].Done; attempts++ { + s.Do(Action{Kind: "events", Count: 1}) + } + if !s.Clients[changed].Done { + t.Fatal("reachable majority failed to publish") + } + if s.Clients[before].Done { + t.Fatal("fixture delivered held old acknowledgments too soon") + } + fresh := s.Submit(old, model.Call{Kind: "read", Actor: "owner", Cell: "a"}) + s.Drain() + if s.Clients[fresh].Done { + t.Fatal("old quorum contexts authorized a fresh minority read") + } + s.Do(Action{Kind: "timeout", Node: fresh}) + finish(t, s) +} + +func TestR3PowerLossDiscardsWrittenAndTornUncommittedSuffix(t *testing.T) { + s := group(t, Config{Seed: 330}) + create(t, s, "a") + leader := s.Leader() + baseline := s.Nodes[leader].Stable.copy() + request := s.Submit(leader, model.Call{Kind: "publish", Actor: "owner", Cell: "a", Fence: 1, OldHead: "head-0", Head: "unflushed"}) + for i := 0; i < 100 && len(s.Nodes[leader].Torn) == 0; i++ { + s.Do(Action{Kind: "events", Count: 1}) + } + n := s.Nodes[leader] + if n.Written == nil || len(n.Torn) == 0 || n.Stable.Hard.GetCommit() != baseline.Hard.GetCommit() || s.Clients[request].Done { + t.Fatal("fixture did not stop between write and flush") + } + s.Do(Action{Kind: "crash", Node: leader, Value: 1}) + if n.Written != nil || len(n.Torn) != 0 || n.Stable.Hard.GetCommit() != baseline.Hard.GetCommit() || n.Stable.App.Cells["a"].Head != "head-0" { + t.Fatal("power loss retained unflushed or partial bytes") + } + s.Do(Action{Kind: "timeout", Node: request}) + s.Do(Action{Kind: "restart", Node: leader}) + s.Drain() + s.Ticks(2) + if n.Stable.App.Cells["a"].Head != "head-0" { + t.Fatal("stale-generation flush completed the lost suffix") + } + finish(t, s) +} + +func TestR3R12ProcessDeathRetainsCompleteWrittenTransactionForReplay(t *testing.T) { + s := group(t, Config{Seed: 331}) + create(t, s, "a") + leader := s.Leader() + request := s.Submit(leader, model.Call{Kind: "publish", Actor: "owner", Cell: "a", Fence: 1, OldHead: "head-0", Head: "process-retained"}) + for i := 0; i < 100 && s.Nodes[leader].Written == nil; i++ { + s.Do(Action{Kind: "events", Count: 1}) + } + n := s.Nodes[leader] + if n.Written == nil || len(n.Written.Entries) <= len(n.Stable.Entries) { + t.Fatal("fixture did not reach a complete written append") + } + written := n.Written.copy() + s.Do(Action{Kind: "crash", Node: leader, Value: 0}) + if len(n.Stable.Entries) != len(written.Entries) { + t.Fatal("process death lost the written transaction") + } + s.Do(Action{Kind: "timeout", Node: request}) + s.Do(Action{Kind: "restart", Node: leader}) + s.Drain() + s.Ticks(2) + finish(t, s) +} + +func TestR1R3R4CounterexampleReplayFixtures(t *testing.T) { + files, err := filepath.Glob("testdata/replay/*.json") + if err != nil || len(files) != 3 { + t.Fatal("missing simulator replay fixtures", err) + } + for _, file := range files { + t.Run(filepath.Base(file), func(t *testing.T) { + data, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + var fixture struct { + Requirement string + Replay Replay + Expected string + } + if err = json.Unmarshal(data, &fixture); err != nil { + t.Fatal(err) + } + s, runErr := Run(fixture.Replay) + if fixture.Requirement == "R3" { + if runErr == nil || !strings.Contains(runErr.Error(), fixture.Expected) { + t.Fatal("flush counterexample escaped", runErr) + } + } else if runErr != nil || string(model.Check(s.History())) != fixture.Expected { + t.Fatal("history counterexample escaped", runErr) + } + again, againErr := Run(fixture.Replay) + if s.Digest() != again.Digest() || fmt.Sprint(runErr) != fmt.Sprint(againErr) { + t.Fatal("stored replay was not deterministic") + } + }) + } +} diff --git a/internal/raftsim/snapshot.go b/internal/raftsim/snapshot.go new file mode 100644 index 0000000000..b8bb43ea1a --- /dev/null +++ b/internal/raftsim/snapshot.go @@ -0,0 +1,110 @@ +package raftsim + +import ( + "bytes" + "fmt" + "slices" + + "github.com/Agent-Field/codeaf/internal/raftcore" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +func (s *Simulator) checkpoint(node int) { + n := s.Nodes[node] + if !n.Alive || n.Written != nil { + s.fail("checkpoint requires an idle actor") + return + } + g, err := n.Stable.checkpoint() + if err != nil { + s.fail("checkpoint: %v", err) + return + } + // Publication is one durable boundary. Chunk/install crash cuts are exercised + // separately, and the portable decoder validates every complete generation. + n.Stable.retain(g) + n.Stable.Snapshot = clonePB(g.Pointer) + n.Stable.Entries = slices.DeleteFunc(n.Stable.Entries, func(e *pb.Entry) bool { return e.GetIndex() <= n.Stable.Applied }) + s.record(node, "compact", 0, raftMessageDigest(g.Pointer.Data)) + s.step(node, raftcore.CompactSnapshot{Snapshot: g.Pointer}) + s.invariants() +} +func (s *Simulator) snapshotFor(node int, pointer *pb.Snapshot) (generation, bool) { + for _, g := range s.Nodes[node].Stable.Generations { + if proto.Equal(g.Pointer, pointer) && g.Complete { + return cloneGeneration(g), true + } + } + return generation{}, false +} +func (s *Simulator) hasComplete(node int, pointer *pb.Snapshot) bool { + _, ok := s.snapshotFor(node, pointer) + return ok +} +func (s *Simulator) snapshotEvent(e event) { + n := s.Nodes[e.Node] + if n.Written != nil { + e.At = s.Time + 1 + s.schedule(e) + return + } + g := e.Snapshot + if g.Pointer.Metadata.GetIndex() <= n.Stable.Applied { + return + } + index := -1 + for i, old := range n.Stable.Generations { + if bytes.Equal(old.Pointer.Data, g.Pointer.Data) { + index = i + break + } + } + switch e.Kind { + case "snapshot-begin": + if index < 0 { + n.Stable.Generations = append(n.Stable.Generations, generation{Pointer: clonePB(g.Pointer)}) + } + e.Kind = "snapshot-chunk" + e.Chunk = 0 + case "snapshot-chunk": + if index < 0 { + s.fail("missing snapshot stage") + return + } + stage := &n.Stable.Generations[index] + if len(stage.Chunks) == e.Chunk { + stage.Chunks = append(stage.Chunks, bytes.Clone(g.Chunks[e.Chunk])) + } + if len(stage.Chunks) <= e.Chunk || !bytes.Equal(stage.Chunks[e.Chunk], g.Chunks[e.Chunk]) { + s.fail("conflicting snapshot chunk") + return + } + e.Chunk++ + if e.Chunk == len(g.Chunks) { + e.Kind = "snapshot-complete" + } + case "snapshot-complete": + if index < 0 { + s.fail("missing snapshot completion") + return + } + stage := n.Stable.Generations[index] + stage.Complete = true + if _, err := decodeGeneration(stage); err != nil { + s.fail("complete snapshot: %v", err) + return + } + n.Stable.retain(stage) + e.Kind = "input" + default: + s.fail("snapshot phase %s", fmt.Sprint(e.Kind)) + return + } + e.At = s.Time + 1 + s.schedule(e) +} + +func (s *Simulator) reportSnapshot(source origin, target int, success bool) { + s.schedule(event{At: s.Time + 1, Node: source.Node, Generation: source.Generation, Kind: "input", Input: raftcore.TransportResult{Peer: uint64(target + 1), Snapshot: true, Success: success}}) +} diff --git a/internal/raftsim/snapshot_test.go b/internal/raftsim/snapshot_test.go new file mode 100644 index 0000000000..cb71bb251a --- /dev/null +++ b/internal/raftsim/snapshot_test.go @@ -0,0 +1,130 @@ +package raftsim + +import ( + "bytes" + "encoding/json" + "fmt" + "slices" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/raftsim/model" + "google.golang.org/protobuf/proto" +) + +func snapshotGroup(t *testing.T) *Simulator { + t.Helper() + s := group(t, Config{Seed: 1301, Nodes: 4, PaddingBytes: 400 << 10}) + leader := s.Leader() + s.Do(Action{Kind: "configure", Node: leader, Other: 3, Count: 1}) + s.Drain() + create(t, s, "a") + create(t, s, "b") + createSession := model.Call{Kind: "acquire", Actor: "b", Cell: "a", Expected: observed(s.Nodes[leader].Stable.App.Cells["a"]), Force: true, Session: "retained", Sequence: 1, Digest: "same"} + submit(t, s, leader, createSession) + submit(t, s, leader, model.Call{Kind: "set_vault", Actor: "owner", NewVault: "opaque-vault"}) + submit(t, s, leader, model.Call{Kind: "revoke", Actor: "owner", Target: "removed"}) + isolate(s, 3, true) + for i := 0; i < 3; i++ { + s.Do(Action{Kind: "snapshot", Node: i}) + } + submit(t, s, leader, model.Call{Kind: "publish", Actor: "b", Cell: "a", Fence: 2, OldHead: "head-0", Head: "after-snapshot"}) + for i := 0; i < 3; i++ { + s.Do(Action{Kind: "snapshot", Node: i}) + } + return s +} + +func TestR13PartialCorruptChunksAndBothRetainedGenerations(t *testing.T) { + s := snapshotGroup(t) + leader := s.Leader() + d := s.Nodes[leader].Stable + if len(d.Generations) != 2 { + t.Fatal("must keep two complete generations") + } + for gi, g := range d.Generations { + if len(g.Chunks) < 2 { + t.Fatal("fixture must cross chunk boundaries") + } + original, err := decodeGeneration(g) + if err != nil { + t.Fatal(err) + } + if len(original.Padding) != 400<<10 || !original.Revoked["removed"] || original.Vault != "opaque-vault" || original.Sessions["b/retained"].Result.Cell.Fence != 2 { + t.Fatal("snapshot lost observable records") + } + for chunk := range g.Chunks { + t.Run(fmt.Sprintf("generation-%d/chunk-%d", gi, chunk), func(t *testing.T) { + for _, kind := range []string{"missing", "corrupt"} { + broken := cloneGeneration(g) + if kind == "missing" { + broken.Chunks = slices.Delete(broken.Chunks, chunk, chunk+1) + } else { + broken.Chunks[chunk][0] ^= 1 + } + if _, err := decodeGeneration(broken); err == nil { + t.Fatal("accepted", kind) + } + corrupted := d.copy() + corrupted.Generations[gi] = broken + if _, err := corrupted.recovered(); err == nil { + t.Fatal("corrupt complete authority was repaired or ignored") + } + } + partial := d.copy() + partial.Generations = append(partial.Generations, generation{Pointer: clonePB(g.Pointer), Chunks: g.Chunks[:chunk], Complete: false}) + before := partial.App.copy() + if _, err := partial.recovered(); err != nil { + t.Fatal(err) + } + if !bytes.Equal(encodedApp(before), encodedApp(partial.App)) { + t.Fatal("partial snapshot changed application authority") + } + }) + } + } + finish(t, s) +} +func encodedApp(a ProbeApplication) []byte { b, _ := json.Marshal(a); return b } + +func TestR13CrashEachSnapshotChunkAndInstallBoundary(t *testing.T) { + cases := []struct { + phase string + occurrence int + }{{"snapshot-begin", 1}, {"snapshot-chunk", 1}, {"snapshot-chunk", 2}, {"snapshot-chunk", 3}, {"snapshot-complete", 1}, {"persist-write", 1}, {"persist-flush", 1}, {"persist-ack", 1}, {"apply-write", 1}, {"apply-flush", 1}, {"apply-ack", 1}} + for _, test := range cases { + t.Run(fmt.Sprintf("%s-%d", test.phase, test.occurrence), func(t *testing.T) { + s := snapshotGroup(t) + leader := s.Leader() + target := 3 + before := encodedApp(s.Nodes[target].Stable.App) + s.Do(Action{Kind: "arm-cut", Node: target, Other: test.occurrence, Count: 1, Request: model.Call{Kind: test.phase}}) + isolate(s, target, false) + s.Ticks(2) + s.Drain() + if !s.CutHit { + t.Fatalf("snapshot cut not reached: %s #%d", test.phase, test.occurrence) + } + if strings.HasPrefix(test.phase, "snapshot-") && !bytes.Equal(before, encodedApp(s.Nodes[target].Stable.App)) { + t.Fatal("incomplete transfer changed application") + } + s.Do(Action{Kind: "restart", Node: target}) + s.Ticks(3) + s.Drain() + if s.Err != nil { + finish(t, s) + } + if !bytes.Equal(encodedApp(s.Nodes[target].Stable.App), encodedApp(s.Nodes[leader].Stable.App)) { + t.Logf("leader=%d target index=%d status=%+v leader index=%d status=%+v queue=%d core=%v", leader, s.Nodes[target].Stable.Applied, s.Nodes[target].Status, s.Nodes[leader].Stable.Applied, s.Nodes[leader].Status, len(s.queue), s.Err) + for _, trace := range s.Trace[max(0, len(s.Trace)-80):] { + t.Logf("%+v", trace) + } + t.Fatal("snapshot recovery lost state/fences/revocations/dedup") + } + if !proto.Equal(s.Nodes[target].Stable.Conf, s.Nodes[leader].Stable.Conf) { + t.Fatal("snapshot recovery lost learner configuration") + } + finish(t, s) + }) + } +} diff --git a/internal/raftsim/testdata/r1-double-take.json b/internal/raftsim/testdata/r1-double-take.json new file mode 100644 index 0000000000..32316f0de1 --- /dev/null +++ b/internal/raftsim/testdata/r1-double-take.json @@ -0,0 +1,77 @@ +{ + "History": [ + { + "ID": 0, + "Call": 1, + "Return": 2, + "Input": { + "Kind": "create", + "Actor": "a", + "Cell": "c", + "Head": "h" + }, + "Output": { + "Code": "ok", + "Cell": { + "Holder": "a", + "Fence": 1, + "Head": "h", + "Until": 90000 + } + } + }, + { + "ID": 1, + "Call": 3, + "Return": 6, + "Input": { + "Kind": "acquire", + "Actor": "b", + "Cell": "c", + "Expected": { + "Holder": "a", + "Fence": 1, + "Head": "h", + "Until": 90000 + }, + "Force": true + }, + "Output": { + "Code": "ok", + "Cell": { + "Holder": "b", + "Fence": 2, + "Head": "h", + "Until": 90000 + } + } + }, + { + "ID": 2, + "Call": 4, + "Return": 7, + "Input": { + "Kind": "acquire", + "Actor": "d", + "Cell": "c", + "Expected": { + "Holder": "a", + "Fence": 1, + "Head": "h", + "Until": 90000 + }, + "Force": true + }, + "Output": { + "Code": "ok", + "Cell": { + "Holder": "d", + "Fence": 2, + "Head": "h", + "Until": 90000 + } + } + } + ], + "Expected": "Illegal" +} diff --git a/internal/raftsim/testdata/r3-timeout-later-commit.json b/internal/raftsim/testdata/r3-timeout-later-commit.json new file mode 100644 index 0000000000..0958ae6bf4 --- /dev/null +++ b/internal/raftsim/testdata/r3-timeout-later-commit.json @@ -0,0 +1,37 @@ +{ + "History": [ + { + "ID": 0, + "Call": 1, + "Return": 2, + "Input": { + "Kind": "create", + "Actor": "a", + "Cell": "c", + "Head": "h" + }, + "Output": {}, + "Unknown": true + }, + { + "ID": 1, + "Call": 3, + "Return": 4, + "Input": { + "Kind": "read", + "Actor": "a", + "Cell": "c" + }, + "Output": { + "Code": "ok", + "Cell": { + "Holder": "a", + "Fence": 1, + "Head": "h", + "Until": 90000 + } + } + } + ], + "Expected": "Ok" +} diff --git a/internal/raftsim/testdata/r4-stale-read.json b/internal/raftsim/testdata/r4-stale-read.json new file mode 100644 index 0000000000..d7aca3498a --- /dev/null +++ b/internal/raftsim/testdata/r4-stale-read.json @@ -0,0 +1,66 @@ +{ + "History": [ + { + "ID": 0, + "Call": 1, + "Return": 2, + "Input": { + "Kind": "create", + "Actor": "a", + "Cell": "c", + "Head": "h" + }, + "Output": { + "Code": "ok", + "Cell": { + "Holder": "a", + "Fence": 1, + "Head": "h", + "Until": 90000 + } + } + }, + { + "ID": 1, + "Call": 3, + "Return": 4, + "Input": { + "Kind": "publish", + "Actor": "a", + "Cell": "c", + "Fence": 1, + "OldHead": "h", + "Head": "new" + }, + "Output": { + "Code": "ok", + "Cell": { + "Holder": "a", + "Fence": 1, + "Head": "new", + "Until": 90000 + } + } + }, + { + "ID": 2, + "Call": 5, + "Return": 6, + "Input": { + "Kind": "read", + "Actor": "a", + "Cell": "c" + }, + "Output": { + "Code": "ok", + "Cell": { + "Holder": "a", + "Fence": 1, + "Head": "h", + "Until": 90000 + } + } + } + ], + "Expected": "Illegal" +} diff --git a/internal/raftsim/testdata/replay/fence.json b/internal/raftsim/testdata/replay/fence.json new file mode 100644 index 0000000000..38f3fd5450 --- /dev/null +++ b/internal/raftsim/testdata/replay/fence.json @@ -0,0 +1,499 @@ +{ + "Requirement": "R1", + "Replay": { + "Config": { + "PaddingBytes": 0, + "CutSnapshot": false, + "RecordDisk": false, + "Seed": 77, + "Nodes": 3, + "MaxEvents": 2000, + "Drop": 0, + "Duplicate": 0, + "Delay": 0, + "CutNode": 0, + "CutPhase": "", + "CutOccurrence": 0, + "Broken": "fence" + }, + "Actions": [ + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "create", + "Actor": "owner", + "Cell": "a", + "At": 100, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "head-0", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "acquire", + "Actor": "b", + "Cell": "a", + "At": 0, + "Expected": { + "Holder": "owner", + "Fence": 1, + "Until": 90100, + "Head": "head-0", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": true, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "acquire", + "Actor": "c", + "Cell": "a", + "At": 0, + "Expected": { + "Holder": "owner", + "Fence": 1, + "Until": 90100, + "Head": "head-0", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": true, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + } + ] + }, + "Expected": "Illegal" +} diff --git a/internal/raftsim/testdata/replay/flush.json b/internal/raftsim/testdata/replay/flush.json new file mode 100644 index 0000000000..abff2c316f --- /dev/null +++ b/internal/raftsim/testdata/replay/flush.json @@ -0,0 +1,57 @@ +{ + "Requirement": "R3", + "Replay": { + "Config": { + "PaddingBytes": 0, + "CutSnapshot": false, + "RecordDisk": false, + "Seed": 77, + "Nodes": 3, + "MaxEvents": 2000, + "Drop": 0, + "Duplicate": 0, + "Delay": 0, + "CutNode": 0, + "CutPhase": "", + "CutOccurrence": 0, + "Broken": "flush" + }, + "Actions": [ + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + } + ] + }, + "Expected": "before flush" +} diff --git a/internal/raftsim/testdata/replay/read.json b/internal/raftsim/testdata/replay/read.json new file mode 100644 index 0000000000..ee898e7268 --- /dev/null +++ b/internal/raftsim/testdata/replay/read.json @@ -0,0 +1,941 @@ +{ + "Requirement": "R4", + "Replay": { + "Config": { + "PaddingBytes": 0, + "CutSnapshot": false, + "RecordDisk": false, + "Seed": 77, + "Nodes": 3, + "MaxEvents": 2000, + "Drop": 0, + "Duplicate": 0, + "Delay": 0, + "CutNode": 0, + "CutPhase": "", + "CutOccurrence": 0, + "Broken": "read" + }, + "Actions": [ + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "create", + "Actor": "owner", + "Cell": "a", + "At": 100, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "head-0", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "block", + "Node": 0, + "Other": 1, + "Count": 0, + "Value": 1, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "block", + "Node": 1, + "Other": 0, + "Count": 0, + "Value": 1, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "block", + "Node": 0, + "Other": 2, + "Count": 0, + "Value": 1, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "block", + "Node": 2, + "Other": 0, + "Count": 0, + "Value": 1, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 1, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "publish", + "Actor": "owner", + "Cell": "a", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 1, + "Head": "new", + "OldHead": "head-0", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "read", + "Actor": "owner", + "Cell": "a", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + } + ] + }, + "Expected": "Illegal" +} From 232cafb680ee2e667d08292cf3b3d49c4d51a6f8 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Fri, 23 Oct 2026 23:58:31 -0400 Subject: [PATCH 013/103] Wait for durable results and repeated election attempts in fault campaigns. A lost proposal can still be uncommitted after draining messages, and randomized elections can split votes beyond a fixed 25-tick wait. Preserve exact replay regressions, check each election timeout separately, and run 500 seeds by default with an explicit larger-soak count. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/raftsim/README.md | 23 +- internal/raftsim/campaign_test.go | 134 + internal/raftsim/simulator_test.go | 63 +- .../campaign/r17-lost-result-1343.json | 1039 ++++++++ .../testdata/campaign/r4-election-292.json | 2263 +++++++++++++++++ 5 files changed, 3513 insertions(+), 9 deletions(-) create mode 100644 internal/raftsim/campaign_test.go create mode 100644 internal/raftsim/testdata/campaign/r17-lost-result-1343.json create mode 100644 internal/raftsim/testdata/campaign/r4-election-292.json diff --git a/internal/raftsim/README.md b/internal/raftsim/README.md index f880d1f2f5..a6ee1e6354 100644 --- a/internal/raftsim/README.md +++ b/internal/raftsim/README.md @@ -20,7 +20,7 @@ sh scripts/raft-proof-footprint.sh nice -n 5 env GOMEMLIMIT=128MiB \ The wrapper admits work only at I/O `some avg10 <= 40`, uses GOMAXPROCS=2, GOFLAGS=-buildvcs=false, one compiler worker and effective nice 15. The campaign -uses seeds 0–99 serially, at most 2,000 processed events per seed, 2 cells and +uses seeds 0–499 serially, at most 2,000 processed events per seed, 2 cells and fewer than 100 operations. One whole-identity Porcupine check gets at most one second; timeout/Unknown fails the proof. No checker partition hides cross-cell vault/revocation/session dependencies. No HTML or large fixture is generated. @@ -28,6 +28,27 @@ Supplementary bounded cases cover all 16 surviving joint-majority subsets, every enter/leave Ready boundary, 100 minority flaps, every snapshot chunk and install boundary, both retained generations and native SQLite parity. +Set `RAFTSIM_SEEDS=` to run seeds 0 through n−1 in both the fault campaign +and the election-timeout campaign. Unset means 500; empty, zero, negative and +noninteger values fail. For the larger serial soak, use the same footprint: + +```sh +sh scripts/raft-proof-footprint.sh nice -n 5 env GOMEMLIMIT=128MiB RAFTSIM_SEEDS=5000 \ + /usr/bin/time -v go test -p=1 ./internal/raftsim/... \ + -run '^TestR' -count=1 -timeout=180s -v +``` + +The 6–12 tick election timeout bounds one attempt. Split votes and omitted +messages can require several attempts; the campaign allows eight maximum +timeout windows while retaining the 2,000-event ceiling. A separate seeded +check asserts one delivered tick per live node per round and each election +attempt within 6–12 ticks. This budget is not a phone-fallback latency claim. +Lost-reply retries first wait for the original request's durable session result +without submitting a second proposal. An absent result after a timeout remains +unknown; it cannot be compared to a later successful retry as an empty result. +Portable replay fixtures retain the original split-vote and uncommitted-timeout +prefixes that exposed these two harness mistakes. + To keep exact negative counterexamples, pass the simulator package alone: ```sh diff --git a/internal/raftsim/campaign_test.go b/internal/raftsim/campaign_test.go new file mode 100644 index 0000000000..6db668dc67 --- /dev/null +++ b/internal/raftsim/campaign_test.go @@ -0,0 +1,134 @@ +package raftsim + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/Agent-Field/codeaf/internal/raftcore" + "go.etcd.io/raft/v3" +) + +func campaignReplay(t *testing.T, name string) *Simulator { + t.Helper() + b, err := os.ReadFile(filepath.Join("testdata", "campaign", name)) + if err != nil { + t.Fatal(err) + } + var replay Replay + if err := json.Unmarshal(b, &replay); err != nil { + t.Fatal(err) + } + s, err := Run(replay) + if err != nil { + t.Fatal(err) + } + return s +} + +// R4/R5 require eventual majority service through randomized elections. A +// split vote does not turn one timeout's upper bound into a total failover cap. +func TestR4R5SplitVoteReplayKeepsProgressing(t *testing.T) { + s := campaignReplay(t, "r4-election-292.json") + if s.Digest() != "6dbfc575ae9dc90f83c8de07a4f05b8a5f679fe6c081fd3d0abbd8df2ca0de61" { + t.Fatal("original election prefix no longer replays exactly") + } + if s.Leader() >= 0 || s.Nodes[0].Status.Role != raft.StatePreCandidate || s.Nodes[2].Status.Role != raft.StateCandidate || s.Nodes[2].Status.Term != 4 { + t.Fatal("expected the unresolved split vote at the old 25-tick boundary") + } + before := s.Events + leader := freshLeader(t, s, 1) + if s.Nodes[leader].Status.Term <= 4 || s.Nodes[leader].Stable.Applied <= 16 || s.Events <= before { + t.Fatal("R4/R5: election did not progress to a committed majority term") + } + cell := create(t, s, "after-split") + if cell.Fence != 1 { + t.Fatal("R4: recovered majority did not accept a fresh operation") + } + finish(t, s) +} + +// R17 applies to committed results. This exact lost request initially has no +// result; its original proposal must commit before testing a lost-success retry. +func TestR17LostReplyReplayWaitsForOriginalCommit(t *testing.T) { + s := campaignReplay(t, "r17-lost-result-1343.json") + id := len(s.Clients) - 1 + c := s.Clients[id] + key := c.History.Input.Actor + "/" + c.History.Input.Session + if _, exists := s.Nodes[c.Node].Stable.App.Sessions[key]; exists || !c.Lost || c.Done { + t.Fatal("expected an uncommitted original whose reply was lost") + } + applied := s.Nodes[c.Node].Stable.Applied + first := lostResult(t, s, id) + if s.Nodes[c.Node].Stable.Applied <= applied || first.Code != "ok" || first.Cell.Fence != 2 { + t.Fatal("R17: original did not durably commit its fence-2 result") + } + if again := submit(t, s, c.Node, c.History.Input); again != first { + t.Fatalf("R17: duplicate changed committed result: %+v %+v", first, again) + } + conflict := c.History.Input + conflict.Digest = "conflicting-digest" + if out := submit(t, s, c.Node, conflict); out.Code != "conflict" { + t.Fatalf("R17: conflicting digest succeeded: %+v", out) + } + if result := s.Nodes[c.Node].Stable.App.Sessions[key].Result; result != first || s.Clients[id].Done { + t.Fatal("R17: retry changed the retained result or delivered the lost reply") + } + finish(t, s) +} + +// R4/R5 and design §7 bound each election timeout at 6–12 ticks. Check the +// observable election messages, separately from eventual election of a leader. +func TestR4R5SeededElectionTimeoutAndTickPacing(t *testing.T) { + for seed := 0; seed < seedCount(t); seed++ { + t.Run(fmt.Sprintf("seed-%03d", seed), func(t *testing.T) { + s, err := New(Config{Seed: uint64(seed)}) + if err != nil { + t.Fatal(err) + } + s.Drain() + for node := range s.Nodes { + isolate(s, node, true) + } + last, attempts := make([]int, len(s.Nodes)), make([]int, len(s.Nodes)) + for tick := 1; tick <= 4*raftcore.ElectionTicks; tick++ { + start := len(s.Trace) + s.Ticks(1) + if s.Err != nil { + t.Fatal(s.Err) + } + ticks, campaign := make([]int, len(s.Nodes)), make([]bool, len(s.Nodes)) + for _, event := range s.Trace[start:] { + if event.Phase == "step-raftcore.Tick" { + ticks[event.Node]++ + } + if event.Phase == "send-MsgPreVote" { + campaign[event.Node] = true + } + } + for node := range s.Nodes { + if ticks[node] != 1 { + t.Fatalf("R4/R5: round %d delivered %d ticks to node %d", tick, ticks[node], node) + } + if campaign[node] { + gap := tick - last[node] + if gap < raftcore.ElectionTicks || gap > 2*raftcore.ElectionTicks { + t.Fatalf("R4/R5: node %d election after %d ticks", node, gap) + } + last[node], attempts[node] = tick, attempts[node]+1 + } else if tick-last[node] > 2*raftcore.ElectionTicks { + t.Fatalf("R4/R5: node %d election timer stopped", node) + } + } + } + for node, count := range attempts { + if count < 2 { + t.Fatalf("R4/R5: node %d did not retry its election", node) + } + } + finish(t, s) + }) + } +} diff --git a/internal/raftsim/simulator_test.go b/internal/raftsim/simulator_test.go index ad248b8a02..2d7fbc2387 100644 --- a/internal/raftsim/simulator_test.go +++ b/internal/raftsim/simulator_test.go @@ -7,9 +7,11 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "testing" + "github.com/Agent-Field/codeaf/internal/raftcore" "github.com/Agent-Field/codeaf/internal/raftsim/model" "github.com/anishathalye/porcupine" "go.etcd.io/raft/v3" @@ -17,6 +19,23 @@ import ( var evidenceDir = flag.String("raftsim-evidence-dir", "", "write synthetic replay evidence outside the repository") +// A randomized timeout bounds one attempt, not the number of split votes or +// omitted election messages. Keep several attempts within the event ceiling. +const electionWaitTicks = 8 * 2 * raftcore.ElectionTicks + +func seedCount(t *testing.T) int { + t.Helper() + value, set := os.LookupEnv("RAFTSIM_SEEDS") + if !set { + return 500 + } + count, err := strconv.Atoi(value) + if err != nil || count <= 0 { + t.Fatalf("RAFTSIM_SEEDS must be a positive integer, got %q", value) + } + return count +} + func group(t *testing.T, c Config) *Simulator { t.Helper() s, err := New(c) @@ -41,7 +60,7 @@ func group(t *testing.T, c Config) *Simulator { } func elect(t *testing.T, s *Simulator) int { t.Helper() - for range 25 { + for range electionWaitTicks { if s.Err != nil { finish(t, s) } @@ -50,7 +69,7 @@ func elect(t *testing.T, s *Simulator) int { } s.Ticks(1) } - t.Fatal("no leader within bounded healthy ticks") + t.Fatalf("no leader within %d election ticks", electionWaitTicks) return -1 } func finish(t *testing.T, s *Simulator) { @@ -107,7 +126,7 @@ func isolate(s *Simulator, node int, blocked bool) { } func freshLeader(t *testing.T, s *Simulator, old int) int { t.Helper() - for range 25 { + for range electionWaitTicks { s.Ticks(1) if l := s.Leader(); l >= 0 && l != old { return l @@ -116,15 +135,44 @@ func freshLeader(t *testing.T, s *Simulator, old int) int { finish(t, s) } } - t.Fatal("healthy majority failed to elect") + t.Fatalf("healthy majority failed to elect within %d ticks", electionWaitTicks) return -1 } +// A timeout is unknown, so draining messages cannot establish a committed +// result. Drive retransmission until the original result is durably applied. +func lostResult(t *testing.T, s *Simulator, id int) model.Outcome { + t.Helper() + c := s.Clients[id] + if !c.Lost || c.Done || c.History.Input.Session == "" { + t.Fatal("R17: expected a lost session request") + } + key := c.History.Input.Actor + "/" + c.History.Input.Session + s.Drain() + for ticks := 0; ticks <= 2*raftcore.ElectionTicks; ticks++ { + if s.Err != nil { + finish(t, s) + } + if result, ok := s.Nodes[c.Node].Stable.App.Sessions[key]; ok { + if result.Sequence != c.History.Input.Sequence || result.Digest != c.History.Input.Digest || s.Clients[id].Done { + t.Fatal("R17: lost request did not retain its exact committed session") + } + return result.Result + } + if ticks < 2*raftcore.ElectionTicks { + s.Ticks(1) + } + } + t.Fatal("R17: original lost request did not durably apply") + return model.Outcome{} +} + // R1–R5/R13/R17 share one whole-identity history per seed, including unknown // outcomes. Each run stays below 2,000 effect events and 100 operations. func TestR1R2R3R4R5R13R17SeededFaultCampaign(t *testing.T) { total, maxEvents := 0, 0 - for seed := uint64(0); seed < 100; seed++ { + count := seedCount(t) + for seed := uint64(0); seed < uint64(count); seed++ { t.Run(fmt.Sprintf("seed-%03d", seed), func(t *testing.T) { s := group(t, Config{Seed: seed, Nodes: 4, Drop: 10, Duplicate: 100, Delay: 3}) a := create(t, s, "a") @@ -153,8 +201,7 @@ func TestR1R2R3R4R5R13R17SeededFaultCampaign(t *testing.T) { take := model.Call{Kind: "acquire", Actor: "retry", Cell: "b", Expected: observed(s.Nodes[leader].Stable.App.Cells["b"]), Force: true, At: 3_600_001, Session: "durable", Sequence: 1, Digest: "opaque-1"} dropped := s.Submit(leader, take) s.Do(Action{Kind: "timeout", Node: dropped}) - s.Drain() - first := s.Nodes[leader].Stable.App.Sessions["retry/durable"].Result + first := lostResult(t, s, dropped) again := submit(t, s, leader, take) if again != first || again.Code != "ok" || again.Cell.Fence != 2 { t.Fatalf("R17: retry changed result %+v %+v", first, again) @@ -243,7 +290,7 @@ func TestR1R2R3R4R5R13R17SeededFaultCampaign(t *testing.T) { } }) } - t.Logf("100 fixed seeds, %d events, maximum %d/2000 per seed", total, maxEvents) + t.Logf("%d seeds (0–%d), %d events, maximum %d/%d per seed", count, count-1, total, maxEvents, MaxEvents) } func TestR5HundredMinorityFlapsKeepHealthyTerm(t *testing.T) { diff --git a/internal/raftsim/testdata/campaign/r17-lost-result-1343.json b/internal/raftsim/testdata/campaign/r17-lost-result-1343.json new file mode 100644 index 0000000000..e568afb1bb --- /dev/null +++ b/internal/raftsim/testdata/campaign/r17-lost-result-1343.json @@ -0,0 +1,1039 @@ +{ + "Config": { + "PaddingBytes": 0, + "CutSnapshot": false, + "RecordDisk": false, + "Seed": 1343, + "Nodes": 4, + "MaxEvents": 2000, + "Drop": 10, + "Duplicate": 100, + "Delay": 3, + "CutNode": 0, + "CutPhase": "", + "CutOccurrence": 0, + "Broken": "" + }, + "Actions": [ + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "create", + "Actor": "owner", + "Cell": "a", + "At": 100, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "head-0", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "create", + "Actor": "owner", + "Cell": "b", + "At": 100, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "head-0", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "acquire", + "Actor": "take-a", + "Cell": "a", + "At": 110, + "Expected": { + "Holder": "owner", + "Fence": 1, + "Until": 90100, + "Head": "head-0", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": true, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "acquire", + "Actor": "take-b", + "Cell": "a", + "At": 90, + "Expected": { + "Holder": "owner", + "Fence": 1, + "Until": 90100, + "Head": "head-0", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "publish", + "Actor": "owner", + "Cell": "a", + "At": 120, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 1, + "Head": "displaced", + "OldHead": "wrong", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "beat", + "Actor": "owner", + "Cell": "a", + "At": 120, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 1, + "Head": "displaced", + "OldHead": "wrong", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "release", + "Actor": "owner", + "Cell": "a", + "At": 120, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 1, + "Head": "displaced", + "OldHead": "wrong", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "advance_time", + "Actor": "owner", + "Cell": "", + "At": 3600000, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "advance_time", + "Actor": "owner", + "Cell": "", + "At": -400, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "durable", + "Sequence": 1, + "Digest": "opaque-1", + "Ack": 0, + "Kind": "acquire", + "Actor": "retry", + "Cell": "b", + "At": 3600001, + "Expected": { + "Holder": "owner", + "Fence": 1, + "Until": 90100, + "Head": "head-0", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": true, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "timeout", + "Node": 9, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + } + ] +} diff --git a/internal/raftsim/testdata/campaign/r4-election-292.json b/internal/raftsim/testdata/campaign/r4-election-292.json new file mode 100644 index 0000000000..aee1156506 --- /dev/null +++ b/internal/raftsim/testdata/campaign/r4-election-292.json @@ -0,0 +1,2263 @@ +{ + "Config": { + "PaddingBytes": 0, + "CutSnapshot": false, + "RecordDisk": false, + "Seed": 292, + "Nodes": 4, + "MaxEvents": 2000, + "Drop": 10, + "Duplicate": 100, + "Delay": 3, + "CutNode": 0, + "CutPhase": "", + "CutOccurrence": 0, + "Broken": "" + }, + "Actions": [ + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 1, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "create", + "Actor": "owner", + "Cell": "a", + "At": 100, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "head-0", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 1, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "create", + "Actor": "owner", + "Cell": "b", + "At": 100, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "head-0", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 1, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "acquire", + "Actor": "take-a", + "Cell": "a", + "At": 110, + "Expected": { + "Holder": "owner", + "Fence": 1, + "Until": 90100, + "Head": "head-0", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": true, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 1, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "acquire", + "Actor": "take-b", + "Cell": "a", + "At": 90, + "Expected": { + "Holder": "owner", + "Fence": 1, + "Until": 90100, + "Head": "head-0", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": true, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 1, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "publish", + "Actor": "owner", + "Cell": "a", + "At": 120, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 1, + "Head": "displaced", + "OldHead": "wrong", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 1, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "beat", + "Actor": "owner", + "Cell": "a", + "At": 120, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 1, + "Head": "displaced", + "OldHead": "wrong", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 1, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "release", + "Actor": "owner", + "Cell": "a", + "At": 120, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 1, + "Head": "displaced", + "OldHead": "wrong", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 1, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "advance_time", + "Actor": "owner", + "Cell": "", + "At": 3600000, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 1, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "advance_time", + "Actor": "owner", + "Cell": "", + "At": -400, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 1, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "durable", + "Sequence": 1, + "Digest": "opaque-1", + "Ack": 0, + "Kind": "acquire", + "Actor": "retry", + "Cell": "b", + "At": 3600001, + "Expected": { + "Holder": "owner", + "Fence": 1, + "Until": 90100, + "Head": "head-0", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": true, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "timeout", + "Node": 9, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 1, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "durable", + "Sequence": 1, + "Digest": "opaque-1", + "Ack": 0, + "Kind": "acquire", + "Actor": "retry", + "Cell": "b", + "At": 3600001, + "Expected": { + "Holder": "owner", + "Fence": 1, + "Until": 90100, + "Head": "head-0", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": true, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "submit", + "Node": 1, + "Other": 0, + "Count": 0, + "Value": 0, + "Request": { + "Session": "durable", + "Sequence": 1, + "Digest": "other-digest", + "Ack": 0, + "Kind": "acquire", + "Actor": "retry", + "Cell": "b", + "At": 3600001, + "Expected": { + "Holder": "owner", + "Fence": 1, + "Until": 90100, + "Head": "head-0", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": true, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "drain", + "Node": 0, + "Other": 0, + "Count": 0, + "Value": 1000, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "block", + "Node": 1, + "Other": 0, + "Count": 0, + "Value": 1, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "block", + "Node": 0, + "Other": 1, + "Count": 0, + "Value": 1, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "block", + "Node": 1, + "Other": 2, + "Count": 0, + "Value": 1, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "block", + "Node": 2, + "Other": 1, + "Count": 0, + "Value": 1, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "block", + "Node": 1, + "Other": 3, + "Count": 0, + "Value": 1, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "block", + "Node": 3, + "Other": 1, + "Count": 0, + "Value": 1, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + }, + { + "Kind": "ticks", + "Node": 0, + "Other": 0, + "Count": 1, + "Value": 0, + "Request": { + "Session": "", + "Sequence": 0, + "Digest": "", + "Ack": 0, + "Kind": "", + "Actor": "", + "Cell": "", + "At": 0, + "Expected": { + "Holder": "", + "Fence": 0, + "Until": 0, + "Head": "", + "Pending": 0, + "ReleasedAt": 0 + }, + "Force": false, + "Fence": 0, + "Head": "", + "OldHead": "", + "Pending": 0, + "Quiet": false, + "OldVault": "", + "NewVault": "", + "Target": "" + } + } + ] +} From 2d76df98b96abe82279943f9eae59b0397f61bf9 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 00:13:54 -0400 Subject: [PATCH 014/103] Authenticate and bound native peer delivery so learner backlog cannot displace voter traffic. Bind relay exchanges to committed identity and member generations, preserve signed opaque bytes across reconnects, and reserve bounded inbox capacity for voters and snapshots. Publish a shared native and JavaScript wire vector. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/rafttransport/README.md | 143 +++++++ internal/rafttransport/envelope.go | 166 ++++++++ internal/rafttransport/exchange.go | 383 +++++++++++++++++ internal/rafttransport/exchange_test.go | 207 +++++++++ internal/rafttransport/router.go | 398 ++++++++++++++++++ internal/rafttransport/router_test.go | 373 ++++++++++++++++ .../rafttransport/testdata/exchange-v1.json | 1 + relay/hosted/test/raft/transport.test.js | 31 ++ 8 files changed, 1702 insertions(+) create mode 100644 internal/rafttransport/README.md create mode 100644 internal/rafttransport/envelope.go create mode 100644 internal/rafttransport/exchange.go create mode 100644 internal/rafttransport/exchange_test.go create mode 100644 internal/rafttransport/router.go create mode 100644 internal/rafttransport/router_test.go create mode 100644 internal/rafttransport/testdata/exchange-v1.json create mode 100644 relay/hosted/test/raft/transport.test.js diff --git a/internal/rafttransport/README.md b/internal/rafttransport/README.md new file mode 100644 index 0000000000..8619407654 --- /dev/null +++ b/internal/rafttransport/README.md @@ -0,0 +1,143 @@ +# Native peer transport + +This is P07's internal relay bridge, not an enrollment service or a directory +writer. `relayserve.Config.Cluster` receives `rafttransport.Handler(router, now)` from +the member host after it loads committed enrollment. An ordinary relay has no peer routes until that assembly +exists. P08 supplies hosted actors, P12 supplies durable enrollment and P17 wires +production client routing. No environment switch selects another writer. + +## Host ordering and authorization + +Create `NewRouter(DefaultLimits())`, then `Install(View)`. A view binds identity, +cluster UUID, epoch, protocol, revision, durable router generation and all member +IDs/incarnations/public transport keys. Install comes **after** the enrollment +transaction commits and **before** acknowledging that change or dispatching more +peer work. Serialize committed registry changes through this seam. It must never +be filled by an HTTP caller, a pairing nameplate, a socket, or a key presented +without committed enrollment. Anchors use protocol-only keys, never identity roots. + +Every request, including an empty poll or acknowledgment, is signed. Every +outbound envelope is independently signed by its actual source. The router checks +sender and destination enrollment/incarnation, scope, digest, signature and the +inner Raft from/to/type. Local control, proposal forwarding and storage-completion +messages are refused. The core repeats its own peer-input checks. A view update +shares the routing lock, invalidates buffered traffic and notifies open sockets; +requests signed before a committed revocation are refused afterward. Retired node +records cannot disappear or regain authority. Key changes require a higher +incarnation. Host policy still owns owner consent, certificate verification, +configuration changes, tombstones and registry persistence. + +The routing inbox is expendable memory. Before restoring service after loss of +that memory, increment and durably store the view's router `Generation`, then +install it. That fences all old delivery cursors and sockets. Member incarnations +also come from durable enrollment. Do not replace either number with a random +connection ID. A router has no automatic bootstrap, singleton fallback or authority +read. P12 must enforce a single durable epoch for each identity; registering two +unrelated scopes is not migration or permission to run two writable clusters. + +## Exchanges and sockets + +`POST /v2/cluster/exchange` takes one canonical `Request` and returns `Response`. +`Client.Exchange` makes one bounded outbound request, refuses HTTP redirects and +starts no background work, listeners, retries or discovery. Hosts enforce paired +and sync-off policy **before** creating/using this client; P14/P17 own that wiring. +Use HTTPS with the configured relay's authenticated TLS endpoint (loopback HTTP is +used by the hermetic tests). Keep protocol admission separate from the public +application-command limiter, and configure an operator ingress rate/connection +budget in addition to these bounded transient queues. + +`GET /v2/cluster/socket` upgrades an outbound connection. `Codeaf-Cluster-Auth` +contains raw-URL-base64 of a signed empty Request, with no outbound messages. +The first binary response is the current inbox. Subsequent signed binary Requests +use the same node/scope/incarnation/router generation and receive responses; +queued traffic also pushes a response without requiring a poll. Each frame and +push rechecks enrollment. The channel carries no lease or append acknowledgment. +Compression is disabled. Initial/body/frame/write/read bounds apply, and at most +eight sockets per member can be open. Send an authenticated frame within 30 seconds +or reconnect using HTTP fallback/backoff. No socket is required on a member. +Hibernation is a hosted-runtime responsibility; the Go server has ordinary sockets. + +Responses are hints to transport delivery, not proof of persistence. Before core +dispatch, the receiving host checks the envelope signature using the **current** +committed peer key and incarnation, its expected identity/cluster/epoch/protocol, +and its own destination/incarnation. Remote hosts must not trust a stale cached +registry to authorize traffic. Recheck after asynchronous waits; discard callbacks +from an old actor generation. Only step a message once the receiver is available +for it. Keep/retry `ErrBusy` input. Keep all P04 persistence/application boundaries: +`Persist -> Persisted -> Apply -> Applied`; a relay receipt completes none of them. + +## Wire version 1 + +`envelope.go` and `exchange.go` define fixed-order JSON structs. All uint64 values, +including the three ack-array elements, are canonical decimal strings. Protocol +and lane are small JSON integers; bytes are padded standard base64. Scope strings +are 1–128 ASCII letters, digits, hyphen, underscore or dot. Digests are lowercase +SHA-256 hex. Strict decoding requires exact re-encoding: unknown/duplicate fields, +extra array elements, alternate numbers, whitespace and trailing bytes are refused. + +Envelope field order is `scope,from,to,incarnation,destination,sequence,chunk,digest, +payload,signature`; scope is `identity,cluster,epoch,protocol`; a non-null chunk is +`generation,offset,hash`. Request order is `scope,node,incarnation,generation, +messages,signature,ack`. Signing replaces signature with JSON null and prepends +`codeaf/raft/envelope/1\n` or `codeaf/raft/exchange/1\n` respectively. No JSON object +key sorting or JS Number conversion is allowed. The committed independently +Node-signed `testdata/exchange-v1.json` is checked by Go and the targeted hosted +Node test. P08 can use that vector before implementing its host-side codec. + +## Bounds, priority and retry + +Normal batches contain at most 64 messages and 64 KiB total raw Raft bytes. A +snapshot request contains exactly one chunk of at most 256 KiB. The encoded request +and response ceiling is 512 KiB. All signature/envelope overhead counts toward +queued memory. Each identity has at most 1 MiB of queued bytes: 384 KiB reserved +for voter-to-voter normal traffic, 128 KiB for traffic involving a learner, and +512 KiB for snapshot chunks. A member inbox has at most eight deliveries, reserved +as four voter, two learner and two snapshot slots. Reservations cannot be borrowed +by other classes. Class comes from committed enrollment, never a sender's claim. +Normal voter traffic drains before learner traffic and snapshot chunks. No messages +are coalesced, including heartbeat read contexts, votes, appends and read barriers. + +`Delivery` has a separate monotonically increasing cursor per lane. Ack only the +contiguous delivered prefix of each lane after the host has accepted those inputs; +copy a cursor into the corresponding Request.Ack slot and sign it. Acks above the +highest actually issued cursor are refused. A lost response redelivers the same +bytes/cursor. Pending exact duplicates consume no extra slot; sequence conflicts +cannot replace a pending packet. After ack or expiry an old sequence may be +redelivered: Raft handles duplicates/reordering. There is no unbounded transport +replay table and no application request-dedup claim. + +A batch with an invalid member/signature admits no outbound message. Backpressure +admits no part of an otherwise valid batch; valid incoming acks still drain that +caller's inbox to avoid a circular queue-full stall. Copy retained bytes before +returning. `ErrCapacity`/HTTP 429 means retain/retry or report failed transport to +the core, never a successful append. Retry with host-owned exponential backoff, +seeded jitter and a 60-second ceiling. Buffers expire after 30 seconds of monotonic +sampled host time, lazily on activity, requiring no idle alarm. Raft retransmits; +lease renewal and snapshots retain their own durable semantics. + +Default operator admission is 1,024 scopes, 4,096 retained enrollment records per +scope, at most 32 live learners and six live voter IDs (room for both sides of a +three-voter joint change). `Limits` makes scope/retained-record/voter ceilings +explicit operator choices; exceeding one refuses growth. Larger topology policy +belongs to P12, not transport. Tombstones and application records are not deleted +to regain admission. `Stats` reports counts/bytes/socket usage only, never payloads. + +## Snapshots and next hosts + +A `MsgSnap` contains P05's small portable manifest pointer. Chunk envelopes carry +that manifest's generation ID, zero-based chunk number and SHA-256. The receiver +must verify generation, offset, expected length and manifest totals before P05's +`BeginSnapshot/PutSnapshotChunk/CompleteSnapshot`. A queued chunk or completed +stream is **not** installation: host success is reported through +`raftcore.TransportResult{Snapshot:true,Success:true}` only after the actual +snapshot Ready has installed durably. A timeout/interruption reports failure so +Raft can retry. Do not feed `MsgSnap` into the core before its chunks are ready. +P08 owns those hosted actor/storage operations and native/wasm lifecycle proof. + +P07's tests use real cores and three SQLite stores behind the actual relay route, +with only outbound member HTTP, intentional duplicate/reorder, create, ReadIndex, +leader death/restart, joint demotion and writes with an absent learner. Transport +unit tests pin auth, generation, socket revocation and all queue reservations. +P06/P06b's 2,000-seed simulator soak remains a required regression gate. Full +R10 enrollment/worker journeys, R11 churn/snapshot interoperability, application +revocation and the R15 full-state sentinel campaign remain later packet gates. diff --git a/internal/rafttransport/envelope.go b/internal/rafttransport/envelope.go new file mode 100644 index 0000000000..3046153cc0 --- /dev/null +++ b/internal/rafttransport/envelope.go @@ -0,0 +1,166 @@ +// Package rafttransport carries authenticated peer traffic without granting +// directory authority. Hosts still own consensus persistence and application. +package rafttransport + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "strings" + + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + "go.etcd.io/raft/v3" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +const ( + Protocol = raftcore.ProtocolVersion + MaxBatchBytes = raftcore.MaxMessageBytes + MaxChunkBytes = raftsnapshot.ChunkBytes + MaxInflight = raftcore.MaxInflightBatches + MaxLearners = 32 + MaxMessages = 64 + MaxQueuedBytes = 1 << 20 + // Encoded envelopes include base64 payloads, signatures and bounded IDs. + MaxExchangeBytes = 2 * MaxChunkBytes +) + +var ( + ErrAuth = errors.New("rafttransport: member authorization refused") + ErrWire = errors.New("rafttransport: invalid peer envelope") + ErrGeneration = errors.New("rafttransport: transport generation changed; reconnect") + ErrCapacity = errors.New("rafttransport: transport is busy; retry with backoff") + ErrCursor = errors.New("rafttransport: invalid delivery cursor") +) + +// Scope binds every signature to an identity, cluster epoch and protocol. All +// uint64 fields cross JavaScript as canonical unsigned decimal strings. +type Scope struct { + Identity string `json:"identity"` + Cluster string `json:"cluster"` + Epoch uint64 `json:"epoch,string"` + Protocol uint16 `json:"protocol"` +} + +func token(s string) bool { + if len(s) == 0 || len(s) > 128 { + return false + } + for _, c := range s { + if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || strings.ContainsRune("-_.", c)) { + return false + } + } + return true +} +func (s Scope) valid() bool { + return token(s.Identity) && token(s.Cluster) && s.Epoch != 0 && s.Protocol == Protocol +} +func node(id uint64) bool { return id != 0 && !raft.IsLocalMsgTarget(id) } + +// Chunk names one portable snapshot generation and zero-based chunk number. +// Hash is checked before routing; installation remains the receiving host's job. +type Chunk struct { + Generation string `json:"generation"` + Offset uint64 `json:"offset,string"` + Hash string `json:"hash"` +} + +type Envelope struct { + Scope Scope `json:"scope"` + From uint64 `json:"from,string"` + To uint64 `json:"to,string"` + Incarnation uint64 `json:"incarnation,string"` + Destination uint64 `json:"destination,string"` + Sequence uint64 `json:"sequence,string"` + Chunk *Chunk `json:"chunk"` + Digest string `json:"digest"` + Payload []byte `json:"payload"` + Signature []byte `json:"signature"` +} + +func digest(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } +func hash(s string) bool { + b, e := hex.DecodeString(s) + return e == nil && len(b) == sha256.Size && s == strings.ToLower(s) +} + +// signingBytes uses a fixed JSON field order and ASCII-only metadata. Payloads +// are base64, so native and JavaScript hosts can sign exactly the same bytes. +func (e Envelope) signingBytes() []byte { + e.Signature = nil + b, _ := json.Marshal(e) + return append([]byte("codeaf/raft/envelope/1\n"), b...) +} +func (e *Envelope) Sign(key ed25519.PrivateKey) error { + if len(key) != ed25519.PrivateKeySize { + return ErrAuth + } + e.Digest = digest(e.Payload) + if err := e.validate(); err != nil { + return err + } + e.Signature = ed25519.Sign(key, e.signingBytes()) + return nil +} +func (e Envelope) Verify(key ed25519.PublicKey) error { + if err := e.validate(); err != nil { + return err + } + if len(key) != ed25519.PublicKeySize || !ed25519.Verify(key, e.signingBytes(), e.Signature) { + return ErrAuth + } + return nil +} +func (e Envelope) validate() error { + if !e.Scope.valid() || !node(e.From) || !node(e.To) || e.From == e.To || e.Incarnation == 0 || e.Destination == 0 || e.Sequence == 0 || e.Digest != digest(e.Payload) { + return ErrWire + } + if e.Chunk != nil { + if len(e.Payload) == 0 || len(e.Payload) > MaxChunkBytes || !hash(e.Chunk.Generation) || e.Chunk.Hash != e.Digest { + return ErrWire + } + return nil + } + if len(e.Payload) == 0 || len(e.Payload) > MaxBatchBytes { + return ErrWire + } + m := &pb.Message{} + if proto.Unmarshal(e.Payload, m) != nil || m.Type == nil || m.GetFrom() != e.From || m.GetTo() != e.To || m.GetTerm() == 0 || len(m.GetResponses()) != 0 || len(m.ProtoReflect().GetUnknown()) != 0 { + return ErrWire + } + switch m.GetType() { + case pb.MsgApp, pb.MsgAppResp, pb.MsgVote, pb.MsgVoteResp, pb.MsgSnap, pb.MsgHeartbeat, pb.MsgHeartbeatResp, pb.MsgPreVote, pb.MsgPreVoteResp, pb.MsgTimeoutNow: + return nil + default: + return ErrWire + } +} + +// Encode is the wire format shared by HTTP and socket frames. Decode rejects +// alternative encodings so duplicate keys cannot disagree across runtimes. +func Encode(v any) ([]byte, error) { return json.Marshal(v) } +func Decode(b []byte, v any) error { + if len(b) > MaxExchangeBytes { + return ErrWire + } + d := json.NewDecoder(bytes.NewReader(b)) + d.DisallowUnknownFields() + if d.Decode(v) != nil { + return ErrWire + } + if d.Decode(new(any)) != io.EOF { + return ErrWire + } + canonical, err := Encode(v) + if err != nil || !bytes.Equal(canonical, b) { + return ErrWire + } + return nil +} diff --git a/internal/rafttransport/exchange.go b/internal/rafttransport/exchange.go new file mode 100644 index 0000000000..549be92059 --- /dev/null +++ b/internal/rafttransport/exchange.go @@ -0,0 +1,383 @@ +package rafttransport + +import ( + "bytes" + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "strconv" + "time" + + "github.com/coder/websocket" +) + +const ( + ExchangePath = "/v2/cluster/exchange" + SocketPath = "/v2/cluster/socket" + SocketAuthHeader = "Codeaf-Cluster-Auth" + socketTimeout = 30 * time.Second +) + +// Request signs polls and acknowledgments as well as outbound digests. A +// stolen unsigned cursor must not allow a third party to drain a peer's inbox. +type Request struct { + Scope Scope `json:"scope"` + Node uint64 `json:"node,string"` + Incarnation uint64 `json:"incarnation,string"` + Generation uint64 `json:"generation,string"` + Ack [laneCount]uint64 `json:"ack"` + Messages []Envelope `json:"messages"` + Signature []byte `json:"signature"` +} + +func decimal(n uint64) string { return strconv.FormatUint(n, 10) } +func parseDecimal(s string) (uint64, error) { + n, err := strconv.ParseUint(s, 10, 64) + if err != nil || decimal(n) != s { + return 0, ErrWire + } + return n, nil +} + +// Ack is encoded through decimal strings even though it is a fixed array. +func (r Request) MarshalJSON() ([]byte, error) { + type raw Request + return json.Marshal(struct { + raw + Ack [laneCount]string `json:"ack"` + }{raw(r), [laneCount]string{decimal(r.Ack[0]), decimal(r.Ack[1]), decimal(r.Ack[2])}}) +} +func (r *Request) UnmarshalJSON(b []byte) error { + type raw Request + var v struct { + raw + Ack [laneCount]string `json:"ack"` + } + d := json.NewDecoder(bytes.NewReader(b)) + d.DisallowUnknownFields() + if d.Decode(&v) != nil { + return ErrWire + } + *r = Request(v.raw) + for i, s := range v.Ack { + n, err := parseDecimal(s) + if err != nil { + return err + } + r.Ack[i] = n + } + return nil +} +func (r Request) signingBytes() []byte { + r.Signature = nil + b, _ := Encode(r) + return append([]byte("codeaf/raft/exchange/1\n"), b...) +} +func (r *Request) Sign(key ed25519.PrivateKey) error { + if len(key) != ed25519.PrivateKeySize { + return ErrAuth + } + if err := r.validate(); err != nil { + return err + } + r.Signature = ed25519.Sign(key, r.signingBytes()) + return nil +} +func (r Request) Verify(key ed25519.PublicKey) error { + if err := r.validate(); err != nil { + return err + } + if len(key) != ed25519.PublicKeySize || !ed25519.Verify(key, r.signingBytes(), r.Signature) { + return ErrAuth + } + return nil +} +func (r Request) validate() error { + if !r.Scope.valid() || !node(r.Node) || r.Incarnation == 0 || r.Generation == 0 || len(r.Messages) > MaxMessages { + return ErrWire + } + normal, chunks := 0, 0 + for _, e := range r.Messages { + if e.Chunk != nil { + chunks++ + } else { + normal += len(e.Payload) + } + } + if normal > MaxBatchBytes || chunks > 1 || chunks != 0 && len(r.Messages) != 1 { + return ErrWire + } + b, err := Encode(r) + if err != nil || len(b) > MaxExchangeBytes { + return ErrWire + } + return nil +} + +// Handler mounts only transport routes. It never discovers or bootstraps an +// identity, and protocol traffic bypasses the public command admission budget. +func Handler(router *Router, now func() time.Time) http.Handler { + if now == nil { + now = time.Now + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + if r.URL.RawQuery != "" { + refuse(w, ErrWire) + return + } + switch r.URL.Path { + case ExchangePath: + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + b, err := io.ReadAll(http.MaxBytesReader(w, r.Body, MaxExchangeBytes)) + if err != nil { + refuse(w, ErrWire) + return + } + var req Request + if Decode(b, &req) != nil { + refuse(w, ErrWire) + return + } + resp, err := router.Exchange(req, now().UnixMilli()) + if err != nil { + refuse(w, err) + return + } + w.Header().Set("Content-Type", "application/json") + data, _ := Encode(resp) + _, _ = w.Write(data) + case SocketPath: + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + auth := r.Header.Get(SocketAuthHeader) + if len(auth) > MaxExchangeBytes { + refuse(w, ErrWire) + return + } + b, err := base64.RawURLEncoding.DecodeString(auth) + var req Request + if err != nil || Decode(b, &req) != nil || len(req.Messages) != 0 { + refuse(w, ErrAuth) + return + } + release, err := router.openSocket(req) + if err != nil { + refuse(w, err) + return + } + defer release() + conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{CompressionMode: websocket.CompressionDisabled}) + if err != nil { + return + } + defer conn.CloseNow() + conn.SetReadLimit(MaxExchangeBytes) + serveSocket(r.Context(), conn, router, req, now) + default: + http.NotFound(w, r) + } + }) +} +func refuse(w http.ResponseWriter, err error) { + code, status := "invalid", http.StatusBadRequest + switch err { + case ErrAuth: + code, status = "unauthorized", http.StatusForbidden + case ErrGeneration: + code, status = "generation", http.StatusConflict + case ErrCapacity: + code, status = "busy", http.StatusTooManyRequests + case ErrCursor: + code, status = "cursor", http.StatusConflict + } + w.Header().Set("Content-Type", "application/json") + if err == ErrCapacity { + w.Header().Set("Retry-After", "1") + } + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"error":"` + code + `"}`)) +} + +func (r *Router) openSocket(req Request) (func(), error) { + r.mu.Lock() + defer r.mu.Unlock() + _, box, err := r.authorize(req) + if err != nil { + return nil, err + } + if box.sockets >= MaxInflight { + return nil, ErrCapacity + } + box.sockets++ + return func() { + r.mu.Lock() + defer r.mu.Unlock() + g := r.groups[req.Scope] + if g != nil && g.view.Generation == req.Generation { + if m, e := g.member(req.Node, req.Incarnation); e == nil { + g.inboxes[m.Node].sockets-- + } + } + }, nil +} +func (r *Router) changed(req Request) (<-chan struct{}, error) { + r.mu.Lock() + defer r.mu.Unlock() + _, box, err := r.authorize(req) + if err != nil { + return nil, err + } + return box.changed, nil +} +func serveSocket(ctx context.Context, conn *websocket.Conn, router *Router, initial Request, now func() time.Time) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + type frame struct { + data []byte + err error + } + frames := make(chan frame, 1) + go func() { + for { + readCtx, stop := context.WithTimeout(ctx, socketTimeout) + kind, data, err := conn.Read(readCtx) + stop() + if kind != websocket.MessageBinary && err == nil { + err = ErrWire + } + select { + case frames <- frame{data, err}: + case <-ctx.Done(): + return + } + if err != nil { + return + } + } + }() + req := initial + changed, err := router.changed(req) + if err != nil { + return + } + response, err := router.exchange(req, now().UnixMilli(), true) + if err != nil || !writeSocket(ctx, conn, response) { + return + } + for { + receiveOnly := false + select { + case <-ctx.Done(): + return + case <-changed: + receiveOnly = true + case f := <-frames: + if f.err != nil { + return + } + var next Request + if Decode(f.data, &next) != nil || next.Scope != initial.Scope || next.Node != initial.Node || next.Incarnation != initial.Incarnation || next.Generation != initial.Generation { + return + } + req = next + } + // Subscribe before draining so an enqueue cannot hide between those steps. + changed, err = router.changed(req) + if err != nil { + return + } + response, err = router.exchange(req, now().UnixMilli(), receiveOnly) + if err != nil || !writeSocket(ctx, conn, response) { + return + } + } +} + +func writeSocket(ctx context.Context, conn *websocket.Conn, resp Response) bool { + data, _ := Encode(resp) + writeCtx, cancel := context.WithTimeout(ctx, socketTimeout) + defer cancel() + return conn.Write(writeCtx, websocket.MessageBinary, data) == nil +} + +// Client performs a single bounded outbound exchange. It does not start timers, +// retry commands or open listeners. Hosts apply quiet-mode policy BEFORE use, +// own exponential backoff (up to 60 s), and verify each delivered peer key. +type Client struct { + URL string + HTTP *http.Client +} + +func (c Client) Exchange(ctx context.Context, req Request) (Response, error) { + b, err := Encode(req) + if err != nil || req.validate() != nil { + return Response{}, ErrWire + } + r, err := http.NewRequestWithContext(ctx, http.MethodPost, c.URL+ExchangePath, bytes.NewReader(b)) + if err != nil { + return Response{}, ErrWire + } + r.Header.Set("Content-Type", "application/json") + client := c.HTTP + if client == nil { + client = &http.Client{Timeout: socketTimeout} + } + // A redirect must not leak a signed peer batch to another origin. + copyClient := *client + if copyClient.Timeout <= 0 || copyClient.Timeout > socketTimeout { + copyClient.Timeout = socketTimeout + } + copyClient.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + resp, err := copyClient.Do(r) + if err != nil { + return Response{}, err + } + defer resp.Body.Close() + data, err := io.ReadAll(io.LimitReader(resp.Body, MaxExchangeBytes+1)) + if err != nil { + return Response{}, err + } + if resp.StatusCode != http.StatusOK { + switch resp.StatusCode { + case http.StatusForbidden: + return Response{}, ErrAuth + case http.StatusTooManyRequests: + return Response{}, ErrCapacity + case http.StatusConflict: + var refusal struct { + Error string `json:"error"` + } + if json.Unmarshal(data, &refusal) != nil { + return Response{}, ErrWire + } + if refusal.Error == "cursor" { + return Response{}, ErrCursor + } + if refusal.Error == "generation" { + return Response{}, ErrGeneration + } + return Response{}, ErrWire + default: + return Response{}, ErrWire + } + } + var result Response + if Decode(data, &result) != nil || result.Generation != req.Generation || len(result.Messages) > MaxMessages { + return Response{}, ErrWire + } + for _, d := range result.Messages { + if d.Lane >= laneCount || d.Cursor == 0 || d.Envelope.Scope != req.Scope || d.Envelope.To != req.Node || d.Envelope.Destination != req.Incarnation { + return Response{}, ErrWire + } + } + return result, nil +} diff --git a/internal/rafttransport/exchange_test.go b/internal/rafttransport/exchange_test.go new file mode 100644 index 0000000000..98e6e3ceca --- /dev/null +++ b/internal/rafttransport/exchange_test.go @@ -0,0 +1,207 @@ +package rafttransport + +import ( + "bytes" + "context" + "crypto/aes" + "crypto/cipher" + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/coder/websocket" + pb "go.etcd.io/raft/v3/raftpb" +) + +// R10/R16 use the same signed exchange on reconnectable outbound sockets and +// HTTP fallback. Every frame rechecks enrollment, including an already open +// socket when its member is revoked or its router generation changes. +func TestR10R16OutboundSocketPushReconnectAndRevoke(t *testing.T) { + r := router(t) + server := httptest.NewServer(Handler(r, func() time.Time { return time.UnixMilli(1000) })) + defer server.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + dial := func(req Request) *websocket.Conn { + t.Helper() + b, err := Encode(req) + must(t, err) + conn, resp, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+SocketPath, &websocket.DialOptions{HTTPHeader: http.Header{SocketAuthHeader: []string{base64.RawURLEncoding.EncodeToString(b)}}}) + if err != nil { + if resp != nil { + t.Log(resp.StatusCode) + } + t.Fatal(err) + } + t.Cleanup(func() { conn.CloseNow() }) + return conn + } + read := func(conn *websocket.Conn) Response { + t.Helper() + kind, data, err := conn.Read(ctx) + must(t, err) + if kind != websocket.MessageBinary { + t.Fatal("R10: socket changed framing") + } + var got Response + must(t, Decode(data, &got)) + return got + } + conn := dial(request(t, 2)) + if got := read(conn); len(got.Messages) != 0 { + t.Fatal("R10: initial poll was not empty") + } + client := Client{URL: server.URL, HTTP: server.Client()} + packet := message(t, 1, 2, 1, pb.MsgApp, 0) + _, err := client.Exchange(ctx, request(t, 1, packet)) + must(t, err) + got := read(conn) + if len(got.Messages) != 1 || !sameEnvelope(got.Messages[0].Envelope, packet) { + t.Fatal("R10: live socket did not receive queued packet") + } + conn.CloseNow() + conn = dial(request(t, 2)) + again := read(conn) + if len(again.Messages) != 1 || again.Messages[0].Cursor != got.Messages[0].Cursor { + t.Fatal("R10: reconnect lost unacknowledged delivery") + } + ack := request(t, 2) + ack.Ack[Voter] = got.Messages[0].Cursor + must(t, ack.Sign(key(2))) + data, err := Encode(ack) + must(t, err) + must(t, conn.Write(ctx, websocket.MessageBinary, data)) + if got = read(conn); len(got.Messages) != 0 { + t.Fatal("R11: socket ack did not release inbox") + } + v := view() + v.Revision++ + v.Members[1].Revoked = true + must(t, r.Install(v)) + if _, _, err = conn.Read(ctx); err == nil { + t.Fatal("R16: revoked socket remained usable") + } + if _, err = client.Exchange(ctx, request(t, 2)); err != ErrAuth { + t.Fatal("R16: revoked socket identity used HTTP fallback", err) + } +} + +// R11/R16/R27 bound both wire ingress and sockets, keep signing material out +// of diagnostics, and stop cross-origin redirects of signed member batches. +func TestR11R16R27WireLimitsAndRedirectRefusal(t *testing.T) { + r := router(t) + h := Handler(r, func() time.Time { return time.UnixMilli(1000) }) + req := request(t, 1) + wire, err := Encode(req) + must(t, err) + for _, test := range []struct { + method, path string + body []byte + status int + }{ + {"POST", ExchangePath, wire, 200}, {"GET", ExchangePath, nil, 405}, {"POST", ExchangePath, append(wire, ' '), 400}, + {"POST", ExchangePath, bytes.Repeat([]byte("x"), MaxExchangeBytes+1), 400}, {"POST", ExchangePath + "?identity=other", wire, 400}, + {"POST", ExchangePath, []byte(`{"secret":"private-sentinel"}`), 400}, + } { + response := httptest.NewRecorder() + h.ServeHTTP(response, httptest.NewRequest(test.method, test.path, bytes.NewReader(test.body))) + if response.Code != test.status || strings.Contains(response.Body.String(), "private-sentinel") { + t.Fatal("R16/R27: incorrect bounded refusal", response.Code, response.Body.String()) + } + } + releases := []func(){} + for range MaxInflight { + release, err := r.openSocket(req) + must(t, err) + releases = append(releases, release) + } + if _, err = r.openSocket(req); err != ErrCapacity { + t.Fatal("R27: socket admission unbounded", err) + } + for _, release := range releases { + release() + } + if r.Stats(testScope, 1000).Sockets != 0 { + t.Fatal("R27: closed socket retained admission") + } + redirected := 0 + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { redirected++; w.WriteHeader(200) })) + defer target.Close() + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusTemporaryRedirect) + })) + defer origin.Close() + _, err = (Client{URL: origin.URL, HTTP: origin.Client()}).Exchange(context.Background(), req) + if err == nil || redirected != 0 { + t.Fatal("R16: signed peer traffic followed a cross-origin redirect") + } +} + +// R14 keeps the native envelope compatible with independently signed Node +// vectors, including all byte values and integers above JavaScript's floor. +func TestR14IndependentJavaScriptWireVectors(t *testing.T) { + data, err := readVector() + must(t, err) + var fixture struct { + PublicKey []byte `json:"public_key"` + Request Request `json:"request"` + } + must(t, Decode(data, &fixture)) + must(t, fixture.Request.Verify(ed25519.PublicKey(fixture.PublicKey))) + for _, e := range fixture.Request.Messages { + must(t, e.Verify(ed25519.PublicKey(fixture.PublicKey))) + } + if fixture.Request.Ack[Voter] != 1<<53 || fixture.Request.Messages[0].Sequence != ^uint64(0) { + t.Fatal("R14: golden vector lost uint64 bits") + } +} + +// readVector intentionally reads a committed cross-runtime fixture, not output +// generated by the production codec during the test. +func readVector() ([]byte, error) { + f, err := http.Dir("testdata").Open("exchange-v1.json") + if err != nil { + return nil, err + } + defer f.Close() + return io.ReadAll(f) +} + +// R15 carries sealed bytes without adding readable names, titles, paths or +// task text to transport metadata, diagnostics or malformed-request refusals. +func TestR15SealedPayloadCaptureAndSafeDiagnostics(t *testing.T) { + r := router(t) + plaintext := []byte("private-title-SENTINEL device-name-SENTINEL /private/path-SENTINEL task-body-SENTINEL secret-SENTINEL") + block, err := aes.NewCipher(bytes.Repeat([]byte{7}, 32)) + must(t, err) + aead, err := cipher.NewGCM(block) + must(t, err) + sealed := aead.Seal(nil, make([]byte, aead.NonceSize()), plaintext, nil) + e := Envelope{Scope: testScope, From: 1, To: 2, Incarnation: 1, Destination: 1, Sequence: 1, Payload: sealed, Chunk: &Chunk{Generation: digest([]byte("snapshot")), Hash: digest(sealed)}} + must(t, e.Sign(key(1))) + req := request(t, 1, e) + wire, err := Encode(req) + must(t, err) + exchange(t, r, req) + got := exchange(t, r, request(t, 2)) + capture, err := Encode(got) + must(t, err) + stats, err := json.Marshal(r.Stats(testScope, 1000)) + must(t, err) + for _, text := range [][]byte{wire, capture, stats} { + if bytes.Contains(text, []byte("SENTINEL")) { + t.Fatal("R15: plaintext appeared in transport capture/diagnostics") + } + } + opened, err := aead.Open(nil, make([]byte, aead.NonceSize()), got.Messages[0].Envelope.Payload, nil) + must(t, err) + if !bytes.Equal(opened, plaintext) { + t.Fatal("R15: transport changed sealed payload") + } +} diff --git a/internal/rafttransport/router.go b/internal/rafttransport/router.go new file mode 100644 index 0000000000..123f2c8a55 --- /dev/null +++ b/internal/rafttransport/router.go @@ -0,0 +1,398 @@ +package rafttransport + +import ( + "bytes" + "crypto/ed25519" + "math" + "sync" +) + +// Lane reservations are charged by encoded envelope bytes, including metadata. +// Learners and snapshots cannot spend the normal voter reservation. +type Lane uint8 + +const ( + Voter Lane = iota + Learner + Snapshot + laneCount + RetentionMillis int64 = 30_000 +) + +// Slot reservations bound the whole member inbox to eight outstanding +// deliveries while leaving space for voter traffic beside stalled learners. +var laneSlots = [laneCount]int{MaxInflight / 2, MaxInflight / 4, MaxInflight / 4} + +var laneBudget = [laneCount]int{3 * MaxQueuedBytes / 8, MaxQueuedBytes / 8, MaxQueuedBytes / 2} + +// Member comes ONLY from committed enrollment. Revoked records are retained as +// tombstones, and their node IDs can never be silently admitted again. +type Member struct { + Node uint64 + Incarnation uint64 + Key ed25519.PublicKey + Voter bool + Revoked bool +} + +// View is installed after durable application and before acknowledging a +// registry change. Generation is durable host state incremented on router +// restart, not a volatile connection number. Queued messages are expendable. +type View struct { + Scope Scope + Revision uint64 + Generation uint64 + Members []Member +} + +// Limits cap operator memory independently of per-identity byte reservations. +// Voters includes both sides of joint consensus, rather than only incoming IDs. +type Limits struct{ Identities, Records, Voters int } + +func DefaultLimits() Limits { return Limits{Identities: 1024, Records: 4096, Voters: 6} } + +type queued struct { + Delivery + size int + expires int64 +} +type inbox struct { + queues [laneCount][]queued + next, issued, acked [laneCount]uint64 + sockets int + changed chan struct{} +} +type group struct { + view View + members map[uint64]Member + inboxes map[uint64]*inbox + bytes [laneCount]int + now int64 +} +type Router struct { + mu sync.Mutex + limits Limits + groups map[Scope]*group +} + +func NewRouter(limits Limits) (*Router, error) { + if limits.Identities <= 0 || limits.Records <= 0 || limits.Voters <= 0 { + return nil, ErrCapacity + } + return &Router{limits: limits, groups: make(map[Scope]*group)}, nil +} + +// Install is a host seam, never an HTTP enrollment endpoint. Updating under +// the exchange lock fences requests authenticated before a committed revoke. +// Hosts must reload durable views before accepting traffic after a restart. +func (r *Router) Install(v View) error { + if !v.Scope.valid() || v.Revision == 0 || v.Generation == 0 || len(v.Members) > r.limits.Records { + return ErrWire + } + members := make(map[uint64]Member, len(v.Members)) + learners, voters := 0, 0 + for _, m := range v.Members { + if !node(m.Node) || m.Incarnation == 0 || len(m.Key) != ed25519.PublicKeySize { + return ErrWire + } + if _, ok := members[m.Node]; ok { + return ErrWire + } + m.Key = bytes.Clone(m.Key) + members[m.Node] = m + if !m.Revoked { + if m.Voter { + voters++ + } else { + learners++ + } + } + } + if learners > MaxLearners || voters > r.limits.Voters { + return ErrCapacity + } + r.mu.Lock() + defer r.mu.Unlock() + old := r.groups[v.Scope] + if old == nil && len(r.groups) >= r.limits.Identities { + return ErrCapacity + } + if old != nil { + if v.Revision <= old.view.Revision || v.Generation < old.view.Generation { + return ErrGeneration + } + for id, before := range old.members { + after, ok := members[id] + if !ok || before.Revoked && !after.Revoked || after.Incarnation < before.Incarnation || after.Incarnation == before.Incarnation && !bytes.Equal(after.Key, before.Key) { + return ErrAuth + } + } + } + g := &group{view: View{Scope: v.Scope, Revision: v.Revision, Generation: v.Generation}, members: members, inboxes: make(map[uint64]*inbox)} + if old != nil { + g.now = old.now + } + for id, m := range members { + if m.Revoked { + continue + } + box := &inbox{changed: make(chan struct{})} + if old != nil { + if prev := old.inboxes[id]; prev != nil { + // Reusing the notification channel lets open sockets notice retirement. + close(prev.changed) + if v.Generation == old.view.Generation && m.Incarnation == old.members[id].Incarnation { + box.next, box.issued, box.acked = prev.next, prev.issued, prev.acked + box.sockets = prev.sockets + } + } + } + g.inboxes[id] = box + } + if old != nil { + for id, box := range old.inboxes { + if m, ok := members[id]; !ok || m.Revoked { + close(box.changed) + } + } + } + r.groups[v.Scope] = g + return nil +} + +func signal(box *inbox) { close(box.changed); box.changed = make(chan struct{}) } +func (g *group) expire(now int64) { + g.now = max(g.now, min(max(now, 0), math.MaxInt64-RetentionMillis)) + for _, box := range g.inboxes { + for lane := Lane(0); lane < laneCount; lane++ { + keep := box.queues[lane][:0] + for _, q := range box.queues[lane] { + if q.expires > g.now { + keep = append(keep, q) + } else { + g.bytes[lane] -= q.size + } + } + clear(box.queues[lane][len(keep):]) + box.queues[lane] = keep + } + } +} +func (g *group) member(id, inc uint64) (Member, error) { + m, ok := g.members[id] + if !ok || m.Revoked || m.Incarnation != inc { + return Member{}, ErrAuth + } + return m, nil +} +func (g *group) lane(e Envelope) Lane { + if e.Chunk != nil { + return Snapshot + } + if g.members[e.From].Voter && g.members[e.To].Voter { + return Voter + } + return Learner +} + +// Delivery.Cursor is scoped to one lane and durable router generation. An ack +// releases transport memory only; it never completes a Raft persistence batch. +type Delivery struct { + Lane Lane `json:"lane"` + Cursor uint64 `json:"cursor,string"` + Envelope Envelope `json:"envelope"` +} +type Response struct { + Generation uint64 `json:"generation,string"` + Messages []Delivery `json:"messages"` +} + +func (r *Router) authorize(req Request) (*group, *inbox, error) { + g := r.groups[req.Scope] + if g == nil { + return nil, nil, ErrAuth + } + if req.Generation != g.view.Generation { + return nil, nil, ErrGeneration + } + m, err := g.member(req.Node, req.Incarnation) + if err != nil { + return nil, nil, err + } + if err = req.Verify(m.Key); err != nil { + return nil, nil, err + } + return g, g.inboxes[req.Node], nil +} + +// Exchange validates the entire outbound batch before admitting any message. +// Valid delivery acks still release memory on backpressure, so a full sender +// cannot prevent its own incoming queue from draining. Retries retain bytes. +func (r *Router) Exchange(req Request, now int64) (Response, error) { + return r.exchange(req, now, false) +} +func (r *Router) exchange(req Request, now int64, receiveOnly bool) (Response, error) { + if now < 0 || now > math.MaxInt64-RetentionMillis { + return Response{}, ErrWire + } + if err := req.validate(); err != nil { + return Response{}, err + } + r.mu.Lock() + defer r.mu.Unlock() + g, box, err := r.authorize(req) + if err != nil { + return Response{}, err + } + for lane, ack := range req.Ack { + if ack > box.issued[lane] { + return Response{}, ErrCursor + } + } + type pending struct { + e Envelope + lane Lane + size int + } + plan := make([]pending, 0, len(req.Messages)) + messages := req.Messages + if receiveOnly { + messages = nil + } + for _, e := range messages { + if e.Scope != req.Scope || e.From != req.Node || e.Incarnation != req.Incarnation { + return Response{}, ErrAuth + } + if _, err = g.member(e.To, e.Destination); err != nil { + return Response{}, err + } + if err = e.Verify(g.members[e.From].Key); err != nil { + return Response{}, err + } + encoded, _ := Encode(e) + plan = append(plan, pending{e, g.lane(e), len(encoded)}) + } + g.expire(now) + for lane, ack := range req.Ack { + if ack <= box.acked[lane] { + continue + } + keep := box.queues[lane][:0] + for _, q := range box.queues[lane] { + if q.Cursor > ack { + keep = append(keep, q) + } else { + g.bytes[lane] -= q.size + } + } + clear(box.queues[lane][len(keep):]) + box.queues[lane] = keep + box.acked[lane] = ack + } + // Pending duplicates are idempotent. Previously acknowledged or expired + // traffic can be redelivered; Raft, rather than an unbounded replay table, + // resolves that duplication. Reordering never invalidates an older sequence. + added := [laneCount]int{} + counts := make(map[[2]uint64]int) + accepted := plan[:0] + for _, p := range plan { + target := g.inboxes[p.e.To] + duplicate := false + for _, queue := range target.queues { + for _, q := range queue { + if q.Envelope.From == p.e.From && q.Envelope.Sequence == p.e.Sequence { + if !sameEnvelope(q.Envelope, p.e) { + return Response{}, ErrAuth + } + duplicate = true + break + } + } + } + for _, q := range accepted { + if q.e.To == p.e.To && q.e.Sequence == p.e.Sequence { + if !sameEnvelope(q.e, p.e) { + return Response{}, ErrAuth + } + duplicate = true + break + } + } + if duplicate { + continue + } + k := [2]uint64{p.e.To, uint64(p.lane)} + counts[k]++ + added[p.lane] += p.size + if len(target.queues[p.lane])+counts[k] > laneSlots[p.lane] || g.bytes[p.lane]+added[p.lane] > laneBudget[p.lane] || target.next[p.lane] > math.MaxUint64-uint64(counts[k]) { + return Response{}, ErrCapacity + } + accepted = append(accepted, p) + } + for _, p := range accepted { + target := g.inboxes[p.e.To] + target.next[p.lane]++ + b, _ := Encode(p.e) + var owned Envelope + _ = Decode(b, &owned) + target.queues[p.lane] = append(target.queues[p.lane], queued{Delivery: Delivery{p.lane, target.next[p.lane], owned}, size: p.size, expires: g.now + RetentionMillis}) + g.bytes[p.lane] += p.size + signal(target) + } + response := Response{Generation: g.view.Generation, Messages: []Delivery{}} + size := 0 + normal := 0 + snapshot := false + for lane := Lane(0); lane < laneCount; lane++ { + for _, q := range box.queues[lane] { + payload := len(q.Envelope.Payload) + if q.Envelope.Chunk != nil { + if snapshot || normal != 0 { + break + } + snapshot = true + } else { + if snapshot || normal+payload > MaxBatchBytes { + break + } + normal += payload + } + b, _ := Encode(q.Delivery) + if size+len(b) > MaxExchangeBytes-1024 { + break + } + var owned Delivery + _ = Decode(b, &owned) + response.Messages = append(response.Messages, owned) + size += len(b) + box.issued[lane] = max(box.issued[lane], q.Cursor) + } + } + return response, nil +} +func sameEnvelope(a, b Envelope) bool { x, _ := Encode(a); y, _ := Encode(b); return bytes.Equal(x, y) } + +// Stats exposes counters without payloads. Expiration uses explicit host time; +// inactive identities need no recurring cleanup alarm or disk write. +type Stats struct { + Bytes [laneCount]int + Queued, Sockets int +} + +func (r *Router) Stats(scope Scope, now int64) Stats { + r.mu.Lock() + defer r.mu.Unlock() + g := r.groups[scope] + if g == nil { + return Stats{} + } + g.expire(now) + s := Stats{Bytes: g.bytes} + for _, box := range g.inboxes { + s.Sockets += box.sockets + for _, q := range box.queues { + s.Queued += len(q) + } + } + return s +} diff --git a/internal/rafttransport/router_test.go b/internal/rafttransport/router_test.go new file mode 100644 index 0000000000..e08d565a50 --- /dev/null +++ b/internal/rafttransport/router_test.go @@ -0,0 +1,373 @@ +package rafttransport + +import ( + "bytes" + "crypto/ed25519" + "math" + "testing" + + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +var testScope = Scope{Identity: "id_test", Cluster: "cluster-test", Epoch: 1, Protocol: Protocol} + +func key(id byte) ed25519.PrivateKey { + return ed25519.NewKeyFromSeed(bytes.Repeat([]byte{id}, ed25519.SeedSize)) +} +func must(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} +func view() View { + v := View{Scope: testScope, Revision: 1, Generation: 9} + for id := uint64(1); id <= 4; id++ { + v.Members = append(v.Members, Member{Node: id, Incarnation: 1, Key: key(byte(id)).Public().(ed25519.PublicKey), Voter: id <= 3}) + } + return v +} +func router(t *testing.T) *Router { + t.Helper() + r, err := NewRouter(DefaultLimits()) + must(t, err) + must(t, r.Install(view())) + return r +} +func message(t *testing.T, from, to, seq uint64, kind pb.MessageType, payload int) Envelope { + t.Helper() + m := &pb.Message{Type: kind.Enum(), From: proto.Uint64(from), To: proto.Uint64(to), Term: proto.Uint64(1), Context: bytes.Repeat([]byte("x"), payload)} + b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m) + must(t, err) + e := Envelope{Scope: testScope, From: from, To: to, Incarnation: 1, Destination: 1, Sequence: seq, Payload: b} + must(t, e.Sign(key(byte(from)))) + return e +} +func request(t *testing.T, id uint64, messages ...Envelope) Request { + t.Helper() + r := Request{Scope: testScope, Node: id, Incarnation: 1, Generation: 9, Messages: messages} + must(t, r.Sign(key(byte(id)))) + return r +} +func exchange(t *testing.T, r *Router, req Request) Response { + t.Helper() + resp, err := r.Exchange(req, 1000) + must(t, err) + return resp +} + +// R10/R14/R16 require wire identities to survive JavaScript and prevent a +// signed sender from impersonating another inner Raft member or local input. +func TestR10R14R16SignedEnvelopesAndCanonicalWire(t *testing.T) { + e := message(t, 1, 2, math.MaxUint64, pb.MsgHeartbeat, 0) + must(t, e.Verify(key(1).Public().(ed25519.PublicKey))) + req := request(t, 1, e) + req.Ack = [laneCount]uint64{math.MaxUint64, 1 << 53, 0} + must(t, req.Sign(key(1))) + b, err := Encode(req) + must(t, err) + var round Request + must(t, Decode(b, &round)) + must(t, round.Verify(key(1).Public().(ed25519.PublicKey))) + if !bytes.Contains(b, []byte(`"18446744073709551615"`)) || !bytes.Contains(b, []byte(`"9007199254740992"`)) { + t.Fatal("R14: lossy integer encoding") + } + for _, bad := range [][]byte{append(b, ' '), bytes.Replace(b, []byte(`"node":"1"`), []byte(`"node":"01"`), 1), bytes.Replace(b, []byte(`"identity":"id_test"`), []byte(`"identity":"id_test","identity":"id_test"`), 1), bytes.Replace(b, []byte(`"ack":[`), []byte(`"ack":["0",`), 1)} { + if Decode(bad, new(Request)) == nil { + t.Fatal("R14: accepted ambiguous wire bytes") + } + } + for _, mutate := range []func(*Envelope){func(e *Envelope) { e.Scope.Identity = "id_other" }, func(e *Envelope) { e.Scope.Cluster = "other" }, func(e *Envelope) { e.Scope.Epoch++ }, func(e *Envelope) { e.To = 3 }, func(e *Envelope) { e.Incarnation++ }, func(e *Envelope) { e.Destination++ }, func(e *Envelope) { e.Sequence++ }, func(e *Envelope) { e.Payload = append(bytes.Clone(e.Payload), 0) }} { + bad := e + mutate(&bad) + if bad.Verify(key(1).Public().(ed25519.PublicKey)) == nil { + t.Fatal("R16: altered envelope signature accepted") + } + } + for _, kind := range []pb.MessageType{pb.MsgHup, pb.MsgBeat, pb.MsgProp, pb.MsgReadIndex, pb.MsgStorageAppend, pb.MsgStorageApply} { + m := &pb.Message{Type: kind.Enum(), From: proto.Uint64(1), To: proto.Uint64(2), Term: proto.Uint64(1)} + b, err := proto.Marshal(m) + must(t, err) + e.Payload = b + if e.Sign(key(1)) != ErrWire { + t.Fatalf("R16: accepted local/forwarded message %v", kind) + } + } + e = message(t, 1, 2, 1, pb.MsgApp, 0) + e.From = 3 + if e.Sign(key(3)) != ErrWire { + t.Fatal("R16: signed inner sender spoof accepted") + } +} + +// R10/R16 permit duplicate/reordered transport while denying cross-identity, +// unknown, revoked or retired-generation authority, including delayed requests. +func TestR10R16DuplicateReorderAndRevocation(t *testing.T) { + r := router(t) + a := message(t, 1, 2, 2, pb.MsgApp, 0) + b := message(t, 1, 2, 1, pb.MsgVote, 0) + req := request(t, 1, a, b, a) + exchange(t, r, req) + exchange(t, r, req) + got := exchange(t, r, request(t, 2)) + if len(got.Messages) != 2 || got.Messages[0].Envelope.Sequence != 2 || got.Messages[1].Envelope.Sequence != 1 { + t.Fatal("R10: duplicate/reorder changed message delivery") + } + got.Messages[0].Envelope.Payload[0] ^= 1 + if next := exchange(t, r, request(t, 2)); next.Messages[0].Envelope.Verify(key(1).Public().(ed25519.PublicKey)) != nil { + t.Fatal("R16: receiver mutated retained queue") + } + ack := request(t, 2) + ack.Ack[Voter] = got.Messages[1].Cursor + must(t, ack.Sign(key(2))) + exchange(t, r, ack) + if s := r.Stats(testScope, 1000); s.Queued != 0 { + t.Fatal("R11: acknowledgment retained messages") + } + exchange(t, r, req) + bad := request(t, 2) + bad.Ack[Voter] = 99 + must(t, bad.Sign(key(2))) + if _, err := r.Exchange(bad, 1000); err != ErrCursor { + t.Fatal("R16: unsigned/unissued cursor drained traffic", err) + } + bad = request(t, 1, a) + bad.Scope.Identity = "id_other" + must(t, bad.Sign(key(1))) + if _, err := r.Exchange(bad, 1000); err != ErrAuth { + t.Fatal("R16: cross-identity admitted", err) + } + other := view() + other.Scope.Identity = "id_other" + must(t, r.Install(other)) + if _, err := r.Exchange(bad, 1000); err != ErrAuth { + t.Fatal("R16: cross-identity payload admitted into registered identity", err) + } + bad = request(t, 1, a) + bad.Signature[0] ^= 1 + if _, err := r.Exchange(bad, 1000); err != ErrAuth { + t.Fatal("R16: forged exchange accepted", err) + } + // The signed request already exists when the durable registry changes. + v := view() + v.Revision++ + v.Members[0].Revoked = true + must(t, r.Install(v)) + if _, err := r.Exchange(req, 1000); err != ErrAuth { + t.Fatal("R16: delayed authenticated request outlived committed revocation", err) + } + if got := exchange(t, r, request(t, 2)); len(got.Messages) != 0 { + t.Fatal("R16: revoked sender retained queued authority") + } + v.Revision++ + v.Members[0].Revoked = false + if r.Install(v) != ErrAuth { + t.Fatal("R16: retired node regained authority") + } + v = view() + v.Revision = 3 + v.Members[0].Revoked = true + v.Members[1].Incarnation = 2 + must(t, r.Install(v)) + if _, err := r.Exchange(request(t, 2), 1000); err != ErrAuth { + t.Fatal("R16: old member incarnation admitted", err) + } + e := message(t, 3, 2, 1, pb.MsgHeartbeat, 0) + if _, err := r.Exchange(request(t, 3, e), 1000); err != ErrAuth { + t.Fatal("R16: old destination generation admitted", err) + } + v.Revision++ + v.Generation++ + must(t, r.Install(v)) + if _, err := r.Exchange(request(t, 3), 1000); err != ErrGeneration { + t.Fatal("R16: old restart cursor admitted", err) + } +} + +// R11/R27 require voter reservation even when an absent learner or snapshot +// receiver fills its budget, with bounded batches, cursors and retention. +func TestR11R27VoterReservationsAndBackpressure(t *testing.T) { + r := router(t) + for i := uint64(1); i <= 2; i++ { + exchange(t, r, request(t, 1, message(t, 1, 4, i, pb.MsgApp, 40<<10))) + } + if _, err := r.Exchange(request(t, 1, message(t, 1, 4, 3, pb.MsgApp, 40<<10)), 1000); err != ErrCapacity { + t.Fatal("R11: unbounded learner queue", err) + } + for i := uint64(1); i <= uint64(laneSlots[Voter]); i++ { + exchange(t, r, request(t, 1, message(t, 1, 2, i, pb.MsgHeartbeat, 0))) + } + if _, err := r.Exchange(request(t, 1, message(t, 1, 2, 9, pb.MsgHeartbeat, 0)), 1000); err != ErrCapacity { + t.Fatal("R27: inflight cap absent", err) + } + // A full-size chunk has a separate reservation and cannot consume votes. + chunk := Envelope{Scope: testScope, From: 2, To: 3, Incarnation: 1, Destination: 1, Sequence: 1, Payload: bytes.Repeat([]byte{9}, MaxChunkBytes), Chunk: &Chunk{Generation: digest([]byte("snapshot"))}} + chunk.Chunk.Hash = digest(chunk.Payload) + must(t, chunk.Sign(key(2))) + exchange(t, r, request(t, 2, chunk)) + if _, err := r.Exchange(request(t, 2, func() Envelope { x := chunk; x.Sequence++; must(t, x.Sign(key(2))); return x }()), 1000); err != ErrCapacity { + t.Fatal("R27: unbounded snapshot queue", err) + } + exchange(t, r, request(t, 3, message(t, 3, 1, 1, pb.MsgVote, 0))) + got := exchange(t, r, request(t, 3)) + if len(got.Messages) != 1 || got.Messages[0].Lane != Snapshot { + t.Fatal("R11: snapshot did not use its own lane") + } + got = exchange(t, r, request(t, 2)) + if len(got.Messages) != laneSlots[Voter] { + t.Fatal("R11: voter traffic was displaced") + } + s := r.Stats(testScope, 1000) + total := 0 + for lane, n := range s.Bytes { + if n > laneBudget[lane] { + t.Fatal("R27: lane budget exceeded") + } + total += n + } + if total > MaxQueuedBytes { + t.Fatal("R27: identity byte bound exceeded") + } + if s = r.Stats(testScope, 1000+RetentionMillis); s.Queued != 0 || s.Bytes != [laneCount]int{} { + t.Fatal("R27: transient buffers did not expire") + } + // Expired cursors are never reused, including under a backward host clock. + again := exchange(t, r, request(t, 1, message(t, 1, 2, 10, pb.MsgApp, 0))) + _ = again + got = exchange(t, r, request(t, 2)) + if len(got.Messages) != 1 || got.Messages[0].Cursor <= uint64(laneSlots[Voter]) { + t.Fatal("R16: expired cursor reused") + } +} + +// R11/R16 require atomic batch admission and safe retry of a queue-full send; +// valid acks must make progress even when the same request hits backpressure. +func TestR11R16AtomicBatchAndAckUnderPressure(t *testing.T) { + r := router(t) + first := message(t, 1, 2, 1, pb.MsgApp, 0) + bad := message(t, 1, 3, 2, pb.MsgApp, 0) + bad.Signature[0] ^= 1 + if _, err := r.Exchange(request(t, 1, first, bad), 1000); err != ErrAuth || r.Stats(testScope, 1000).Queued != 0 { + t.Fatal("R16: partial invalid batch admitted", err) + } + for i := uint64(1); i <= uint64(laneSlots[Voter]); i++ { + exchange(t, r, request(t, 1, message(t, 1, 2, i, pb.MsgApp, 0))) + } + exchange(t, r, request(t, 2, message(t, 2, 1, 1, pb.MsgApp, 0))) + got := exchange(t, r, request(t, 1)) + req := request(t, 1, message(t, 1, 2, 9, pb.MsgApp, 0)) + req.Ack[Voter] = got.Messages[0].Cursor + must(t, req.Sign(key(1))) + if _, err := r.Exchange(req, 1000); err != ErrCapacity { + t.Fatal("R11: expected outbound backpressure", err) + } + if got = exchange(t, r, request(t, 1)); len(got.Messages) != 0 { + t.Fatal("R11: full outbound blocked incoming ack") + } + // A conflicting sequence while pending cannot replace an authenticated packet. + conflict := message(t, 1, 2, 1, pb.MsgVote, 0) + if _, err := r.Exchange(request(t, 1, conflict), 1000); err != ErrAuth { + t.Fatal("R16: conflicting transport sequence replaced pending message", err) + } +} + +// R11/R27 bound live learners and operator registration without turning a +// signed poll into enrollment, or permitting history deletion to regain space. +func TestR11R27RegistryAndOperatorBounds(t *testing.T) { + r, err := NewRouter(Limits{1, 40, 6}) + must(t, err) + v := view() + for id := uint64(5); id <= 35; id++ { + v.Members = append(v.Members, Member{Node: id, Incarnation: 1, Key: key(byte(id)).Public().(ed25519.PublicKey)}) + } + must(t, r.Install(v)) + v.Revision++ + v.Members = append(v.Members, Member{Node: 36, Incarnation: 1, Key: key(36).Public().(ed25519.PublicKey)}) + if r.Install(v) != ErrCapacity { + t.Fatal("R11: learner admission unbounded") + } + second := view() + second.Scope.Cluster = "second" + if r.Install(second) != ErrCapacity { + t.Fatal("R27: operator identity cap ignored") + } + v = view() + v.Revision = 3 + if r.Install(v) != ErrAuth { + t.Fatal("R16: node tombstones silently deleted") + } + if _, err := r.Exchange(request(t, 99), 1000); err != ErrAuth { + t.Fatal("R16: signed unknown key enrolled itself", err) + } +} + +// R11 preserves voter ordering and read-barrier contexts even when older +// learner/chunk traffic was queued first; lane acknowledgments stay separate. +func TestR11VoterPriorityPreservesReadContexts(t *testing.T) { + r := router(t) + exchange(t, r, request(t, 4, message(t, 4, 2, 1, pb.MsgApp, 0))) + chunk := Envelope{Scope: testScope, From: 3, To: 2, Incarnation: 1, Destination: 1, Sequence: 1, Payload: []byte("opaque-chunk"), Chunk: &Chunk{Generation: digest([]byte("generation")), Hash: digest([]byte("opaque-chunk"))}} + must(t, chunk.Sign(key(3))) + exchange(t, r, request(t, 3, chunk)) + first := message(t, 1, 2, 1, pb.MsgHeartbeat, 11) + second := message(t, 1, 2, 2, pb.MsgHeartbeat, 12) + exchange(t, r, request(t, 1, first, second)) + got := exchange(t, r, request(t, 2)) + if len(got.Messages) != 3 || got.Messages[0].Lane != Voter || got.Messages[1].Lane != Voter || got.Messages[2].Lane != Learner || !sameEnvelope(got.Messages[0].Envelope, first) || !sameEnvelope(got.Messages[1].Envelope, second) { + t.Fatal("R11: learner backlog overtook votes or coalesced read barriers") + } + ack := request(t, 2) + ack.Ack[Voter] = got.Messages[1].Cursor + must(t, ack.Sign(key(2))) + got = exchange(t, r, ack) + if len(got.Messages) != 1 || got.Messages[0].Lane != Learner { + t.Fatal("R11: voter ack consumed another lane") + } + ack.Ack[Learner] = got.Messages[0].Cursor + must(t, ack.Sign(key(2))) + got = exchange(t, r, ack) + if len(got.Messages) != 1 || got.Messages[0].Lane != Snapshot || !sameEnvelope(got.Messages[0].Envelope, chunk) { + t.Fatal("R11: drained normal traffic did not release snapshot delivery") + } +} + +// R11/R27 refuse oversized normal batches and chunks before admission, while +// a maximum-size chunk remains deliverable with its hash/offset/generation. +func TestR11R27BatchAndChunkBoundaries(t *testing.T) { + r := router(t) + req := Request{Scope: testScope, Node: 1, Incarnation: 1, Generation: 9, Messages: []Envelope{message(t, 1, 2, 1, pb.MsgApp, 40<<10), message(t, 1, 2, 2, pb.MsgApp, 40<<10)}} + if req.Sign(key(1)) != ErrWire { + t.Fatal("R27: normal batch byte cap missing") + } + chunk := Envelope{Scope: testScope, From: 1, To: 2, Incarnation: 1, Destination: 1, Sequence: 1, Payload: bytes.Repeat([]byte{3}, MaxChunkBytes), Chunk: &Chunk{Generation: digest([]byte("generation")), Offset: math.MaxUint64}} + chunk.Chunk.Hash = digest(chunk.Payload) + must(t, chunk.Sign(key(1))) + exchange(t, r, request(t, 1, chunk)) + got := exchange(t, r, request(t, 2)) + if len(got.Messages) != 1 || !sameEnvelope(got.Messages[0].Envelope, chunk) { + t.Fatal("R14/R27: maximum chunk lost bytes or full-width offset") + } + chunk.Payload = append(chunk.Payload, 0) + chunk.Chunk.Hash = digest(chunk.Payload) + if chunk.Sign(key(1)) != ErrWire { + t.Fatal("R27: oversized chunk admitted") + } +} + +// R16 prevents a pending transport sequence from being replaced under a +// different queue class, and preserves its original signed bytes on refusal. +func TestR16PendingSequenceCannotChangeLane(t *testing.T) { + r := router(t) + original := message(t, 1, 2, 1, pb.MsgApp, 0) + exchange(t, r, request(t, 1, original)) + e := Envelope{Scope: testScope, From: 1, To: 2, Incarnation: 1, Destination: 1, Sequence: 1, Payload: []byte("sealed-chunk"), Chunk: &Chunk{Generation: digest([]byte("generation")), Hash: digest([]byte("sealed-chunk"))}} + must(t, e.Sign(key(1))) + if _, err := r.Exchange(request(t, 1, e), 1000); err != ErrAuth { + t.Fatal("R16: pending sequence changed class", err) + } + got := exchange(t, r, request(t, 2)) + if len(got.Messages) != 1 || !sameEnvelope(got.Messages[0].Envelope, original) { + t.Fatal("R16: refused replacement damaged original packet") + } +} diff --git a/internal/rafttransport/testdata/exchange-v1.json b/internal/rafttransport/testdata/exchange-v1.json new file mode 100644 index 0000000000..33dd5e4304 --- /dev/null +++ b/internal/rafttransport/testdata/exchange-v1.json @@ -0,0 +1 @@ +{"public_key":"iojj3XQJ8ZX9UtstPLpdcspnCb8dlBIb83SIAbQPb1w=","request":{"scope":{"identity":"id_test","cluster":"cluster-test","epoch":"18446744073709551615","protocol":1},"node":"1","incarnation":"9007199254740992","generation":"18446744073709551615","messages":[{"scope":{"identity":"id_test","cluster":"cluster-test","epoch":"18446744073709551615","protocol":1},"from":"1","to":"2","incarnation":"9007199254740992","destination":"1","sequence":"18446744073709551615","chunk":null,"digest":"f51bf3b4b51ff22a780b8782cc491cff17da1f99d668caf785dcd540533f4972","payload":"CAgQAhgBIAFigAIAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIiMkJSYnKCkqKywtLi8wMTIzNDU2Nzg5Ojs8PT4/QEFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaW1xdXl9gYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXp7fH1+f4CBgoOEhYaHiImKi4yNjo+QkZKTlJWWl5iZmpucnZ6foKGio6SlpqeoqaqrrK2ur7CxsrO0tba3uLm6u7y9vr/AwcLDxMXGx8jJysvMzc7P0NHS09TV1tfY2drb3N3e3+Dh4uPk5ebn6Onq6+zt7u/w8fLz9PX29/j5+vv8/f7/","signature":"jo3NopLnkdQRafpcAlyxYh5WLeNAWkckTTo5cvWEEwCOf71gdMJvaSZob4s0c0azgnQLAsvfAaZr3vGgPE72BQ=="}],"signature":"Vsa6T0vktNg0fEL8zth0J+yLzyPE7CCTlEk30aDWBlTyu2dHn0RkMnHh9/RVZpBheUqiG1K6Oi3r4AOXJ+SDBA==","ack":["9007199254740992","0","18446744073709551615"]}} \ No newline at end of file diff --git a/relay/hosted/test/raft/transport.test.js b/relay/hosted/test/raft/transport.test.js new file mode 100644 index 0000000000..4e43447a0d --- /dev/null +++ b/relay/hosted/test/raft/transport.test.js @@ -0,0 +1,31 @@ +import assert from 'node:assert/strict'; +import { createPublicKey, createHash, verify } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { test } from 'node:test'; + +// R14 checks independently authored Node signatures against the exact wire +// fixture also decoded by native Go; no Worker or hosted service is contacted. +test('R14/R16 native peer envelopes preserve bytes, integers and signature domains', () => { + const text = readFileSync(new URL('../../../../internal/rafttransport/testdata/exchange-v1.json', import.meta.url), 'utf8'); + const fixture = JSON.parse(text); + assert.equal(JSON.stringify(fixture), text); + const key = createPublicKey({ key: Buffer.concat([Buffer.from('302a300506032b6570032100', 'hex'), Buffer.from(fixture.public_key, 'base64')]), format: 'der', type: 'spki' }); + const check = (value, domain) => { + const signature = Buffer.from(value.signature, 'base64'); + const unsigned = { ...value, signature: null }; + assert(verify(null, Buffer.from(domain + JSON.stringify(unsigned)), key, signature)); + assert(!verify(null, Buffer.from('wrong-domain\n' + JSON.stringify(unsigned)), key, signature)); + }; + check(fixture.request, 'codeaf/raft/exchange/1\n'); + const envelope = fixture.request.messages[0]; + check(envelope, 'codeaf/raft/envelope/1\n'); + const payload = Buffer.from(envelope.payload, 'base64'); + assert.equal(createHash('sha256').update(payload).digest('hex'), envelope.digest); + assert.deepEqual(payload.subarray(-256), Buffer.from(Array.from({ length: 256 }, (_, i) => i))); + assert.equal(BigInt(envelope.sequence), (1n << 64n) - 1n); + assert.equal(BigInt(fixture.request.ack[0]), 1n << 53n); + assert.equal(BigInt(envelope.scope.epoch), (1n << 64n) - 1n); + assert.equal(BigInt(envelope.incarnation), 1n << 53n); + const altered = { ...envelope, to: '3', signature: null }; + assert(!verify(null, Buffer.from('codeaf/raft/envelope/1\n' + JSON.stringify(altered)), key, Buffer.from(envelope.signature, 'base64'))); +}); From 5d6666e217861fd4c8a75862fae02f6a4788a409 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 00:13:55 -0400 Subject: [PATCH 015/103] Prove native quorum progress through outbound relay exchanges before production host assembly. Mount the peer handler through the relay host boundary and drive three durable native members through election, committed application, ReadIndex, restart and absent-learner progress without inbound member listeners. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/relayserve/cluster.go | 14 ++ internal/relayserve/cluster_test.go | 313 ++++++++++++++++++++++++++++ internal/relayserve/service.go | 6 + 3 files changed, 333 insertions(+) create mode 100644 internal/relayserve/cluster.go create mode 100644 internal/relayserve/cluster_test.go diff --git a/internal/relayserve/cluster.go b/internal/relayserve/cluster.go new file mode 100644 index 0000000000..d9bcd7c10a --- /dev/null +++ b/internal/relayserve/cluster.go @@ -0,0 +1,14 @@ +package relayserve + +import "net/http" + +// mountCluster keeps peer transport separate from pairing mailboxes, public +// command admission and the directory writer. A host with no enrolled members +// has no peer capability to advertise. The handler boundary avoids starting or +// linking a consensus runtime before the persistent host is assembled. +func mountCluster(mux *http.ServeMux, handler http.Handler) { + if handler == nil { + return + } + mux.Handle("/v2/cluster/", handler) +} diff --git a/internal/relayserve/cluster_test.go b/internal/relayserve/cluster_test.go new file mode 100644 index 0000000000..a04211efe3 --- /dev/null +++ b/internal/relayserve/cluster_test.go @@ -0,0 +1,313 @@ +package relayserve + +import ( + "bytes" + "context" + "crypto/ed25519" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "slices" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftabi" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/rafttransport" + "go.etcd.io/raft/v3" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// clusterFixture drives actual native stores and cores. The only listener is +// the relay; members can only make outbound exchange requests. Application is +// P03's narrow real-rule probe, never the future authorized directory writer. +type clusterFixture struct { + t *testing.T + ctx context.Context + router *rafttransport.Router + scope rafttransport.Scope + client rafttransport.Client + cores [3]*raftcore.Core + stores [3]*raftstore.SQLite + paths [3]string + apps [3]raftstore.Application + status [3]raftcore.Status + row [3][]byte + pending [3][]rafttransport.Envelope + ack [3][3]uint64 + seq [3]uint64 + alive [3]bool + events int + exchanges int + reads []raftcore.ReadBarrier +} + +func clusterMust(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} +func clusterKey(id byte) ed25519.PrivateKey { + return ed25519.NewKeyFromSeed(bytes.Repeat([]byte{id}, 32)) +} +func newClusterFixture(t *testing.T) *clusterFixture { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + t.Cleanup(cancel) + router, err := rafttransport.NewRouter(rafttransport.DefaultLimits()) + clusterMust(t, err) + scope := rafttransport.Scope{Identity: "id_native", Cluster: "native-outbound", Epoch: 1, Protocol: rafttransport.Protocol} + view := rafttransport.View{Scope: scope, Revision: 1, Generation: 1} + for id := uint64(1); id <= 3; id++ { + view.Members = append(view.Members, rafttransport.Member{Node: id, Incarnation: 1, Key: clusterKey(byte(id)).Public().(ed25519.PublicKey), Voter: true}) + } + clusterMust(t, router.Install(view)) + service := New(Config{Cluster: rafttransport.Handler(router, func() time.Time { return time.UnixMilli(1000) }), Now: func() time.Time { return time.UnixMilli(1000) }}) + t.Cleanup(func() { clusterMust(t, service.Close()) }) + server := httptest.NewServer(service.Handler) + t.Cleanup(server.Close) + f := &clusterFixture{t: t, ctx: ctx, router: router, scope: scope, client: rafttransport.Client{URL: server.URL, HTTP: server.Client()}} + for i := range f.cores { + f.paths[i] = filepath.Join(t.TempDir(), "member.db") + f.start(i) + } + return f +} +func (f *clusterFixture) start(i int) { + f.t.Helper() + id := raftstore.Identity{ClusterUUID: f.scope.Cluster, NodeID: uint64(i + 1), Protocol: raftcore.ProtocolVersion} + s, err := raftstore.Open(f.paths[i], id) + clusterMust(f.t, err) + f.t.Cleanup(func() { s.Close() }) + recovered, err := s.Recover(f.ctx) + clusterMust(f.t, err) + cfg := raftcore.Config{ClusterUUID: id.ClusterUUID, NodeID: id.NodeID, Seed: uint64(100 + i + f.events), Protocol: id.Protocol} + if recovered.Fresh { + cfg.Bootstrap = []uint64{1, 2, 3} + } + f.cores[i], err = raftcore.New(cfg, recovered.Core) + clusterMust(f.t, err) + f.stores[i], f.apps[i], f.alive[i] = s, recovered.Application, true + records, _, err := s.Read(f.ctx, recovered.Application.Applied, []raftstore.Key{{Namespace: raftstore.State, Key: "cell"}}) + clusterMust(f.t, err) + f.row[i] = records[raftstore.Key{Namespace: raftstore.State, Key: "cell"}] + f.call(i, raftcore.Poll{}) +} +func (f *clusterFixture) call(i int, input raftcore.Input) { + f.t.Helper() + if !f.alive[i] { + return + } + for depth := 0; depth < 100; depth++ { + f.events++ + if f.events > 2000 { + f.t.Fatal("R10: outbound fixture exceeded 2,000 events") + } + effects, err := f.cores[i].Step(input) + clusterMust(f.t, err) + f.status[i] = effects.Status + for _, out := range effects.Send { + f.seq[i]++ + e := rafttransport.Envelope{Scope: f.scope, From: uint64(i + 1), To: out.To, Incarnation: 1, Destination: 1, Sequence: f.seq[i], Payload: out.Bytes} + clusterMust(f.t, e.Sign(clusterKey(byte(i+1)))) + f.pending[i] = append(f.pending[i], e) + } + for _, read := range effects.Reads { + if f.apps[i].Applied < read.Index { + f.t.Fatal("R4: transport escaped durable application") + } + f.reads = append(f.reads, read) + } + if effects.Persist != nil { + if len(effects.Send) != 0 { + f.t.Fatal("R3: transport saw a pre-flush append response") + } + clusterMust(f.t, f.stores[i].Persist(f.ctx, *effects.Persist)) + input = raftcore.Persisted{BatchID: effects.Persist.ID} + continue + } + if effects.Apply != nil { + a := f.apps[i] + u := raftstore.Update{ExpectedApplied: a.Applied, VisibleVersion: a.VisibleVersion, LogicalTime: a.LogicalTime} + for _, entry := range effects.Apply.Entries { + if entry.GetType() != pb.EntryNormal || len(entry.Data) == 0 { + continue + } + f.row[i], err = raftabi.ProbeApply(entry.Data, f.row[i]) + clusterMust(f.t, err) + command, err := clusterstate.DecodeCommand(entry.Data) + clusterMust(f.t, err) + u.LogicalTime = max(u.LogicalTime, command.Now) + u.VisibleVersion++ + } + if !bytes.Equal(f.row[i], nil) { + u.Writes = []raftstore.Write{{Key: raftstore.Key{Namespace: raftstore.State, Key: "cell"}, Value: f.row[i]}} + } + clusterMust(f.t, f.stores[i].Apply(f.ctx, *effects.Apply, u)) + recovered, err := f.stores[i].Recover(f.ctx) + clusterMust(f.t, err) + f.apps[i] = recovered.Application + input = raftcore.Applied{BatchID: effects.Apply.ID} + continue + } + return + } + f.t.Fatal("R10: unbounded completion chain") +} +func (f *clusterFixture) pump() { + f.t.Helper() + for round := 0; round < 40; round++ { + activity := false + for i := range f.cores { + if !f.alive[i] { + continue + } + // Deliberately reverse and duplicate batches before relay admission. No + // member can rely on an inbound socket, FIFO delivery or unique arrival. + outgoing := slices.Clone(f.pending[i]) + slices.Reverse(outgoing) + if len(outgoing) > 0 { + outgoing = []rafttransport.Envelope{outgoing[0], outgoing[0]} + } + req := rafttransport.Request{Scope: f.scope, Node: uint64(i + 1), Incarnation: 1, Generation: 1, Ack: f.ack[i], Messages: outgoing} + clusterMust(f.t, req.Sign(clusterKey(byte(i+1)))) + resp, err := f.client.Exchange(f.ctx, req) + if len(outgoing) > 0 { + f.pending[i] = f.pending[i][:len(f.pending[i])-1] + } + if err == rafttransport.ErrCapacity { + // Failed delivery is reported to the core, which owns retry. + // A blocked destination cannot hold healthy peers behind it. + f.call(i, raftcore.TransportResult{Peer: outgoing[0].To, Success: false}) + req.Messages = nil + clusterMust(f.t, req.Sign(clusterKey(byte(i+1)))) + resp, err = f.client.Exchange(f.ctx, req) + } + clusterMust(f.t, err) + f.exchanges++ + if len(outgoing) != 0 || len(resp.Messages) != 0 { + activity = true + } + for _, delivery := range resp.Messages { + e := delivery.Envelope + clusterMust(f.t, e.Verify(clusterKey(byte(e.From)).Public().(ed25519.PublicKey))) + if e.Scope != f.scope || e.To != uint64(i+1) || e.Incarnation != 1 || e.Destination != 1 { + f.t.Fatal("R16: wrong dispatch authority") + } + f.call(i, raftcore.PeerMessage{Bytes: e.Payload}) + f.ack[i][delivery.Lane] = max(f.ack[i][delivery.Lane], delivery.Cursor) + } + } + if !activity { + return + } + } + f.t.Fatal("R10: relay did not drain within bounded exchanges") +} +func (f *clusterFixture) tick() { + for i := range f.cores { + f.call(i, raftcore.Tick{}) + } + f.pump() +} +func (f *clusterFixture) leader() int { + f.t.Helper() + for range 40 { + f.tick() + for i, s := range f.status { + if f.alive[i] && s.Role == raft.StateLeader { + return i + } + } + } + f.t.Fatal("R10: outbound majority could not elect") + return -1 +} +func (f *clusterFixture) propose(i int, seq uint64, change clusterstate.Mutation) { + f.t.Helper() + command, err := clusterstate.EncodeCommand(clusterstate.Command{Schema: clusterstate.SchemaVersion, Cluster: f.scope.Cluster, Actor: "device", Session: "probe", Sequence: seq, RequestDigest: fmt.Sprint(seq), Now: int64(1000 + seq), Change: change}) + clusterMust(f.t, err) + f.call(i, raftcore.Propose{Command: command}) + f.pump() +} + +// R3/R4/R10 prove three outbound-only members elect, commit a real chat rule, +// pass ReadIndex and recover through the relay after loss of the old leader. +func TestR3R4R10OutboundOnlyNativeRelayProgressAndRecovery(t *testing.T) { + f := newClusterFixture(t) + leader := f.leader() + f.propose(leader, 1, clusterstate.Mutation{Create: &clusterstate.CreateCell{Cell: "cell", Init: clusterstate.CellInit{Head: "sealed-head"}, Receipt: "uploaded"}}) + f.call(leader, raftcore.Read{RequestID: "outbound-read"}) + f.pump() + if len(f.reads) != 1 { + t.Fatal("R4/R10: outbound quorum read did not finish") + } + before := bytes.Clone(f.row[leader]) + for i := range f.cores { + if !bytes.Equal(before, f.row[i]) { + t.Fatal("R3: connected member missed durable create") + } + } + f.alive[leader] = false + clusterMust(t, f.stores[leader].Close()) + f.cores[leader] = nil + next := f.leader() + f.call(next, raftcore.Read{RequestID: "fallback-read"}) + f.pump() + if len(f.reads) != 2 || !bytes.Equal(before, f.row[next]) { + t.Fatal("R3/R4/R10: outbound fallback lost committed row or read authority") + } + f.start(leader) + for range 2 { + f.tick() + } + if !bytes.Equal(before, f.row[leader]) || f.status[leader].Role == raft.StateLeader { + t.Fatal("R3/R10: restarted peer did not recover as follower") + } + // Demote one node through the real joint mechanism, then lose that learner. + leader = next + change, err := protoChange(3) + clusterMust(t, err) + f.call(leader, raftcore.Configure{Change: change}) + f.pump() + empty, err := protoChange(0) + clusterMust(t, err) + f.call(leader, raftcore.Configure{Change: empty}) + f.pump() + f.alive[2] = false + leader = f.leader() + f.propose(leader, 2, clusterstate.Mutation{Release: &clusterstate.ReleaseCell{Cell: "cell", Fence: 1}}) + row, err := clusterstate.DecodeRow(f.row[leader]) + clusterMust(t, err) + if row.Cell.Lease.Expires != 0 || row.Cell.Lease.ReleasedAt != 1002 { + t.Fatal("R11: departed learner blocked a committed release") + } + t.Logf("R3/R4/R10/R11: %d core inputs, %d outbound HTTP exchanges; only relay owns a listener", f.events, f.exchanges) +} +func protoChange(target uint64) ([]byte, error) { + change := &pb.ConfChangeV2{} + if target != 0 { + change.Transition = pb.ConfChangeTransitionJointExplicit.Enum() + change.Changes = []*pb.ConfChangeSingle{{Type: pb.ConfChangeAddLearnerNode.Enum(), NodeId: &target}} + } + return proto.MarshalOptions{Deterministic: true}.Marshal(change) +} + +// R10/R30 keep unassembled peer capability absent from a blind relay and route +// signed peer exchange separately from legacy pairing and directory requests. +func TestR10R30ClusterRouteAbsentUntilHostAssembly(t *testing.T) { + service := New(Config{}) + defer service.Close() + for _, path := range []string{rafttransport.ExchangePath, rafttransport.SocketPath} { + got := httptest.NewRecorder() + service.Handler.ServeHTTP(got, httptest.NewRequest(http.MethodPost, path, nil)) + if got.Code != 404 { + t.Fatal("R30: unassembled cluster transport advertised", path, got.Code) + } + } +} diff --git a/internal/relayserve/service.go b/internal/relayserve/service.go index d12ee0d6dd..c87ff7c2c6 100644 --- a/internal/relayserve/service.go +++ b/internal/relayserve/service.go @@ -19,6 +19,11 @@ import ( // Config is everything a relay is told. The zero Store is the blind pipe. type Config struct { + // Cluster is supplied by the persistent member host after loading its + // committed enrollment view. Its handler is an internal assembly seam, not a + // rollout switch or an alternative directory writer. + Cluster http.Handler + Store string // directory holding directory/ and blobs/; empty serves the pipe only Status bool // answer GET /status with what is connected right now Now func() time.Time // the relay's clock; nil is the wall clock @@ -103,6 +108,7 @@ func New(cfg Config) *Service { } mux := pipeMux(&relay.Server{Note: cfg.Note}, cfg.Status) mountPairing(mux, cfg, now) + mountCluster(mux, cfg.Cluster) mux.HandleFunc("/p/", servePairPage) svc := &Service{Handler: stamped(now, mux)} if cfg.Store == "" { From b2c1d328e8a7cc6adb5a124e3edd4dd0bef242d9 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 00:30:31 -0400 Subject: [PATCH 016/103] Keep hosted storage framing on the shared Go core. Expose bounded protobuf inspection, streaming hashes and portable snapshot pointers so hosted storage can persist consensus without a second application writer. Exercise the same protocol through native SQLite proof peers. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/codeaf-raft-probe/actor.go | 290 +++++++++++++++++++++++++++++++++ cmd/codeaf-raft-probe/main.go | 7 + internal/raftabi/README.md | 9 + internal/raftabi/abi.go | 4 + internal/raftabi/host.go | 111 +++++++++++++ internal/raftabi/host_test.go | 48 ++++++ 6 files changed, 469 insertions(+) create mode 100644 cmd/codeaf-raft-probe/actor.go create mode 100644 internal/raftabi/host.go create mode 100644 internal/raftabi/host_test.go diff --git a/cmd/codeaf-raft-probe/actor.go b/cmd/codeaf-raft-probe/actor.go new file mode 100644 index 0000000000..ff8c1998df --- /dev/null +++ b/cmd/codeaf-raft-probe/actor.go @@ -0,0 +1,290 @@ +package main + +import ( + "context" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + + "github.com/Agent-Field/codeaf/internal/raftabi" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + "github.com/Agent-Field/codeaf/internal/raftstore" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +type actorRequest struct { + Op string `json:"op"` + Cluster string `json:"cluster"` + ID uint64 `json:"id,string"` + Tag uint32 `json:"tag"` + Value json.RawMessage `json:"value"` + Data []byte `json:"data"` + Generation string `json:"generation"` + Offset uint64 `json:"offset"` + Hash string `json:"hash"` +} +type probeActor struct { + core *raftcore.Core + store *raftstore.SQLite + path string + cfg raftcore.Config +} + +var actorWire = proto.MarshalOptions{Deterministic: true} + +func (a *probeActor) input(in raftcore.Input) (any, error) { + ctx := context.Background() + e, err := a.core.Step(in) + if err != nil { + return nil, err + } + type send struct { + To string `json:"to"` + Bytes []byte `json:"bytes"` + } + type barrier struct { + ID string `json:"id"` + Index string `json:"index"` + } + var sends = []send{} + var reads = []barrier{} + for count := 0; count < 256; count++ { + for _, m := range e.Send { + sends = append(sends, send{strconv.FormatUint(m.To, 10), m.Bytes}) + } + for _, r := range e.Reads { + reads = append(reads, barrier{r.RequestID, strconv.FormatUint(r.Index, 10)}) + } + if e.Persist != nil { + if err = a.store.Persist(ctx, *e.Persist); err != nil { + return nil, err + } + e, err = a.core.Step(raftcore.Persisted{BatchID: e.Persist.ID}) + } else if e.Apply != nil { + recovered, err := a.store.Recover(ctx) + if err != nil { + return nil, err + } + rows, app, err := a.store.Read(ctx, recovered.Application.Applied, []raftstore.Key{{Namespace: raftstore.State, Key: "cell"}}) + if err != nil { + return nil, err + } + update := raftstore.Update{ExpectedApplied: app.Applied, VisibleVersion: app.VisibleVersion, LogicalTime: app.LogicalTime} + row := rows[raftstore.Key{Namespace: raftstore.State, Key: "cell"}] + for _, entry := range e.Apply.Entries { + if entry.GetType() == pb.EntryNormal && len(entry.Data) > 0 { + row, err = raftabi.ProbeApply(entry.Data, row) + if err != nil { + return nil, err + } + update.VisibleVersion++ + update.Writes = []raftstore.Write{{Key: raftstore.Key{Namespace: raftstore.State, Key: "cell"}, Value: row}} + } + } + if err = a.store.Apply(ctx, *e.Apply, update); err != nil { + return nil, err + } + e, err = a.core.Step(raftcore.Applied{BatchID: e.Apply.ID}) + } else { + s := e.Status + return map[string]any{"send": sends, "reads": reads, "status": map[string]any{"id": fmt.Sprint(s.NodeID), "leader": fmt.Sprint(s.Leader), "term": fmt.Sprint(s.Term), "commit": fmt.Sprint(s.Commit), "applied": fmt.Sprint(s.Applied), "role": uint32(s.Role)}}, nil + } + if err != nil { + return nil, err + } + } + return nil, fmt.Errorf("probe actor completion budget") +} +func (a *probeActor) open() error { + var err error + a.store, err = raftstore.Open(a.path, raftstore.Identity{ClusterUUID: a.cfg.ClusterUUID, NodeID: a.cfg.NodeID, Protocol: 1}) + if err != nil { + return err + } + recovered, err := a.store.Recover(context.Background()) + if err != nil { + return err + } + cfg := a.cfg + if !recovered.Fresh { + cfg.Bootstrap = nil + } + a.core, err = raftcore.New(cfg, recovered.Core) + return err +} +func (a *probeActor) call(r actorRequest, dir string) (any, error) { + ctx := context.Background() + switch r.Op { + case "init": + if a.store != nil { + if err := a.store.Close(); err != nil { + return nil, err + } + } + a.cfg = raftcore.Config{ClusterUUID: r.Cluster, NodeID: r.ID, Seed: 100 + r.ID, Protocol: 1, Bootstrap: []uint64{1, 2, 3}} + a.path = filepath.Join(dir, fmt.Sprintf("%s-%d.db", r.Cluster, r.ID)) + if err := a.open(); err != nil { + return nil, err + } + return a.input(raftcore.Poll{}) + case "restart": + if err := a.store.Close(); err != nil { + return nil, err + } + if err := a.open(); err != nil { + return nil, err + } + return a.input(raftcore.Poll{}) + case "input": + var in raftcore.Input + var b []byte + var s string + switch r.Tag { + case 0: + in = raftcore.Poll{} + case 1: + in = raftcore.Tick{} + case 2: + json.Unmarshal(r.Value, &b) + in = raftcore.PeerMessage{Bytes: b} + case 3: + json.Unmarshal(r.Value, &b) + in = raftcore.Propose{Command: b} + case 4: + json.Unmarshal(r.Value, &s) + in = raftcore.Read{RequestID: s} + case 7: + json.Unmarshal(r.Value, &s) + id, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return nil, err + } + in = raftcore.Transfer{Target: id} + case 8: + json.Unmarshal(r.Value, &b) + in = raftcore.Configure{Change: b} + case 12: + var v struct { + Peer string `json:"peer"` + Snapshot bool `json:"snapshot"` + Success bool `json:"success"` + } + if err := json.Unmarshal(r.Value, &v); err != nil { + return nil, err + } + id, err := strconv.ParseUint(v.Peer, 10, 64) + if err != nil { + return nil, err + } + in = raftcore.TransportResult{Peer: id, Snapshot: v.Snapshot, Success: v.Success} + default: + return nil, fmt.Errorf("probe tag") + } + return a.input(in) + case "row": + recovered, err := a.store.Recover(ctx) + if err != nil { + return nil, err + } + rows, _, err := a.store.Read(ctx, recovered.Application.Applied, []raftstore.Key{{Namespace: raftstore.State, Key: "cell"}}) + return rows[raftstore.Key{Namespace: raftstore.State, Key: "cell"}], err + case "checkpoint": + p, err := a.store.CreateSnapshot(ctx) + if err != nil { + return nil, err + } + if err = a.store.Compact(ctx, p.GetMetadata().GetIndex()); err != nil { + return nil, err + } + if _, err = a.core.Step(raftcore.CompactSnapshot{Snapshot: p}); err != nil { + return nil, err + } + return actorWire.Marshal(p) + case "begin": + return a.store.BeginSnapshot(ctx, r.Data) + case "chunk": + return nil, a.store.PutSnapshotChunk(ctx, r.Generation, r.Offset, r.Data, r.Hash) + case "complete": + p, err := a.store.CompleteSnapshot(ctx, r.Generation) + if err != nil { + return nil, err + } + return actorWire.Marshal(p) + case "read-chunk": + return a.store.ReadSnapshotChunk(ctx, r.Generation, r.Offset) + case "describe": + p := new(pb.Snapshot) + if err := proto.Unmarshal(r.Data, p); err != nil { + return nil, err + } + m, err := raftsnapshot.Decode(p.Data) + return m, err + default: + return nil, fmt.Errorf("probe operation") + } +} + +// actorProof uses actual native SQLite stores beside hosted actors. The only +// application is the explicit development lease probe, never production policy. +func actorProof(dir string) error { + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + actors := map[uint64]*probeActor{} + defer func() { + for _, a := range actors { + if a.store != nil { + a.store.Close() + } + } + }() + for { + var length uint32 + if err := binary.Read(os.Stdin, binary.LittleEndian, &length); err != nil { + if err == io.EOF { + return nil + } + return err + } + if length == 0 || length > raftabi.MaxBuffer { + return fmt.Errorf("probe frame capacity") + } + b := make([]byte, length) + if _, err := io.ReadFull(os.Stdin, b); err != nil { + return err + } + var request actorRequest + var out any + if err := json.Unmarshal(b, &request); err != nil { + out = map[string]string{"error": err.Error()} + } else { + a := actors[request.ID] + if a == nil { + a = &probeActor{} + actors[request.ID] = a + } + result, err := a.call(request, dir) + if err != nil { + out = map[string]string{"error": err.Error()} + } else { + out = result + } + } + b, err := json.Marshal(out) + if err != nil { + return err + } + if err = binary.Write(os.Stdout, binary.LittleEndian, uint32(len(b))); err != nil { + return err + } + if _, err = os.Stdout.Write(b); err != nil { + return err + } + } +} diff --git a/cmd/codeaf-raft-probe/main.go b/cmd/codeaf-raft-probe/main.go index 2ba88dfe42..63227add60 100644 --- a/cmd/codeaf-raft-probe/main.go +++ b/cmd/codeaf-raft-probe/main.go @@ -11,6 +11,13 @@ import ( ) func main() { + if len(os.Args) == 3 && os.Args[1] == "actors" { + if err := actorProof(os.Args[2]); err != nil { + os.Stderr.WriteString(err.Error() + "\n") + os.Exit(1) + } + return + } if len(os.Args) == 5 && os.Args[1] == "measure" { if err := measureRows(os.Args[2], os.Args[3], os.Args[4]); err != nil { os.Stderr.WriteString(err.Error() + "\n") diff --git a/internal/raftabi/README.md b/internal/raftabi/README.md index ae0b524bbb..8930fe30ed 100644 --- a/internal/raftabi/README.md +++ b/internal/raftabi/README.md @@ -142,3 +142,12 @@ stores and snapshot/compaction, P08 supplies production actors/admission and serialization, and P09/P11 supply complete policy and migration. P08/P18 must repeat the size/memory gate with those later components. Application rows are paged; no production route or feature toggle was added by this proof. + +P08 adds stateless host codecs without changing the existing operations: op 7 +inspects Entry/HardState/Snapshot/peer Message (kind 1/2/3/4); op 8 hashes a +bounded block with an explicit serialized SHA-256 state and returns the next +state/digest; op 9 validates a canonical portable manifest and constructs its +protobuf snapshot pointer. Step tag 12 carries peer ID plus snapshot/success +flags for transport completion. These helpers provide framing and integrity, +never authorization or application policy. The same codecs run in native proof +peers and the bundled reactor. diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index 4c165201ac..aba4367262 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -239,6 +239,8 @@ func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { in = raftcore.StorageFailed{BatchID: d.u64()} case 10: in = raftcore.CancelRead{RequestID: d.str()} + case 12: + in = raftcore.TransportResult{Peer: d.u64(), Snapshot: d.u32() != 0, Success: d.u32() != 0} case 11: p := &pb.Snapshot{} d.message(p) @@ -293,6 +295,8 @@ func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { e.u64(point.Index) e.u64(point.Term) } + case 7, 8, 9: + return hostCodec(op, d) default: return nil, errFrame } diff --git a/internal/raftabi/host.go b/internal/raftabi/host.go new file mode 100644 index 0000000000..a266348a8f --- /dev/null +++ b/internal/raftabi/host.go @@ -0,0 +1,111 @@ +package raftabi + +import ( + "crypto/sha256" + "encoding" + + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// Host codecs keep protobuf validation and portable snapshot interpretation in +// the shared Go binary. They evaluate no application or authorization policy. +func hostCodec(op uint32, d *decoder) ([]byte, error) { + e := encoder{} + switch op { + case 7: + kind, data := d.u32(), d.blob() + if err := d.end(); err != nil { + return nil, err + } + switch kind { + case 1: + v := new(pb.Entry) + if proto.Unmarshal(data, v) != nil || len(v.ProtoReflect().GetUnknown()) != 0 { + return nil, errFrame + } + e.u32(uint32(v.GetType())) + e.u64(v.GetIndex()) + e.u64(v.GetTerm()) + e.blob(v.GetData()) + case 2: + v := new(pb.HardState) + if proto.Unmarshal(data, v) != nil || len(v.ProtoReflect().GetUnknown()) != 0 { + return nil, errFrame + } + e.u64(v.GetTerm()) + e.u64(v.GetVote()) + e.u64(v.GetCommit()) + case 3: + v := new(pb.Snapshot) + if proto.Unmarshal(data, v) != nil || len(v.ProtoReflect().GetUnknown()) != 0 { + return nil, errFrame + } + m, err := raftsnapshot.Decode(v.GetData()) + if err != nil || m.Index != v.GetMetadata().GetIndex() || m.Term != v.GetMetadata().GetTerm() || len(v.GetMetadata().ProtoReflect().GetUnknown()) != 0 { + return nil, errFrame + } + conf, err := m.Configuration() + if err != nil || !proto.Equal(conf, v.GetMetadata().GetConfState()) { + return nil, errFrame + } + e.u64(m.Index) + e.u64(m.Term) + e.blob(m.Conf) + e.blob(v.Data) + case 4: + v := new(pb.Message) + if proto.Unmarshal(data, v) != nil || v.Type == nil || v.GetTerm() == 0 || len(v.GetResponses()) != 0 || len(v.ProtoReflect().GetUnknown()) != 0 { + return nil, errFrame + } + switch v.GetType() { + case pb.MsgApp, pb.MsgAppResp, pb.MsgVote, pb.MsgVoteResp, pb.MsgSnap, pb.MsgHeartbeat, pb.MsgHeartbeatResp, pb.MsgPreVote, pb.MsgPreVoteResp, pb.MsgTimeoutNow: + default: + return nil, errFrame + } + e.u32(uint32(v.GetType())) + e.u64(v.GetFrom()) + e.u64(v.GetTo()) + e.message(v.GetSnapshot()) + default: + return nil, errFrame + } + case 8: + state, block := d.blob(), d.blob() + if err := d.end(); err != nil { + return nil, err + } + if len(block) > raftsnapshot.RecordBytes+1024 { + return nil, errFrame + } + h := sha256.New() + if len(state) > 0 { + if err := h.(encoding.BinaryUnmarshaler).UnmarshalBinary(state); err != nil { + return nil, errFrame + } + } + h.Write(block) + state, err := h.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + e.blob(state) + e.blob(h.Sum(nil)) + case 9: + data := d.blob() + if err := d.end(); err != nil { + return nil, err + } + m, err := raftsnapshot.Decode(data) + if err != nil { + return nil, err + } + conf, err := m.Configuration() + if err != nil { + return nil, err + } + e.message(&pb.Snapshot{Data: data, Metadata: &pb.SnapshotMetadata{Index: proto.Uint64(m.Index), Term: proto.Uint64(m.Term), ConfState: conf}}) + } + return e.b, nil +} diff --git a/internal/raftabi/host_test.go b/internal/raftabi/host_test.go new file mode 100644 index 0000000000..3c41b6c086 --- /dev/null +++ b/internal/raftabi/host_test.go @@ -0,0 +1,48 @@ +package raftabi + +import ( + "bytes" + "crypto/sha256" + "testing" + + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +func TestR13R14HostStreamingHashAndProtoAdmission(t *testing.T) { + r := New() + var state []byte + all := bytes.Repeat([]byte("sealed"), 100000) + for at := 0; at < len(all); { + end := min(at+256<<10, len(all)) + w := encoder{} + w.u32(Version) + w.u32(8) + w.blob(state) + w.blob(all[at:end]) + d := decoder{b: r.Call(w.b)} + if d.u32() != Version || d.str() != "" { + t.Fatal("R14 hash ABI refused") + } + state = d.blob() + digest := d.blob() + expected := sha256.Sum256(all[:end]) + if !bytes.Equal(digest, expected[:]) || d.end() != nil { + t.Fatal("R13 streamed checksum differs") + } + at = end + } + for _, kind := range []pb.MessageType{pb.MsgHup, pb.MsgProp, pb.MsgReadIndex, pb.MsgStorageAppend} { + b, _ := wire.Marshal(&pb.Message{Type: kind.Enum(), From: proto.Uint64(1), To: proto.Uint64(2), Term: proto.Uint64(1)}) + w := encoder{} + w.u32(Version) + w.u32(7) + w.u32(4) + w.blob(b) + d := decoder{b: r.Call(w.b)} + d.u32() + if d.str() == "" { + t.Fatalf("R14 untrusted local verb %v admitted", kind) + } + } +} From f56bd5497ac4fa8cc493b0b53f10ebddc6ceb55b Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 00:30:31 -0400 Subject: [PATCH 017/103] Persist hosted members before their coordination effects escape. Serialize each actor across durability waits, authenticate bounded peer inboxes and stream retained snapshots through the shared reactor. Add the local compiled-module entry and independent object binding while leaving the directory cutover to a later change. Co-Authored-By: Claude Opus 5.5 (1M context) --- .gitignore | 3 + relay/hosted/src/raft-worker.js | 5 + relay/hosted/src/raft/README.md | 68 +++++++++ relay/hosted/src/raft/abi.js | 22 +++ relay/hosted/src/raft/actor.js | 98 +++++++++++++ relay/hosted/src/raft/inbox.js | 60 ++++++++ relay/hosted/src/raft/member.js | 93 ++++++++++++ relay/hosted/src/raft/storage.js | 242 +++++++++++++++++++++++++++++++ relay/hosted/src/worker.js | 2 + relay/hosted/wrangler.toml | 13 +- scripts/raft-hosted-build.sh | 6 + 11 files changed, 611 insertions(+), 1 deletion(-) create mode 100644 relay/hosted/src/raft-worker.js create mode 100644 relay/hosted/src/raft/README.md create mode 100644 relay/hosted/src/raft/actor.js create mode 100644 relay/hosted/src/raft/inbox.js create mode 100644 relay/hosted/src/raft/member.js create mode 100644 relay/hosted/src/raft/storage.js create mode 100644 scripts/raft-hosted-build.sh diff --git a/.gitignore b/.gitignore index ab5b3da550..cb377aab29 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,6 @@ relay/hosted/.wrangler/ # Native Windows test executables are local acceptance artifacts. .wintest/ + +# The hosted reactor is generated from the shared, pinned Go core. +/relay/hosted/src/raft/core.wasm diff --git a/relay/hosted/src/raft-worker.js b/relay/hosted/src/raft-worker.js new file mode 100644 index 0000000000..a5bf794b0a --- /dev/null +++ b/relay/hosted/src/raft-worker.js @@ -0,0 +1,5 @@ +import core from './raft/core.wasm'; +import { installRaftModule } from './raft/member.js'; +installRaftModule(core); +export * from './worker.js'; +export { default } from './worker.js'; diff --git a/relay/hosted/src/raft/README.md b/relay/hosted/src/raft/README.md new file mode 100644 index 0000000000..55e06b8ae4 --- /dev/null +++ b/relay/hosted/src/raft/README.md @@ -0,0 +1,68 @@ +# Hosted consensus actors + +`RaftMemberDO` is exported by the existing Worker. The build entry +`src/raft-worker.js` statically imports the generated `core.wasm` and installs +one shared reactor per isolate. Wrangler's local build command compiles that +artifact from the pinned modified Go Raft source. The bindings add independent +SQL objects; they do not route the legacy identity directory to a new writer. +P09 owns the application adapter and P11 the irrevocable migration gate. + +The internal namespace provisions an immutable cluster/node/bootstrap/seed +configuration before bootstrap writes. All anchors use the same initial voter +set. An owner/enrollment adapter must supply durable committed enrollment and +an application evaluator before there is an application proposal door. The +current production class has neither a public proposal endpoint nor a fallback +JavaScript lease writer. The test subclass explicitly supplies ProbeApply. + +The actor queue spans every storage await and admits at most 64 callbacks. +Core calls synchronously copy/free their binary buffers. SQL uses +`transactionSync`; `storage.sync()` finishes before Persisted, Applied, any +reply or outbound dependent message. Failures stop participation and preserve +the database. Initialization/recovery is serialized too. Recovery starts as a +follower and never serves an authority read from remembered status. + +Application writes and index/configuration/version/time commit atomically. +The evaluator is synchronous with an immutable applied view and bounded 1 MiB +read/write sets. It must implement committed policy, declared dependencies and +request/session semantics; storage supplies atomicity, not that policy. Rows +cover state, sessions, membership, enrollment, tombstones and outboxes. The +persisted application ceiling defaults to 8 MiB, including framed keys/values; +an explicit larger imported ceiling is preserved. Pressure refuses growth, +never deletes application rows. P11 owns actual inventory/grandfathering. + +Snapshots use `internal/raftsnapshot` format 1 without a second JS schema. +Rows stream in SQL binary order into 256 KiB chunks. The shared Go codec +validates protobuf and manifest bytes and incrementally hashes the stream. +Each chunk, complete stream and keyed record aggregate is checked. Missing or +corrupt complete evidence refuses recovery; an incomplete stage has no +application authority. Two complete generations survive. Snapshot completion +is distinct from installation: send signed MsgSnap to stage its manifest, +stream signed chunks, then retry MsgSnap. Only its durable Ready install may +return `installed: true`. A transport sender must report failure/timeout to its +core if delivery is interrupted. The receiver can resume after eviction. + +Every peer envelope is the exact P07 wire/signature domain. Current committed +source/destination incarnations and revocation are checked after cryptographic +awaits and again inside the actor queue. Queues reserve voter/learner/snapshot +slots (4/2/2 per member) and encoded bytes across an identity (512/128/384 KiB). +They do not coalesce votes, appends or read barriers. There is no durable cursor +in this direct internal delivery path: a lost call is failed delivery and Raft +retransmits. P12 supplies durable enrollment; public outbound exchange/router +assembly reuses P07's authenticated generation-fenced bridge. + +Each active handle reserves 6 MiB before creation; total reservations plus +observed reactor memory must fit 64 MiB. Retained core log cache is capped at +2 MiB, with lossless checkpoints at 1 MiB or 1,024 applied entries. This earlier +local checkpoint keeps the cache below the shared 4 MiB retained-log ceiling. +A leader is never evicted to meet admission. Only an explicitly quiescent, +nonleader, drained actor may close; P14 must coordinate actual quiescence/wake +and alarm scheduling. `drainRecovery` exposes recovered dependent effects for +an internal router to deliver rather than silently dropping them. + +Run the bounded local proof with `sh relay/hosted/test/raft/actors-run.sh +/tmp/codeaf-p08`. It exercises real workerd actors and native SQLite peers, +signed messages, ReadIndex/minority refusal, both snapshot directions, +process death, full joint configuration restart and co-resident identities. +Unit tests cover durability waits, queue bounds, all snapshot namespaces, +partial/corrupt generations, revocation during an await and native wire bytes. +This is internal host proof, not the final client journeys or a deployment. diff --git a/relay/hosted/src/raft/abi.js b/relay/hosted/src/raft/abi.js index 80f6953c06..80d14e540c 100644 --- a/relay/hosted/src/raft/abi.js +++ b/relay/hosted/src/raft/abi.js @@ -44,6 +44,7 @@ export function stepFrame(handle, tag, value) { if ([2, 3, 8, 11].includes(tag)) w.blob(value); else if ([4, 10].includes(tag)) w.str(value); else if ([5, 6, 7, 9].includes(tag)) w.u64(value); + if (tag === 12) w.u64(value.peer).u32(value.snapshot ? 1 : 0).u32(value.success ? 1 : 0); return w.finish(); } export function effects(bytes) { @@ -99,3 +100,24 @@ export function snapshotNeeded(bytes) { const point = r.u32() ? { index: r.u64(), term: r.u64() } : null; r.end(); return point; } + +// Storage uses the Go codec to inspect consensus bytes without implementing +// protobuf or directory policy in JavaScript. Each call owns its result. +export class HostCodec { + constructor(runtime) { this.runtime = runtime; } + call(w) { return response(this.runtime.call(w.finish())); } + inspect(kind, bytes) { + const r = this.call(frame(7).u32(kind).blob(bytes)); + let v; + if (kind === 1) v = { type: r.u32(), index: r.u64(), term: r.u64(), data: r.blob() }; + if (kind === 2) v = { term: r.u64(), vote: r.u64(), commit: r.u64() }; + if (kind === 3) v = { index: r.u64(), term: r.u64(), conf: r.blob(), data: r.blob() }; + if (kind === 4) v = { type: r.u32(), from: r.u64(), to: r.u64(), snapshot: r.blob() }; + r.end(); return v; + } + hash(block, state = new Uint8Array()) { + const r = this.call(frame(8).blob(state).blob(block)); + const v = { state: r.blob(), digest: r.blob() }; r.end(); return v; + } + snapshot(data) { const r = this.call(frame(9).blob(data)); const b = r.blob(); r.end(); return b; } +} diff --git a/relay/hosted/src/raft/actor.js b/relay/hosted/src/raft/actor.js new file mode 100644 index 0000000000..a575e110df --- /dev/null +++ b/relay/hosted/src/raft/actor.js @@ -0,0 +1,98 @@ +import { CORE_MEMORY_BUDGET, Reactor, HostCodec, createFrame, response, effects, frame, stepFrame } from './abi.js'; +import { MemberStorage, DEFAULT_ROWS_BYTES } from './storage.js'; + +const RESERVATION = 6 * 1024 * 1024; +const CACHE_BYTES = 2 * 1024 * 1024; +const QUEUED_INPUTS = 64; +let module, runtime; +const active = new Set(); + +// A compiled module and runtime are shared by every object in this isolate. +// Reservations precede allocation; pressure refuses activation, never evicts a +// serving leader. Quiescent eviction is an explicit later scheduler decision. +export function installRaftModule(compiled) { + if (module && module !== compiled) throw new Error('raft member: core module already installed'); + module = compiled; +} +export function sharedRuntime() { if (!module) throw new Error('raft member: core unavailable'); return runtime ??= new Reactor(module); } +export function runtimeMetrics() { return { ...sharedRuntime().metrics(), active: active.size, reservedBytes: active.size * RESERVATION }; } + +// A member queue spans awaits and remains bounded even when storage stalls. +// Rejected inputs are retryable; a failed persistence stops this handle. +export class MemberActor { + constructor(storage, reactor, evaluate) { + this.reactor = reactor; this.codec = new HostCodec(reactor); this.store = new MemberStorage(storage, this.codec); + this.evaluate = evaluate; this.tail = Promise.resolve(); this.pending = 0; this.handle = 0; this.stopped = false; this.generation = 0; this.checkpointIndex = 0n; + } + serial(fn) { + if (this.pending >= QUEUED_INPUTS) return Promise.reject(new Error('raft member: capacity; retry with backoff')); + this.pending++; + const p = this.tail.then(fn); this.tail = p.catch(() => {}); + return p.finally(() => { this.pending--; }); + } + init(config) { config = structuredClone(config); return this.serial(async () => { + if (this.handle) throw new Error('raft member: already active'); + if (active.size * RESERVATION + RESERVATION + this.reactor.memory > CORE_MEMORY_BUDGET) throw new Error('raft member: isolate capacity; retry with backoff'); + active.add(this); + try { + const recovered = await this.store.open({ cluster: config.cluster, id: String(config.id), protocol: 1 }, config.ceiling ?? DEFAULT_ROWS_BYTES); + const cache = recovered.entries.reduce((n, b) => n + b.length, 0); + if (cache > CACHE_BYTES) throw new Error('raft member: log cache capacity; recover with checkpoint'); + const r = response(this.reactor.call(createFrame({ ...config, bootstrap: recovered.entries.length || recovered.snapshot.length || recovered.hard.length ? [] : config.bootstrap, ...recovered }))); + this.handle = r.u32(); r.end(); this.config = config; this.generation++; this.cache = cache; + return this.drive(0); + } catch (e) { + this.stopped = true; + if (this.handle) { try { this.reactor.call(frame(2).u32(this.handle).finish()); } catch {} this.handle = 0; } + active.delete(this); throw e; + } + }); } + async drive(tag, value) { + if (this.stopped || this.store.stopped || !this.handle) throw new Error('raft member: stopped; recover from durable storage'); + const sends = [], reads = []; let e; + try { e = effects(this.reactor.call(stepFrame(this.handle, tag, value))); } + catch (error) { if (this.reactor.poisoned) this.stopped = true; throw error; } + for (let budget = 0; ; budget++) { + if (budget > 256) { this.stopped = true; throw new Error('raft member: completion budget'); } + this.status = e.status; + sends.push(...e.send); reads.push(...e.reads); + try { + if (e.persist) { + await this.store.persist(e.persist); + this.cache = this.store.one('SELECT coalesce(sum(length(data)),0) AS n FROM journal').n; + e = effects(this.reactor.call(stepFrame(this.handle, 5, e.persist.id))); continue; + } + if (e.apply) { await this.store.apply(e.apply, this.evaluate); e = effects(this.reactor.call(stepFrame(this.handle, 6, e.apply.id))); continue; } + } catch (error) { + this.stopped = true; + try { this.reactor.call(stepFrame(this.handle, 9, e.persist?.id ?? e.apply?.id ?? 0n)); } catch {} + throw error; + } + // Checkpoint sooner than the retained-log ceiling so the active core's + // cache stays bounded. This compacts only durable applied history. + const app = this.store.application(); + if ((this.cache >= CACHE_BYTES / 2 || this.status.applied - this.checkpointIndex >= 1024n) && BigInt(app.applied) > 0n && BigInt(app.applied) === this.status.commit) { + let snapshot; + try { snapshot = await this.store.checkpoint(); e = effects(this.reactor.call(stepFrame(this.handle, 11, snapshot))); } + catch (error) { this.stopped = true; throw error; } + this.checkpointIndex = BigInt(app.applied); this.cache = this.store.one('SELECT coalesce(sum(length(data)),0) AS n FROM journal').n; continue; + } + if (this.cache > CACHE_BYTES) { this.stopped = true; throw new Error('raft member: log cache capacity; recover'); } + return { send: sends, reads, status: this.status }; + } + } + input(tag, value) { value = value instanceof Uint8Array ? value.slice() : value && typeof value === 'object' ? structuredClone(value) : value; return this.serial(() => this.drive(tag, value)); } + checkpoint() { return this.serial(async () => { + try { const snapshot = await this.store.checkpoint(); await this.drive(11, snapshot); this.checkpointIndex = this.status.applied; return snapshot; } + catch (error) { this.stopped = true; throw error; } + }); } + begin(data) { data = data.slice(); return this.serial(() => this.store.beginSnapshot(data)); } + chunk(id, offset, data, hash) { data = data.slice(); return this.serial(() => this.store.putChunk(id, offset, data, hash)); } + complete(id) { return this.serial(() => this.store.completeSnapshot(id)); } + close() { return this.serial(() => { + if (this.status?.role === 2 || this.pending > 1) throw new Error('raft member: active member cannot be evicted'); + if (this.handle) { const r = response(this.reactor.call(frame(2).u32(this.handle).finish())); r.end(); } + this.handle = 0; this.generation++; active.delete(this); + }); } +} + diff --git a/relay/hosted/src/raft/inbox.js b/relay/hosted/src/raft/inbox.js new file mode 100644 index 0000000000..8836706767 --- /dev/null +++ b/relay/hosted/src/raft/inbox.js @@ -0,0 +1,60 @@ +import { b64, unb64, equal } from './storage.js'; + +const text = new TextEncoder(); +const uint = (n, nonzero = true) => typeof n === 'string' && /^(0|[1-9][0-9]*)$/.test(n) && BigInt(n) <= (1n << 64n) - 1n && (!nonzero || BigInt(n) !== 0n); +const token = s => typeof s === 'string' && /^[a-zA-Z0-9_.-]{1,128}$/.test(s); +const hash = s => typeof s === 'string' && /^[a-f0-9]{64}$/.test(s); +const check = v => { if (!v) throw new Error('raft inbox: peer authorization refused'); }; +export const INBOX_BYTES = 1024 * 1024; +const budgets = [INBOX_BYTES - 128 * 1024 - 384 * 1024, 128 * 1024, 384 * 1024]; +const slots = [4, 2, 2]; +const shared = new Map(); + +// Authentication is repeated at dispatch, after every async signature wait. +// A queued packet never conveys application authority or a disk acknowledgment. +export class MemberInbox { + constructor(scope, id, codec, registry, enqueue) { + this.scope = scope; this.id = String(id); this.codec = codec; this.registry = registry; this.enqueue = enqueue; + this.identity = JSON.stringify(scope); this.counts = [0, 0, 0]; + this.pending = new Map(); + } + members(e) { + check(JSON.stringify(e.scope) === JSON.stringify(this.scope) && e.to === this.id && e.from !== e.to && uint(e.from) && uint(e.to) && uint(e.incarnation) && uint(e.destination) && uint(e.sequence)); + const view = this.registry(), from = view[e.from], to = view[e.to]; + check(from && to && !from.revoked && !to.revoked && from.incarnation === e.incarnation && to.incarnation === e.destination); + return { from, to }; + } + async receive(raw, dispatch) { + check(raw instanceof Uint8Array && raw.length <= 512 * 1024); + const e = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(raw)); + const { from } = this.members(e); + check(token(e.scope.identity) && token(e.scope.cluster) && uint(e.scope.epoch) && e.scope.protocol === 1); + const payload = unb64(e.payload); check(b64(payload) === e.payload && hash(e.digest) && this.codec.hash(payload).digest.every((x, i) => x === parseInt(e.digest.slice(i * 2, i * 2 + 2), 16))); + const unsigned = { scope: { identity: e.scope.identity, cluster: e.scope.cluster, epoch: e.scope.epoch, protocol: e.scope.protocol }, from: e.from, to: e.to, incarnation: e.incarnation, destination: e.destination, sequence: e.sequence, chunk: e.chunk, digest: e.digest, payload: e.payload, signature: null }; + const canonical = { ...unsigned, signature: e.signature }; check(JSON.stringify(canonical) === new TextDecoder().decode(raw)); + if (e.chunk !== null) { + check(Object.keys(e.chunk).join(',') === 'generation,offset,hash' && hash(e.chunk.generation) && uint(e.chunk.offset, false) && e.chunk.hash === e.digest && payload.length > 0 && payload.length <= 256 * 1024); + } else { + check(payload.length > 0 && payload.length <= 64 * 1024); + const m = this.codec.inspect(4, payload); check(String(m.from) === e.from && String(m.to) === e.to); + } + const key = await crypto.subtle.importKey('raw', unb64(from.key), 'Ed25519', false, ['verify']); + check(await crypto.subtle.verify('Ed25519', key, unb64(e.signature), text.encode('codeaf/raft/envelope/1\n' + JSON.stringify(unsigned)))); + this.members(e); + const lane = e.chunk !== null ? 2 : from.voter ? 0 : 1; + const id = e.from + '/' + e.sequence; + const old = this.pending.get(id); + if (old) { check(equal(old.raw, raw)); return old.promise; } + if (!shared.has(this.identity)) shared.set(this.identity, { used: [0, 0, 0] }); + const budget = shared.get(this.identity); + if (this.counts[lane] >= slots[lane] || budget.used[lane] + raw.length > budgets[lane]) throw new Error('raft inbox: capacity; retry with backoff'); + const owned = raw.slice(); this.counts[lane]++; budget.used[lane] += owned.length; + const promise = this.enqueue(async () => { + // The durable registry may have changed while another member persisted. + this.members(e); return dispatch(e, payload); + }); + this.pending.set(id, { raw: owned, promise }); + try { return await promise; } + finally { this.pending.delete(id); this.counts[lane]--; budget.used[lane] -= owned.length; if (budget.used.every(n => n === 0)) shared.delete(this.identity); } + } +} diff --git a/relay/hosted/src/raft/member.js b/relay/hosted/src/raft/member.js new file mode 100644 index 0000000000..57ef80f9a3 --- /dev/null +++ b/relay/hosted/src/raft/member.js @@ -0,0 +1,93 @@ +import { DurableObject } from 'cloudflare:workers'; +import { MemberActor, sharedRuntime } from './actor.js'; +import { MemberInbox } from './inbox.js'; +export { installRaftModule, runtimeMetrics } from './actor.js'; + +// Only internal namespace calls can initialize a provisioned member. The +// public Worker exports this class but does not route directory traffic here. +// P12 supplies the durable committed enrollment view and P09 the evaluator. +export class RaftMemberDO extends DurableObject { + constructor(ctx, env) { super(ctx, env); this.actor = null; this.loading = null; this.ctx = ctx; this.env = env; } + async initialize(config) { + config = { ...config, id: String(config.id), seed: String(config.seed), bootstrap: config.bootstrap.map(String) }; + if (this.loading) { + if (this.loadingConfig !== JSON.stringify(config)) throw new Error('raft member: different activation is in progress'); + return this.loading; + } + this.loadingConfig = JSON.stringify(config); + this.loading = this.activate(config); + try { return await this.loading; } finally { this.loading = null; this.loadingConfig = null; } + } + async activate(config) { + if (this.actor) { + if (JSON.stringify(this.config) !== JSON.stringify(config)) throw new Error('raft member: provisioned identity cannot change'); + return this.actor.input(0); + } + const canonical = { ...config, id: String(config.id), seed: String(config.seed), bootstrap: config.bootstrap.map(String) }; + const previous = await this.ctx.storage.get('raft-config'); + if (previous && JSON.stringify(previous) !== JSON.stringify(canonical)) throw new Error('raft member: provisioned identity cannot change'); + if (!previous) { await this.ctx.storage.put('raft-config', canonical); await this.ctx.storage.sync(); } + const actor = new MemberActor(this.ctx.storage, sharedRuntime(), this.env.raftEvaluate); + const result = await actor.init(canonical); + this.config = canonical; this.registry = await this.ctx.storage.get('raft-registry'); this.actor = actor; + return result; + } + + async load() { + if (this.actor) return; + if (this.loading) return this.loading; + this.loading = this.restore(); + try { return await this.loading; } finally { this.loading = null; } + } + async restore() { + const config = await this.ctx.storage.get('raft-config'); if (!config) throw new Error('raft member: not provisioned'); + const actor = new MemberActor(this.ctx.storage, sharedRuntime(), this.env.raftEvaluate); + this.recoveredEffects = await actor.init(config); this.config = config; this.registry = await this.ctx.storage.get('raft-registry'); this.actor = actor; + } + async drainRecovery() { + await this.load(); + return this.actor.serial(() => { const effects = this.recoveredEffects; this.recoveredEffects = null; return effects; }); + } + async peer(raw) { + await this.load(); + const config = this.config; + if (!this.registry) throw new Error('raft member: enrollment is absent'); + this.inbox ??= new MemberInbox(config.scope, config.id, this.actor.codec, () => this.registry, fn => this.actor.serial(fn)); + return this.inbox.receive(raw, async (envelope, payload) => { + if (envelope.chunk) { + if (!this.actor.store.one('SELECT id FROM snapshots WHERE id=?', envelope.chunk.generation)) return { staged: false }; + await this.actor.store.putChunk(envelope.chunk.generation, Number(BigInt(envelope.chunk.offset)), payload, envelope.chunk.hash); + return { staged: true }; + } + const m = this.actor.codec.inspect(4, payload); + if (m.snapshot.length) { + const snapshot = this.actor.codec.inspect(3, m.snapshot); + if (snapshot.index > this.actor.status.applied) { + const id = await this.actor.store.beginSnapshot(snapshot.data); + const manifest = this.actor.store.generation(id).manifest; + const chunks = this.actor.store.one('SELECT count(*) AS n FROM chunks WHERE generation=?', id).n; + if (chunks !== Number(manifest.chunks)) return { staged: true, installed: false }; + await this.actor.store.completeSnapshot(id); + } + const result = await this.actor.drive(2, payload); + return { ...result, installed: this.actor.status.applied >= snapshot.index }; + } + return this.actor.drive(2, payload); + }); + } + // A registry update is trusted assembly input derived from committed state. + // Serialize it with dispatch and persist before acknowledging revocation. + async enrollment(view, applied) { + view = structuredClone(view); + await this.load(); + return this.actor.serial(async () => { + if (BigInt(applied) > this.actor.status.applied) throw new Error('raft member: enrollment barrier not applied'); + const old = await this.ctx.storage.get('raft-registry') ?? {}; + for (const [id, member] of Object.entries(old)) { + const next = view[id]; + if (!next || (member.revoked && !next.revoked) || BigInt(next.incarnation) < BigInt(member.incarnation) || (next.incarnation === member.incarnation && next.key !== member.key)) throw new Error('raft member: enrollment regression'); + } + await this.ctx.storage.put('raft-registry', view); await this.ctx.storage.sync(); this.registry = view; + }); + } +} diff --git a/relay/hosted/src/raft/storage.js b/relay/hosted/src/raft/storage.js new file mode 100644 index 0000000000..2e36ee4656 --- /dev/null +++ b/relay/hosted/src/raft/storage.js @@ -0,0 +1,242 @@ +import { Writer } from './abi.js'; + +export const CHUNK_BYTES = 256 * 1024; +export const ROW_BYTES = 1024 * 1024; +export const DEFAULT_ROWS_BYTES = 8 * ROW_BYTES; +const EMPTY = new Uint8Array(); +const enc = new TextEncoder(), dec = new TextDecoder('utf-8', { fatal: true }); +export const hex = b => Array.from(b, x => x.toString(16).padStart(2, '0')).join(''); +export const equal = (a, b) => a.length === b.length && a.every((x, i) => x === b[i]); +export const bytes = b => b instanceof Uint8Array ? b : new Uint8Array(b); +export const b64 = b => { let s = ''; for (const x of b) s += String.fromCharCode(x); return btoa(s); }; +export const unb64 = s => Uint8Array.from(atob(s), c => c.charCodeAt(0)); +const namespaces = ['state', 'sessions', 'membership', 'enrollment', 'tombstones', 'outbox']; +const key = n => BigInt(n).toString(16).padStart(16, '0'); +const check = (v, message = 'raft storage: invalid or damaged durable state') => { if (!v) throw new Error(message); }; +const xor = (a, b) => { for (let i = 0; i < 32; i++) a[i] ^= b[i]; }; +const compare = (a, b) => { const x = enc.encode(a), y = enc.encode(b); for (let i = 0; i < Math.min(x.length, y.length); i++) if (x[i] !== y[i]) return x[i] - y[i]; return x.length - y.length; }; +const rowSize = (ns, name, data) => 12 + enc.encode(ns).length + enc.encode(name).length + data.length; +const scalar = v => enc.encode(JSON.stringify(v)); + +// SQL transactions are synchronous so no other event can enter the immutable +// read-set. sync() is awaited before a persistence completion or reply escapes. +export class MemberStorage { + constructor(storage, codec) { this.storage = storage; this.sql = storage.sql; this.codec = codec; this.stopped = false; } + rows(query, ...bindings) { return this.sql.exec(query, ...bindings); } + one(query, ...bindings) { return this.rows(query, ...bindings).toArray()[0]; } + hash(b) { return hex(this.codec.hash(b).digest); } + checksum(...parts) { + let state = EMPTY, digest; + for (const part of parts) { + const b = typeof part === 'string' ? enc.encode(part) : bytes(part); + const length = new Uint8Array(8); new DataView(length.buffer).setBigUint64(0, BigInt(b.length)); + state = this.codec.hash(length, state).state; + ({ state, digest } = this.codec.hash(b, state)); + } + return digest; + } + put(name, value) { const data = scalar(value); this.rows('INSERT OR REPLACE INTO meta VALUES (?,?,?)', name, data, this.hash(data)); } + get(name) { const v = this.one('SELECT data,hash FROM meta WHERE name=?', name); check(v); const b = bytes(v.data); check(this.hash(b) === v.hash); return JSON.parse(dec.decode(b)); } + async transaction(fn) { + check(!this.stopped, 'raft storage: member stopped; recover'); + try { const result = this.storage.transactionSync(fn); await this.storage.sync(); return result; } + catch (e) { this.stopped = true; throw e; } + } + async open(identity, ceiling = DEFAULT_ROWS_BYTES) { + const exists = this.one("SELECT name FROM sqlite_master WHERE type='table' AND name='meta'"); + if (!exists) { + check(this.one("SELECT count(*) AS n FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' AND name NOT LIKE '_cf_%'").n === 0); + await this.transaction(() => { + this.rows('CREATE TABLE meta (name TEXT PRIMARY KEY, data BLOB NOT NULL, hash TEXT NOT NULL)'); + this.rows('CREATE TABLE journal (idx TEXT PRIMARY KEY, data BLOB NOT NULL, hash TEXT NOT NULL)'); + this.rows('CREATE TABLE records (namespace TEXT NOT NULL, key TEXT NOT NULL, data BLOB NOT NULL, hash TEXT NOT NULL, PRIMARY KEY(namespace,key))'); + this.rows('CREATE TABLE snapshots (id TEXT PRIMARY KEY, data BLOB NOT NULL, hash TEXT NOT NULL, complete INTEGER NOT NULL)'); + this.rows('CREATE TABLE chunks (generation TEXT NOT NULL, offset INTEGER NOT NULL, data BLOB NOT NULL, hash TEXT NOT NULL, PRIMARY KEY(generation,offset))'); + this.put('identity', { format: 1, ...identity, ceiling }); + this.put('hard', ''); this.put('snapshot', ''); this.put('tip', '0'); + this.put('application', { applied: '0', conf: '', version: '0', now: '0', count: 0, size: 0, hash: '00'.repeat(32) }); + }); + } + this.identity = this.get('identity'); + check(this.identity.format === 1 && this.identity.cluster === identity.cluster && this.identity.id === identity.id && this.identity.protocol === 1 && this.identity.ceiling >= DEFAULT_ROWS_BYTES); + check(this.one('SELECT count(*) AS n FROM meta').n === 5); + return this.recover(); + } + application() { return this.get('application'); } + read(namespace, name) { + const r = this.one('SELECT data,hash FROM records WHERE namespace=? AND key=?', namespace, name); + if (!r) return null; + const b = bytes(r.data); check(hex(this.checksum('records', namespace, name, b)) === r.hash); return b; + } + *recordRows() { + for (const r of this.rows('SELECT namespace,key,data,hash FROM records ORDER BY namespace,key')) { + const data = bytes(r.data); check(hex(this.checksum('records', r.namespace, r.key, data)) === r.hash); + yield { namespace: r.namespace, key: r.key, data }; + } + } + recover() { + const app = this.application(), entries = [], aggregate = new Uint8Array(32); let count = 0, size = 0; + for (const r of this.recordRows()) { xor(aggregate, this.checksum('records', r.namespace, r.key, r.data)); count++; size += rowSize(r.namespace, r.key, r.data); } + check(count === app.count && size === app.size && hex(aggregate) === app.hash); + const snapshot = unb64(this.get('snapshot')); + let previous = snapshot.length ? this.codec.inspect(3, snapshot).index : 0n; + if (snapshot.length) { const s = this.codec.inspect(3, snapshot); this.walk(this.hash(s.data)); check(BigInt(app.applied) >= s.index); } + for (const r of this.rows('SELECT idx,data,hash FROM journal ORDER BY idx')) { + const b = bytes(r.data), e = this.codec.inspect(1, b); check(this.hash(b) === r.hash && key(e.index) === r.idx && e.index === previous + 1n); previous = e.index; entries.push(b); + } + const hard = unb64(this.get('hard')), h = hard.length ? this.codec.inspect(2, hard) : { commit: 0n }; + check(previous === BigInt(this.get('tip')) && h.commit <= previous && BigInt(app.applied) <= h.commit); + // Every retained complete generation is evidence. Corruption never falls + // back silently to an older apparently usable pointer. + for (const r of this.rows('SELECT id FROM snapshots WHERE complete=1')) this.walk(r.id); + return { hard, conf: unb64(app.conf), snapshot, entries, applied: BigInt(app.applied) }; + } + async persist(p) { + await this.transaction(() => { + let app = this.application(); const oldBytes = unb64(this.get('hard')); + const old = oldBytes.length ? this.codec.inspect(2, oldBytes) : { term: 0n, vote: 0n, commit: 0n }; + const h = p.hard.length ? this.codec.inspect(2, p.hard) : old; + check(h.term >= old.term && h.commit >= old.commit && !(h.term === old.term && old.vote !== 0n && h.vote !== old.vote)); + if (p.snapshot.length) { + const s = this.codec.inspect(3, p.snapshot), m = JSON.parse(dec.decode(s.data)); + check(m.cluster === this.identity.cluster && m.protocol === '1' && s.index > BigInt(app.applied)); + const g = this.generation(this.hash(s.data)); check(g.complete === 1 && equal(g.data, s.data)); + this.rows('DELETE FROM records'); + this.walk(g.id, r => this.rows('INSERT INTO records VALUES (?,?,?,?)', r.namespace, r.key, r.data, hex(this.checksum('records', r.namespace, r.key, r.data)))); + check(BigInt(m.version) >= BigInt(app.version) && BigInt(m.now) >= BigInt(app.now)); + app = { applied: m.index, conf: b64(s.conf), version: m.version, now: m.now, count: Number(m.records), size: Number(m.bytes), hash: m.records_hash }; + this.put('application', app); this.put('snapshot', b64(p.snapshot)); this.rows('DELETE FROM journal'); this.put('tip', m.index); + } + let tip = BigInt(this.get('tip')); + if (p.entries.length) { + const first = this.codec.inspect(1, p.entries[0]).index; check(first <= tip + 1n); + for (const b of p.entries) { + const e = this.codec.inspect(1, b); + if (e.index <= old.commit) { const r = this.one('SELECT data FROM journal WHERE idx=?', key(e.index)); check(r && equal(bytes(r.data), b)); } + } + this.rows('DELETE FROM journal WHERE idx>=?', key(first)); tip = first - 1n; + for (const b of p.entries) { const e = this.codec.inspect(1, b); check(e.index === tip + 1n && e.term <= h.term); this.rows('INSERT INTO journal VALUES (?,?,?)', key(e.index), b, this.hash(b)); tip = e.index; } + this.put('tip', String(tip)); + } + check(h.commit <= tip && BigInt(app.applied) <= h.commit); + if (p.hard.length) this.put('hard', b64(p.hard)); + }); + } + async apply(a, evaluate) { + await this.transaction(() => { + const app = this.application(), original = BigInt(app.applied); let index = original; + const aggregate = Uint8Array.from(app.hash.match(/../g), h => parseInt(h, 16)); + for (const b of a.entries) { + const e = this.codec.inspect(1, b); check(e.index === index + 1n); + const persisted = this.one('SELECT data FROM journal WHERE idx=?', key(e.index)); check(persisted && equal(bytes(persisted.data), b)); + if (e.type === 0 && e.data.length) { + check(evaluate, 'raft storage: application evaluator is absent'); + let readBytes = 0, writeBytes = 0; + const change = evaluate(e.data, (ns, k) => { const data = this.read(ns, k); readBytes += data?.length ?? 0; check(readBytes <= ROW_BYTES, 'raft storage: evaluation read-set capacity'); return data; }, { ...app }); + check(change && !change.then && Array.isArray(change.writes)); + for (const r of change.writes) { + check(namespaces.includes(r.namespace) && typeof r.key === 'string' && enc.encode(r.key).length > 0 && enc.encode(r.key).length <= ROW_BYTES && r.data instanceof Uint8Array && r.data.length <= ROW_BYTES); + writeBytes += rowSize(r.namespace, r.key, r.data); check(writeBytes <= ROW_BYTES, 'raft storage: evaluation write-set capacity'); + const before = this.read(r.namespace, r.key); + if (before) { xor(aggregate, this.checksum('records', r.namespace, r.key, before)); app.size -= rowSize(r.namespace, r.key, before); } else app.count++; + xor(aggregate, this.checksum('records', r.namespace, r.key, r.data)); app.size += rowSize(r.namespace, r.key, r.data); + this.rows('INSERT OR REPLACE INTO records VALUES (?,?,?,?)', r.namespace, r.key, r.data, hex(this.checksum('records', r.namespace, r.key, r.data))); + } + check(app.size <= this.identity.ceiling || app.size <= this.application().size, 'raft storage: application capacity; retry after capacity is raised'); + if (change.version !== undefined) { check(BigInt(change.version) >= BigInt(app.version)); app.version = String(change.version); } + if (change.now !== undefined) { check(BigInt(change.now) >= BigInt(app.now)); app.now = String(change.now); } + } + index = e.index; + } + const hard = this.codec.inspect(2, unb64(this.get('hard'))); + check(a.index === index && index <= hard.commit && (index >= original)); + app.applied = String(index); app.conf = b64(a.conf); app.hash = hex(aggregate); this.put('application', app); + }); + } + generation(id) { + const r = this.one('SELECT data,hash,complete FROM snapshots WHERE id=?', id); check(r); + const data = bytes(r.data); check(this.hash(data) === id && this.hash(data) === r.hash); + this.codec.snapshot(data); check(r.complete === 0 || r.complete === 1); return { ...r, id, data, manifest: JSON.parse(dec.decode(data)) }; + } + async beginSnapshot(data) { + const p = this.codec.snapshot(data), s = this.codec.inspect(3, p), m = JSON.parse(dec.decode(data)), id = this.hash(data); + check(m.cluster === this.identity.cluster && m.protocol === '1' && BigInt(m.bytes) <= BigInt(this.identity.ceiling), 'raft storage: snapshot capacity or identity mismatch'); + await this.transaction(() => { + const active = unb64(this.get('snapshot')), at = active.length ? this.codec.inspect(3, active).index : 0n; + check(s.index >= at); + for (const r of this.rows('SELECT id FROM snapshots WHERE complete=1')) { const g = this.generation(r.id); check(BigInt(g.manifest.index) <= at || r.id === id); } + if (this.one('SELECT id FROM snapshots WHERE id=?', id)) { this.generation(id); return; } + this.rows('DELETE FROM chunks WHERE generation IN (SELECT id FROM snapshots WHERE complete=0)'); this.rows('DELETE FROM snapshots WHERE complete=0'); + this.rows('INSERT INTO snapshots VALUES (?,?,?,0)', id, data, id); + }); return id; + } + async putChunk(id, offset, data, digest) { + check(Number.isSafeInteger(offset) && offset >= 0 && data.length > 0 && data.length <= CHUNK_BYTES && this.hash(data) === digest); + await this.transaction(() => { + const g = this.generation(id), m = g.manifest; + check(BigInt(offset) < BigInt(m.chunks) && data.length === Math.min(CHUNK_BYTES, Number(BigInt(m.bytes) - BigInt(offset) * BigInt(CHUNK_BYTES)))); + const old = this.one('SELECT data FROM chunks WHERE generation=? AND offset=?', id, offset); + if (old) { check(equal(bytes(old.data), data)); return; } + check(!g.complete); this.rows('INSERT INTO chunks VALUES (?,?,?,?)', id, offset, data, digest); + }); + } + chunk(id, offset) { const r = this.one('SELECT data,hash FROM chunks WHERE generation=? AND offset=?', id, offset); check(r); const b = bytes(r.data); check(this.hash(b) === r.hash); return b; } + walk(id, visit) { + const { manifest: m } = this.generation(id); + let state = EMPTY, digest = this.codec.hash(EMPTY).digest, pending = EMPTY, count = 0, total = 0, size = 0, lastNS = '', lastKey = ''; + const aggregate = new Uint8Array(32); + const chunks = Number(m.chunks); check(Number.isSafeInteger(chunks) && chunks >= 0); + check(this.one('SELECT count(*) AS n FROM chunks WHERE generation=?', id).n === chunks); + for (let i = 0; i < chunks; i++) { + const block = this.chunk(id, i); check(block.length === Math.min(CHUNK_BYTES, Number(m.bytes) - total)); total += block.length; + ({ state, digest } = this.codec.hash(block, state)); + const combined = new Uint8Array(pending.length + block.length); combined.set(pending); combined.set(block, pending.length); pending = combined; + while (pending.length >= 12) { + const v = new DataView(pending.buffer, pending.byteOffset, pending.byteLength), n = v.getUint32(0), k = v.getUint32(4), d = v.getUint32(8), length = 12 + n + k + d; + check(n <= 16 && k > 0 && k <= ROW_BYTES && d <= ROW_BYTES); + if (pending.length < length) break; + const namespace = dec.decode(pending.subarray(12, 12 + n)), name = dec.decode(pending.subarray(12 + n, 12 + n + k)); + check(namespaces.includes(namespace) && (count === 0 || compare(namespace, lastNS) > 0 || (namespace === lastNS && compare(name, lastKey) > 0))); + const data = pending.slice(12 + n + k, length); xor(aggregate, this.checksum('records', namespace, name, data)); count++; size += rowSize(namespace, name, data); + visit?.({ namespace, key: name, data }); lastNS = namespace; lastKey = name; pending = pending.slice(length); + } + check(pending.length <= 2 * ROW_BYTES + 28); + } + check(pending.length === 0 && String(count) === m.records && String(total) === m.bytes && hex(digest) === m.digest && hex(aggregate) === m.records_hash); + check(size <= this.identity.ceiling, 'raft storage: snapshot exceeds retained identity capacity'); + } + retain(id) { + const others = this.rows('SELECT id FROM snapshots WHERE complete=1 AND id<>?', id).toArray().map(r => this.generation(r.id)).sort((a, b) => BigInt(a.manifest.index) > BigInt(b.manifest.index) ? -1 : 1); + for (const g of others.slice(1)) { this.rows('DELETE FROM chunks WHERE generation=?', g.id); this.rows('DELETE FROM snapshots WHERE id=?', g.id); } + } + async completeSnapshot(id) { + return this.transaction(() => { this.walk(id); this.rows('UPDATE snapshots SET complete=1 WHERE id=?', id); this.retain(id); return this.codec.snapshot(this.generation(id).data); }); + } + async checkpoint() { + return this.transaction(() => { + const app = this.application(), active = unb64(this.get('snapshot')); + if (active.length && this.codec.inspect(3, active).index === BigInt(app.applied)) { this.walk(this.hash(this.codec.inspect(3, active).data)); return active; } + const entry = this.one('SELECT data FROM journal WHERE idx=?', key(app.applied)); check(entry); + const term = this.codec.inspect(1, bytes(entry.data)).term; + this.rows('DELETE FROM chunks WHERE generation=?', 'creating'); + let buffer = new Uint8Array(CHUNK_BYTES), used = 0, offset = 0, total = 0, state = EMPTY, digest = this.codec.hash(EMPTY).digest; + const flush = () => { const b = buffer.slice(0, used); this.rows('INSERT INTO chunks VALUES (?,?,?,?)', 'creating', offset++, b, this.hash(b)); used = 0; }; + for (const r of this.recordRows()) { + const ns = enc.encode(r.namespace), name = enc.encode(r.key), header = new Uint8Array(12), v = new DataView(header.buffer); + v.setUint32(0, ns.length); v.setUint32(4, name.length); v.setUint32(8, r.data.length); + for (const part of [header, ns, name, r.data]) { + ({ state, digest } = this.codec.hash(part, state)); total += part.length; + for (let at = 0; at < part.length;) { const n = Math.min(CHUNK_BYTES - used, part.length - at); buffer.set(part.subarray(at, at + n), used); used += n; at += n; if (used === CHUNK_BYTES) flush(); } + } + } + if (used) flush(); + const data = scalar({ format: '1', cluster: this.identity.cluster, protocol: '1', index: app.applied, term: String(term), conf: app.conf || null, version: app.version, now: app.now, records: String(app.count), records_hash: app.hash, bytes: String(total), chunks: String(offset), digest: hex(digest) }); + const snapshot = this.codec.snapshot(data), id = this.hash(data); + const existing = this.one('SELECT id FROM snapshots WHERE id=?', id); + if (existing) { this.walk(id); this.rows('DELETE FROM chunks WHERE generation=?', 'creating'); } + else { this.rows('INSERT INTO snapshots VALUES (?,?,?,1)', id, data, id); this.rows('UPDATE chunks SET generation=? WHERE generation=?', id, 'creating'); } + this.walk(id); this.retain(id); this.put('snapshot', b64(snapshot)); this.rows('DELETE FROM journal WHERE idx<=?', key(app.applied)); + return snapshot; + }); + } +} diff --git a/relay/hosted/src/worker.js b/relay/hosted/src/worker.js index 0e8ba989ec..1b6dc3bfcc 100644 --- a/relay/hosted/src/worker.js +++ b/relay/hosted/src/worker.js @@ -16,3 +16,5 @@ export { Mailbox } from './pair/mailbox.js'; export default { fetch: (request, env) => respond(() => dispatch(request, env)), }; + +export { RaftMemberDO } from './raft/member.js'; diff --git a/relay/hosted/wrangler.toml b/relay/hosted/wrangler.toml index 7ad35ea140..c6be7524bf 100644 --- a/relay/hosted/wrangler.toml +++ b/relay/hosted/wrangler.toml @@ -1,5 +1,5 @@ name = "codeaf-hosted-relay" -main = "src/worker.js" +main = "src/raft-worker.js" compatibility_date = "2026-09-01" # Local development and the base of every deployment: `wrangler dev --local` runs this in workerd @@ -41,6 +41,17 @@ new_sqlite_classes = ["IdentityDO", "PairGate", "Mailbox", "NewcomerGate"] tag = "v2" new_sqlite_classes = ["LinkGate", "LinkRequest"] +[build] +command = "sh ../../scripts/raft-hosted-build.sh" + [vars] # "amended": 90-second leases, renewed by publish; "stage1": 30-second leases. LEASE_POLICY = "amended" + +[[durable_objects.bindings]] +name = "RAFT_MEMBERS" +class_name = "RaftMemberDO" + +[[migrations]] +tag = "v3-raft-members" +new_sqlite_classes = ["RaftMemberDO"] diff --git a/scripts/raft-hosted-build.sh b/scripts/raft-hosted-build.sh new file mode 100644 index 0000000000..60e6a964f2 --- /dev/null +++ b/scripts/raft-hosted-build.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# Wrangler invokes this from relay/hosted. The generated reactor is bundled +# through a static import and is never an independently deployed service. +set -eu +cd ../.. +sh scripts/raft-proof-footprint.sh env GOOS=wasip1 GOARCH=wasm GOMEMLIMIT=192MiB go build -p=1 -trimpath -buildmode=c-shared -ldflags='-s -w' -o relay/hosted/src/raft/core.wasm ./cmd/codeaf-raft-wasm From 5b329b2aacbf936a3c75e47507c1242edc058f79 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 00:30:31 -0400 Subject: [PATCH 018/103] Prove hosted recovery beside real native peers. Drive signed mixed-runtime commits, quorum reads, both snapshot directions and joint-state restarts in local workerd. Bound admission, retained state and co-resident memory before the directory adapter uses these actors. Co-Authored-By: Claude Opus 5.5 (1M context) --- relay/hosted/test/raft/actor-unit.test.js | 148 ++++++++++++++++++ relay/hosted/test/raft/actor-worker.js | 57 +++++++ relay/hosted/test/raft/actors-run.sh | 11 ++ relay/hosted/test/raft/actors.test.js | 178 ++++++++++++++++++++++ relay/hosted/test/raft/build.mjs | 9 +- relay/hosted/test/raft/workerd.js | 5 +- 6 files changed, 403 insertions(+), 5 deletions(-) create mode 100644 relay/hosted/test/raft/actor-unit.test.js create mode 100644 relay/hosted/test/raft/actor-worker.js create mode 100644 relay/hosted/test/raft/actors-run.sh create mode 100644 relay/hosted/test/raft/actors.test.js diff --git a/relay/hosted/test/raft/actor-unit.test.js b/relay/hosted/test/raft/actor-unit.test.js new file mode 100644 index 0000000000..4b5ec0b52b --- /dev/null +++ b/relay/hosted/test/raft/actor-unit.test.js @@ -0,0 +1,148 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { DatabaseSync } from 'node:sqlite'; +import { createHash, generateKeyPairSync, sign } from 'node:crypto'; +import { Reactor, HostCodec } from '../../src/raft/abi.js'; +import { MemberStorage, b64 } from '../../src/raft/storage.js'; +import { MemberActor } from '../../src/raft/actor.js'; +import { MemberInbox } from '../../src/raft/inbox.js'; + +const reactor = new Reactor(new WebAssembly.Module(readFileSync((process.env.CODEAF_RAFT_PROOF_DIR ?? '/tmp/codeaf-p08') + '/core.wasm'))); +const codec = new HostCodec(reactor), empty = new Uint8Array(); +const sha = b => createHash('sha256').update(b).digest('hex'); +const text = new TextEncoder(); +class Host { + constructor() { this.db = new DatabaseSync(':memory:'); this.sql = { exec: (q, ...p) => { const rows = this.db.prepare(q).all(...p); return { toArray: () => rows, [Symbol.iterator]: () => rows[Symbol.iterator]() }; } }; this.gate = null; } + transactionSync(fn) { this.db.exec('BEGIN'); try { const v = fn(); this.db.exec('COMMIT'); return v; } catch (e) { this.db.exec('ROLLBACK'); throw e; } } + async sync() { if (this.gate) await this.gate; } +} +const identity = id => ({ cluster: 'unit', id: String(id), protocol: 1 }); +const conf = Uint8Array.of(8,1,8,2,8,3); +const entry = (index, data = Uint8Array.of(1)) => Uint8Array.of(8,0,16,1,24,index,34,data.length,...data); +const hard = commit => Uint8Array.of(8,1,16,0,24,commit); +async function seeded(host, id = 1) { + const s = new MemberStorage(host, codec); await s.open(identity(id)); + const entries = [entry(1)]; await s.persist({ hard: hard(1), entries, snapshot: empty }); + const writes = ['state','sessions','membership','enrollment','tombstones','outbox'].map((namespace, i) => ({ namespace, key: 'opaque-' + i, data: new Uint8Array(i === 0 ? 600000 : 4096).fill(i + 1) })); + await s.apply({ index: 1n, conf, entries, snapshot: empty }, () => ({ writes, version: '8', now: '1000' })); return s; +} + +test('R3 R27 serial actor waits for durable host completion and bounds stalled admission', async () => { + const host = new Host(), actor = new MemberActor(host, reactor); + let release; host.gate = new Promise(resolve => { release = resolve; }); + let completed = false; + const first = actor.init({ cluster: 'queue', id: '1', seed: '101', bootstrap: ['1','2','3'] }).then(v => { completed = true; return v; }); + await new Promise(resolve => setImmediate(resolve)); assert.equal(completed, false); + const tick = actor.input(1); const queued = []; + for (let i = 0; i < 62; i++) queued.push(actor.serial(() => i)); + await assert.rejects(actor.serial(() => 100), /capacity/); + release(); await first; await tick; assert.deepEqual(await Promise.all(queued), Array.from({ length: 62 }, (_, i) => i)); + assert.equal(actor.status.applied, 3n); await actor.close(); host.db.close(); +}); + +test('R12 R13 R17 R28 portable host snapshots retain every namespace, incomplete stages and both generations', async () => { + const sourceHost = new Host(), destHost = new Host(); + const source = await seeded(sourceHost), snapshot = await source.checkpoint(), info = codec.inspect(3, snapshot), manifest = JSON.parse(new TextDecoder().decode(info.data)), id = sha(info.data); + assert(BigInt(manifest.chunks) >= 3n); assert.equal(manifest.records, '6'); + let dest = new MemberStorage(destHost, codec); await dest.open(identity(2)); await dest.beginSnapshot(info.data); + const chunk = source.chunk(id, 0); await dest.putChunk(id, 0, chunk, sha(chunk)); + await assert.rejects(dest.completeSnapshot(id), /invalid|damaged/); + // Reopening discards no authoritative state and does not adopt the partial + // generation. All namespace bytes remain at the source throughout. + dest = new MemberStorage(destHost, codec); const recovered = await dest.open(identity(2)); assert.equal(recovered.applied, 0n); + for (let n = 1; n < Number(manifest.chunks); n++) { const b = source.chunk(id, n); await dest.putChunk(id, n, b, sha(b)); } + assert.deepEqual(await dest.completeSnapshot(id), snapshot); + await dest.persist({ hard: hard(1), entries: [], snapshot }); + for (const r of source.recordRows()) assert.deepEqual(dest.read(r.namespace, r.key), r.data); + assert.equal(dest.application().version, '8'); assert.equal(dest.application().now, '1000'); + assert.deepEqual(dest.recover().conf, info.conf); + assert.deepEqual(await source.checkpoint(), snapshot, 'repeated checkpoint is idempotent'); + for (let index = 2; index <= 3; index++) { + const entries = [entry(index)]; await source.persist({ hard: hard(index), entries, snapshot: empty }); + await source.apply({ index: BigInt(index), conf, entries, snapshot: empty }, () => ({ writes: [], version: String(index + 8) })); await source.checkpoint(); + } + assert.equal(source.one('SELECT count(*) AS n FROM snapshots WHERE complete=1').n, 2); + assert.equal(source.one('SELECT count(*) AS n FROM records').n, 6); + assert.equal(source.one('SELECT count(*) AS n FROM journal').n, 0); + for (const r of source.rows('SELECT id FROM snapshots')) source.walk(r.id); + sourceHost.db.close(); destHost.db.close(); +}); + +test('R3 R13 hosted transaction failure and corrupt complete bytes refuse participation without resetting records', async () => { + const host = new Host(), source = await seeded(host), snapshot = await source.checkpoint(), info = codec.inspect(3, snapshot), id = sha(info.data); + host.db.prepare('UPDATE chunks SET hash=? WHERE generation=? AND offset=0').run('damaged', id); + const reopened = new MemberStorage(host, codec); + await assert.rejects(reopened.open(identity(1)), /damaged/); + assert.equal(host.db.prepare('SELECT count(*) AS n FROM records').get().n, 6); + assert.equal(host.db.prepare('SELECT count(*) AS n FROM snapshots').get().n, 1); + host.db.close(); + const failingHost = new Host(), actor = new MemberActor(failingHost, reactor); + failingHost.db.exec("CREATE TABLE evidence (body TEXT)"); + await assert.rejects(actor.init({ cluster: 'refuse', id: '1', seed: '1', bootstrap: ['1','2','3'] }), /damaged/); + assert.equal(failingHost.db.prepare('SELECT count(*) AS n FROM evidence').get().n, 0); + failingHost.db.close(); +}); + +test('R14 R16 hosted inbox accepts native wire and rechecks revocation after queued signature admission', async () => { + const f = JSON.parse(readFileSync(new URL('../../../../internal/rafttransport/testdata/exchange-v1.json', import.meta.url), 'utf8')); + const e = f.request.messages[0], scope = e.scope; let view = { 1: { incarnation: e.incarnation, key: f.public_key, voter: true }, 2: { incarnation: '1', key: f.public_key, voter: true } }; + const queue = [], inbox = new MemberInbox(scope, '2', codec, () => view, fn => new Promise((resolve, reject) => queue.push(() => Promise.resolve().then(fn).then(resolve, reject)))); + let dispatched = 0; const first = inbox.receive(text.encode(JSON.stringify(e)), () => { dispatched++; }); + while (queue.length === 0) await new Promise(resolve => setImmediate(resolve)); + view = { ...view, 1: { ...view[1], revoked: true } }; queue.shift()(); + await assert.rejects(first, /authorization/); assert.equal(dispatched, 0); + view = { ...view, 1: { ...view[1], revoked: false } }; + const accepted = inbox.receive(text.encode(JSON.stringify(e)), () => { dispatched++; }); + while (queue.length === 0) await new Promise(resolve => setImmediate(resolve)); queue.shift()(); await accepted; assert.equal(dispatched, 1); + await assert.rejects(inbox.receive(text.encode(JSON.stringify({ ...e, to: '3' })), () => {}), /authorization/); +}); + +test('R27 R28 hosted quotas refuse growth and excessive evaluation sets without deleting retained rows', async () => { + const host = new Host(), s = await seeded(host); + const entries = [entry(2)]; await s.persist({ hard: hard(2), entries, snapshot: empty }); + await assert.rejects(s.apply({ index: 2n, entries, conf, snapshot: empty }, (_, read) => { read('state', 'opaque-0'); read('state', 'opaque-0'); return { writes: [] }; }), /read-set capacity/); + assert.equal(s.one('SELECT count(*) AS n FROM records').n, 6); assert.equal(s.application().applied, '1'); host.db.close(); + const growingHost = new Host(), growing = await seeded(growingHost); let refused = false; + for (let index = 2; index < 16; index++) { + const entries = [entry(index)]; await growing.persist({ hard: hard(index), entries, snapshot: empty }); + const before = growing.one('SELECT count(*) AS n FROM records').n; + try { await growing.apply({ index: BigInt(index), entries, conf, snapshot: empty }, () => ({ writes: [{ namespace: 'sessions', key: `retained-${index}`, data: new Uint8Array(900000).fill(index) }] })); } + catch (e) { assert.match(e.message, /application capacity/); assert.equal(growing.one('SELECT count(*) AS n FROM records').n, before); refused = true; break; } + } + assert(refused); assert(growing.application().size <= 8 * 1024 * 1024); + assert.equal(growing.read('state', 'opaque-0').length, 600000); growingHost.db.close(); +}); + +test('R11 R16 R27 hosted inbox reservations preserve voter delivery when a learner stalls', async () => { + const keys = generateKeyPairSync('ed25519'), key = keys.publicKey.export({ format: 'der', type: 'spki' }).subarray(-32).toString('base64'); + const scope = { identity: 'bounded', cluster: 'bounded', epoch: '1', protocol: 1 }; + const view = { 1: { incarnation: '1', key, voter: true }, 2: { incarnation: '1', key, voter: true }, 3: { incarnation: '1', key, voter: false } }; + const queue = [], inbox = new MemberInbox(scope, '2', codec, () => view, fn => new Promise((resolve, reject) => queue.push(() => Promise.resolve().then(fn).then(resolve, reject)))); + const packet = (from, sequence) => { + const payload = Uint8Array.of(8,8,16,2,24,from,32,1); + const e = { scope, from: String(from), to: '2', incarnation: '1', destination: '1', sequence: String(sequence), chunk: null, digest: sha(payload), payload: b64(payload), signature: null }; + e.signature = sign(null, text.encode('codeaf/raft/envelope/1\n' + JSON.stringify(e)), keys.privateKey).toString('base64'); return text.encode(JSON.stringify(e)); + }; + const learner = [inbox.receive(packet(3, 1), () => 'learner'), inbox.receive(packet(3, 2), () => 'learner')]; + while (queue.length < 2) await new Promise(resolve => setImmediate(resolve)); + await assert.rejects(inbox.receive(packet(3, 3), () => {}), /capacity/); + const voter = inbox.receive(packet(1, 1), () => 'voter'); + while (queue.length < 3) await new Promise(resolve => setImmediate(resolve)); + queue.pop()(); assert.equal(await voter, 'voter'); + for (const deliver of queue) deliver(); await Promise.all(learner); + assert.deepEqual(inbox.counts, [0,0,0]); +}); + +test('R27 isolate activation refuses before allocating a new member store and retains all active followers', async () => { + const actors = [], hosts = []; let refused = false; + try { + for (let i = 0; i < 16; i++) { + const host = new Host(), actor = new MemberActor(host, reactor); hosts.push(host); + try { await actor.init({ cluster: `capacity-${i}`, id: '1', seed: '101', bootstrap: ['1','2','3'] }); actors.push(actor); } + catch (e) { assert.match(e.message, /isolate capacity/); assert.equal(host.db.prepare("SELECT count(*) AS n FROM sqlite_master WHERE name='meta'").get().n, 0); refused = true; break; } + } + assert(refused); assert(actors.length >= 3); + for (const actor of actors) assert.equal((await actor.input(0)).status.applied, 3n); + } finally { for (const actor of actors) await actor.close(); for (const host of hosts) host.db.close(); } +}); diff --git a/relay/hosted/test/raft/actor-worker.js b/relay/hosted/test/raft/actor-worker.js new file mode 100644 index 0000000000..48a950679b --- /dev/null +++ b/relay/hosted/test/raft/actor-worker.js @@ -0,0 +1,57 @@ +// Local-only RPC controls drive the production actor/store. No production +// request router exposes ticks, bootstrap, proposals or checkpoint controls. +import core from './core.wasm'; +import { installRaftModule, runtimeMetrics, MemberActor, sharedRuntime } from '../../src/raft/actor.js'; +import { RaftMemberDO } from '../../src/raft/member.js'; +import { frame, response } from '../../src/raft/abi.js'; +import { b64, unb64 } from '../../src/raft/storage.js'; +export { IdentityDO, NewcomerGate, LinkGate, LinkRequest, PairGate, Mailbox } from '../../src/worker.js'; +installRaftModule(core); +const json = v => Response.json(v, { headers: { 'content-type': 'application/json' } }); +const encode = v => JSON.stringify(v, (_, x) => typeof x === 'bigint' ? String(x) : x instanceof Uint8Array ? b64(x) : x); +const reply = v => new Response(encode(v ?? null), { headers: { 'content-type': 'application/json' } }); +const evaluate = (command, read, app) => { + const reactor = sharedRuntime(), r = response(reactor.call(frame(4).blob(command).blob(read('state', 'cell') ?? new Uint8Array()).finish())); + const data = r.blob(); r.end(); return { writes: [{ namespace: 'state', key: 'cell', data }], version: BigInt(app.version) + 1n }; +}; +export class ActorFixture extends RaftMemberDO { + constructor(ctx, env) { super(ctx, { ...env, raftEvaluate: evaluate }); } + async fetch(request) { + const url = new URL(request.url), b = request.method === 'POST' ? await request.json() : {}; + try { + if (url.pathname === '/init') return reply(await this.initialize(b)); + await this.load(); + if (url.pathname === '/input') return reply(await this.actor.input(b.tag, [2,3,8,11].includes(b.tag) ? unb64(b.value) : [5,6,7,9].includes(b.tag) ? BigInt(b.value) : b.value)); + if (url.pathname === '/row') return reply(this.actor.store.read('state', 'cell')); + if (url.pathname === '/application') return reply(this.actor.store.application()); + if (url.pathname === '/checkpoint') return reply(await this.actor.checkpoint()); + if (url.pathname === '/begin') return reply(await this.actor.begin(unb64(b.data))); + if (url.pathname === '/chunk') return reply(await this.actor.chunk(b.id, b.offset, unb64(b.data), b.hash)); + if (url.pathname === '/complete') return reply(await this.actor.complete(b.id)); + if (url.pathname === '/read-chunk') return reply(this.actor.store.chunk(b.id, b.offset)); + if (url.pathname === '/peer') return reply(await this.peer(unb64(b.data))); + if (url.pathname === '/enrollment') return reply(await this.enrollment(b.view, b.applied)); + if (url.pathname === '/transport') return reply(await this.actor.serial(() => this.actor.drive(12, b))); + if (url.pathname === '/corrupt') { this.actor.store.rows('UPDATE chunks SET hash=? WHERE generation=? AND offset=?', 'damaged', b.id, b.offset); await this.ctx.storage.sync(); return reply(true); } + if (url.pathname === '/storage-fail') { this.actor.store.rows("CREATE TRIGGER fail_write BEFORE INSERT ON journal BEGIN SELECT RAISE(ABORT, 'injected storage failure'); END"); return reply(true); } + if (url.pathname === '/concurrent') { + const seen = []; let release; + const wait = new Promise(resolve => { release = resolve; }); + const first = this.actor.serial(async () => { seen.push('first-start'); await wait; seen.push('first-end'); }); + const second = this.actor.serial(() => { seen.push('second'); }); + await Promise.resolve(); release(); await Promise.all([first, second]); return reply(seen); + } + return new Response('fixture path', { status: 404 }); + } catch (e) { return json({ error: e.message }); } + } +} +export default { + async fetch(request, env) { + const u = new URL(request.url); + if (u.pathname === '/ready') return new Response(null, { status: 204 }); + if (u.pathname === '/metrics') return reply(runtimeMetrics()); + const [, , name, ...parts] = u.pathname.split('/'); + const stub = env.ACTORS.get(env.ACTORS.idFromName(name)); + return stub.fetch(new Request('http://actor/' + parts.join('/'), request)); + }, +}; diff --git a/relay/hosted/test/raft/actors-run.sh b/relay/hosted/test/raft/actors-run.sh new file mode 100644 index 0000000000..d87971a572 --- /dev/null +++ b/relay/hosted/test/raft/actors-run.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# All phases are serial, pressure-admitted and local. A watchdog bounds each +# runtime phase without weakening resource or correctness assertions. +set -eu +out=${1:-/tmp/codeaf-p08} +sh scripts/raft-proof-build.sh "$out" +sh scripts/raft-proof-footprint.sh go test -p=1 ./internal/raftabi ./internal/raftcore ./internal/raftsnapshot -run '^TestR' -count=1 -timeout=120s +cd relay/hosted +sh ../../scripts/raft-proof-footprint.sh sh test/local.sh env CODEAF_RAFT_ACTORS=1 node test/raft/build.mjs "$out" +sh ../../scripts/raft-proof-footprint.sh sh test/local.sh env CODEAF_RAFT_PROOF_DIR="$out" node --max-old-space-size=32 test/raft/actor-unit.test.js +sh ../../scripts/raft-proof-footprint.sh sh test/local.sh env CODEAF_RAFT_PROOF_DIR="$out" timeout 120s node --max-old-space-size=24 test/raft/actors.test.js diff --git a/relay/hosted/test/raft/actors.test.js b/relay/hosted/test/raft/actors.test.js new file mode 100644 index 0000000000..09b75efdbd --- /dev/null +++ b/relay/hosted/test/raft/actors.test.js @@ -0,0 +1,178 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn, execFileSync } from 'node:child_process'; +import { mkdtemp, readFile, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { createHash, generateKeyPairSync, sign } from 'node:crypto'; +import { openWorkerd } from './workerd.js'; +import { encodeCanonical } from '../../src/raft/codec.js'; + +const out = resolve(process.env.CODEAF_RAFT_PROOF_DIR ?? '/tmp/codeaf-p08'); +const sha = b => createHash('sha256').update(b).digest('hex'); +const b64 = b => Buffer.from(b).toString('base64'); +const unb64 = s => Buffer.from(s, 'base64'); +const clockTicks = Number(execFileSync('getconf', ['CLK_TCK'], { encoding: 'utf8' }).trim()); +const report = { calls: [], requirements: ['R3','R4','R10','R12','R13','R14','R16','R27'], samples: [], phases: [] }; +async function sample() { + const ids = [process.pid, ...(await readFile(`/proc/${process.pid}/task/${process.pid}/children`, 'utf8')).trim().split(/\s+/).filter(Boolean).map(Number)]; + const processes = []; + for (const pid of ids) { try { const s = await readFile(`/proc/${pid}/status`, 'utf8'), stat = (await readFile(`/proc/${pid}/stat`, 'utf8')).split(') ').slice(1).join(') ').split(' '); processes.push({ pid, cpuSeconds: (Number(stat[11]) + Number(stat[12])) / clockTicks, rss: Number(s.match(/^VmRSS:\s+(\d+)/m)?.[1] ?? 0) * 1024 }); } catch {} } + const value = { processes, combinedRSS: processes.reduce((n, p) => n + p.rss, 0) }; report.samples.push(value); return value; +} +class Native { + constructor(dir) { + this.child = spawn(join(out, 'native-peer'), ['actors', dir], { stdio: ['pipe','pipe','inherit'], env: { ...process.env, GOMEMLIMIT: '32MiB' } }); + this.buffer = Buffer.alloc(0); this.waiter = null; + this.child.stdout.on('data', b => { this.buffer = Buffer.concat([this.buffer, b]); this.pump(); }); + this.child.on('exit', code => this.waiter?.reject(new Error(`native exit ${code}`))); + } + pump() { if (!this.waiter || this.buffer.length < 4) return; const n = this.buffer.readUInt32LE(0); assert(n < 2 * 1024 * 1024); if (this.buffer.length < n + 4) return; const value = JSON.parse(this.buffer.subarray(4, n + 4)); this.buffer = this.buffer.subarray(n + 4); const p = this.waiter; this.waiter = null; clearTimeout(p.timer); p.resolve(value); } + call(value) { + assert(!this.waiter); return new Promise((resolve, reject) => { this.waiter = { resolve, reject, timer: setTimeout(() => reject(new Error('native actor timeout')), 5000) }; const b = Buffer.from(JSON.stringify(value)), h = Buffer.alloc(4); h.writeUInt32LE(b.length); this.child.stdin.write(h); this.child.stdin.write(b); this.pump(); }); + } + async close() { this.child.stdin.end(); if (this.child.exitCode === null) await new Promise(resolve => this.child.once('exit', resolve)); } +} +// This parser only identifies snapshot transport in the test scheduler. Both +// production receivers use Go's strict protobuf admission, never this parser. +function fields(b) { + let at = 0; const map = new Map(); + const uint = () => { let n = 0n, shift = 0n; for (let i = 0; i < 10; i++) { assert(at < b.length); const x = b[at++]; n |= BigInt(x & 127) << shift; if (!(x & 128)) return n; shift += 7n; } throw new Error('varint'); }; + while (at < b.length) { const k = uint(), kind = Number(k & 7n); let v; if (kind === 0) v = uint(); else { assert.equal(kind, 2); const n = Number(uint()); v = b.subarray(at, at + n); at += n; } map.set(Number(k >> 3n), v); } return map; +} +class Group { + constructor(worker, native, cluster, wasm) { + this.worker = worker; this.native = native; this.cluster = cluster; this.wasm = wasm; this.nodes = new Map(); this.queue = []; this.reads = []; this.block = new Set(); this.sequence = 0n; + this.scope = { identity: cluster, cluster, epoch: '1', protocol: 1 }; this.keys = {}; this.view = {}; + for (let i = 1; i <= 4; i++) { const k = generateKeyPairSync('ed25519'); this.keys[i] = k.privateKey; this.view[i] = { key: k.publicKey.export({ format: 'der', type: 'spki' }).subarray(-32).toString('base64'), incarnation: '1', revoked: false, voter: true }; } + } + async call(id, op, value = {}) { + let r; const start = performance.now(); + if (this.wasm.includes(id)) { + const res = await this.worker.dispatchFetch(`http://proof/actor/${this.cluster}-${id}/${op}`, { method: 'POST', body: JSON.stringify(value), signal: AbortSignal.timeout(5000) }); assert.equal(res.status, 200, await res.clone().text()); r = await res.json(); + } else r = await this.native.call({ op, ...value, id: String(id) }); + report.calls.push({ op, wasm: this.wasm.includes(id), ms: performance.now() - start }); + if (r?.error) throw new Error(`node ${id} ${op}: ${r.error}`); + return r; + } + accept(id, r) { if (!r) return; if (r.status) this.nodes.set(id, r.status); this.queue.push(...(r.send ?? []).map(m => ({ ...m, from: id }))); this.reads.push(...(r.reads ?? []).map(r => ({ ...r, node: id }))); } + async init() { + for (let id = 1; id <= 3; id++) this.accept(id, await this.call(id, 'init', { cluster: this.cluster, id: String(id), seed: String(100 + id), bootstrap: ['1','2','3'], scope: this.scope })); + for (const id of this.wasm) await this.call(id, 'enrollment', { view: this.view, applied: this.nodes.get(id).applied }); + await this.pump(); await this.tick(16); assert(this.leader(), JSON.stringify([...this.nodes])); + } + envelope(from, to, payload, chunk = null) { + const e = { scope: this.scope, from: String(from), to: String(to), incarnation: '1', destination: '1', sequence: String(++this.sequence), chunk, digest: sha(payload), payload: b64(payload), signature: null }; + e.signature = sign(null, Buffer.from('codeaf/raft/envelope/1\n' + JSON.stringify(e)), this.keys[from]).toString('base64'); return b64(Buffer.from(JSON.stringify(e))); + } + async step(id, tag, value) { this.accept(id, await this.call(id, 'input', { tag, value })); } + async deliver(m) { + const to = Number(m.to), payload = unb64(m.bytes), f = fields(payload), snapshot = f.get(9); + if (snapshot?.length) { + const data = fields(snapshot).get(1), manifest = JSON.parse(data.toString()), generation = sha(data); + await this.call(to, 'begin', { data: b64(data) }); + for (let offset = 0; offset < Number(manifest.chunks); offset++) { + const chunk = await this.call(m.from, 'read-chunk', { generation, id: generation, offset }); + if (this.wasm.includes(to)) await this.call(to, 'peer', { data: this.envelope(m.from, to, unb64(chunk), { generation, offset: String(offset), hash: sha(unb64(chunk)) }) }); + else await this.call(to, 'chunk', { generation, offset, data: chunk, hash: sha(unb64(chunk)) }); + } + await this.call(to, 'complete', { generation, id: generation }); + report.phases.push(`${this.wasm.includes(m.from) ? 'wasm' : 'native'} snapshot -> ${this.wasm.includes(to) ? 'wasm' : 'native'}`); + } + if (this.wasm.includes(to)) this.accept(to, await this.call(to, 'peer', { data: this.envelope(m.from, to, payload) })); + else await this.step(to, 2, m.bytes); + if (snapshot?.length) await this.step(m.from, 12, { peer: String(to), snapshot: true, success: true }); + } + async pump() { for (let events = 0; this.queue.length; events++) { assert(events < 2000); const m = this.queue.shift(); if (this.block.has(m.from) || this.block.has(Number(m.to))) continue; await this.deliver(m); } } + async tick(rounds = 1) { for (let i = 0; i < rounds; i++) { for (const id of this.nodes.keys()) if (!this.block.has(id)) await this.step(id, 1); await this.pump(); } } + leader() { return [...this.nodes.entries()].find(([id, s]) => !this.block.has(id) && s.role === 2)?.[0]; } + async transfer(to) { const leader = this.leader(); if (leader !== to) { await this.step(leader, 7, String(to)); await this.pump(); await this.tick(2); } assert.equal(this.leader(), to); } + async read(id, name) { await this.step(id, 4, name); await this.pump(); return this.reads.find(r => r.id === name); } + command(change, sequence = 1) { return b64(Buffer.from(encodeCanonical({ schema: 1, cluster: this.cluster, actor: 'device-a', session: 'actor-probe', sequence: String(sequence), request_digest: 'opaque-digest', now: 1000, change, signature: 'AQID' }))); } + async create() { await this.step(this.leader(), 3, this.command({ create: { cell: 'cell', init: { head: 'sealed-head', class: 'work', size: '7', title: 'sealed-title', keys: {} }, receipt: 'durable-receipt' } })); await this.pump(); } + async mutate(sequence) { const change = sequence === 2 ? { release: { cell: 'cell', fence: '1', quiet: false } } : { acquire: { cell: 'cell', force: false, expect: { device: 'device-a', fence: '1' } } }; await this.step(this.leader(), 3, this.command(change, sequence)); await this.pump(); } + async sameRows() { const rows = []; for (const id of this.nodes.keys()) rows.push(await this.call(id, 'row')); assert(rows[0]); assert.equal(rows[0], rows[1]); assert.equal(rows[0], rows[2]); for (const row of rows) assert.equal(row, rows[0]); return rows[0]; } +} + +test('R3 R4 R10 R13 R14 R27 hosted actors commit, ReadIndex, crash and snapshot in both directions', { timeout: 120000 }, async () => { + const dir = await mkdtemp(join(tmpdir(), 'p08-')), native = new Native(join(dir, 'native')); + let worker, inspector, sampling = false; const heaps = []; + const timer = setInterval(async () => { if (sampling) return; sampling = true; try { await sample(); if (inspector) heaps.push(await inspector.call('Runtime.getHeapUsage')); } finally { sampling = false; } }, 100); + try { + worker = await openWorkerd(out, join(dir, 'do'), true); inspector = await inspect(worker); + const g = new Group(worker, native, 'mixed', [1]); await g.init(); await g.transfer(1); await g.create(); + assert(await g.read(1, 'after-create')); const row = await g.sameRows(); + await g.call(1, 'init', { cluster: 'mixed', id: '1', seed: '101', bootstrap: ['1','2','3'], scope: g.scope }); + await assert.rejects(g.call(1, 'init', { cluster: 'wrong', id: '1', seed: '101', bootstrap: ['1','2','3'], scope: g.scope }), /identity cannot change/); + assert.deepEqual(await g.call(1, 'concurrent'), ['first-start','first-end','second']); + g.block.add(1); assert.equal(await g.read(1, 'minority'), undefined); await g.tick(16); assert(g.leader() !== 1); assert(await g.read(g.leader(), 'majority')); + g.block.clear(); await g.tick(14); await g.transfer(1); + // Node 3 misses a suffix which the hosted leader compacts. Its recovery + // must install the streamed snapshot before it can acknowledge MsgSnap. + g.block.add(3); await g.mutate(2); await g.call(1, 'checkpoint'); g.block.clear(); await g.tick(4); await g.sameRows(); + assert(report.phases.includes('wasm snapshot -> native')); + await g.transfer(2); g.block.add(1); await g.mutate(3); await g.call(2, 'checkpoint'); g.block.clear(); await g.tick(4); const latest = await g.sameRows(); + assert(report.phases.includes('native snapshot -> wasm')); + await g.read(g.leader(), 'after-snapshots'); await g.transfer(1); await sample(); heaps.push(await inspector.call('Runtime.getHeapUsage')); + inspector.close(); inspector = null; await worker.dispose(); worker = null; + worker = await openWorkerd(out, join(dir, 'do'), true); g.worker = worker; inspector = await inspect(worker); + // A new workerd process loads the actor and validates durable rows and + // snapshot chunks. Recovery does not restore a leader or prior read lease. + assert.equal(await g.call(1, 'row'), latest); + g.accept(1, await g.call(1, 'input', { tag: 0 })); assert.equal(g.nodes.get(1).role, 0); + g.accept(2, await g.call(2, 'restart')); g.accept(3, await g.call(3, 'restart')); await g.tick(16); + assert(await g.read(g.leader(), 'recovered')); await g.sameRows(); + report.phases.push('killed workerd + reopened native SQLite + quorum barrier'); + // A second mixed topology exercises two wasm members, full joint state, + // learners-next, checkpoint and a process restart before explicit leave. + const j = new Group(worker, native, 'joint', [1,2]); await j.init(); await j.create(); + j.accept(4, await j.call(4, 'init', { cluster: 'joint', id: '4', seed: '104', bootstrap: ['1','2','3'], scope: j.scope })); + await j.step(j.leader(), 8, b64(Uint8Array.of(18,4,8,3,16,4))); await j.pump(); await j.tick(2); + await j.step(j.leader(), 8, b64(Uint8Array.of(8,2,18,4,8,3,16,3,18,4,8,0,16,4))); await j.pump(); await j.tick(2); + const jointSnapshot = unb64(await j.call(1, 'checkpoint')); + const jointManifest = JSON.parse(fields(jointSnapshot).get(1).toString()); + const jointConf = fields(unb64(jointManifest.conf)); assert(jointConf.has(3)); assert(jointConf.has(4)); + inspector.close(); inspector = null; await worker.dispose(); worker = null; + worker = await openWorkerd(out, join(dir, 'do'), true); inspector = await inspect(worker); j.worker = worker; g.worker = worker; + for (const id of [1,2]) { await j.call(id, 'row'); j.accept(id, await j.call(id, 'input', { tag: 0 })); } + for (const id of [3,4]) j.accept(id, await j.call(id, 'restart')); + await j.tick(16); assert(await j.read(j.leader(), 'joint-restart')); + const restored = unb64(await j.call(1, 'checkpoint')); assert.equal(JSON.parse(fields(restored).get(1).toString()).conf, jointManifest.conf); + await j.step(j.leader(), 8, ''); await j.pump(); await j.tick(2); assert(await j.read(j.leader(), 'left-joint')); await j.sameRows(); + const left = unb64(await j.call(1, 'checkpoint')); const leftConf = fields(unb64(JSON.parse(fields(left).get(1).toString()).conf)); assert(!leftConf.has(3)); assert(!leftConf.has(4)); + assert.notEqual(j.leader(), 3, 'removed voter remains a learner after leave/restart'); + report.phases.push('two wasm + native peers; full joint checkpoint/restart/leave'); + // All three anchors plus two other identities share exactly one runtime. + const h = new Group(worker, native, 'co-resident', [1,2,3]); await h.init(); await h.create(); assert(await h.read(h.leader(), 'all-hosted')); await h.sameRows(); + assert.equal(await g.call(1, 'row'), latest); assert.notEqual(row, latest); + await h.transfer(1); const prior = await h.call(3, 'row'); await h.call(3, 'storage-fail'); + await h.step(1, 3, h.command({ release: { cell: 'cell', fence: '1', quiet: false } }, 2)); + await assert.rejects(h.pump(), /injected storage failure/); h.block.add(3); await h.pump(); + assert.equal(await h.call(3, 'row'), prior, 'failed hosted persist retains prior application'); + await assert.rejects(h.call(3, 'input', { tag: 1 }), /stopped/); + assert(await h.read(1, 'after-store-failure'), 'healthy hosted majority still coordinates'); + report.phases.push('SQL failure stops hosted participation while healthy majority commits'); + const metrics = await (await worker.dispatchFetch('http://proof/metrics')).json(); heaps.push(await inspector.call('Runtime.getHeapUsage')); + report.metrics = metrics; report.heapSamples = heaps; + report.isolateBytes = Math.max(...heaps.map(h => h.totalSize + (h.embedderHeapUsedSize ?? 0) + (h.backingStorageSize ?? 0))) + metrics.peakMemory; + assert(report.isolateBytes <= 96 * 1024 * 1024); assert(metrics.peakMemory <= 64 * 1024 * 1024); assert.equal(metrics.active, 6); + report.phases.push('three hosted anchors plus mixed identity in same isolate'); + } finally { + clearInterval(timer); await sample(); inspector?.close(); if (worker) await worker.dispose(); await native.close(); + report.combinedRSS = Math.max(...report.samples.map(s => s.combinedRSS)); + const cpu = new Map(); for (const sample of report.samples) for (const p of sample.processes) cpu.set(p.pid, Math.max(cpu.get(p.pid) ?? 0, p.cpuSeconds)); report.sampledCPUSeconds = [...cpu.values()].reduce((a, b) => a + b, 0); + const times = report.calls.filter(c => c.wasm).map(c => c.ms).sort((a, b) => a - b); report.callMs = { p50: times[Math.floor(times.length * .5)], p95: times[Math.floor(times.length * .95)], p99: times[Math.floor(times.length * .99)] }; + report.coldFirstInitMs = report.calls.find(c => c.wasm && c.op === 'init')?.ms; + await writeFile(join(out, 'actors-runtime.json'), JSON.stringify(report, null, 2) + '\n'); + } + assert(report.coldFirstInitMs < 1000, 'R27 cold member activation stays within one second'); + assert(report.combinedRSS <= 256 * 1024 * 1024, `R27 combined RSS ${report.combinedRSS}`); +}); +async function inspect(worker) { + const base = await worker.getInspectorURL(), listing = await (await fetch(new URL('/json/list', base))).json(); + const target = listing.find(x => x.title?.includes('raft-proof')) ?? listing[0]; + const ws = new WebSocket(target.webSocketDebuggerUrl); await new Promise((resolve, reject) => { ws.onopen = resolve; ws.onerror = reject; }); + let serial = 0; const waiting = new Map(); + ws.onmessage = e => { const m = JSON.parse(e.data); if (!m.id) return; const p = waiting.get(m.id); if (!p) return; waiting.delete(m.id); clearTimeout(p.timer); m.error ? p.reject(new Error(JSON.stringify(m.error))) : p.resolve(m.result); }; + return { close: () => ws.close(), call: (method) => new Promise((resolve, reject) => { const id = ++serial, timer = setTimeout(() => { waiting.delete(id); reject(new Error('inspector timeout')); }, 5000); waiting.set(id, { resolve, reject, timer }); ws.send(JSON.stringify({ id, method })); }) }; +} diff --git a/relay/hosted/test/raft/build.mjs b/relay/hosted/test/raft/build.mjs index da795bd607..85c8abaa9d 100644 --- a/relay/hosted/test/raft/build.mjs +++ b/relay/hosted/test/raft/build.mjs @@ -13,8 +13,11 @@ await mkdir(out, { recursive: true }); const require=createRequire(import.meta.url); await writeFile(resolve(out,'workerd-path'),require('workerd').default+'\n'); await writeFile(resolve(out,'workerd-schema-path'),resolve(dirname(require.resolve('workerd/package.json')),'workerd.capnp')+'\n'); -const result = await build({ entryPoints: ['test/raft/worker.js'], bundle: true, format: 'esm', target: 'es2022', external: ['*.wasm', 'cloudflare:*'], write: false }); +const result = await build({ entryPoints: [process.env.CODEAF_RAFT_ACTORS ? 'test/raft/actor-worker.js' : 'test/raft/worker.js'], bundle: true, format: 'esm', target: 'es2022', external: ['*.wasm', 'cloudflare:*'], write: false }); const js = result.outputFiles[0].contents; +const production = await build({ entryPoints: ['src/raft-worker.js'], bundle: true, format: 'esm', target: 'es2022', external: ['*.wasm', 'cloudflare:*'], write: false }); +const productionJS = production.outputFiles[0].contents; +await writeFile(resolve(out, 'production-worker.js'), productionJS); await writeFile(resolve(out, 'worker.js'), js); const storage = await readFile('test/raft/storage-worker.js'); await writeFile(resolve(out,'storage-worker.js'),storage); @@ -37,8 +40,8 @@ for (const p of ['../../go.mod','../../go.sum']) source.push([relative(resolve(' source.sort((a,b)=>a[0].localeCompare(b[0])); const manifest = { abi: 1, coreProtocol: 1, rowSchema: 1, snapshotSchema: 1, snapshotFormatSHA256: sha(await readFile('../../internal/raftsnapshot/format.go')), go: 'go1.26.5', sourceHash: sha(JSON.stringify(source)), wasmSHA256: sha(wasm), sources: source }; await writeFile(resolve(out, 'manifest.json'), JSON.stringify(manifest, null, 2)+'\n'); -const compressed = { wasm: gzipSync(wasm,{level:9}).length, javascript: gzipSync(js,{level:9}).length, storageFixture:gzipSync(storage,{level:9}).length, manifest: gzipSync(JSON.stringify(manifest),{level:9}).length }; -const report = { imports, rawWasm: wasm.length, rawJavascript: js.length, compressed, totalCompressed: Object.values(compressed).reduce((a,b)=>a+b,0), limit: 3*1024*1024, wasmSHA256: manifest.wasmSHA256, sourceHash: manifest.sourceHash }; +const compressed = { wasm: gzipSync(wasm,{level:9}).length, javascript: Math.max(gzipSync(js,{level:9}).length, gzipSync(productionJS,{level:9}).length), storageFixture:gzipSync(storage,{level:9}).length, manifest: gzipSync(JSON.stringify(manifest),{level:9}).length }; +const report = { imports, rawWasm: wasm.length, rawJavascript: js.length, rawProductionJavascript: productionJS.length, productionJavascriptCompressed: gzipSync(productionJS,{level:9}).length, compressed, totalCompressed: Object.values(compressed).reduce((a,b)=>a+b,0), limit: 3*1024*1024, wasmSHA256: manifest.wasmSHA256, sourceHash: manifest.sourceHash }; await writeFile(resolve(out, 'size.json'), JSON.stringify(report, null, 2)+'\n'); console.log(JSON.stringify(report)); if (report.totalCompressed >= report.limit) throw new Error('R27 full package exceeds 3 MiB'); diff --git a/relay/hosted/test/raft/workerd.js b/relay/hosted/test/raft/workerd.js index ea8d2362df..48cca4ec76 100644 --- a/relay/hosted/test/raft/workerd.js +++ b/relay/hosted/test/raft/workerd.js @@ -13,7 +13,7 @@ async function port() { const s=createServer();await new Promise((resolve,reject)=>{s.on('error',reject);s.listen(0,'127.0.0.1',resolve);}); const n=s.address().port;await new Promise(resolve=>s.close(resolve));return n; } -export async function openWorkerd(out,persist) { +export async function openWorkerd(out,persist, actors = false) { await mkdir(persist,{recursive:true}); const httpPort=await port(),inspectorPort=await port(); const quote=JSON.stringify; @@ -25,7 +25,8 @@ const config :Workerd.Config = ( compatibilityDate = "2026-09-01", modules = [(name = "worker.js", esModule = embed ${quote(join(out,'worker.js'))}), (name = "core.wasm", wasm = embed ${quote(join(out,'core.wasm'))})], - bindings = [(name = "PROOF_STORE", durableObjectNamespace = (className = "ProofStore", serviceName = "proof-storage"))], + ${actors ? `durableObjectNamespaces = [(className = "ActorFixture", uniqueKey = "p08-actors", enableSql = true)], durableObjectStorage = (localDisk = "storage"),` : ""} + bindings = [${actors ? `(name = "ACTORS", durableObjectNamespace = (className = "ActorFixture")),` : ""}(name = "PROOF_STORE", durableObjectNamespace = (className = "ProofStore", serviceName = "proof-storage"))], globalOutbound = "blocked" )), (name = "proof-storage", worker = ( From 94c78b6dec4c917c84a249cc23a449b53e64b6c5 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 00:48:14 -0400 Subject: [PATCH 019/103] Decide directory mutations and retry results in the shared Go state machine. Verify committed enrollment and signed requests at application, retain exact sequence results, and expose bounded dependency evaluation through the same native and WASI ABI. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterstate/apply.go | 627 ++++++++++++++++++++++++++ internal/clusterstate/apply_test.go | 354 +++++++++++++++ internal/clusterstate/codec.go | 2 +- internal/clusterstate/imports_test.go | 2 + internal/clusterstate/types.go | 81 +++- internal/raftabi/abi.go | 2 + internal/raftabi/application.go | 74 +++ 7 files changed, 1116 insertions(+), 26 deletions(-) create mode 100644 internal/clusterstate/apply.go create mode 100644 internal/clusterstate/apply_test.go create mode 100644 internal/raftabi/application.go diff --git a/internal/clusterstate/apply.go b/internal/clusterstate/apply.go new file mode 100644 index 0000000000..251d43066c --- /dev/null +++ b/internal/clusterstate/apply.go @@ -0,0 +1,627 @@ +package clusterstate + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "encoding/hex" + "errors" + "reflect" + "sort" +) + +const ( + MaxEvaluationBytes = 1 << 20 + MaxCommandBytes = 64 << 10 + MaxLiveSessions = 128 + MaxRetainedSessions = 4096 + DefaultApplicationBytes = 8 << 20 + MinGraceMillis int64 = 3_600_000 + MaxGraceMillis int64 = 30 * 24 * MinGraceMillis + DefaultGraceMillis int64 = 7 * 24 * MinGraceMillis +) + +var ErrDependency = errors.New("clusterstate: incomplete or invalid committed read-set") + +// RecordKey separates durable policy, application rows and retained responses. +// Hex segments keep arbitrary opaque IDs from aliasing another record's key. +type RecordKey struct { + Namespace string `json:"namespace"` + Key string `json:"key"` +} +type RecordWrite struct { + RecordKey + Data []byte `json:"data"` +} +type ReadRecord func(RecordKey) ([]byte, error) + +func segment(s string) string { return hex.EncodeToString([]byte(s)) } +func IdentityKey() RecordKey { return RecordKey{"state", "identity"} } +func DeviceKey(id string) RecordKey { return RecordKey{"state", "device/" + segment(id)} } +func CellKey(id string) RecordKey { return RecordKey{"state", "cell/" + segment(id)} } +func AuthorityKey(id string) RecordKey { return RecordKey{"enrollment", "actor/" + segment(id)} } +func SessionKey(actor, session string) RecordKey { + return RecordKey{"sessions", segment(actor) + "/" + segment(session)} +} +func SessionsKey() RecordKey { return RecordKey{"sessions", "counts"} } +func JoinKey(code string) RecordKey { return RecordKey{"outbox", "join/" + segment(code)} } + +// Authority is installed by authenticated genesis/enrollment, never PutDevice. +// Scope is deliberately personal-only until delegated workers land in P15. +type Authority struct { + Cluster string `json:"cluster"` + Key []byte `json:"key"` + Personal bool `json:"personal"` +} +type session struct { + Sequence uint64 `json:"sequence"` + Digest string `json:"digest"` + Closed bool `json:"closed"` + Result Result `json:"result"` +} +type sessionCounts struct { + Live uint32 `json:"live"` + Retained uint32 `json:"retained"` +} + +// JoinRequest is a committed external-effect dependency, provisioned by P10. +// An absent dependency cannot be manufactured by a decision request. +type JoinRequest struct { + Device string `json:"device"` + Expires int64 `json:"expires"` + Decision *JoinDecision `json:"decision,omitempty"` +} + +// ApplicationMeta is an immutable view at one applied index. Byte accounting +// includes record framing and keys; capacity is a refusal, never an eviction. +type ApplicationMeta struct { + Cluster string `json:"cluster"` + Now int64 `json:"now"` + Version uint64 `json:"version"` + Bytes uint64 `json:"bytes"` + Ceiling uint64 `json:"ceiling"` +} +type Evaluation struct { + Now int64 `json:"now"` + Version uint64 `json:"version"` + Writes []RecordWrite `json:"writes"` + Result Result `json:"result"` +} + +// Digest excludes time, signature and acknowledgement. A retry may arrive at a +// newer ingress time; its operation and durable sequence must remain identical. +func Digest(c Command) (string, error) { + b, err := Encode(struct { + Cluster string `json:"cluster"` + Actor string `json:"actor"` + Session string `json:"session"` + Sequence uint64 `json:"sequence"` + Change Mutation `json:"change"` + }{c.Cluster, c.Actor, c.Session, c.Sequence, c.Change}) + if err != nil { + return "", err + } + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]), nil +} + +// SigningBytes binds all routing, sequence and time fields to this protocol. +func SigningBytes(c Command) ([]byte, error) { + c.Signature = nil + b, err := EncodeCommand(c) + return append([]byte("codeaf.directory.command.v1\x00"), b...), err +} +func Sign(c Command, key ed25519.PrivateKey) (Command, error) { + var err error + c.RequestDigest, err = Digest(c) + if err != nil { + return c, err + } + b, err := SigningBytes(c) + if err != nil { + return c, err + } + c.Signature = ed25519.Sign(key, b) + return c, nil +} + +// Dependencies declares every possible row before evaluation. Hosts may page +// these immutable records, but must distinguish absent rows from unread rows. +func Dependencies(c Command) []RecordKey { + keys := []RecordKey{IdentityKey(), DeviceKey(c.Actor), AuthorityKey(c.Actor), SessionKey(c.Actor, c.Session), SessionsKey()} + m := c.Change + if m.PutDevice != nil { + keys = append(keys, DeviceKey(m.PutDevice.ID)) + } + if m.Revoke != nil { + keys = append(keys, DeviceKey(m.Revoke.Device)) + } + if m.DeviceSeen != nil { + keys = append(keys, DeviceKey(m.DeviceSeen.Device)) + } + if m.DecideJoin != nil { + keys = append(keys, JoinKey(m.DecideJoin.Code), DeviceKey(m.DecideJoin.Device)) + } + for _, id := range cellIDs(m) { + keys = append(keys, CellKey(id)) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].Namespace != keys[j].Namespace { + return keys[i].Namespace < keys[j].Namespace + } + return keys[i].Key < keys[j].Key + }) + out := keys[:0] + for _, k := range keys { + if len(out) == 0 || out[len(out)-1] != k { + out = append(out, k) + } + } + return out +} +func cellIDs(m Mutation) []string { + switch { + case m.Create != nil: + return []string{m.Create.Cell} + case m.Acquire != nil: + return []string{m.Acquire.Cell} + case m.Beat != nil: + return []string{m.Beat.Cell} + case m.Publish != nil: + return []string{m.Publish.Cell} + case m.Release != nil: + return []string{m.Release.Cell} + case m.Archive != nil: + return []string{m.Archive.Cell} + case m.BeatBatch != nil: + out := []string{} + for _, b := range m.BeatBatch.Items { + out = append(out, b.Cell) + } + return out + } + return nil +} +func RecordBytes(k RecordKey, b []byte) uint64 { + return uint64(12 + len(k.Namespace) + len(k.Key) + len(b)) +} + +// Evaluate runs only after commitment against the host's immutable applied view. +// Policy refusals are ordinary persisted results. Invalid schemas/read-sets are +// host errors and stop application rather than pretending an entry succeeded. +func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, error) { + out := Evaluation{Now: meta.Now, Version: meta.Version, Writes: []RecordWrite{}, Result: Result{Schema: SchemaVersion, Code: Unauthorized, Now: meta.Now, Version: meta.Version}} + if len(raw) > MaxCommandBytes { + return out, ErrCodec + } + c, err := DecodeCommand(raw) + if err != nil { + return out, err + } + rows := map[RecordKey][]byte{} + n := 0 + for _, k := range Dependencies(c) { + b, e := read(k) + if e != nil { + return out, e + } + n += len(k.Namespace) + len(k.Key) + len(b) + if n > MaxEvaluationBytes { + return out, ErrDependency + } + rows[k] = bytes.Clone(b) + } + + // Corrupt dependency bytes are storage evidence, never a not-found or policy + // refusal. Validate every declared present row before interpreting any rule. + for k, b := range rows { + if b == nil { + continue + } + var target any + switch { + case k == IdentityKey(): + target = new(IdentityRec) + case k.Namespace == "state" && len(k.Key) >= 7 && k.Key[:7] == "device/": + target = new(Device) + case k.Namespace == "state" && len(k.Key) >= 5 && k.Key[:5] == "cell/": + target = new(Cell) + case k.Namespace == "enrollment": + target = new(Authority) + case k == SessionsKey(): + target = new(sessionCounts) + case k.Namespace == "sessions": + target = new(session) + case k.Namespace == "outbox": + target = new(JoinRequest) + default: + return out, ErrDependency + } + if Decode(b, target) != nil { + return out, ErrDependency + } + } + if meta.Now < 0 || meta.Now > int64(MaxSafeInteger)-LeaseTTLMillis { + return out, ErrDependency + } + decode := func(k RecordKey, v any) error { + b, ok := rows[k] + if !ok { + return ErrDependency + } + if b == nil { + return nil + } + return Decode(b, v) + } + var auth Authority + var actor Device + var identity IdentityRec + if decode(AuthorityKey(c.Actor), &auth) != nil || decode(DeviceKey(c.Actor), &actor) != nil || decode(IdentityKey(), &identity) != nil { + return out, ErrDependency + } + digest, _ := Digest(c) + signed, e := SigningBytes(c) + if e != nil || c.Cluster != meta.Cluster || auth.Cluster != c.Cluster || !auth.Personal || len(auth.Key) != ed25519.PublicKeySize || !ed25519.Verify(auth.Key, signed, c.Signature) || digest != c.RequestDigest { + return out, nil + } + // Revocation precedes dedup: a saved old response cannot authorize a revoked + // actor to resume activity. Other accepted retries preserve exact old bytes. + if actor.Revoked { + out.Result.Code = Revoked + return out, nil + } + var last session + var counts sessionCounts + if decode(SessionKey(c.Actor, c.Session), &last) != nil || decode(SessionsKey(), &counts) != nil { + return out, ErrDependency + } + known := rows[SessionKey(c.Actor, c.Session)] != nil + refuse := func(code Code) (Evaluation, error) { out.Result.Code = code; return out, nil } + if known && (last.Closed || c.Sequence < last.Sequence || c.Sequence == last.Sequence && c.RequestDigest != last.Digest) { + return refuse(Conflict) + } + if known && c.Sequence == last.Sequence { + out.Result = last.Result + return out, nil + } + if !known && (c.Sequence != 1 || c.Ack != 0) || known && (last.Sequence == ^uint64(0) || c.Sequence != last.Sequence+1 || c.Ack != last.Sequence) { + return refuse(Conflict) + } + if !known && (counts.Live >= MaxLiveSessions || counts.Retained >= MaxRetainedSessions) { + return refuse(Capacity) + } + if c.Now < 0 || c.Now > int64(MaxSafeInteger)-LeaseTTLMillis { + return refuse(BadRequest) + } + out.Now = max(meta.Now, c.Now) + out.Result.Now = out.Now + writes := map[RecordKey][]byte{} + put := func(k RecordKey, v any) { + b, e := Encode(v) + if e != nil { + err = e + return + } + writes[k] = b + } + code := OK + if identity.Rotation != nil && c.Change.Rotate == nil && c.Change.AdvanceTime == nil && c.Change.DeviceSeen == nil && c.Change.CloseSession == nil { + code = Rotated + } else { + code = applyMutation(c, out.Now, identity, rows, put, &out.Result) + } + if err != nil { + return out, err + } + changed := false + for k, b := range writes { + if k.Namespace == "state" && visibleChange(k, rows[k], b, out.Now) { + changed = true + } + } + if c.Change.DeviceSeen != nil { + changed = false + } + if changed { + if meta.Version == ^uint64(0) { + return refuse(Capacity) + } + out.Version++ + } + out.Result.Code = code + out.Result.Version = out.Version + if code != OK { + writes = map[RecordKey][]byte{} + out.Version = meta.Version + out.Result.Version = meta.Version + } + if !known { + counts.Live++ + counts.Retained++ + } + if c.Change.CloseSession != nil { + counts.Live-- + } + save := func() { + put(SessionKey(c.Actor, c.Session), session{Sequence: c.Sequence, Digest: c.RequestDigest, Closed: c.Change.CloseSession != nil, Result: out.Result}) + put(SessionsKey(), counts) + } + save() + + fits := func() bool { + size := meta.Bytes + budget := 0 + for k, b := range writes { + cost := RecordBytes(k, b) + budget += int(cost) + if old := rows[k]; old != nil { + oldSize := RecordBytes(k, old) + if oldSize > size { + return false + } + size -= oldSize + } + size += cost + } + ceiling := meta.Ceiling + if ceiling == 0 { + ceiling = DefaultApplicationBytes + } + return budget <= MaxEvaluationBytes && (size <= ceiling || size <= meta.Bytes) + } + if !fits() { + writes = map[RecordKey][]byte{} + out.Version = meta.Version + out.Result = Result{Schema: SchemaVersion, Code: Capacity, Now: out.Now, Version: meta.Version} + save() + if !fits() { + return out, ErrDependency + } + } + for k, b := range writes { + out.Writes = append(out.Writes, RecordWrite{k, b}) + } + sort.Slice(out.Writes, func(i, j int) bool { + a, b := out.Writes[i].RecordKey, out.Writes[j].RecordKey + if a.Namespace != b.Namespace { + return a.Namespace < b.Namespace + } + return a.Key < b.Key + }) + return out, err +} + +func applyMutation(c Command, now int64, identity IdentityRec, rows map[RecordKey][]byte, put func(RecordKey, any), result *Result) Code { + m := c.Change + device := func(id string) (Device, bool) { + var d Device + b := rows[DeviceKey(id)] + if b == nil { + return d, false + } + if Decode(b, &d) != nil { + return d, false + } + return d, true + } + switch { + case m.AdvanceTime != nil, m.CloseSession != nil: + return OK + case m.PutDevice != nil: + v := m.PutDevice + old, had := device(v.ID) + put(DeviceKey(v.ID), Stamped(old, had, v.Device, now)) + return OK + case m.Revoke != nil: + v := m.Revoke + old, had := device(v.Device) + if !had { + return NotFound + } + next, err := RevokeOf(old, c.Actor, v.Device) + if err != nil { + return SelfRevoke + } + put(DeviceKey(v.Device), next) + return OK + case m.SetVault != nil: + v := m.SetVault + if identity.Vault != v.Old { + return CAS + } + identity.Vault = v.New + put(IdentityKey(), identity) + return OK + case m.DeviceSeen != nil: + if m.DeviceSeen.Device != c.Actor { + return Unauthorized + } + d, had := device(c.Actor) + if had { + d.LastSeen = now + put(DeviceKey(c.Actor), d) + } + return OK + case m.Rotate != nil: + v := m.Rotate + r := identity.Rotation + switch v.Op { + case "freeze": + if r == nil { + r = &Rotation{State: "frozen", By: c.Actor, At: now} + } else if r.By != c.Actor { + return Rotated + } + case "thaw": + if r != nil && r.State == "retired" { + return Rotated + } + r = nil + case "retire": + if r == nil { + return RotationStep + } + if r.By != c.Actor { + return Rotated + } + if r.State != "retired" { + g := v.GraceMS + if g == 0 { + g = DefaultGraceMillis + } + if g < MinGraceMillis || g > MaxGraceMillis || now > int64(MaxSafeInteger)-g { + return BadGrace + } + r = &Rotation{State: "retired", By: r.By, At: r.At, RetireAt: now + g} + } + default: + return BadRequest + } + identity.Rotation = r + put(IdentityKey(), identity) + return OK + case m.DecideJoin != nil: + v := m.DecideJoin + var request JoinRequest + b := rows[JoinKey(v.Code)] + if b == nil { + return RequestGone + } + if Decode(b, &request) != nil { + return BadRequest + } + if request.Decision != nil { + if reflect.DeepEqual(*request.Decision, *v) { + return OK + } + return AlreadyDecided + } + if request.Expires <= now { + return RequestGone + } + if !v.Deny && (request.Device != v.Device || v.Grant == "") { + return BadRequest + } + if !v.Deny { + old, had := device(v.Device) + put(DeviceKey(v.Device), Stamped(old, had, v.Record, now)) + } + request.Decision = v + put(JoinKey(v.Code), request) + return OK + case m.BeatBatch != nil: + batchRows := map[RecordKey][]byte{} + for k, b := range rows { + batchRows[k] = b + } + for _, b := range m.BeatBatch.Items { + sub := c + sub.Change = Mutation{Beat: &b} + r := Result{} + code := applyMutation(sub, now, identity, batchRows, func(k RecordKey, v any) { put(k, v); data, _ := Encode(v); batchRows[k] = data }, &r) + result.Items = append(result.Items, ItemResult{Cell: b.Cell, Code: code, View: r.View}) + } + return OK + } + ids := cellIDs(m) + if len(ids) != 1 || ids[0] == "" { + return BadRequest + } + id := ids[0] + var cell Cell + b := rows[CellKey(id)] + if b != nil && Decode(b, &cell) != nil { + return BadRequest + } + if m.Create != nil { + if b != nil { + return Exists + } + cell = Created(m.Create.Init, c.Actor, now) + } else { + if b == nil { + return NotFound + } + var err error + switch { + case m.Acquire != nil: + v := m.Acquire + var expect *LeaseExpectation = &v.Expect + force := v.Force + if v.Legacy { + expect = nil + force = false + } + oldFence := cell.Lease.Fence + cell, err = Acquire(cell, c.Actor, now, force, expect) + if err == nil && (oldFence == ^uint64(0) || v.V1 && oldFence >= MaxSafeInteger) { + return Capacity + } + case m.Beat != nil: + cell, err = Heartbeat(cell, c.Actor, m.Beat.Beat, now) + case m.Publish != nil: + cell, err = PublishTo(cell, c.Actor, m.Publish.Publish, now) + case m.Release != nil: + at := now + if m.Release.Quiet { + at = 0 + } + cell, err = ReleaseOf(cell, c.Actor, m.Release.Fence, at) + case m.Archive != nil: + cell.Archived = true + default: + return BadRequest + } + switch err { + case ErrLeaseHeld: + return LeaseHeld + case ErrFenceStale: + return FenceStale + case ErrHeadMoved: + return HeadMoved + } + if err != nil { + return BadRequest + } + } + put(CellKey(id), cell) + result.View = &CellView{Now: now, Cell: cell} + return OK +} + +// Visible versions follow the existing projection: renewed expiry within the +// same held state and advisory last-seen timestamps do not repaint Home. +func visibleChange(k RecordKey, old, next []byte, now int64) bool { + if old == nil { + return true + } + switch { + case len(k.Key) > 5 && k.Key[:5] == "cell/": + var a, b Cell + if Decode(old, &a) != nil || Decode(next, &b) != nil { + return true + } + if a.Lease.Expires > now { + a.Lease.Expires = 1 + } else { + a.Lease.Expires = 0 + } + if b.Lease.Expires > now { + b.Lease.Expires = 1 + } else { + b.Lease.Expires = 0 + } + return !reflect.DeepEqual(a, b) + case len(k.Key) > 7 && k.Key[:7] == "device/": + var a, b Device + if Decode(old, &a) != nil || Decode(next, &b) != nil { + return true + } + a.LastSeen = 0 + b.LastSeen = 0 + return !reflect.DeepEqual(a, b) + default: + return !bytes.Equal(old, next) + } +} diff --git a/internal/clusterstate/apply_test.go b/internal/clusterstate/apply_test.go new file mode 100644 index 0000000000..94f8030e88 --- /dev/null +++ b/internal/clusterstate/apply_test.go @@ -0,0 +1,354 @@ +package clusterstate_test + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "fmt" + "math/rand" + "os" + "strconv" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftsim/model" + "github.com/anishathalye/porcupine" +) + +type application struct { + t *testing.T + rows map[cs.RecordKey][]byte + meta cs.ApplicationMeta + seq map[string]uint64 +} + +func actorKey(id string) ed25519.PrivateKey { + h := sha256.Sum256([]byte(id)) + return ed25519.NewKeyFromSeed(h[:]) +} +func newApplication(t *testing.T) *application { + a := &application{t: t, rows: map[cs.RecordKey][]byte{}, meta: cs.ApplicationMeta{Cluster: "test", Ceiling: cs.DefaultApplicationBytes}, seq: map[string]uint64{}} + for _, id := range []string{"a", "b"} { + a.put(cs.AuthorityKey(id), cs.Authority{Cluster: "test", Key: actorKey(id).Public().(ed25519.PublicKey), Personal: true}) + a.put(cs.DeviceKey(id), cs.Device{V: 1}) + } + a.put(cs.IdentityKey(), cs.IdentityRec{V: 1}) + return a +} +func (a *application) put(k cs.RecordKey, v any) { + b, e := cs.Encode(v) + if e != nil { + a.t.Fatal(e) + } + a.rows[k] = b +} +func (a *application) prepare(actor string, now int64, m cs.Mutation) cs.Command { + a.seq[actor]++ + seq := a.seq[actor] + c, e := cs.Sign(cs.Command{Schema: 1, Cluster: "test", Actor: actor, Session: "test", Sequence: seq, Ack: seq - 1, Now: now, Change: m}, actorKey(actor)) + if e != nil { + a.t.Fatal(e) + } + return c +} +func (a *application) evaluate(c cs.Command) cs.Evaluation { + a.meta.Bytes = 0 + for k, b := range a.rows { + a.meta.Bytes += cs.RecordBytes(k, b) + } + raw, e := cs.EncodeCommand(c) + if e != nil { + a.t.Fatal(e) + } + out, e := cs.Evaluate(raw, a.meta, func(k cs.RecordKey) ([]byte, error) { return a.rows[k], nil }) + if e != nil { + a.t.Fatal(e) + } + for _, w := range out.Writes { + a.rows[w.RecordKey] = w.Data + } + a.meta.Now = out.Now + a.meta.Version = out.Version + return out +} +func (a *application) run(actor string, now int64, m cs.Mutation) cs.Result { + return a.evaluate(a.prepare(actor, now, m)).Result +} +func (a *application) cell() cs.Cell { + var c cs.Cell + if b := a.rows[cs.CellKey("chat")]; b != nil { + if e := cs.Decode(b, &c); e != nil { + a.t.Fatal(e) + } + } + return c +} +func small(c cs.Cell) model.Cell { + return model.Cell{Holder: c.Lease.Device, Fence: c.Lease.Fence, Until: c.Lease.Expires, Head: c.Head, Pending: c.Lease.Pending, ReleasedAt: c.Lease.ReleasedAt} +} +func seedCount(t *testing.T) int { + t.Helper() + if s, ok := os.LookupEnv("RAFTSIM_SEEDS"); ok { + n, e := strconv.Atoi(s) + if e != nil || n < 1 || n > 10000 { + t.Fatal("invalid RAFTSIM_SEEDS") + } + return n + } + return 500 +} + +// R1/R2/R16/R17 compare the actual signed production evaluator with the +// independent sequential ownership/vault/revocation model, including errors. +func TestR1R2R16R17AuthoritativeApplyIndependentCampaign(t *testing.T) { + for seed := 0; seed < seedCount(t); seed++ { + t.Run(fmt.Sprintf("seed-%04d", seed), func(t *testing.T) { + a := newApplication(t) + rng := rand.New(rand.NewSource(int64(seed))) + state := model.State{} + history := []model.Operation{} + now := int64(1000000) + for step := 0; step < 24; step++ { + actor := []string{"a", "b"}[rng.Intn(2)] + op := model.Call{Actor: actor, Cell: "chat", At: now} + m := cs.Mutation{} + cell := a.cell() + f := cell.Lease.Fence + if rng.Intn(4) == 0 && f > 0 { + f-- + } + switch { + case step == 0: + op.Kind = "create" + op.Head = "sealed-one" + m.Create = &cs.CreateCell{Cell: "chat", Init: cs.CellInit{Head: op.Head}} + case step == 23: + op.Kind = "revoke" + op.Target = "b" + m.Revoke = &cs.DeviceID{Device: op.Target} + default: + switch rng.Intn(6) { + case 0: + op.Kind = "acquire" + op.Force = rng.Intn(2) == 0 + op.Expected = model.Cell{Holder: cell.Lease.Device, Fence: f} + m.Acquire = &cs.AcquireCell{Cell: "chat", Force: op.Force, Expect: cs.LeaseExpectation{Device: op.Expected.Holder, Fence: f}} + case 1: + op.Kind = "publish" + op.Fence = f + op.OldHead = cell.Head + if rng.Intn(4) == 0 { + op.OldHead = "old" + } + op.Head = fmt.Sprint("sealed-", step) + m.Publish = &cs.PublishCell{Cell: "chat", Publish: cs.Publish{Fence: f, OldHead: op.OldHead, Head: op.Head}} + case 2: + op.Kind = "beat" + op.Fence = f + op.Pending = uint32(rng.Intn(4)) + m.Beat = &cs.BeatCell{Cell: "chat", Beat: cs.Beat{Fence: f, Pending: op.Pending}} + case 3: + op.Kind = "release" + op.Fence = f + op.Quiet = rng.Intn(2) == 0 + m.Release = &cs.ReleaseCell{Cell: "chat", Fence: f, Quiet: op.Quiet} + case 4: + op.Kind = "set_vault" + op.OldVault = state.Vault + if rng.Intn(3) == 0 { + op.OldVault = "old" + } + op.NewVault = fmt.Sprint("sealed-vault-", step) + m.SetVault = &cs.VaultCAS{Old: op.OldVault, New: op.NewVault} + case 5: + op.Kind = "advance_time" + m.AdvanceTime = &cs.AdvanceTime{} + } + } + next, want := model.Step(state, op) + got := a.run(actor, now, m) + if string(got.Code) != want.Code { + t.Fatalf("step %d %s: got %s want %s", step, op.Kind, got.Code, want.Code) + } + if want.Code == "ok" && got.View != nil && small(got.View.Cell) != want.Cell { + t.Fatalf("step %d result differs: %+v %+v", step, small(got.View.Cell), want.Cell) + } + if c, ok := next.Cells["chat"]; ok && small(a.cell()) != c { + t.Fatalf("step %d persisted cell differs", step) + } + var identity cs.IdentityRec + if e := cs.Decode(a.rows[cs.IdentityKey()], &identity); e != nil { + t.Fatal(e) + } + if identity.Vault != next.Vault { + t.Fatalf("step %d vault differs", step) + } + observed := model.Outcome{Code: string(got.Code)} + if got.View != nil { + observed.Cell = small(got.View.Cell) + } + history = append(history, model.Operation{ID: step, Call: int64(step * 2), Return: int64(step*2 + 1), Input: op, Output: observed}) + state = next + now += int64(rng.Intn(150000) - 30000) + } + if result := model.Check(history); result != porcupine.Ok { + t.Fatalf("independent history check: %v", result) + } + }) + } +} +func TestR16R17SignedAuthorizationAndExactRetry(t *testing.T) { + a := newApplication(t) + c := a.prepare("a", 1000, cs.Mutation{Create: &cs.CreateCell{Cell: "chat", Init: cs.CellInit{Head: "sealed"}}}) + first := a.evaluate(c) + retry := c + retry.Now = 9000 + retry, e := cs.Sign(retry, actorKey("a")) + if e != nil { + t.Fatal(e) + } + again := a.evaluate(retry) + x, _ := cs.EncodeResult(first.Result) + y, _ := cs.EncodeResult(again.Result) + if !bytes.Equal(x, y) || len(again.Writes) != 0 { + t.Fatal("retry changed its committed result") + } + bad := c + bad.Change = cs.Mutation{Archive: &cs.CellID{Cell: "chat"}} + bad, e = cs.Sign(bad, actorKey("a")) + if e != nil { + t.Fatal(e) + } + if r := a.evaluate(bad); r.Result.Code != cs.Conflict { + t.Fatal(r) + } + old := a.prepare("b", 1001, cs.Mutation{Acquire: &cs.AcquireCell{Cell: "chat", Force: true, Expect: cs.LeaseExpectation{Device: "a", Fence: 1}}}) + if r := a.run("a", 1002, cs.Mutation{Revoke: &cs.DeviceID{Device: "b"}}); r.Code != cs.OK { + t.Fatal(r) + } + if r := a.evaluate(old); r.Result.Code != cs.Revoked { + t.Fatal("already authenticated request escaped revocation", r) + } + tampered := a.prepare("a", 1003, cs.Mutation{Archive: &cs.CellID{Cell: "chat"}}) + tampered.Now++ + if r := a.evaluate(tampered); r.Result.Code != cs.Unauthorized || len(r.Writes) != 0 { + t.Fatal(r) + } +} +func TestR17SessionCloseRetainsTombstoneAndAcknowledgedAdvance(t *testing.T) { + a := newApplication(t) + first := a.prepare("a", 1000, cs.Mutation{AdvanceTime: &cs.AdvanceTime{}}) + a.evaluate(first) + next := a.prepare("a", 1001, cs.Mutation{CloseSession: &cs.CloseSession{}}) + bad := next + bad.Ack = 0 + bad, _ = cs.Sign(bad, actorKey("a")) + if a.evaluate(bad).Result.Code != cs.Conflict { + t.Fatal("missing acknowledgement accepted") + } + if a.evaluate(next).Result.Code != cs.OK { + t.Fatal("close failed") + } + if a.evaluate(first).Result.Code != cs.Conflict { + t.Fatal("closed session resurrected") + } +} +func TestR1R2R22BatchItemResultsAndVersions(t *testing.T) { + a := newApplication(t) + a.run("a", 1000, cs.Mutation{Create: &cs.CreateCell{Cell: "chat", Init: cs.CellInit{Head: "sealed"}}}) + version := a.meta.Version + r := a.run("a", 2000, cs.Mutation{BeatBatch: &cs.BeatBatch{Items: []cs.BeatCell{{Cell: "chat", Beat: cs.Beat{Fence: 1, Pending: 3}}, {Cell: "missing", Beat: cs.Beat{Fence: 1}}, {Cell: "chat", Beat: cs.Beat{Fence: 0}}}}}) + if len(r.Items) != 3 || r.Items[0].Code != cs.OK || r.Items[1].Code != cs.NotFound || r.Items[2].Code != cs.FenceStale || a.cell().Lease.Pending != 3 || a.meta.Version != version+1 { + t.Fatal(r, a.meta) + } + version = a.meta.Version + a.run("a", 3000, cs.Mutation{Beat: &cs.BeatCell{Cell: "chat", Beat: cs.Beat{Fence: 1, Pending: 3}}}) + if a.meta.Version != version { + t.Fatal("expiry-only renewal moved visible version") + } +} +func TestR27CapacityRefusalIsDurableAndDoesNotDeleteState(t *testing.T) { + a := newApplication(t) + a.run("a", 1000, cs.Mutation{Create: &cs.CreateCell{Cell: "chat", Init: cs.CellInit{Head: "sealed"}}}) + a.meta.Bytes = 0 + for k, b := range a.rows { + a.meta.Bytes += cs.RecordBytes(k, b) + } + a.meta.Ceiling = a.meta.Bytes + 1000 + r := a.run("a", 2000, cs.Mutation{Publish: &cs.PublishCell{Cell: "chat", Publish: cs.Publish{Fence: 1, OldHead: "sealed", Head: "new", Title: string(bytes.Repeat([]byte("x"), 4000))}}}) + if r.Code != cs.Capacity || a.cell().Head != "sealed" { + t.Fatal(r) + } +} + +func TestR14R25FullWidthFencesAndV1Refusal(t *testing.T) { + a := newApplication(t) + a.put(cs.CellKey("chat"), cs.Cell{V: 1, Head: "sealed", Lease: cs.Lease{Device: "a", Fence: cs.MaxSafeInteger, Expires: 90000}}) + r := a.run("a", 1000, cs.Mutation{Acquire: &cs.AcquireCell{Cell: "chat", V1: true, Expect: cs.LeaseExpectation{Device: "a", Fence: cs.MaxSafeInteger}}}) + if r.Code != cs.Capacity { + t.Fatal(r) + } + r = a.run("a", 1001, cs.Mutation{Acquire: &cs.AcquireCell{Cell: "chat", Expect: cs.LeaseExpectation{Device: "a", Fence: cs.MaxSafeInteger}}}) + if r.Code != cs.OK || r.View.Cell.Lease.Fence != cs.MaxSafeInteger+1 { + t.Fatal(r) + } + a.put(cs.CellKey("chat"), cs.Cell{V: 1, Head: "sealed", Lease: cs.Lease{Device: "a", Fence: ^uint64(0), Expires: 90000}}) + r = a.run("a", 1002, cs.Mutation{Acquire: &cs.AcquireCell{Cell: "chat", Expect: cs.LeaseExpectation{Device: "a", Fence: ^uint64(0)}}}) + if r.Code != cs.Capacity || a.cell().Lease.Fence != ^uint64(0) { + t.Fatal("overflow changed fence", r) + } +} + +func TestR16R17R27DependencyAndSessionAdmissionFailures(t *testing.T) { + a := newApplication(t) + raw, e := cs.EncodeCommand(a.prepare("a", 1000, cs.Mutation{AdvanceTime: &cs.AdvanceTime{}})) + if e != nil { + t.Fatal(e) + } + if _, e := cs.Evaluate(raw, a.meta, func(k cs.RecordKey) ([]byte, error) { + if k == cs.SessionsKey() { + return nil, cs.ErrDependency + } + return a.rows[k], nil + }); e == nil { + t.Fatal("unread dependency accepted") + } + a.rows[cs.DeviceKey("b")] = []byte("corrupt") + raw, e = cs.EncodeCommand(a.prepare("a", 1000, cs.Mutation{Revoke: &cs.DeviceID{Device: "b"}})) + if e != nil { + t.Fatal(e) + } + if _, e := cs.Evaluate(raw, a.meta, func(k cs.RecordKey) ([]byte, error) { return a.rows[k], nil }); e == nil { + t.Fatal("corrupt target became a policy refusal") + } +} + +func TestR16R30CommittedPairDecisionIsAtomicAndFirstWins(t *testing.T) { + a := newApplication(t) + a.put(cs.JoinKey("request"), cs.JoinRequest{Device: "new", Expires: 10000}) + decision := cs.JoinDecision{Code: "request", Device: "new", Record: cs.Device{V: 1, Name: "sealed"}, Grant: "sealed-grant", Cert: "public-cert"} + if r := a.run("a", 1000, cs.Mutation{DecideJoin: &decision}); r.Code != cs.OK { + t.Fatal(r) + } + var request cs.JoinRequest + if e := cs.Decode(a.rows[cs.JoinKey("request")], &request); e != nil { + t.Fatal(e) + } + var device cs.Device + if e := cs.Decode(a.rows[cs.DeviceKey("new")], &device); e != nil { + t.Fatal(e) + } + if request.Decision == nil || request.Decision.Grant != "sealed-grant" || device.Created != 1000 { + t.Fatal(request, device) + } + version := a.meta.Version + if r := a.run("a", 1001, cs.Mutation{DecideJoin: &decision}); r.Code != cs.OK || a.meta.Version != version { + t.Fatal(r) + } + if r := a.run("b", 1002, cs.Mutation{DecideJoin: &cs.JoinDecision{Code: "request", Deny: true}}); r.Code != cs.AlreadyDecided { + t.Fatal(r) + } + if a.rows[cs.AuthorityKey("new")] != nil { + t.Fatal("pair decision bypassed P10 certificate admission") + } +} diff --git a/internal/clusterstate/codec.go b/internal/clusterstate/codec.go index 2a2f42e502..525a0408a4 100644 --- a/internal/clusterstate/codec.go +++ b/internal/clusterstate/codec.go @@ -336,7 +336,7 @@ func (r Result) Validate() error { func (c Code) valid() bool { switch c { case OK, NotFound, Exists, LeaseHeld, FenceStale, HeadMoved, CAS, Revoked, - SelfRevoke, Unauthorized, Rotated, Conflict, Capacity: + SelfRevoke, Unauthorized, Rotated, Conflict, Capacity, RotationStep, BadGrace, RequestGone, AlreadyDecided, BadRequest: return true } return false diff --git a/internal/clusterstate/imports_test.go b/internal/clusterstate/imports_test.go index baeb95a97d..efcf92950d 100644 --- a/internal/clusterstate/imports_test.go +++ b/internal/clusterstate/imports_test.go @@ -15,6 +15,8 @@ import ( func TestR14PureImportBoundary(t *testing.T) { allowed := map[string]bool{ "bytes": true, "encoding/json": true, "errors": true, "fmt": true, + // Signature verification, hashing, key encoding and ordering are pure. + "crypto/ed25519": true, "crypto/sha256": true, "encoding/hex": true, "sort": true, "reflect": true, "strconv": true, "strings": true, "unicode/utf8": true, } files, err := os.ReadDir(".") diff --git a/internal/clusterstate/types.go b/internal/clusterstate/types.go index e3836cb3c6..a469368074 100644 --- a/internal/clusterstate/types.go +++ b/internal/clusterstate/types.go @@ -34,6 +34,7 @@ type Command struct { Session string `json:"session"` Sequence uint64 `json:"sequence"` RequestDigest string `json:"request_digest"` + Ack uint64 `json:"ack,omitempty"` Now int64 `json:"now"` Change Mutation `json:"change"` Signature []byte `json:"signature"` @@ -43,18 +44,21 @@ type Command struct { // their typed operations here; unknown fields fail decoding on older members. // There is no arbitrary JSON payload in which task text or secrets can hide. type Mutation struct { - Create *CreateCell `json:"create,omitempty"` - Acquire *AcquireCell `json:"acquire,omitempty"` - Beat *BeatCell `json:"beat,omitempty"` - BeatBatch *BeatBatch `json:"beat_batch,omitempty"` - Publish *PublishCell `json:"publish,omitempty"` - Release *ReleaseCell `json:"release,omitempty"` - Archive *CellID `json:"archive,omitempty"` - PutDevice *PutDevice `json:"put_device,omitempty"` - Revoke *DeviceID `json:"revoke,omitempty"` - SetVault *VaultCAS `json:"set_vault,omitempty"` - AdvanceTime *AdvanceTime `json:"advance_time,omitempty"` - DeviceSeen *DeviceSeen `json:"device_seen,omitempty"` + Rotate *RotateRequest `json:"rotate,omitempty"` + CloseSession *CloseSession `json:"close_session,omitempty"` + DecideJoin *JoinDecision `json:"decide_join,omitempty"` + Create *CreateCell `json:"create,omitempty"` + Acquire *AcquireCell `json:"acquire,omitempty"` + Beat *BeatCell `json:"beat,omitempty"` + BeatBatch *BeatBatch `json:"beat_batch,omitempty"` + Publish *PublishCell `json:"publish,omitempty"` + Release *ReleaseCell `json:"release,omitempty"` + Archive *CellID `json:"archive,omitempty"` + PutDevice *PutDevice `json:"put_device,omitempty"` + Revoke *DeviceID `json:"revoke,omitempty"` + SetVault *VaultCAS `json:"set_vault,omitempty"` + AdvanceTime *AdvanceTime `json:"advance_time,omitempty"` + DeviceSeen *DeviceSeen `json:"device_seen,omitempty"` } type CellID struct { @@ -77,6 +81,8 @@ type AcquireCell struct { Cell string `json:"cell"` Force bool `json:"force"` Expect LeaseExpectation `json:"expect"` + Legacy bool `json:"legacy,omitempty"` + V1 bool `json:"v1,omitempty"` } type BeatCell struct { @@ -140,19 +146,24 @@ type ItemResult struct { type Code string const ( - OK Code = "ok" - NotFound Code = "not_found" - Exists Code = "exists" - LeaseHeld Code = "lease_held" - FenceStale Code = "fence_stale" - HeadMoved Code = "head_moved" - CAS Code = "cas" - Revoked Code = "revoked" - SelfRevoke Code = "self_revoke" - Unauthorized Code = "unauthorized" - Rotated Code = "rotated" - Conflict Code = "conflict" - Capacity Code = "capacity" + OK Code = "ok" + NotFound Code = "not_found" + Exists Code = "exists" + LeaseHeld Code = "lease_held" + FenceStale Code = "fence_stale" + HeadMoved Code = "head_moved" + CAS Code = "cas" + Revoked Code = "revoked" + SelfRevoke Code = "self_revoke" + Unauthorized Code = "unauthorized" + Rotated Code = "rotated" + Conflict Code = "conflict" + RotationStep Code = "rotation_step" + BadGrace Code = "bad_grace" + RequestGone Code = "request_gone" + AlreadyDecided Code = "already_decided" + BadRequest Code = "bad_request" + Capacity Code = "capacity" ) // Row encodes one application record for paged reads and streamed snapshots. @@ -165,3 +176,23 @@ type Row struct { Device *Device `json:"device,omitempty"` Cell *Cell `json:"cell,omitempty"` } + +// CloseSession retains a tombstone so an old sequence can never execute again. +type CloseSession struct{} + +// RotateRequest uses milliseconds so the shared evaluator needs no time package. +type RotateRequest struct { + Op string `json:"op"` + GraceMS int64 `json:"grace_ms,omitempty"` +} + +// JoinDecision records only sealed pairing data. P10 supplies a committed request +// dependency and delivers the outbox after this transaction has committed. +type JoinDecision struct { + Code string `json:"code"` + Device string `json:"device"` + Record Device `json:"record"` + Grant string `json:"grant"` + Cert string `json:"cert"` + Deny bool `json:"deny"` +} diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index aba4367262..582a1ec0cd 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -164,6 +164,8 @@ func (r *Runtime) Call(b []byte) (out []byte) { func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { e := encoder{} switch op { + case 10, 11: + return applicationCodec(op, d) case 1: cfg := raftcore.Config{ClusterUUID: d.str(), NodeID: d.u64(), Seed: d.u64()} protocol := d.u32() diff --git a/internal/raftabi/application.go b/internal/raftabi/application.go new file mode 100644 index 0000000000..3786a121da --- /dev/null +++ b/internal/raftabi/application.go @@ -0,0 +1,74 @@ +package raftabi + +import "github.com/Agent-Field/codeaf/internal/clusterstate" + +func applicationCodec(op uint32, d *decoder) ([]byte, error) { + raw := d.blob() + c, err := clusterstate.DecodeCommand(raw) + if err != nil { + return nil, err + } + keys := clusterstate.Dependencies(c) + e := encoder{} + if op == 11 { + if err := d.end(); err != nil { + return nil, err + } + e.u32(uint32(len(keys))) + for _, k := range keys { + e.str(k.Namespace) + e.str(k.Key) + } + return e.b, nil + } + meta := clusterstate.ApplicationMeta{Cluster: d.str(), Now: int64(d.u64()), Version: d.u64(), Bytes: d.u64(), Ceiling: d.u64()} + count := d.u32() + if int(count) != len(keys) { + return nil, clusterstate.ErrDependency + } + rows := map[clusterstate.RecordKey][]byte{} + size := 0 + for i := uint32(0); i < count; i++ { + k := clusterstate.RecordKey{Namespace: d.str(), Key: d.str()} + present := d.u32() + b := d.blob() + if _, ok := rows[k]; ok || present > 1 || present == 0 && len(b) > 0 { + return nil, clusterstate.ErrDependency + } + if present == 0 { + b = nil + } + rows[k] = b + size += len(b) + len(k.Key) + len(k.Namespace) + } + if err := d.end(); err != nil { + return nil, err + } + if size > clusterstate.MaxEvaluationBytes { + return nil, clusterstate.ErrDependency + } + v, err := clusterstate.Evaluate(raw, meta, func(k clusterstate.RecordKey) ([]byte, error) { + b, ok := rows[k] + if !ok { + return nil, clusterstate.ErrDependency + } + return b, nil + }) + if err != nil { + return nil, err + } + e.u64(uint64(v.Now)) + e.u64(v.Version) + result, err := clusterstate.EncodeResult(v.Result) + if err != nil { + return nil, err + } + e.blob(result) + e.u32(uint32(len(v.Writes))) + for _, w := range v.Writes { + e.str(w.Namespace) + e.str(w.Key) + e.blob(w.Data) + } + return e.b, nil +} From 76aa41494bdd5d3ed0d125e7befe4af62a39d327 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 00:48:14 -0400 Subject: [PATCH 020/103] Commit native directory responses only after quorum and durable application. Adapt every Client verb to the consensus boundary, assemble immutable native read-sets and atomic write-sets, and prove the public directory and HTTP doors with real stores and the independent ownership model. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../directory/directorytest/conditional.go | 29 +- internal/directory/directorytest/run.go | 4 +- internal/directory/raft.go | 174 +++++++ internal/directory/rotation.go | 2 +- internal/raftdirectory/README.md | 51 ++ internal/raftdirectory/apply.go | 109 ++++ internal/raftdirectory/directory_test.go | 470 ++++++++++++++++++ internal/raftdirectory/routing_test.go | 30 ++ internal/raftstore/view.go | 73 +++ 9 files changed, 936 insertions(+), 6 deletions(-) create mode 100644 internal/directory/raft.go create mode 100644 internal/raftdirectory/README.md create mode 100644 internal/raftdirectory/apply.go create mode 100644 internal/raftdirectory/directory_test.go create mode 100644 internal/raftdirectory/routing_test.go create mode 100644 internal/raftstore/view.go diff --git a/internal/directory/directorytest/conditional.go b/internal/directory/directorytest/conditional.go index 4c9189dace..f7c158979d 100644 --- a/internal/directory/directorytest/conditional.go +++ b/internal/directory/directorytest/conditional.go @@ -19,7 +19,7 @@ func RunConditionalTakes(t *testing.T, factory RigFactory) { // conditionalForcedTakeOneWins proves C7: two people who saw A's live lease // cannot both win, even if the loser asks by force after the winner finishes. -func conditionalForcedTakeOneWins(t *testing.T, e env) { +func conditionalForcedTakeRace(t *testing.T, e env) { v := start(t, e) seen := &directory.LeaseExpectation{Device: v.Cell.Lease.Device, Fence: v.Cell.Lease.Fence} win, err := e.as(devB).Acquire(ctx, cell, directory.AcquireOpts{Force: true, Expect: seen}) @@ -29,11 +29,34 @@ func conditionalForcedTakeOneWins(t *testing.T, e env) { if now := get(t, e); now.Lease != win.Cell.Lease { t.Fatalf("the losing continuation changed the winner's lease: %+v", now.Lease) } - // An older client that omits the expectation keeps its original behavior. - _, err = e.as(devA).Acquire(ctx, cell, directory.AcquireOpts{Force: true}) +} + +// Legacy stores still prove their historical blind-force behavior until P11 +// freezes them. The consensus suite below proves R25's safe normalization. +func conditionalForcedTakeOneWins(t *testing.T, e env) { + conditionalForcedTakeRace(t, e) + _, err := e.as(devA).Acquire(ctx, cell, directory.AcquireOpts{Force: true}) must(t, err) } +// RunConsensus uses every existing verb case, with the R25 compatibility +// contract at the one place where blind legacy force deliberately changes. +func RunConsensus(t *testing.T, factory Factory) { + for name, fn := range cases { + t.Run(name, func(t *testing.T) { + clock := NewFakeClock() + e := envOf(Rig{Clock: clock, Devices: factory(t, clock)}) + if name == "C7ConditionalForcedTakeOneWins" { + conditionalForcedTakeRace(t, e) + _, err := e.as(devA).Acquire(ctx, cell, directory.AcquireOpts{Force: true}) + wantErr(t, err, directory.ErrLeaseHeld) + } else { + fn(t, e) + } + }) + } +} + // conditionalTakeChecksHolderAndFence proves C7: neither a matching holder // alone nor a matching fence alone is the lease the person chose. func conditionalTakeChecksHolderAndFence(t *testing.T, e env) { diff --git a/internal/directory/directorytest/run.go b/internal/directory/directorytest/run.go index 728c3eea0a..8f56dacfe6 100644 --- a/internal/directory/directorytest/run.go +++ b/internal/directory/directorytest/run.go @@ -199,8 +199,8 @@ func acquireOwnLeaseRaisesFence(t *testing.T, e env) { // forcedAcquireBeatsLiveLease: a person who continues a chat where it runs now // takes the lease at once, and the displaced holder is refused as in L7. func forcedAcquireBeatsLiveLease(t *testing.T, e env) { - start(t, e) - v, err := e.as(devB).Acquire(ctx, cell, directory.AcquireOpts{Force: true}) + seen := start(t, e).Cell.Lease + v, err := e.as(devB).Acquire(ctx, cell, directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: seen.Device, Fence: seen.Fence}}) must(t, err) if l := v.Cell.Lease; l.Device != e.id(devB) || l.Fence != 2 || l.Pending != 0 { t.Fatalf("lease = %+v", l) diff --git a/internal/directory/raft.go b/internal/directory/raft.go new file mode 100644 index 0000000000..a74e923e16 --- /dev/null +++ b/internal/directory/raft.go @@ -0,0 +1,174 @@ +package directory + +import ( + "context" + "errors" + + "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +// RaftBackend is the host's consensus boundary. Change returns only a durable +// committed result. Listing and ReadCell pass a fresh current-term ReadIndex, +// await durable application, and recheck committed authorization at that view. +// The host authenticates the original ingress and preserves its request ID; +// forwarding through this facade never chooses a new expectation or retries a +// tool. P11 installs this boundary only after the legacy writer is frozen. +type RaftBackend interface { + Change(context.Context, string, clusterstate.Mutation) (clusterstate.Result, error) + Listing(context.Context, string) (Listing, error) + ReadCell(context.Context, string, string) (CellView, error) +} + +type Raft struct{ backend RaftBackend } + +func NewRaft(backend RaftBackend) *Raft { return &Raft{backend: backend} } +func (r *Raft) For(device string) Client { return &raftClient{r: r, device: device} } + +type raftClient struct { + r *Raft + device string +} + +var _ Directory = (*Raft)(nil) +var _ Client = (*raftClient)(nil) + +func (c *raftClient) change(ctx context.Context, m clusterstate.Mutation) (CellView, error) { + r, err := c.r.backend.Change(ctx, c.device, m) + if err != nil { + return CellView{}, err + } + if err = resultError(r.Code); err != nil { + return CellView{}, err + } + if r.View != nil { + return *r.View, nil + } + return CellView{}, nil +} +func (c *raftClient) List(ctx context.Context) (Listing, error) { + l, e := c.r.backend.Listing(ctx, c.device) + return withoutFrames(l), e +} +func (c *raftClient) Cell(ctx context.Context, id string) (CellView, error) { + return c.r.backend.ReadCell(ctx, c.device, id) +} +func (c *raftClient) PutDevice(ctx context.Context, id string, d Device) error { + _, e := c.change(ctx, clusterstate.Mutation{PutDevice: &clusterstate.PutDevice{ID: id, Device: d}}) + return e +} +func (c *raftClient) Revoke(ctx context.Context, id string) error { + _, e := c.change(ctx, clusterstate.Mutation{Revoke: &clusterstate.DeviceID{Device: id}}) + return e +} +func (c *raftClient) SetVault(ctx context.Context, old, next string) error { + _, e := c.change(ctx, clusterstate.Mutation{SetVault: &clusterstate.VaultCAS{Old: old, New: next}}) + return e +} +func (c *raftClient) Create(ctx context.Context, id string, in CellInit) (CellView, error) { + return c.change(ctx, clusterstate.Mutation{Create: &clusterstate.CreateCell{Cell: id, Init: in}}) +} +func (c *raftClient) Acquire(ctx context.Context, id string, o AcquireOpts) (CellView, error) { + v := &clusterstate.AcquireCell{Cell: id, Force: o.Force, V1: true} + if o.Expect != nil { + v.Expect = *o.Expect + } else { + if o.RequireConditional { + return CellView{}, ErrConditionalTake + } + v.Legacy = true + } + return c.change(ctx, clusterstate.Mutation{Acquire: v}) +} +func (c *raftClient) Heartbeat(ctx context.Context, id string, b Beat) (CellView, error) { + return c.change(ctx, clusterstate.Mutation{Beat: &clusterstate.BeatCell{Cell: id, Beat: b}}) +} +func (c *raftClient) Publish(ctx context.Context, id string, p Publish) (CellView, error) { + return c.change(ctx, clusterstate.Mutation{Publish: &clusterstate.PublishCell{Cell: id, Publish: p}}) +} +func (c *raftClient) Release(ctx context.Context, id string, f uint64) error { + _, e := c.change(ctx, clusterstate.Mutation{Release: &clusterstate.ReleaseCell{Cell: id, Fence: f}}) + return e +} +func (c *raftClient) ReleaseQuiet(ctx context.Context, id string, f uint64) error { + _, e := c.change(ctx, clusterstate.Mutation{Release: &clusterstate.ReleaseCell{Cell: id, Fence: f, Quiet: true}}) + return e +} +func (c *raftClient) Archive(ctx context.Context, id string) error { + _, e := c.change(ctx, clusterstate.Mutation{Archive: &clusterstate.CellID{Cell: id}}) + return e +} +func (c *raftClient) Presence(ctx context.Context) (PresenceView, error) { + l, e := c.List(ctx) + return presenceOf(l, nil), e +} +func (c *raftClient) Rotate(ctx context.Context, req RotationReq) (RotationView, error) { + _, e := c.change(ctx, clusterstate.Mutation{Rotate: &clusterstate.RotateRequest{Op: string(req.Op), GraceMS: req.GraceMS}}) + if e != nil { + return RotationView{}, e + } + return c.Rotation(ctx) +} +func (c *raftClient) Rotation(ctx context.Context) (RotationView, error) { + l, e := c.List(ctx) + return viewOf(l.Identity, l.Now, DefaultGraceBounds), e +} + +// Pairing commits a sealed decision dependency; delivery belongs to P10's +// durable outbox. There is no callback that can expose a grant before commit. +func (c *raftClient) ApproveRequest(ctx context.Context, code string, a Approval) error { + d, e := decisionOf(a) + if e != nil { + return e + } + _, e = c.change(ctx, clusterstate.Mutation{DecideJoin: &clusterstate.JoinDecision{Code: NormalizeCode(code), Device: d.Device, Record: a.Device, Grant: a.Grant, Cert: a.Cert}}) + return e +} +func (c *raftClient) DenyRequest(ctx context.Context, code string) error { + _, e := c.change(ctx, clusterstate.Mutation{DecideJoin: &clusterstate.JoinDecision{Code: NormalizeCode(code), Deny: true}}) + return e +} + +var errRaftConflict = errors.New("directory: request sequence conflict") + +func resultError(code clusterstate.Code) error { + switch code { + case clusterstate.OK: + return nil + case clusterstate.NotFound: + return ErrNotFound + case clusterstate.Exists: + return ErrExists + case clusterstate.LeaseHeld: + return ErrLeaseHeld + case clusterstate.FenceStale: + return ErrFenceStale + case clusterstate.HeadMoved: + return ErrHeadMoved + case clusterstate.CAS: + return ErrCAS + case clusterstate.Revoked: + return ErrRevoked + case clusterstate.SelfRevoke: + return ErrSelfRevoke + case clusterstate.Rotated: + return ErrRotated + case clusterstate.Unauthorized: + return ErrUnauthorized + case clusterstate.RotationStep: + return ErrRotationStep + case clusterstate.BadGrace: + return ErrBadGrace + case clusterstate.RequestGone: + return ErrRequestGone + case clusterstate.AlreadyDecided: + return ErrAlreadyDecided + case clusterstate.BadRequest: + return ErrBadRequest + case clusterstate.Capacity: + return ErrFull + case clusterstate.Conflict: + return errRaftConflict + default: + return ErrUnreachable + } +} diff --git a/internal/directory/rotation.go b/internal/directory/rotation.go index 3e331ae532..07778b4b0f 100644 --- a/internal/directory/rotation.go +++ b/internal/directory/rotation.go @@ -38,7 +38,7 @@ type RotationReq struct { type GraceBounds struct{ Min, Max, Default time.Duration } // DefaultGraceBounds is one hour to thirty days, a week unless told. -var DefaultGraceBounds = GraceBounds{Min: time.Hour, Max: 30 * 24 * time.Hour, Default: 7 * 24 * time.Hour} +var DefaultGraceBounds = GraceBounds{Min: time.Duration(clusterstate.MinGraceMillis) * time.Millisecond, Max: time.Duration(clusterstate.MaxGraceMillis) * time.Millisecond, Default: time.Duration(clusterstate.DefaultGraceMillis) * time.Millisecond} // RotationView is the answer to both rotation calls: the state now, the // directory's time, and the bounds it enforces, so a conformance run can test diff --git a/internal/raftdirectory/README.md b/internal/raftdirectory/README.md new file mode 100644 index 0000000000..2dd7f99cb5 --- /dev/null +++ b/internal/raftdirectory/README.md @@ -0,0 +1,51 @@ +# Committed directory application + +`clusterstate.Evaluate` is the single directory application evaluator for native and +WASI members. It takes a signed canonical command, immutable application metadata +and the complete declared dependency read-set. Missing and unread records differ. +The evaluator verifies the original signature against committed personal enrollment, +checks revocation at that log position, then applies the directory rules. Metadata +upserts cannot create signing authority. Delegated worker policy remains P15. + +Every accepted session keeps its exact last result until an acknowledged sequence +advance or explicit close; closed IDs remain tombstoned. There are 128 live sessions +and 4,096 retained session IDs. Capacity refuses growth and deletes nothing. A +read-set/corrupt-store failure stops application; it is not a successful refusal. +Hosts must reject oversized ingress before proposing and reserve capacity for +coordination responses. If even a capacity response cannot fit, evaluation fails +closed rather than acknowledge an unrecorded result. This is backpressure, never +permission to retry a tool or erase application records. + +Native `Apply` evaluates committed entries over an overlay in order and commits +rows, responses, version, monotonic time, applied index and complete configuration +in one FULL transaction. Serialize the actor across dependency reads and that +transaction. `ApplicationView` and `ScanPage` are local storage operations. Before +serving Listing/Cell/authority/rotation, commit current ingress time if needed, +pass a fresh current-term quorum ReadIndex, await durable application, and check +`AuthorizedRead` at that view. No local status or successful proposal admission is +an authority response. The integration harness proves this through `directory.Raft` +and its real HTTP handler; the backend interface is an assembly boundary. + +The hosted member chooses `directoryEvaluator`, which calls Go through additive +ABI-1 operations 11 (dependencies) and 10 (evaluation). JS copies immutable bytes, +persists the returned writes and advances the core after durable completion. It +contains no lease, CAS, revocation, clock or sequence decisions. `RaftDirectoryFacade` +forwards original signed bytes to a host's commit and barrierRead operations. The +old hosted directory is still solely the pre-migration path; P11 must irrevocably +freeze it before routing a namespace to this facade. No production cutover or +feature toggle is introduced here. + +The Client facade exposes all existing directory verbs. Its v1 acquire normalization +never manufactures an observed tuple: a missing expectation disables force, and a +required conditional take without an observation refuses. v1 fences cannot exceed +the exact JSON-integer limit; direct v2 commands retain full uint64 fences and refuse +overflow. Pure rules retain their legacy behavior for pre-cutover stores. + +Pairing commands require an existing committed request dependency, atomically write +the device metadata and sealed decision, and retain the first decision. The native +and hosted tests provision genesis/request dependencies explicitly. P10 still owns +external request/certificate admission, authorization installation, outbox delivery, +blob receipts and upload-before-head. P11 owns actual genesis/import; P12 owns peer +registry; P14 owns watch presence and committed renewal scheduling. No socket can +extend a lease through this evaluator. P17 supplies original v1 authentication proof +and production client wiring. These boundaries are internal, not new manual claims. diff --git a/internal/raftdirectory/apply.go b/internal/raftdirectory/apply.go new file mode 100644 index 0000000000..30a0876bcf --- /dev/null +++ b/internal/raftdirectory/apply.go @@ -0,0 +1,109 @@ +// Package raftdirectory assembles deterministic directory evaluation with the +// durable native member store. It is not a second consensus or policy writer. +package raftdirectory + +import ( + "cmp" + "context" + "errors" + "slices" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftstore" + pb "go.etcd.io/raft/v3/raftpb" +) + +// Apply evaluates a complete committed batch under the actor's serialization. +// Dependencies see prior entries in this batch; the host persists the final +// overlay, all exact results and the applied header in one FULL transaction. +// Returned results may escape only after this function returns successfully. +func Apply(ctx context.Context, store *raftstore.SQLite, b raftcore.ApplyBatch, cluster string, ceiling uint64) ([]clusterstate.Result, error) { + app, size, err := store.ApplicationView(ctx) + if err != nil { + return nil, err + } + update := raftstore.Update{ExpectedApplied: app.Applied, VisibleVersion: app.VisibleVersion, LogicalTime: app.LogicalTime} + overlay := map[clusterstate.RecordKey][]byte{} + original := map[clusterstate.RecordKey][]byte{} + read := func(k clusterstate.RecordKey) ([]byte, error) { + if v, ok := overlay[k]; ok { + return v, nil + } + if v, ok := original[k]; ok { + return v, nil + } + records, _, e := store.Read(ctx, app.Applied, []raftstore.Key{{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key}}) + if e != nil { + return nil, e + } + v := records[raftstore.Key{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key}] + original[k] = v + return v, nil + } + results := []clusterstate.Result{} + for _, entry := range b.Entries { + if entry.GetType() != pb.EntryNormal || len(entry.GetData()) == 0 { + continue + } + e, err := clusterstate.Evaluate(entry.GetData(), clusterstate.ApplicationMeta{Cluster: cluster, Now: update.LogicalTime, Version: update.VisibleVersion, Bytes: size, Ceiling: ceiling}, read) + if err != nil { + return nil, err + } + for _, w := range e.Writes { + old, err := read(w.RecordKey) + if err != nil { + return nil, err + } + if old != nil { + size -= clusterstate.RecordBytes(w.RecordKey, old) + } + size += clusterstate.RecordBytes(w.RecordKey, w.Data) + overlay[w.RecordKey] = w.Data + } + update.LogicalTime = e.Now + update.VisibleVersion = e.Version + results = append(results, e.Result) + } + for k, v := range overlay { + update.Writes = append(update.Writes, raftstore.Write{Key: raftstore.Key{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key}, Value: v}) + } + slices.SortFunc(update.Writes, func(a, b raftstore.Write) int { + if v := cmp.Compare(a.Namespace, b.Namespace); v != 0 { + return v + } + return cmp.Compare(a.Key.Key, b.Key.Key) + }) + if err = store.Apply(ctx, b, update); err != nil { + return nil, err + } + return results, nil +} + +// AuthorizedRead runs only within the actor after its current-term quorum +// barrier. Enrollment is not inferred from a device metadata upsert. +func AuthorizedRead(ctx context.Context, store *raftstore.SQLite, cluster, actor string) (raftstore.Application, error) { + app, _, err := store.ApplicationView(ctx) + if err != nil { + return app, err + } + keys := []raftstore.Key{{Namespace: raftstore.Enrollment, Key: clusterstate.AuthorityKey(actor).Key}, {Namespace: raftstore.State, Key: clusterstate.DeviceKey(actor).Key}} + rows, _, err := store.Read(ctx, app.Applied, keys) + if err != nil { + return app, err + } + var auth clusterstate.Authority + var device clusterstate.Device + if clusterstate.Decode(rows[keys[0]], &auth) != nil || auth.Cluster != cluster || !auth.Personal { + return app, errors.New("raftdirectory: unauthorized") + } + if raw := rows[keys[1]]; raw != nil { + if err := clusterstate.Decode(raw, &device); err != nil { + return app, err + } + if device.Revoked { + return app, errors.New("raftdirectory: revoked") + } + } + return app, nil +} diff --git a/internal/raftdirectory/directory_test.go b/internal/raftdirectory/directory_test.go new file mode 100644 index 0000000000..ab414fdb3a --- /dev/null +++ b/internal/raftdirectory/directory_test.go @@ -0,0 +1,470 @@ +package raftdirectory_test + +import ( + "context" + "crypto/ed25519" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "github.com/Agent-Field/codeaf/internal/wireauth" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/directory/directorytest" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/raftstore" + "go.etcd.io/raft/v3" +) + +var ctx = context.Background() + +const cluster = "p09-directory" +const devA = "dev_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +const devB = "dev_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +func must(t *testing.T, e error) { + t.Helper() + if e != nil { + t.Fatal(e) + } +} +func encoded(t *testing.T, v any) []byte { + t.Helper() + b, e := clusterstate.Encode(v) + must(t, e) + return b +} +func key(actor string) ed25519.PrivateKey { + h := sha256.Sum256([]byte(actor)) + return ed25519.NewKeyFromSeed(h[:]) +} + +type group struct { + t *testing.T + mu sync.Mutex + clock func() time.Time + paths [3]string + stores [3]*raftstore.SQLite + nodes [3]*raftcore.Core + status [3]raftcore.Status + queue []raftcore.Outbound + results [3][]clusterstate.Result + reads map[string]bool + seq map[string]uint64 + genesis []raftstore.Write + blocked bool + readID uint64 +} + +func newGroup(t *testing.T, clock func() time.Time, extra ...raftstore.Write) *group { + g := &group{t: t, clock: clock, reads: map[string]bool{}, seq: map[string]uint64{}} + actors := []string{devA, devB, "a", "b"} + for i := range 50 { + actors = append(actors, fmt.Sprintf("dev_%032x", i+1)) + } + g.genesis = append(g.genesis, raftstore.Write{Key: raftstore.Key{Namespace: raftstore.State, Key: clusterstate.IdentityKey().Key}, Value: encoded(t, clusterstate.IdentityRec{V: 1})}) + for _, actor := range actors { + k := clusterstate.AuthorityKey(actor) + g.genesis = append(g.genesis, raftstore.Write{Key: raftstore.Key{Namespace: raftstore.Enrollment, Key: k.Key}, Value: encoded(t, clusterstate.Authority{Cluster: cluster, Key: key(actor).Public().(ed25519.PublicKey), Personal: true})}) + } + g.genesis = append(g.genesis, extra...) + for i := range 3 { + g.paths[i] = filepath.Join(t.TempDir(), "member.db") + s, e := raftstore.Open(g.paths[i], raftstore.Identity{ClusterUUID: cluster, NodeID: uint64(i + 1), Protocol: 1}) + must(t, e) + g.stores[i] = s + n, e := raftcore.New(raftcore.Config{ClusterUUID: cluster, NodeID: uint64(i + 1), Seed: uint64(i + 13), Protocol: 1, Bootstrap: []uint64{1, 2, 3}}, raftcore.RecoveredState{}) + must(t, e) + g.nodes[i] = n + g.drive(i, raftcore.Poll{}) + } + g.drain() + for n := 0; n < 40 && g.leader() < 0; n++ { + for i := range 3 { + g.drive(i, raftcore.Tick{}) + } + g.drain() + } + if g.leader() < 0 { + t.Fatal("no leader") + } + t.Cleanup(func() { + for _, s := range g.stores { + must(t, s.Close()) + } + }) + return g +} +func (g *group) drive(i int, in raftcore.Input) { + e, err := g.nodes[i].Step(in) + must(g.t, err) + for n := 0; n < 100; n++ { + g.status[i] = e.Status + g.queue = append(g.queue, e.Send...) + for _, r := range e.Reads { + app, _, err := g.stores[i].ApplicationView(ctx) + must(g.t, err) + if app.Applied < r.Index { + g.t.Fatal("read before apply") + } + g.reads[r.RequestID] = true + } + if e.Persist != nil { + must(g.t, g.stores[i].Persist(ctx, *e.Persist)) + e, err = g.nodes[i].Step(raftcore.Persisted{BatchID: e.Persist.ID}) + } else if e.Apply != nil { + app, _, err := g.stores[i].ApplicationView(ctx) + must(g.t, err) + if app.Applied == 0 { + must(g.t, g.stores[i].Apply(ctx, *e.Apply, raftstore.Update{ExpectedApplied: 0, Writes: g.genesis})) + } else { + r, err := raftdirectory.Apply(ctx, g.stores[i], *e.Apply, cluster, clusterstate.DefaultApplicationBytes) + must(g.t, err) + g.results[i] = append(g.results[i], r...) + } + e, err = g.nodes[i].Step(raftcore.Applied{BatchID: e.Apply.ID}) + } else { + return + } + must(g.t, err) + } + g.t.Fatal("completion budget") +} +func (g *group) drain() { + for n := 0; len(g.queue) > 0; n++ { + if n > 10000 { + g.t.Fatal("network budget") + } + m := g.queue[0] + g.queue = g.queue[1:] + if !g.blocked { + g.drive(int(m.To)-1, raftcore.PeerMessage{Bytes: m.Bytes}) + } + } +} +func (g *group) leader() int { + for i, s := range g.status { + if s.Role == raft.StateLeader { + return i + } + } + return -1 +} +func (g *group) submit(c clusterstate.Command) (clusterstate.Result, error) { + i := g.leader() + if i < 0 { + return clusterstate.Result{}, directory.ErrUnreachable + } + raw, e := clusterstate.EncodeCommand(c) + if e != nil { + return clusterstate.Result{}, e + } + before := len(g.results[i]) + g.drive(i, raftcore.Propose{Command: raw}) + g.drain() + if len(g.results[i]) == before { + return clusterstate.Result{}, directory.ErrUnreachable + } + return g.results[i][before], nil +} +func (g *group) Change(_ context.Context, actor string, m clusterstate.Mutation) (clusterstate.Result, error) { + g.mu.Lock() + defer g.mu.Unlock() + g.seq[actor]++ + seq := g.seq[actor] + c, e := clusterstate.Sign(clusterstate.Command{Schema: 1, Cluster: cluster, Actor: actor, Session: "api", Sequence: seq, Ack: seq - 1, Now: g.clock().UnixMilli(), Change: m}, key(actor)) + if e != nil { + return clusterstate.Result{}, e + } + return g.submit(c) +} +func (g *group) barrier(actor string) (int, raftstore.Application, error) { + i := g.leader() + if i < 0 { + return i, raftstore.Application{}, directory.ErrUnreachable + } + // Public expiry reads commit the ingress clock before their fresh barrier. + g.seq[actor]++ + seq := g.seq[actor] + c, e := clusterstate.Sign(clusterstate.Command{Schema: 1, Cluster: cluster, Actor: actor, Session: "api", Sequence: seq, Ack: seq - 1, Now: g.clock().UnixMilli(), Change: clusterstate.Mutation{AdvanceTime: &clusterstate.AdvanceTime{}}}, key(actor)) + if e != nil { + return i, raftstore.Application{}, e + } + result, e := g.submit(c) + if e != nil { + return i, raftstore.Application{}, e + } + if result.Code == clusterstate.Revoked { + return i, raftstore.Application{}, directory.ErrRevoked + } + if result.Code != clusterstate.OK { + return i, raftstore.Application{}, directory.ErrUnauthorized + } + g.readID++ + id := fmt.Sprint(g.readID) + g.drive(i, raftcore.Read{RequestID: id}) + g.drain() + if !g.reads[id] { + return i, raftstore.Application{}, directory.ErrUnreachable + } + app, e := raftdirectory.AuthorizedRead(ctx, g.stores[i], cluster, actor) + return i, app, e +} +func (g *group) Listing(_ context.Context, actor string) (directory.Listing, error) { + g.mu.Lock() + defer g.mu.Unlock() + i, app, e := g.barrier(actor) + if e != nil { + return directory.Listing{}, e + } + l := directory.Listing{Now: app.LogicalTime, Version: app.VisibleVersion, Devices: map[string]directory.Device{}, Cells: map[string]directory.Cell{}} + after := "" + for { + rows, _, err := g.stores[i].ScanPage(ctx, app.Applied, raftstore.State, after) + if err != nil { + return l, err + } + if len(rows) == 0 { + break + } + for _, r := range rows { + after = r.Key.Key + switch { + case after == "identity": + must(g.t, clusterstate.Decode(r.Value, &l.Identity)) + case strings.HasPrefix(after, "device/"): + var d directory.Device + must(g.t, clusterstate.Decode(r.Value, &d)) + b, e := hex.DecodeString(strings.TrimPrefix(after, "device/")) + must(g.t, e) + l.Devices[string(b)] = d + case strings.HasPrefix(after, "cell/"): + var c directory.Cell + must(g.t, clusterstate.Decode(r.Value, &c)) + b, e := hex.DecodeString(strings.TrimPrefix(after, "cell/")) + must(g.t, e) + l.Cells[string(b)] = c + } + } + } + return l, nil +} +func (g *group) ReadCell(_ context.Context, actor, id string) (directory.CellView, error) { + g.mu.Lock() + defer g.mu.Unlock() + i, app, e := g.barrier(actor) + if e != nil { + return directory.CellView{}, e + } + k := raftstore.Key{Namespace: raftstore.State, Key: clusterstate.CellKey(id).Key} + rows, _, e := g.stores[i].Read(ctx, app.Applied, []raftstore.Key{k}) + if e != nil { + return directory.CellView{}, e + } + if rows[k] == nil { + return directory.CellView{}, directory.ErrNotFound + } + var c directory.Cell + e = clusterstate.Decode(rows[k], &c) + return directory.CellView{Now: app.LogicalTime, Cell: c}, e +} + +// R1/R2/R16/R25/R30 exercise the existing Client verbs through three real +// durable stores and the real quorum/persistence/application protocol. +func TestR1R2R16R25R30RaftDirectoryConformance(t *testing.T) { + directorytest.RunConsensus(t, func(t *testing.T, clock *directorytest.FakeClock) directorytest.Devices { + g := newGroup(t, clock.Now) + return directory.NewRaft(g).For + }) +} +func TestR4MinorityDirectoryCannotReadOrWrite(t *testing.T) { + clock := directorytest.NewFakeClock() + g := newGroup(t, clock.Now) + cl := directory.NewRaft(g).For("a") + _, e := cl.Create(ctx, "chat", directory.CellInit{Head: "one"}) + must(t, e) + g.blocked = true + _, e = cl.Cell(ctx, "chat") + if !errors.Is(e, directory.ErrUnreachable) { + t.Fatalf("minority read: %v", e) + } +} + +func TestR3R13R17DirectoryRetryAfterSnapshotAndNativeRestart(t *testing.T) { + clock := directorytest.NewFakeClock() + g := newGroup(t, clock.Now) + cl := directory.NewRaft(g).For("a") + _, err := cl.Create(ctx, "chat", directory.CellInit{Head: "sealed"}) + must(t, err) + c, err := clusterstate.Sign(clusterstate.Command{Schema: 1, Cluster: cluster, Actor: "b", Session: "lost-reply", Sequence: 1, Now: clock.Now().UnixMilli(), Change: clusterstate.Mutation{Acquire: &clusterstate.AcquireCell{Cell: "chat", Force: true, Expect: clusterstate.LeaseExpectation{Device: "a", Fence: 1}}}}, key("b")) + must(t, err) + first, err := g.submit(c) + must(t, err) + if first.Code != clusterstate.OK || first.View.Cell.Lease.Fence != 2 { + t.Fatal(first) + } + for i := range 3 { + p, err := g.stores[i].CreateSnapshot(ctx) + must(t, err) + must(t, g.stores[i].Compact(ctx, p.GetMetadata().GetIndex())) + g.drive(i, raftcore.CompactSnapshot{Snapshot: p}) + } + g.drain() + for i := range 3 { + must(t, g.stores[i].Close()) + s, err := raftstore.Open(g.paths[i], raftstore.Identity{ClusterUUID: cluster, NodeID: uint64(i + 1), Protocol: 1}) + must(t, err) + g.stores[i] = s + recovered, err := s.Recover(ctx) + must(t, err) + node, err := raftcore.New(raftcore.Config{ClusterUUID: cluster, NodeID: uint64(i + 1), Protocol: 1, Seed: uint64(i + 13)}, recovered.Core) + must(t, err) + g.nodes[i] = node + g.drive(i, raftcore.Poll{}) + } + g.drain() + for n := 0; n < 40 && g.leader() < 0; n++ { + for i := range 3 { + g.drive(i, raftcore.Tick{}) + } + g.drain() + } + c.Now += 5000 + c, err = clusterstate.Sign(c, key("b")) + must(t, err) + retry, err := g.submit(c) + must(t, err) + firstBytes, err := clusterstate.EncodeResult(first) + must(t, err) + retryBytes, err := clusterstate.EncodeResult(retry) + must(t, err) + if string(firstBytes) != string(retryBytes) { + t.Fatal("lost result changed after snapshot/restart") + } +} + +// R30 covers the remaining Client verbs at the actual committed adapter door. +// External LinkRequest delivery and certificate admission remain P10's gate. +func TestR16R30PresenceAndPairDecisionDependencies(t *testing.T) { + clock := directorytest.NewFakeClock() + g := newGroup(t, clock.Now) + cl := directory.NewRaft(g).For("a") + p, err := cl.Presence(ctx) + must(t, err) + if len(p.Devices) != 0 { + t.Fatal(p) + } + if err := cl.DenyRequest(ctx, "unknown"); !errors.Is(err, directory.ErrRequestGone) { + t.Fatal(err) + } + if err := cl.ApproveRequest(ctx, "unknown", directory.Approval{}); !errors.Is(err, directory.ErrBadRequest) { + t.Fatal(err) + } + must(t, cl.PutDevice(ctx, "b", directory.Device{V: 1, Name: "sealed-name"})) + must(t, cl.Revoke(ctx, "b")) + revoked := directory.NewRaft(g).For("b") + if _, err := revoked.Presence(ctx); !errors.Is(err, directory.ErrRevoked) { + t.Fatal(err) + } + if err := revoked.DenyRequest(ctx, "unknown"); !errors.Is(err, directory.ErrRevoked) { + t.Fatal(err) + } +} + +func TestR16R30ApproveAndDenyCommitBeforeAnswer(t *testing.T) { + clock := directorytest.NewFakeClock() + pub := key("joiner").Public().(ed25519.PublicKey) + sum := sha256.Sum256(pub) + device := "dev_" + hex.EncodeToString(sum[:16]) + writes := []raftstore.Write{} + for _, code := range []string{"approve", "deny"} { + k := clusterstate.JoinKey(code) + writes = append(writes, raftstore.Write{Key: raftstore.Key{Namespace: raftstore.Outbox, Key: k.Key}, Value: encoded(t, clusterstate.JoinRequest{Device: device, Expires: clock.Now().UnixMilli() + 600000})}) + } + g := newGroup(t, clock.Now, writes...) + cl := directory.NewRaft(g).For("a") + raw, err := json.Marshal(map[string]any{"V": 1, "device": hex.EncodeToString(pub), "created": 1, "sig": "00"}) + must(t, err) + approval := directory.Approval{Device: directory.Device{V: 1, Name: "sealed"}, Cert: base64.RawURLEncoding.EncodeToString(raw), Grant: "sealed-grant"} + must(t, cl.ApproveRequest(ctx, "approve", approval)) + must(t, cl.ApproveRequest(ctx, "approve", approval)) + if err := cl.DenyRequest(ctx, "approve"); !errors.Is(err, directory.ErrAlreadyDecided) { + t.Fatal(err) + } + must(t, cl.DenyRequest(ctx, "deny")) + must(t, cl.DenyRequest(ctx, "deny")) + l, err := cl.List(ctx) + must(t, err) + if l.Devices[device].Name != "sealed" { + t.Fatal(l) + } + for _, store := range g.stores { + app, _, err := store.ApplicationView(ctx) + must(t, err) + k := raftstore.Key{Namespace: raftstore.Outbox, Key: clusterstate.JoinKey("approve").Key} + rows, _, err := store.Read(ctx, app.Applied, []raftstore.Key{k}) + must(t, err) + var request clusterstate.JoinRequest + must(t, clusterstate.Decode(rows[k], &request)) + if request.Decision == nil || request.Decision.Grant != "sealed-grant" { + t.Fatal("answer preceded committed sealed decision") + } + } +} + +func TestR1R25R30DirectoryHTTPFacadeUsesConsensus(t *testing.T) { + clock := directorytest.NewFakeClock() + g := newGroup(t, clock.Now) + adapter := directory.NewRaft(g) + auth := func(r *http.Request, _ []byte) (string, string, error) { + actor := r.Header.Get("X-Test-Actor") + if actor != "a" && actor != "b" { + return "", "", wireauth.ErrUnauthorized + } + return cluster, actor, nil + } + server := httptest.NewServer(directory.Handler(auth, func(id string) (directory.Directory, error) { + if id != cluster { + return nil, directory.ErrUnauthorized + } + return adapter, nil + })) + defer server.Close() + client := func(actor string) *directory.HTTP { + return directory.NewHTTP(server.URL, func(r *http.Request, _ []byte) { r.Header.Set("X-Test-Actor", actor) }, server.Client()) + } + a, b := client("a"), client("b") + id := "01J0000000000000000000000A" + head := strings.Repeat("1", 64) + created, err := a.Create(ctx, id, directory.CellInit{Head: head, Class: "chat", Size: 1}) + must(t, err) + expect := directory.LeaseExpectation{Device: created.Cell.Lease.Device, Fence: created.Cell.Lease.Fence} + taken, err := b.Acquire(ctx, id, directory.AcquireOpts{Force: true, Expect: &expect}) + must(t, err) + if taken.Cell.Lease.Fence != 2 { + t.Fatal(taken) + } + if _, err := a.Acquire(ctx, id, directory.AcquireOpts{Force: true}); !errors.Is(err, directory.ErrLeaseHeld) { + t.Fatal("blind v1 force changed the holder", err) + } + _, err = a.Publish(ctx, id, directory.Publish{Fence: 1, OldHead: head, Head: strings.Repeat("2", 64)}) + if !errors.Is(err, directory.ErrFenceStale) { + t.Fatal(err) + } + l, err := a.List(ctx) + must(t, err) + if l.Cells[id].Lease.Device != "b" { + t.Fatal(l) + } +} diff --git a/internal/raftdirectory/routing_test.go b/internal/raftdirectory/routing_test.go new file mode 100644 index 0000000000..e6c262f051 --- /dev/null +++ b/internal/raftdirectory/routing_test.go @@ -0,0 +1,30 @@ +package raftdirectory_test + +import ( + "os" + "strings" + "testing" +) + +// R30 forbids another lease writer on the authoritative hosted path. Existing +// pre-cutover legacy rules remain tested until P11 freezes that source. +func TestR30HostedDirectoryRoutesOnlyThroughGoPolicy(t *testing.T) { + b, err := os.ReadFile("../../relay/hosted/src/raft/directory.js") + must(t, err) + s := string(b) + for _, needle := range []string{"frame(10)", "frame(11)", "this.host.commit(originalSignedBytes)", "this.host.barrierRead(authenticatedRequest)"} { + if !strings.Contains(s, needle) { + t.Fatalf("missing policy boundary %s", needle) + } + } + for _, needle := range []string{"rules.js", "rotateBy", "LeaseTTL", "expires =", "fence++"} { + if strings.Contains(s, needle) { + t.Fatalf("JS policy writer %s", needle) + } + } + b, err = os.ReadFile("../../relay/hosted/src/raft/member.js") + must(t, err) + if strings.Contains(string(b), "env.raftEvaluate") || !strings.Contains(string(b), "directoryEvaluator(sharedRuntime()") { + t.Fatal("production member lacks the shared Go evaluator") + } +} diff --git a/internal/raftstore/view.go b/internal/raftstore/view.go new file mode 100644 index 0000000000..b57900d23a --- /dev/null +++ b/internal/raftstore/view.go @@ -0,0 +1,73 @@ +package raftstore + +import ( + "bytes" + "context" + "database/sql" +) + +// ApplicationView supplies only committed metadata and keyed byte accounting. +// It is a local dependency read; the actor must first pass ReadIndex before +// exposing any of this view to a client. No identity-sized buffer is created. +func (s *SQLite) ApplicationView(ctx context.Context) (Application, uint64, error) { + s.mu.Lock() + defer s.mu.Unlock() + var app Application + var size uint64 + err := s.transaction(ctx, func(tx *sql.Tx) error { + var err error + app, err = getApplication(ctx, tx) + if err != nil { + return err + } + return tx.QueryRowContext(ctx, `SELECT coalesce(sum(12+length(CAST(namespace AS BLOB))+length(CAST(key AS BLOB))+length(data)),0) FROM records`).Scan(&size) + }) + return app, size, err +} + +// ScanPage bounds listings at one immutable applied view. Callers serialize +// the complete multi-page read with application and discard it on CAS failure. +// A page carries at most MaxRecordBytes payload bytes, including keys. +func (s *SQLite) ScanPage(ctx context.Context, expected uint64, namespace Namespace, after string) ([]Write, Application, error) { + s.mu.Lock() + defer s.mu.Unlock() + var app Application + out := []Write{} + err := s.transaction(ctx, func(tx *sql.Tx) error { + var err error + app, err = getApplication(ctx, tx) + if err != nil { + return err + } + if app.Applied != expected { + return ErrConflict + } + rows, err := tx.QueryContext(ctx, `SELECT key,data,checksum FROM records WHERE namespace=? AND key>? ORDER BY key`, namespace, after) + if err != nil { + return err + } + defer rows.Close() + size := 0 + for rows.Next() { + var k string + var data, hash []byte + if err := rows.Scan(&k, &data, &hash); err != nil { + return err + } + if !bytes.Equal(hash, checksum([]byte("records"), []byte(namespace), []byte(k), data)) { + return ErrCorrupt + } + cost := len(k) + len(data) + if size+cost > MaxRecordBytes { + if len(out) == 0 { + return ErrBatch + } + break + } + size += cost + out = append(out, Write{Key: Key{Namespace: namespace, Key: k}, Value: data}) + } + return rows.Err() + }) + return out, app, err +} From 97068fd4eb1965de89d3980c919997ccd6132296 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 00:48:14 -0400 Subject: [PATCH 021/103] Route hosted member application through the shared Go evaluator. Keep JavaScript limited to dependency bytes and durable writes, expose the migration-ready facade, and prove signed hosted decisions with native ABI parity while preserving the earlier workerd mechanism fixture. Co-Authored-By: Claude Opus 5.5 (1M context) --- relay/hosted/src/raft/directory.js | 24 ++++++ relay/hosted/src/raft/member.js | 7 +- relay/hosted/src/tenant.js | 5 ++ relay/hosted/test/raft/actor-worker.js | 2 +- relay/hosted/test/raft/actors.test.js | 24 ++++-- relay/hosted/test/raft/directory.test.js | 98 ++++++++++++++++++++++++ 6 files changed, 149 insertions(+), 11 deletions(-) create mode 100644 relay/hosted/src/raft/directory.js create mode 100644 relay/hosted/test/raft/directory.test.js diff --git a/relay/hosted/src/raft/directory.js b/relay/hosted/src/raft/directory.js new file mode 100644 index 0000000000..2ae57de7ba --- /dev/null +++ b/relay/hosted/src/raft/directory.js @@ -0,0 +1,24 @@ +import { frame, response } from './abi.js'; + +// JavaScript supplies immutable dependency bytes and persists the Go write-set. +// It never interprets leases, clocks, revocations, CAS or request sequences. +export function directoryEvaluator(runtime, cluster, ceiling) { + return (command, read, app) => { + let r = response(runtime.call(frame(11).blob(command).finish())); + const keys = Array.from({ length: r.u32() }, () => ({ namespace: r.str(), key: r.str() })); r.end(); + const w = frame(10).blob(command).str(cluster).u64(app.now).u64(app.version).u64(app.size).u64(ceiling).u32(keys.length); + for (const k of keys) { const data = read(k.namespace, k.key); w.str(k.namespace).str(k.key).u32(data == null ? 0 : 1).blob(data ?? new Uint8Array()); } + r = response(runtime.call(w.finish())); + const now = r.u64(), version = r.u64(), result = r.blob(); + const writes = Array.from({ length: r.u32() }, () => ({ namespace: r.str(), key: r.str(), data: r.blob() })); r.end(); + return { now, version, result, writes }; + }; +} + +// The consensus host implements commit and barrier, not a local-success cache. +// Calls return only committed/durably applied bytes; a leader hint is routing. +export class RaftDirectoryFacade { + constructor(host) { this.host = host; } + async command(originalSignedBytes) { return this.host.commit(originalSignedBytes); } + async read(authenticatedRequest) { return this.host.barrierRead(authenticatedRequest); } +} diff --git a/relay/hosted/src/raft/member.js b/relay/hosted/src/raft/member.js index 57ef80f9a3..c55bd9eea9 100644 --- a/relay/hosted/src/raft/member.js +++ b/relay/hosted/src/raft/member.js @@ -1,6 +1,8 @@ import { DurableObject } from 'cloudflare:workers'; import { MemberActor, sharedRuntime } from './actor.js'; import { MemberInbox } from './inbox.js'; +import { directoryEvaluator } from './directory.js'; +import { DEFAULT_ROWS_BYTES } from './storage.js'; export { installRaftModule, runtimeMetrics } from './actor.js'; // Only internal namespace calls can initialize a provisioned member. The @@ -8,6 +10,7 @@ export { installRaftModule, runtimeMetrics } from './actor.js'; // P12 supplies the durable committed enrollment view and P09 the evaluator. export class RaftMemberDO extends DurableObject { constructor(ctx, env) { super(ctx, env); this.actor = null; this.loading = null; this.ctx = ctx; this.env = env; } + evaluator(config) { return directoryEvaluator(sharedRuntime(), config.cluster, config.ceiling ?? DEFAULT_ROWS_BYTES); } async initialize(config) { config = { ...config, id: String(config.id), seed: String(config.seed), bootstrap: config.bootstrap.map(String) }; if (this.loading) { @@ -27,7 +30,7 @@ export class RaftMemberDO extends DurableObject { const previous = await this.ctx.storage.get('raft-config'); if (previous && JSON.stringify(previous) !== JSON.stringify(canonical)) throw new Error('raft member: provisioned identity cannot change'); if (!previous) { await this.ctx.storage.put('raft-config', canonical); await this.ctx.storage.sync(); } - const actor = new MemberActor(this.ctx.storage, sharedRuntime(), this.env.raftEvaluate); + const actor = new MemberActor(this.ctx.storage, sharedRuntime(), this.evaluator(config)); const result = await actor.init(canonical); this.config = canonical; this.registry = await this.ctx.storage.get('raft-registry'); this.actor = actor; return result; @@ -41,7 +44,7 @@ export class RaftMemberDO extends DurableObject { } async restore() { const config = await this.ctx.storage.get('raft-config'); if (!config) throw new Error('raft member: not provisioned'); - const actor = new MemberActor(this.ctx.storage, sharedRuntime(), this.env.raftEvaluate); + const actor = new MemberActor(this.ctx.storage, sharedRuntime(), this.evaluator(config)); this.recoveredEffects = await actor.init(config); this.config = config; this.registry = await this.ctx.storage.get('raft-registry'); this.actor = actor; } async drainRecovery() { diff --git a/relay/hosted/src/tenant.js b/relay/hosted/src/tenant.js index 9dd254bded..ad14dcd51f 100644 --- a/relay/hosted/src/tenant.js +++ b/relay/hosted/src/tenant.js @@ -150,3 +150,8 @@ export class Tenant { return this.dir.rotation()?.retire_at; } } + +// This facade is assembled only for a Raft-owned namespace after migration. +// Keeping it separate prevents constructing or invoking the legacy SQL writer +// while the integration harness exercises the authoritative application path. +export { RaftDirectoryFacade } from './raft/directory.js'; diff --git a/relay/hosted/test/raft/actor-worker.js b/relay/hosted/test/raft/actor-worker.js index 48a950679b..05235d8290 100644 --- a/relay/hosted/test/raft/actor-worker.js +++ b/relay/hosted/test/raft/actor-worker.js @@ -15,7 +15,7 @@ const evaluate = (command, read, app) => { const data = r.blob(); r.end(); return { writes: [{ namespace: 'state', key: 'cell', data }], version: BigInt(app.version) + 1n }; }; export class ActorFixture extends RaftMemberDO { - constructor(ctx, env) { super(ctx, { ...env, raftEvaluate: evaluate }); } + evaluator() { return evaluate; } async fetch(request) { const url = new URL(request.url), b = request.method === 'POST' ? await request.json() : {}; try { diff --git a/relay/hosted/test/raft/actors.test.js b/relay/hosted/test/raft/actors.test.js index 09b75efdbd..bc8500332e 100644 --- a/relay/hosted/test/raft/actors.test.js +++ b/relay/hosted/test/raft/actors.test.js @@ -96,10 +96,17 @@ class Group { test('R3 R4 R10 R13 R14 R27 hosted actors commit, ReadIndex, crash and snapshot in both directions', { timeout: 120000 }, async () => { const dir = await mkdtemp(join(tmpdir(), 'p08-')), native = new Native(join(dir, 'native')); - let worker, inspector, sampling = false; const heaps = []; - const timer = setInterval(async () => { if (sampling) return; sampling = true; try { await sample(); if (inspector) heaps.push(await inspector.call('Runtime.getHeapUsage')); } finally { sampling = false; } }, 100); + let worker, inspector, samplingTask, samplingError, samplingPaused = false; const heaps = []; + // A sample owns its inspector call until completion. Drain it before each + // intentional process death so a late timeout cannot escape the test. + const timer = setInterval(() => { + if (samplingTask || samplingPaused) return; + samplingTask = (async () => { await sample(); if (inspector) heaps.push(await inspector.call('Runtime.getHeapUsage')); })() + .catch(error => { samplingError ??= error; }).finally(() => { samplingTask = null; }); + }, 100); + const closeInspector = async () => { samplingPaused = true; await samplingTask; inspector?.close(); inspector = null; }; try { - worker = await openWorkerd(out, join(dir, 'do'), true); inspector = await inspect(worker); + worker = await openWorkerd(out, join(dir, 'do'), true); inspector = await inspect(worker); samplingPaused = false; const g = new Group(worker, native, 'mixed', [1]); await g.init(); await g.transfer(1); await g.create(); assert(await g.read(1, 'after-create')); const row = await g.sameRows(); await g.call(1, 'init', { cluster: 'mixed', id: '1', seed: '101', bootstrap: ['1','2','3'], scope: g.scope }); @@ -114,8 +121,8 @@ test('R3 R4 R10 R13 R14 R27 hosted actors commit, ReadIndex, crash and snapshot await g.transfer(2); g.block.add(1); await g.mutate(3); await g.call(2, 'checkpoint'); g.block.clear(); await g.tick(4); const latest = await g.sameRows(); assert(report.phases.includes('native snapshot -> wasm')); await g.read(g.leader(), 'after-snapshots'); await g.transfer(1); await sample(); heaps.push(await inspector.call('Runtime.getHeapUsage')); - inspector.close(); inspector = null; await worker.dispose(); worker = null; - worker = await openWorkerd(out, join(dir, 'do'), true); g.worker = worker; inspector = await inspect(worker); + await closeInspector(); await worker.dispose(); worker = null; + worker = await openWorkerd(out, join(dir, 'do'), true); g.worker = worker; inspector = await inspect(worker); samplingPaused = false; // A new workerd process loads the actor and validates durable rows and // snapshot chunks. Recovery does not restore a leader or prior read lease. assert.equal(await g.call(1, 'row'), latest); @@ -132,8 +139,8 @@ test('R3 R4 R10 R13 R14 R27 hosted actors commit, ReadIndex, crash and snapshot const jointSnapshot = unb64(await j.call(1, 'checkpoint')); const jointManifest = JSON.parse(fields(jointSnapshot).get(1).toString()); const jointConf = fields(unb64(jointManifest.conf)); assert(jointConf.has(3)); assert(jointConf.has(4)); - inspector.close(); inspector = null; await worker.dispose(); worker = null; - worker = await openWorkerd(out, join(dir, 'do'), true); inspector = await inspect(worker); j.worker = worker; g.worker = worker; + await closeInspector(); await worker.dispose(); worker = null; + worker = await openWorkerd(out, join(dir, 'do'), true); inspector = await inspect(worker); samplingPaused = false; j.worker = worker; g.worker = worker; for (const id of [1,2]) { await j.call(id, 'row'); j.accept(id, await j.call(id, 'input', { tag: 0 })); } for (const id of [3,4]) j.accept(id, await j.call(id, 'restart')); await j.tick(16); assert(await j.read(j.leader(), 'joint-restart')); @@ -158,13 +165,14 @@ test('R3 R4 R10 R13 R14 R27 hosted actors commit, ReadIndex, crash and snapshot assert(report.isolateBytes <= 96 * 1024 * 1024); assert(metrics.peakMemory <= 64 * 1024 * 1024); assert.equal(metrics.active, 6); report.phases.push('three hosted anchors plus mixed identity in same isolate'); } finally { - clearInterval(timer); await sample(); inspector?.close(); if (worker) await worker.dispose(); await native.close(); + clearInterval(timer); await closeInspector(); await sample(); if (worker) await worker.dispose(); await native.close(); report.combinedRSS = Math.max(...report.samples.map(s => s.combinedRSS)); const cpu = new Map(); for (const sample of report.samples) for (const p of sample.processes) cpu.set(p.pid, Math.max(cpu.get(p.pid) ?? 0, p.cpuSeconds)); report.sampledCPUSeconds = [...cpu.values()].reduce((a, b) => a + b, 0); const times = report.calls.filter(c => c.wasm).map(c => c.ms).sort((a, b) => a - b); report.callMs = { p50: times[Math.floor(times.length * .5)], p95: times[Math.floor(times.length * .95)], p99: times[Math.floor(times.length * .99)] }; report.coldFirstInitMs = report.calls.find(c => c.wasm && c.op === 'init')?.ms; await writeFile(join(out, 'actors-runtime.json'), JSON.stringify(report, null, 2) + '\n'); } + if (samplingError) throw samplingError; assert(report.coldFirstInitMs < 1000, 'R27 cold member activation stays within one second'); assert(report.combinedRSS <= 256 * 1024 * 1024, `R27 combined RSS ${report.combinedRSS}`); }); diff --git a/relay/hosted/test/raft/directory.test.js b/relay/hosted/test/raft/directory.test.js new file mode 100644 index 0000000000..9f5ef55666 --- /dev/null +++ b/relay/hosted/test/raft/directory.test.js @@ -0,0 +1,98 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { spawn } from 'node:child_process'; +import { DatabaseSync } from 'node:sqlite'; +import { createHash, generateKeyPairSync, sign } from 'node:crypto'; +import { Reactor } from '../../src/raft/abi.js'; +import { encodeCanonical, decodeCanonical } from '../../src/raft/codec.js'; +import { MemberActor } from '../../src/raft/actor.js'; +import { directoryEvaluator, RaftDirectoryFacade } from '../../src/raft/directory.js'; + +const out = process.env.CODEAF_RAFT_PROOF_DIR ?? '/tmp/codeaf-p09'; +const enc = new TextEncoder(), dec = new TextDecoder(); +const bytes = value => enc.encode(encodeCanonical(value)); +const sha = value => createHash('sha256').update(value).digest('hex'); +const segment = value => Buffer.from(value).toString('hex'); +class Host { + constructor() { this.db = new DatabaseSync(':memory:'); this.sql = { exec: (q, ...p) => { const rows = this.db.prepare(q).all(...p); return { toArray: () => rows, [Symbol.iterator]: () => rows[Symbol.iterator]() }; } }; } + transactionSync(fn) { this.db.exec('BEGIN'); try { const result = fn(); this.db.exec('COMMIT'); return result; } catch (e) { this.db.exec('ROLLBACK'); throw e; } } + async sync() {} +} +class Native { + constructor() { + this.child = spawn(out + '/native-peer', [], { stdio: ['pipe','pipe','inherit'], env: { ...process.env, GOMEMLIMIT: '32MiB' } }); this.buffer = Buffer.alloc(0); + this.child.stdout.on('data', b => { this.buffer = Buffer.concat([this.buffer, b]); this.pump(); }); + this.child.on('exit', code => { if (this.waiter) { clearTimeout(this.waiter.timer); this.waiter.reject(new Error(`native exited ${code}`)); } }); + } + pump() { if (!this.waiter || this.buffer.length < 4) return; const n = this.buffer.readUInt32LE(0); assert(n <= 2 * 1024 * 1024); if (this.buffer.length < n + 4) return; const b = this.buffer.subarray(4, n + 4); this.buffer = this.buffer.subarray(n + 4); const w = this.waiter; this.waiter = null; clearTimeout(w.timer); w.resolve(new Uint8Array(b)); } + call(b) { assert(!this.waiter); return new Promise((resolve, reject) => { this.waiter = { resolve, reject, timer: setTimeout(() => reject(new Error('native ABI timeout')), 5000) }; const h = Buffer.alloc(4); h.writeUInt32LE(b.length); this.child.stdin.write(h); this.child.stdin.write(b); this.pump(); }); } + async close() { this.child.stdin.end(); if (this.child.exitCode === null) await new Promise(resolve => this.child.once('exit', resolve)); } +} + +test('R1 R2 R4 R14 R16 R17 hosted facade uses committed Go policy and native-identical application bytes', { timeout: 30000 }, async () => { + const reactor = new Reactor(new WebAssembly.Module(readFileSync(out + '/core.wasm'))), native = new Native(); + const commands = [], calls = [], hostStores = [], actors = [], queue = [], reads = []; + const cluster = 'directory-p09', keys = Object.fromEntries(['a','b'].map(id => [id, generateKeyPairSync('ed25519')])), sequences = { a: 0, b: 0 }; + const initial = [{ namespace: 'state', key: 'identity', data: bytes({ V: 1, identity: '' }) }]; + for (const id of ['a','b']) initial.push({ namespace: 'enrollment', key: 'actor/' + segment(id), data: bytes({ cluster, key: keys[id].publicKey.export({ format: 'der', type: 'spki' }).subarray(-32).toString('base64'), personal: true }) }); + const size = writes => writes.reduce((n, r) => n + 12 + enc.encode(r.namespace).length + enc.encode(r.key).length + r.data.length, 0); + const evaluate = directoryEvaluator(reactor, cluster, 8 * 1024 * 1024); + const results = [[],[],[]]; + const accept = (id, effects) => { queue.push(...effects.send); reads.push(...effects.reads); }; + const pump = async blocked => { for (let count = 0; queue.length; count++) { assert(count < 2000); const m = queue.shift(); if (blocked) continue; accept(Number(m.to), await actors[Number(m.to) - 1].input(2, m.bytes)); } }; + const leader = () => actors.findIndex(a => a.status.role === 2) + 1; + const command = (actor, change, now = 1000) => { + const sequence = ++sequences[actor], c = { schema: 1, cluster, actor, session: 'api', sequence: String(sequence), request_digest: '', now, change, signature: null }; + if (sequence > 1) c.ack = String(sequence - 1); + c.request_digest = sha(bytes({ cluster, actor, session: c.session, sequence: c.sequence, change })); + c.signature = sign(null, Buffer.concat([Buffer.from('codeaf.directory.command.v1\0'), bytes(c)]), keys[actor].privateKey).toString('base64'); + return bytes(c); + }; + try { + for (let i = 0; i < 3; i++) { + const host = new Host(); hostStores.push(host); + // P11 owns genesis import. This test-only genesis is supplied at the + // first committed application entry, identically on all three stores. + const evaluator = (raw, read, app) => { + let base = []; + if (!read('state', 'identity')) base = initial; + const local = (ns, key) => base.find(r => r.namespace === ns && r.key === key)?.data ?? read(ns, key); + const traced = { call(b) { const result = reactor.call(b); calls.push([b.slice(), result.slice()]); return result; } }; + const result = directoryEvaluator(traced, cluster, 8 * 1024 * 1024)(raw, local, { ...app, size: app.size + size(base) }); + results[i].push(decodeCanonical(dec.decode(result.result))); + return { ...result, writes: [...base, ...result.writes] }; + }; + const actor = new MemberActor(host, reactor, evaluator); actors.push(actor); + accept(i + 1, await actor.init({ cluster, id: String(i + 1), seed: String(i + 19), bootstrap: ['1','2','3'] })); + } + await pump(); for (let n = 0; n < 20 && !leader(); n++) { for (let i = 0; i < 3; i++) accept(i + 1, await actors[i].input(1)); await pump(); } + assert(leader()); + let blocked = false; + const facade = new RaftDirectoryFacade({ + async commit(raw) { const id = leader(), before = results[id - 1].length; commands.push(raw); accept(id, await actors[id - 1].input(3, raw)); await pump(blocked); if (results[id - 1].length === before) throw new Error('unavailable'); return results[id - 1].at(-1); }, + async barrierRead(id) { const l = leader(); accept(l, await actors[l - 1].input(4, id)); await pump(blocked); if (!reads.find(r => r.id === id)) throw new Error('unavailable'); return actors[l - 1].store.read('state', 'cell/' + segment('chat')); }, + }); + let r = await facade.command(command('a', { create: { cell: 'chat', init: { head: 'sealed-one', class: 'chat', size: '1', keys: {} }, receipt: '' } })); assert.equal(r.code, 'ok'); assert.equal(r.view.cell.lease.fence, '1'); + const take = command('b', { acquire: { cell: 'chat', force: true, expect: { device: 'a', fence: '1' } } }); + r = await facade.command(take); assert.equal(r.code, 'ok'); assert.equal(r.view.cell.lease.fence, '2'); + assert.deepEqual(await facade.command(take), r, 'exact retry'); + r = await facade.command(command('a', { publish: { cell: 'chat', publish: { fence: '1', old_head: 'wrong', head: 'bad', size: '0', class: '', pending: 0 }, receipt: '' } })); assert.equal(r.code, 'fence_stale'); + r = await facade.command(command('a', { set_vault: { old: '', new: 'sealed-vault' } })); assert.equal(r.code, 'ok'); + r = await facade.command(command('b', { set_vault: { old: '', new: 'wrong' } })); assert.equal(r.code, 'cas'); + const delayed = command('b', { release: { cell: 'chat', fence: '2', quiet: false } }); + r = await facade.command(command('a', { revoke: { device: 'b' } })); assert.equal(r.code, 'not_found'); + // Enrollment and device metadata are distinct; the device's first metadata + // write preserves relay-owned fields before revocation is committed. + r = await facade.command(command('a', { put_device: { id: 'b', device: { V: 1, name: 'c2VhbGVk', added_by: '', revoked: false, caps: { os: '', arch: '', sandbox: null, container: null, gpu: null, cow: '' } } } })); assert.equal(r.code, 'ok'); + r = await facade.command(command('a', { revoke: { device: 'b' } })); assert.equal(r.code, 'ok'); + r = await facade.command(delayed); assert.equal(r.code, 'revoked'); + const row = await facade.read('fresh'); assert.equal(decodeCanonical(dec.decode(row)).head, 'sealed-one'); + for (const actor of actors) assert.deepEqual(actor.store.read('state', 'cell/' + segment('chat')), row); + blocked = true; await assert.rejects(facade.read('minority'), /unavailable/); + // Replay every complete application ABI call against the exact native Go + // binary. This includes dependency requests, signatures and refusal bytes. + for (const [input, expected] of calls) assert.deepEqual(await native.call(input), expected); + assert(calls.length > 50); assert.equal(reactor.metrics().wasi.poll_oneoff ?? 0, 0); + } finally { await native.close(); for (const h of hostStores) h.db.close(); } +}); From a882e9de0dab6a2f49cf7d0f2b6a5e48527baf5e Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 01:12:29 -0400 Subject: [PATCH 022/103] Require committed upload evidence and install pairing authority before delivering grants. Keep effect signing separate from personal authority, bind approvals to authenticated request keys, and retain exact outbox outcomes across crashes. Preserve original v1 upload ordering while requiring evidence for v2 publication. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/blobstore/receipt.go | 128 ++++++++++ internal/blobstore/receipt_test.go | 55 ++++ internal/clusterstate/apply.go | 82 +++++- internal/clusterstate/apply_test.go | 31 ++- internal/clusterstate/effects.go | 166 ++++++++++++ internal/clusterstate/effects_test.go | 165 ++++++++++++ internal/clusterstate/imports_test.go | 2 +- internal/clusterstate/records.go | 7 +- internal/clusterstate/types.go | 11 +- internal/directory/linkeffects.go | 51 ++++ internal/directory/linkeffects_test.go | 49 ++++ internal/directory/links.go | 13 +- internal/directory/raft.go | 8 +- internal/directory/requests.go | 14 +- .../manual/chat/add-or-remove-a-computer.md | 8 + .../continuing-a-chat-on-another-computer.md | 8 + internal/raftdirectory/README.md | 50 +++- internal/raftdirectory/apply.go | 12 +- internal/raftdirectory/directory_test.go | 19 +- internal/raftdirectory/effects.go | 160 ++++++++++++ internal/raftdirectory/effects_test.go | 238 ++++++++++++++++++ internal/relayserve/blob_authority_test.go | 39 +++ internal/relayserve/namespaces.go | 54 ++++ 23 files changed, 1322 insertions(+), 48 deletions(-) create mode 100644 internal/blobstore/receipt.go create mode 100644 internal/blobstore/receipt_test.go create mode 100644 internal/clusterstate/effects.go create mode 100644 internal/clusterstate/effects_test.go create mode 100644 internal/directory/linkeffects.go create mode 100644 internal/directory/linkeffects_test.go create mode 100644 internal/raftdirectory/effects.go create mode 100644 internal/raftdirectory/effects_test.go create mode 100644 internal/relayserve/blob_authority_test.go diff --git a/internal/blobstore/receipt.go b/internal/blobstore/receipt.go new file mode 100644 index 0000000000..479ef3bed9 --- /dev/null +++ b/internal/blobstore/receipt.go @@ -0,0 +1,128 @@ +package blobstore + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" +) + +// CheckReferences reads the indexed head object and each declared frame before +// the relay attests an upload. The head's remote ID is supplied separately from +// its local turn ID: sealed object IDs are keyed and cannot be derived here. +// The encrypted closure remains the publishing client's responsibility; frame +// plans are incremental hints, so an empty plan never means an empty closure. +func CheckReferences(ctx context.Context, store Store, headRID string, frames []string) error { + if durable, ok := store.(interface { + ConfirmReferences(context.Context, string, []string) error + }); ok { + return durable.ConfirmReferences(ctx, headRID, frames) + } + locations, err := store.Locate(ctx, []string{headRID}) + if err != nil { + return err + } + head, ok := locations[headRID] + if !ok { + return ErrNotFound + } + return checkReferences(ctx, store, headRID, frames, head.Frame) +} +func checkReferences(ctx context.Context, store Store, headRID string, frames []string, headFrame string) error { + if _, err := store.Get(ctx, headRID); err != nil { + return err + } + seen := map[string]bool{} + for _, id := range append([]string{headFrame}, frames...) { + if err := ctx.Err(); err != nil { + return err + } + if seen[id] { + continue + } + seen[id] = true + b, err := store.GetFrame(ctx, id) + if err != nil { + return err + } + if IDOf(b) != id { + return ErrDamaged + } + _, objects, err := Decode(b) + if err != nil { + return err + } + for _, o := range objects { + have, err := store.Get(ctx, o.RID) + if err != nil { + return err + } + if !bytes.Equal(have, o.Bytes) { + return fmt.Errorf("%w: indexed upload", ErrDamaged) + } + } + } + return nil +} + +// ConfirmReferences repairs a lost directory-flush completion before issuing a +// receipt. File contents precede every rename; syncing all ancestor directories +// also covers freshly created object shards and identity namespace directories. +func (d *Disk) ConfirmReferences(ctx context.Context, headRID string, frames []string) error { + d.mu.Lock() + defer d.mu.Unlock() + if err := checkGet(headRID); err != nil { + return err + } + line, err := os.ReadFile(d.pointerPath(headRID)) + if errors.Is(err, os.ErrNotExist) { + return ErrNotFound + } + if err != nil { + return err + } + location, err := parsePointer(line) + if err != nil { + return err + } + if err := checkReferences(ctx, d, headRID, frames, location.Frame); err != nil { + return err + } + paths := []string{d.frames, d.objects, filepath.Dir(d.pointerPath(headRID))} + // Object paths have only 256 possible shards. Sync those directories + // directly instead of retaining one duplicate path per referenced object. + shards, err := os.ReadDir(d.objects) + if err != nil { + return err + } + for _, shard := range shards { + if shard.IsDir() && isLowerHex(shard.Name(), 2) { + paths = append(paths, filepath.Join(d.objects, shard.Name())) + } + } + seen := map[string]bool{} + var dirs []*os.File + defer func() { + for _, f := range dirs { + _ = f.Close() + } + }() + for _, path := range paths { + for !seen[path] { + seen[path] = true + f, err := os.Open(path) + if err != nil { + return err + } + dirs = append(dirs, f) + parent := filepath.Dir(path) + if parent == path { + break + } + path = parent + } + } + return d.sync(dirs) +} diff --git a/internal/blobstore/receipt_test.go b/internal/blobstore/receipt_test.go new file mode 100644 index 0000000000..d9e430a701 --- /dev/null +++ b/internal/blobstore/receipt_test.go @@ -0,0 +1,55 @@ +package blobstore + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" +) + +// R21 requires durable bytes AND a retrievable index before a head receipt. +func TestR21ReceiptRefusesEveryIncompleteUploadBoundary(t *testing.T) { + ctx := context.Background() + for _, cut := range []string{"frame-sync", "after-frame", "pointer-sync", "directory-sync"} { + t.Run(cut, func(t *testing.T) { + d, root, frame, o := newCrashDisk(t) + calls := 0 + failure := errors.New("cut") + actual := d.sync + d.sync = func(files []*os.File) error { + calls++ + if cut == "frame-sync" && calls == 1 || cut == "pointer-sync" && calls == 3 || cut == "directory-sync" && calls == 4 { + return failure + } + return actual(files) + } + if cut == "after-frame" { + d.afterFrame = func() error { return failure } + } + if _, e := d.PutFrame(ctx, frame); e == nil { + t.Fatal("fault did not fire") + } + // An indexed-but-unflushed pointer is repaired only by a successful + // durability completion, never by observing that its pathname exists. + d.sync = func([]*os.File) error { return failure } + if e := CheckReferences(ctx, d, o.RID, []string{IDOf(frame)}); e == nil { + t.Fatal("incomplete upload produced a receipt") + } + d.sync = actual + d.afterFrame = func() error { return nil } + if _, e := d.PutFrame(ctx, frame); e != nil { + t.Fatal(e) + } + if e := CheckReferences(ctx, d, o.RID, []string{IDOf(frame)}); e != nil { + t.Fatal(e) + } + if e := os.WriteFile(filepath.Join(root, "frames", IDOf(frame)), append(append([]byte(nil), frame[:len(frame)-1]...), frame[len(frame)-1]^1), 0600); e != nil { + t.Fatal(e) + } + if e := CheckReferences(ctx, d, o.RID, nil); !errors.Is(e, ErrDamaged) { + t.Fatal(e) + } + }) + } +} diff --git a/internal/clusterstate/apply.go b/internal/clusterstate/apply.go index 251d43066c..873010be55 100644 --- a/internal/clusterstate/apply.go +++ b/internal/clusterstate/apply.go @@ -47,11 +47,13 @@ func SessionsKey() RecordKey { return RecordKey{"sessions", "counts"} } func JoinKey(code string) RecordKey { return RecordKey{"outbox", "join/" + segment(code)} } // Authority is installed by authenticated genesis/enrollment, never PutDevice. -// Scope is deliberately personal-only until delegated workers land in P15. +// Personal directory authority and restricted effect authority are exclusive. +// Delegated worker scopes are admitted separately. type Authority struct { Cluster string `json:"cluster"` Key []byte `json:"key"` Personal bool `json:"personal"` + Effects bool `json:"effects,omitempty"` } type session struct { Sequence uint64 `json:"sequence"` @@ -64,12 +66,15 @@ type sessionCounts struct { Retained uint32 `json:"retained"` } -// JoinRequest is a committed external-effect dependency, provisioned by P10. +// JoinRequest is a committed dependency from a reserved public request. // An absent dependency cannot be manufactured by a decision request. type JoinRequest struct { - Device string `json:"device"` - Expires int64 `json:"expires"` - Decision *JoinDecision `json:"decision,omitempty"` + Device string `json:"device"` + Expires int64 `json:"expires"` + Platform string `json:"platform,omitempty"` + PublicKey []byte `json:"public_key,omitempty"` + Decision *JoinDecision `json:"decision,omitempty"` + Outcome string `json:"outcome,omitempty"` } // ApplicationMeta is an immutable view at one applied index. Byte accounting @@ -139,8 +144,23 @@ func Dependencies(c Command) []RecordKey { if m.DeviceSeen != nil { keys = append(keys, DeviceKey(m.DeviceSeen.Device)) } + if m.RecordUpload != nil { + keys = append(keys, UploadKey(UploadID(*m.RecordUpload)), DeviceKey(m.RecordUpload.Device), AuthorityKey(m.RecordUpload.Device)) + } + if m.Create != nil { + keys = append(keys, UploadKey(m.Create.Receipt)) + } + if m.Publish != nil { + keys = append(keys, UploadKey(m.Publish.Receipt)) + } + if m.AdmitJoin != nil { + keys = append(keys, JoinKey(m.AdmitJoin.Code)) + } + if m.CompleteJoin != nil { + keys = append(keys, JoinKey(m.CompleteJoin.Code)) + } if m.DecideJoin != nil { - keys = append(keys, JoinKey(m.DecideJoin.Code), DeviceKey(m.DecideJoin.Device)) + keys = append(keys, JoinKey(m.DecideJoin.Code), DeviceKey(m.DecideJoin.Device), AuthorityKey(m.DecideJoin.Device)) } for _, id := range cellIDs(m) { keys = append(keys, CellKey(id)) @@ -232,6 +252,8 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er target = new(sessionCounts) case k.Namespace == "sessions": target = new(session) + case k.Namespace == "outbox" && len(k.Key) >= 7 && k.Key[:7] == "upload/": + target = new(UploadReceipt) case k.Namespace == "outbox": target = new(JoinRequest) default: @@ -262,7 +284,7 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er } digest, _ := Digest(c) signed, e := SigningBytes(c) - if e != nil || c.Cluster != meta.Cluster || auth.Cluster != c.Cluster || !auth.Personal || len(auth.Key) != ed25519.PublicKeySize || !ed25519.Verify(auth.Key, signed, c.Signature) || digest != c.RequestDigest { + if e != nil || c.Cluster != meta.Cluster || auth.Cluster != c.Cluster || (!auth.Personal && !auth.Effects) || len(auth.Key) != ed25519.PublicKeySize || !ed25519.Verify(auth.Key, signed, c.Signature) || digest != c.RequestDigest { return out, nil } // Revocation precedes dedup: a saved old response cannot authorize a revoked @@ -271,6 +293,10 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er out.Result.Code = Revoked return out, nil } + service := c.Change.RecordUpload != nil || c.Change.AdmitJoin != nil || c.Change.CompleteJoin != nil + if service != auth.Effects || auth.Personal && auth.Effects { + return out, nil + } var last session var counts sessionCounts if decode(SessionKey(c.Actor, c.Session), &last) != nil || decode(SessionsKey(), &counts) != nil { @@ -285,10 +311,10 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er out.Result = last.Result return out, nil } - if !known && (c.Sequence != 1 || c.Ack != 0) || known && (last.Sequence == ^uint64(0) || c.Sequence != last.Sequence+1 || c.Ack != last.Sequence) { + if !service && (!known && (c.Sequence != 1 || c.Ack != 0) || known && (last.Sequence == ^uint64(0) || c.Sequence != last.Sequence+1 || c.Ack != last.Sequence)) { return refuse(Conflict) } - if !known && (counts.Live >= MaxLiveSessions || counts.Retained >= MaxRetainedSessions) { + if !service && !known && (counts.Live >= MaxLiveSessions || counts.Retained >= MaxRetainedSessions) { return refuse(Capacity) } if c.Now < 0 || c.Now > int64(MaxSafeInteger)-LeaseTTLMillis { @@ -306,10 +332,14 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er writes[k] = b } code := OK - if identity.Rotation != nil && c.Change.Rotate == nil && c.Change.AdvanceTime == nil && c.Change.DeviceSeen == nil && c.Change.CloseSession == nil { + if identity.Rotation != nil && c.Change.Rotate == nil && c.Change.AdvanceTime == nil && c.Change.DeviceSeen == nil && c.Change.CloseSession == nil && c.Change.CompleteJoin == nil { code = Rotated } else { - code = applyMutation(c, out.Now, identity, rows, put, &out.Result) + if service { + code = applyEffect(c, out.Now, rows, put) + } else { + code = applyMutation(c, out.Now, identity, rows, put, &out.Result) + } } if err != nil { return out, err @@ -344,6 +374,9 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er counts.Live-- } save := func() { + if service { + return + } put(SessionKey(c.Actor, c.Session), session{Sequence: c.Sequence, Digest: c.RequestDigest, Closed: c.Change.CloseSession != nil, Result: out.Result}) put(SessionsKey(), counts) } @@ -505,7 +538,26 @@ func applyMutation(c Command, now int64, identity IdentityRec, rows map[RecordKe } if !v.Deny { old, had := device(v.Device) - put(DeviceKey(v.Device), Stamped(old, had, v.Record, now)) + if old.Revoked { + return Revoked + } + public, e := JoinCertificate(identity, request, v.Cert) + if e != nil || validGrant(v.Grant) == false { + return BadRequest + } + var prior Authority + if b := rows[AuthorityKey(v.Device)]; b != nil { + if Decode(b, &prior) != nil || prior.Cluster != c.Cluster || !prior.Personal || !bytes.Equal(prior.Key, public) { + return Unauthorized + } + } + put(AuthorityKey(v.Device), Authority{Cluster: c.Cluster, Key: public, Personal: true}) + record := v.Record + if record.Platform == "" { + record.Platform = request.Platform + } + record.AddedBy = identity.Identity + put(DeviceKey(v.Device), Stamped(old, had, record, now)) } request.Decision = v put(JoinKey(v.Code), request) @@ -538,6 +590,9 @@ func applyMutation(c Command, now int64, identity IdentityRec, rows map[RecordKe if b != nil { return Exists } + if !m.Create.V1 && !validReceipt(rows, m.Create.Receipt, c.Actor, m.Create.Init.Head, m.Create.Init.Frames) { + return BadRequest + } cell = Created(m.Create.Init, c.Actor, now) } else { if b == nil { @@ -562,6 +617,9 @@ func applyMutation(c Command, now int64, identity IdentityRec, rows map[RecordKe cell, err = Heartbeat(cell, c.Actor, m.Beat.Beat, now) case m.Publish != nil: cell, err = PublishTo(cell, c.Actor, m.Publish.Publish, now) + if err == nil && !m.Publish.V1 && !validReceipt(rows, m.Publish.Receipt, c.Actor, m.Publish.Publish.Head, m.Publish.Publish.Frames) { + return BadRequest + } case m.Release != nil: at := now if m.Release.Quiet { diff --git a/internal/clusterstate/apply_test.go b/internal/clusterstate/apply_test.go index 94f8030e88..4385988ebb 100644 --- a/internal/clusterstate/apply_test.go +++ b/internal/clusterstate/apply_test.go @@ -32,7 +32,7 @@ func newApplication(t *testing.T) *application { a.put(cs.AuthorityKey(id), cs.Authority{Cluster: "test", Key: actorKey(id).Public().(ed25519.PublicKey), Personal: true}) a.put(cs.DeviceKey(id), cs.Device{V: 1}) } - a.put(cs.IdentityKey(), cs.IdentityRec{V: 1}) + a.put(cs.IdentityKey(), cs.IdentityRec{V: 1, Identity: cs.IdentityID(actorKey("root").Public().(ed25519.PublicKey)), SigningKey: actorKey("root").Public().(ed25519.PublicKey)}) return a } func (a *application) put(k cs.RecordKey, v any) { @@ -43,6 +43,21 @@ func (a *application) put(k cs.RecordKey, v any) { a.rows[k] = b } func (a *application) prepare(actor string, now int64, m cs.Mutation) cs.Command { + // The independent ownership campaign assumes completed uploads. Supply the + // corresponding committed receipt fixture; missing uploads have separate R21 vectors. + if m.Create != nil && m.Create.Receipt == "" { + r := cs.UploadReceipt{Device: actor, Head: m.Create.Init.Head, HeadRID: "fixture", Frames: m.Create.Init.Frames} + m.Create.Receipt = cs.UploadID(r) + a.put(cs.UploadKey(m.Create.Receipt), r) + } + if m.Publish != nil && m.Publish.Receipt == "" { + r := cs.UploadReceipt{Device: actor, Head: m.Publish.Publish.Head, HeadRID: "fixture", Frames: m.Publish.Publish.Frames} + m.Publish.Receipt = cs.UploadID(r) + a.put(cs.UploadKey(m.Publish.Receipt), r) + } + return a.prepareBare(actor, now, m) +} +func (a *application) prepareBare(actor string, now int64, m cs.Mutation) cs.Command { a.seq[actor]++ seq := a.seq[actor] c, e := cs.Sign(cs.Command{Schema: 1, Cluster: "test", Actor: actor, Session: "test", Sequence: seq, Ack: seq - 1, Now: now, Change: m}, actorKey(actor)) @@ -325,8 +340,10 @@ func TestR16R17R27DependencyAndSessionAdmissionFailures(t *testing.T) { func TestR16R30CommittedPairDecisionIsAtomicAndFirstWins(t *testing.T) { a := newApplication(t) - a.put(cs.JoinKey("request"), cs.JoinRequest{Device: "new", Expires: 10000}) - decision := cs.JoinDecision{Code: "request", Device: "new", Record: cs.Device{V: 1, Name: "sealed"}, Grant: "sealed-grant", Cert: "public-cert"} + public := actorKey("new").Public().(ed25519.PublicKey) + newID := cs.DeviceIDOfKey(public) + a.put(cs.JoinKey("request"), cs.JoinRequest{Device: newID, PublicKey: public, Expires: 10000}) + decision := cs.JoinDecision{Code: "request", Device: newID, Record: cs.Device{V: 1, Name: "sealed"}, Grant: "c2VhbGVkLWdyYW50", Cert: joinCert(t, "new")} if r := a.run("a", 1000, cs.Mutation{DecideJoin: &decision}); r.Code != cs.OK { t.Fatal(r) } @@ -335,10 +352,10 @@ func TestR16R30CommittedPairDecisionIsAtomicAndFirstWins(t *testing.T) { t.Fatal(e) } var device cs.Device - if e := cs.Decode(a.rows[cs.DeviceKey("new")], &device); e != nil { + if e := cs.Decode(a.rows[cs.DeviceKey(newID)], &device); e != nil { t.Fatal(e) } - if request.Decision == nil || request.Decision.Grant != "sealed-grant" || device.Created != 1000 { + if request.Decision == nil || request.Decision.Grant != "c2VhbGVkLWdyYW50" || device.Created != 1000 { t.Fatal(request, device) } version := a.meta.Version @@ -348,7 +365,7 @@ func TestR16R30CommittedPairDecisionIsAtomicAndFirstWins(t *testing.T) { if r := a.run("b", 1002, cs.Mutation{DecideJoin: &cs.JoinDecision{Code: "request", Deny: true}}); r.Code != cs.AlreadyDecided { t.Fatal(r) } - if a.rows[cs.AuthorityKey("new")] != nil { - t.Fatal("pair decision bypassed P10 certificate admission") + if a.rows[cs.AuthorityKey(newID)] == nil { + t.Fatal("pair decision omitted committed certificate authorization") } } diff --git a/internal/clusterstate/effects.go b/internal/clusterstate/effects.go new file mode 100644 index 0000000000..42baa8a07b --- /dev/null +++ b/internal/clusterstate/effects.go @@ -0,0 +1,166 @@ +package clusterstate + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "reflect" +) + +const ( + MaxUploadFrames = 512 + MaxJoinGrant = 4096 +) + +// UploadReceipt names only durable references. The host checks the encrypted +// store before proposing it; neither a personal device nor a worker can mint it. +// Frames remain the existing incremental hint, not a plaintext closure manifest. +type UploadReceipt struct { + Device string `json:"device"` + Head string `json:"head"` + HeadRID string `json:"head_rid"` + Frames []string `json:"frames,omitempty"` +} +type AdmitJoin struct { + Code string `json:"code"` + PublicKey []byte `json:"public_key"` + Expires int64 `json:"expires"` + Platform string `json:"platform,omitempty"` +} +type CompleteJoin struct { + Code string `json:"code"` + Decision string `json:"decision"` + Outcome string `json:"outcome"` +} + +func UploadKey(id string) RecordKey { return RecordKey{"outbox", "upload/" + segment(id)} } +func UploadID(r UploadReceipt) string { + b, _ := Encode(r) + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]) +} +func DecisionID(d JoinDecision) string { + b, _ := Encode(d) + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]) +} +func DeviceIDOfKey(public []byte) string { + h := sha256.Sum256(public) + return "dev_" + hex.EncodeToString(h[:16]) +} +func IdentityID(public []byte) string { + h := sha256.Sum256(public) + return "id_" + hex.EncodeToString(h[:16]) +} + +func validReceipt(rows map[RecordKey][]byte, id, actor, head string, frames []string) bool { + var r UploadReceipt + return id != "" && Decode(rows[UploadKey(id)], &r) == nil && UploadID(r) == id && r.Device == actor && r.Head == head && reflect.DeepEqual(normalFrames(r.Frames), normalFrames(frames)) +} +func normalFrames(f []string) []string { + if len(f) == 0 { + return nil + } + return f +} + +// JoinCertificate verifies the original identity certificate at the committed +// decision, rather than trusting ingress that may have raced revocation. +func JoinCertificate(identity IdentityRec, request JoinRequest, text string) ([]byte, error) { + var cert struct { + V uint16 `json:"V"` + Device string `json:"device"` + Created int64 `json:"created"` + Sig string `json:"sig"` + } + raw, e := base64.RawURLEncoding.DecodeString(text) + if e != nil || len(raw) > 1024 || json.Unmarshal(raw, &cert) != nil || cert.V != 1 || cert.Created < 0 || cert.Created > int64(MaxSafeInteger) { + return nil, ErrCodec + } + public, e := hex.DecodeString(cert.Device) + sig, se := hex.DecodeString(cert.Sig) + if e != nil || se != nil || len(public) != ed25519.PublicKeySize || !bytes.Equal(public, request.PublicKey) || DeviceIDOfKey(public) != request.Device || len(identity.SigningKey) != ed25519.PublicKeySize || IdentityID(identity.SigningKey) != identity.Identity { + return nil, ErrCodec + } + cert.Sig = "" + body, _ := json.Marshal(cert) + if !ed25519.Verify(identity.SigningKey, append([]byte("codeaf/device-cert/v1\n"), body...), sig) { + return nil, ErrCodec + } + return public, nil +} +func validGrant(text string) bool { + if len(text) > base64.RawURLEncoding.EncodedLen(MaxJoinGrant) { + return false + } + b, e := base64.RawURLEncoding.DecodeString(text) + return e == nil && len(b) > 0 && len(b) <= MaxJoinGrant +} + +// Effects use content-addressed, immutable keys rather than expiring request +// sessions. Replaying delivery after a lost acknowledgement changes no grant. +func applyEffect(c Command, now int64, rows map[RecordKey][]byte, put func(RecordKey, any)) Code { + switch { + case c.Change.RecordUpload != nil: + r := *c.Change.RecordUpload + if r.Device == "" || r.Head == "" || r.HeadRID == "" || len(r.Frames) > MaxUploadFrames { + return BadRequest + } + var d Device + var auth Authority + if b := rows[DeviceKey(r.Device)]; b != nil { + if Decode(b, &d) != nil { + return BadRequest + } + } + if d.Revoked { + return Revoked + } + if Decode(rows[AuthorityKey(r.Device)], &auth) != nil || !auth.Personal || auth.Cluster != c.Cluster { + return Unauthorized + } + k := UploadKey(UploadID(r)) + if rows[k] == nil { + put(k, r) + } + return OK + case c.Change.AdmitJoin != nil: + v := c.Change.AdmitJoin + if v.Code == "" || len(v.PublicKey) != ed25519.PublicKeySize || v.Expires <= now || v.Expires > int64(MaxSafeInteger) { + return BadRequest + } + k := JoinKey(v.Code) + if raw := rows[k]; raw != nil { + var r JoinRequest + if Decode(raw, &r) != nil { + return BadRequest + } + if bytes.Equal(r.PublicKey, v.PublicKey) && r.Expires == v.Expires && r.Platform == PlatformOf(v.Platform) { + return OK + } + return Conflict + } + put(k, JoinRequest{Device: DeviceIDOfKey(v.PublicKey), PublicKey: v.PublicKey, Expires: v.Expires, Platform: PlatformOf(v.Platform)}) + return OK + case c.Change.CompleteJoin != nil: + v := c.Change.CompleteJoin + k := JoinKey(v.Code) + var r JoinRequest + if Decode(rows[k], &r) != nil || r.Decision == nil { + return RequestGone + } + if DecisionID(*r.Decision) != v.Decision || v.Outcome != "delivered" && v.Outcome != "gone" { + return Conflict + } + if r.Outcome != "" && r.Outcome != v.Outcome { + return Conflict + } + r.Outcome = v.Outcome + put(k, r) + return OK + } + return Unauthorized +} diff --git a/internal/clusterstate/effects_test.go b/internal/clusterstate/effects_test.go new file mode 100644 index 0000000000..2623761f7d --- /dev/null +++ b/internal/clusterstate/effects_test.go @@ -0,0 +1,165 @@ +package clusterstate_test + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "math/rand" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +func joinCert(t *testing.T, actor string) string { + t.Helper() + c := struct { + V uint16 `json:"V"` + Device string `json:"device"` + Created int64 `json:"created"` + Sig string `json:"sig"` + }{1, hex.EncodeToString(actorKey(actor).Public().(ed25519.PublicKey)), 1, ""} + b, _ := json.Marshal(c) + c.Sig = hex.EncodeToString(ed25519.Sign(actorKey("root"), append([]byte("codeaf/device-cert/v1\n"), b...))) + b, _ = json.Marshal(c) + return base64.RawURLEncoding.EncodeToString(b) +} +func service(a *application, now int64, m cs.Mutation) cs.Result { + a.put(cs.AuthorityKey("effects"), cs.Authority{Cluster: "test", Key: actorKey("effects").Public().(ed25519.PublicKey), Effects: true}) + c, e := cs.Sign(cs.Command{Schema: 1, Cluster: "test", Actor: "effects", Session: "effects", Sequence: 1, Now: now, Change: m}, actorKey("effects")) + if e != nil { + a.t.Fatal(e) + } + return a.evaluate(c).Result +} +func TestR21UploadReceiptBeforeHeadAndFenceBeforeReceipt(t *testing.T) { + a := newApplication(t) + create := cs.Mutation{Create: &cs.CreateCell{Cell: "chat", Init: cs.CellInit{Head: "sealed"}}} + if r := a.evaluate(a.prepareBare("a", 1000, create)).Result; r.Code != cs.BadRequest || a.rows[cs.CellKey("chat")] != nil { + t.Fatal(r) + } + upload := cs.UploadReceipt{Device: "a", Head: "sealed", HeadRID: "remote", Frames: []string{"frame"}} + if r := a.run("a", 1001, cs.Mutation{RecordUpload: &upload}); r.Code != cs.Unauthorized { + t.Fatal("personal caller minted upload", r) + } + a.seq["a"]-- // An unauthorized request did not consume its directory sequence. + if r := service(a, 1002, cs.Mutation{RecordUpload: &upload}); r.Code != cs.OK { + t.Fatal(r) + } + create.Create.Receipt = cs.UploadID(upload) + if r := a.evaluate(a.prepareBare("a", 1003, create)).Result; r.Code != cs.BadRequest { + t.Fatal("receipt accepted a changed plan", r) + } + create.Create.Init.Frames = upload.Frames + if r := a.evaluate(a.prepareBare("b", 1004, create)).Result; r.Code != cs.BadRequest { + t.Fatal("cross-device receipt", r) + } + if r := a.evaluate(a.prepareBare("a", 1005, create)).Result; r.Code != cs.OK { + t.Fatal(r) + } + p := cs.Mutation{Publish: &cs.PublishCell{Cell: "chat", Publish: cs.Publish{Fence: 1, OldHead: "sealed", Head: "next"}}} + if r := a.evaluate(a.prepareBare("a", 1006, p)).Result; r.Code != cs.BadRequest || a.cell().Head != "sealed" { + t.Fatal(r) + } + p.Publish.Publish.Fence = 0 + p.Publish.Publish.OldHead = "wrong" + if r := a.evaluate(a.prepareBare("a", 1007, p)).Result; r.Code != cs.FenceStale { + t.Fatal("receipt masked stale fence", r) + } + if r := service(a, 1008, cs.Mutation{Revoke: &cs.DeviceID{Device: "b"}}); r.Code != cs.Unauthorized { + t.Fatal("effect key acquired directory authority", r) + } +} + +// R16/R17/R21 enumerate authorization orderings with independently selected +// expected outcomes. Every seed honors the wave's 2,000-seed soak lever. +func TestR16R17R21EffectsAuthorizationCampaign(t *testing.T) { + for seed := 0; seed < seedCount(t); seed++ { + t.Run(fmt.Sprintf("seed-%04d", seed), func(t *testing.T) { + a := newApplication(t) + rng := rand.New(rand.NewSource(int64(seed))) + public := actorKey("joiner").Public().(ed25519.PublicKey) + id := cs.DeviceIDOfKey(public) + if r := service(a, 1000, cs.Mutation{AdmitJoin: &cs.AdmitJoin{Code: "request", PublicKey: public, Expires: 10000}}); r.Code != cs.OK { + t.Fatal(r) + } + decision := cs.JoinDecision{Code: "request", Device: id, Cert: joinCert(t, "joiner"), Grant: "c2VhbGVk", Record: cs.Device{V: 1, Name: "c2VhbGVk"}} + c := a.prepareBare("b", 1001, cs.Mutation{DecideJoin: &decision}) + revoke := rng.Intn(2) == 0 + rotate := rng.Intn(2) == 0 + if revoke { + a.run("a", 1002, cs.Mutation{Revoke: &cs.DeviceID{Device: "b"}}) + } + if rotate { + a.run("a", 1003, cs.Mutation{Rotate: &cs.RotateRequest{Op: "freeze"}}) + } + want := cs.OK + if rotate { + want = cs.Rotated + } + if revoke { + want = cs.Revoked + } + r := a.evaluate(c).Result + if r.Code != want { + t.Fatal(seed, r.Code, want) + } + if (a.rows[cs.AuthorityKey(id)] != nil) != (want == cs.OK) { + t.Fatal("authorization escaped committed decision") + } + if want == cs.OK { + retry := a.evaluate(c).Result + if fmt.Sprint(retry) != fmt.Sprint(r) { + t.Fatal("decision retry changed") + } + wrong := decision + wrong.Grant = "b3RoZXI" + if r := a.run("a", 1004, cs.Mutation{DecideJoin: &wrong}); r.Code != cs.AlreadyDecided { + t.Fatal(r) + } + done := cs.CompleteJoin{Code: "request", Decision: cs.DecisionID(decision), Outcome: "delivered"} + if r := service(a, 1005, cs.Mutation{CompleteJoin: &done}); r.Code != cs.OK { + t.Fatal(r) + } + if r := service(a, 1006, cs.Mutation{CompleteJoin: &done}); r.Code != cs.OK { + t.Fatal(r) + } + } + }) + } +} +func TestR16PairingCertificateAndStickyRevocation(t *testing.T) { + a := newApplication(t) + pub := actorKey("joiner").Public().(ed25519.PublicKey) + id := cs.DeviceIDOfKey(pub) + service(a, 1000, cs.Mutation{AdmitJoin: &cs.AdmitJoin{Code: "request", PublicKey: pub, Expires: 10000}}) + d := cs.JoinDecision{Code: "request", Device: id, Cert: joinCert(t, "different"), Grant: "c2VhbGVk"} + if r := a.run("a", 1001, cs.Mutation{DecideJoin: &d}); r.Code != cs.BadRequest { + t.Fatal(r) + } + d.Cert = joinCert(t, "joiner") + a.put(cs.DeviceKey(id), cs.Device{V: 1, Revoked: true}) + if r := a.run("a", 1002, cs.Mutation{DecideJoin: &d}); r.Code != cs.Revoked || a.rows[cs.AuthorityKey(id)] != nil { + t.Fatal(r) + } +} + +// R25 preserves real old-client requests that cannot name an encrypted head +// RID, while R21 requires evidence at the distinct v2 command door. Ingress +// supplies protocol provenance; P17 must never copy compatibility flags from v2. +func TestR21R25LegacyProtocolKeepsExistingUploadOrdering(t *testing.T) { + a := newApplication(t) + m := cs.Mutation{Create: &cs.CreateCell{Cell: "chat", Init: cs.CellInit{Head: "legacy"}, V1: true}} + if r := a.evaluate(a.prepareBare("a", 1000, m)).Result; r.Code != cs.OK { + t.Fatal(r) + } + p := cs.Mutation{Publish: &cs.PublishCell{Cell: "chat", Publish: cs.Publish{Fence: 1, OldHead: "legacy", Head: "legacy-next"}}} + if r := a.evaluate(a.prepareBare("a", 1001, p)).Result; r.Code != cs.BadRequest { + t.Fatal("v2 publish accepted absent evidence", r) + } + p.Publish.V1 = true + if r := a.evaluate(a.prepareBare("a", 1002, p)).Result; r.Code != cs.OK { + t.Fatal("legacy request changed behaviour", r) + } +} diff --git a/internal/clusterstate/imports_test.go b/internal/clusterstate/imports_test.go index efcf92950d..e1115b04af 100644 --- a/internal/clusterstate/imports_test.go +++ b/internal/clusterstate/imports_test.go @@ -16,7 +16,7 @@ func TestR14PureImportBoundary(t *testing.T) { allowed := map[string]bool{ "bytes": true, "encoding/json": true, "errors": true, "fmt": true, // Signature verification, hashing, key encoding and ordering are pure. - "crypto/ed25519": true, "crypto/sha256": true, "encoding/hex": true, "sort": true, + "encoding/base64": true, "crypto/ed25519": true, "crypto/sha256": true, "encoding/hex": true, "sort": true, "reflect": true, "strconv": true, "strings": true, "unicode/utf8": true, } files, err := os.ReadDir(".") diff --git a/internal/clusterstate/records.go b/internal/clusterstate/records.go index e079b1392f..77737bf131 100644 --- a/internal/clusterstate/records.go +++ b/internal/clusterstate/records.go @@ -5,9 +5,10 @@ package clusterstate // IdentityRec is the per-identity root record. type IdentityRec struct { - V uint16 `json:"V"` - Identity string `json:"identity"` // id_…, derived from the signing key - Vault string `json:"vault,omitempty"` // rid of the current vault object + V uint16 `json:"V"` + Identity string `json:"identity"` // id_…, derived from the signing key + SigningKey []byte `json:"signing_key,omitempty"` // Public identity key for committed certificate checks. + Vault string `json:"vault,omitempty"` // rid of the current vault object // Rotation is set once the identity has been replaced by a rotation: it // takes no writes from then on. Nil means live. Rotation *Rotation `json:"rotation,omitempty"` diff --git a/internal/clusterstate/types.go b/internal/clusterstate/types.go index a469368074..a48c868724 100644 --- a/internal/clusterstate/types.go +++ b/internal/clusterstate/types.go @@ -44,6 +44,9 @@ type Command struct { // their typed operations here; unknown fields fail decoding on older members. // There is no arbitrary JSON payload in which task text or secrets can hide. type Mutation struct { + RecordUpload *UploadReceipt `json:"record_upload,omitempty"` + AdmitJoin *AdmitJoin `json:"admit_join,omitempty"` + CompleteJoin *CompleteJoin `json:"complete_join,omitempty"` Rotate *RotateRequest `json:"rotate,omitempty"` CloseSession *CloseSession `json:"close_session,omitempty"` DecideJoin *JoinDecision `json:"decide_join,omitempty"` @@ -69,7 +72,10 @@ type DeviceID struct { Device string `json:"device"` } +// V1 is protocol provenance supplied by authenticated v1 ingress; v2 request +// bodies must never select it. Old clients retain their upload ordering. type CreateCell struct { + V1 bool `json:"v1,omitempty"` Cell string `json:"cell"` Init CellInit `json:"init"` Receipt string `json:"receipt"` @@ -97,6 +103,7 @@ type BeatBatch struct { } type PublishCell struct { + V1 bool `json:"v1,omitempty"` Cell string `json:"cell"` Publish Publish `json:"publish"` Receipt string `json:"receipt"` @@ -186,8 +193,8 @@ type RotateRequest struct { GraceMS int64 `json:"grace_ms,omitempty"` } -// JoinDecision records only sealed pairing data. P10 supplies a committed request -// dependency and delivers the outbox after this transaction has committed. +// JoinDecision records only sealed pairing data. The committed request binds +// its certificate key; outbox delivery follows the authorization transaction. type JoinDecision struct { Code string `json:"code"` Device string `json:"device"` diff --git a/internal/directory/linkeffects.go b/internal/directory/linkeffects.go new file mode 100644 index 0000000000..3876dd8665 --- /dev/null +++ b/internal/directory/linkeffects.go @@ -0,0 +1,51 @@ +package directory + +// Bind reserves a temporary request for one cluster before it can enter that +// cluster's committed outbox. Legacy decisions cannot write a bound request. +// A failed admission leaves the reservation until expiry; it never permits two +// identities to acknowledge different decisions for the same public code. +func (l *Links) Bind(code, cluster string) (Request, error) { + l.mu.Lock() + defer l.mu.Unlock() + l.sweep(l.now()) + e := l.reqs[NormalizeCode(code)] + if e == nil { + return Request{}, ErrRequestGone + } + if cluster == "" || e.cluster != "" && e.cluster != cluster || e.cluster == "" && e.rec.State != RequestPending { + return Request{}, ErrAlreadyDecided + } + e.cluster = cluster + return e.rec, nil +} + +// Deliver is an internal outbox destination, not a public pairing decision. +// The caller has already read the exact committed decision through a barrier. +func (l *Links) Deliver(code, cluster, id string, d Decision) error { + l.mu.Lock() + defer l.mu.Unlock() + l.sweep(l.now()) + e := l.reqs[NormalizeCode(code)] + if e == nil { + return ErrRequestGone + } + if id == "" || cluster == "" || e.cluster != cluster || d.Device != e.rec.Device { + return ErrAlreadyDecided + } + if e.delivery != "" { + if e.delivery == id && (d.State == e.rec.State && d.sameBody(e.rec)) { + return nil + } + return ErrAlreadyDecided + } + if d.State != RequestApproved && d.State != RequestDenied || d.State == RequestApproved && (d.Device != e.rec.Device || d.Grant == "") { + return ErrBadRequest + } + if e.rec.State != RequestPending { + return ErrAlreadyDecided + } + e.rec, e.until = Decided(e.rec, d), l.now()+DecidedKeep.Milliseconds() + e.delivery = id + close(e.done) + return nil +} diff --git a/internal/directory/linkeffects_test.go b/internal/directory/linkeffects_test.go new file mode 100644 index 0000000000..cb2e4fc979 --- /dev/null +++ b/internal/directory/linkeffects_test.go @@ -0,0 +1,49 @@ +package directory_test + +import ( + "context" + "encoding/base64" + "errors" + "testing" + + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/directory/directorytest" +) + +func TestR16R17PairingReservationExcludesOtherNamespacesAndLegacyDecisions(t *testing.T) { + clock := directorytest.NewFakeClock() + links := directory.NewLinks(clock.Now, directory.DefaultLinkLimits) + b64 := base64.RawURLEncoding.EncodeToString + pub := make([]byte, 32) + pub[0] = 1 + opened, e := links.Open("fixture", directory.NewRequest{Pubkey: b64(pub), X25519: b64(pub), NameSealed: b64(make([]byte, 40))}) + if e != nil { + t.Fatal(e) + } + if _, e = links.Bind(opened.Code, "one"); e != nil { + t.Fatal(e) + } + if _, e = links.Bind(opened.Code, "two"); !errors.Is(e, directory.ErrAlreadyDecided) { + t.Fatal(e) + } + d := directory.Decision{State: directory.RequestApproved, Device: opened.Device, Grant: "c2VhbGVk"} + if e = links.Decide(opened.Code, d, func(directory.Request) error { t.Fatal("legacy writer ran"); return nil }); !errors.Is(e, directory.ErrAlreadyDecided) { + t.Fatal(e) + } + if e = links.Deliver(opened.Code, "two", "decision", d); !errors.Is(e, directory.ErrAlreadyDecided) { + t.Fatal(e) + } + r, e := links.Read(context.Background(), "fixture", opened.Code, 0) + if e != nil || r.State != directory.RequestPending { + t.Fatal(r, e) + } + for range 2 { + if e = links.Deliver(opened.Code, "one", "decision", d); e != nil { + t.Fatal(e) + } + } + d.Grant = "b3RoZXI" + if e = links.Deliver(opened.Code, "one", "decision", d); !errors.Is(e, directory.ErrAlreadyDecided) { + t.Fatal(e) + } +} diff --git a/internal/directory/links.go b/internal/directory/links.go index 6985f3c365..3cb7823ed7 100644 --- a/internal/directory/links.go +++ b/internal/directory/links.go @@ -52,10 +52,12 @@ type Links struct { // entry is a request, the moment it stops being readable, and a channel that // closes when it is decided. type entry struct { - rec Request - peer string - until int64 - done chan struct{} + rec Request + peer string + until int64 + cluster string + delivery string + done chan struct{} } // NewLinks returns an empty store that reads time from clock. @@ -184,6 +186,9 @@ func (l *Links) Decide(code string, d Decision, apply func(Request) error) error if !ok { return ErrRequestGone } + if e.cluster != "" { + return ErrAlreadyDecided + } switch err := Settle(e.rec, d); err { case errRepeat: return nil diff --git a/internal/directory/raft.go b/internal/directory/raft.go index a74e923e16..842a4f7e00 100644 --- a/internal/directory/raft.go +++ b/internal/directory/raft.go @@ -65,7 +65,7 @@ func (c *raftClient) SetVault(ctx context.Context, old, next string) error { return e } func (c *raftClient) Create(ctx context.Context, id string, in CellInit) (CellView, error) { - return c.change(ctx, clusterstate.Mutation{Create: &clusterstate.CreateCell{Cell: id, Init: in}}) + return c.change(ctx, clusterstate.Mutation{Create: &clusterstate.CreateCell{Cell: id, Init: in, V1: true}}) } func (c *raftClient) Acquire(ctx context.Context, id string, o AcquireOpts) (CellView, error) { v := &clusterstate.AcquireCell{Cell: id, Force: o.Force, V1: true} @@ -83,7 +83,7 @@ func (c *raftClient) Heartbeat(ctx context.Context, id string, b Beat) (CellView return c.change(ctx, clusterstate.Mutation{Beat: &clusterstate.BeatCell{Cell: id, Beat: b}}) } func (c *raftClient) Publish(ctx context.Context, id string, p Publish) (CellView, error) { - return c.change(ctx, clusterstate.Mutation{Publish: &clusterstate.PublishCell{Cell: id, Publish: p}}) + return c.change(ctx, clusterstate.Mutation{Publish: &clusterstate.PublishCell{Cell: id, Publish: p, V1: true}}) } func (c *raftClient) Release(ctx context.Context, id string, f uint64) error { _, e := c.change(ctx, clusterstate.Mutation{Release: &clusterstate.ReleaseCell{Cell: id, Fence: f}}) @@ -113,8 +113,8 @@ func (c *raftClient) Rotation(ctx context.Context) (RotationView, error) { return viewOf(l.Identity, l.Now, DefaultGraceBounds), e } -// Pairing commits a sealed decision dependency; delivery belongs to P10's -// durable outbox. There is no callback that can expose a grant before commit. +// Pairing commits a sealed decision dependency; delivery follows through the +// durable effects outbox. There is no callback that can expose a grant before commit. func (c *raftClient) ApproveRequest(ctx context.Context, code string, a Approval) error { d, e := decisionOf(a) if e != nil { diff --git a/internal/directory/requests.go b/internal/directory/requests.go index a4408aadc2..cc85786a5d 100644 --- a/internal/directory/requests.go +++ b/internal/directory/requests.go @@ -29,13 +29,13 @@ const ( // The numbers every implementation shares (contract 1, 3). const ( - RequestTTL = 10 * time.Minute // a request lives this long, counted by the relay's clock - DecidedKeep = 2 * time.Minute // a decided request stays readable this long - MaxWait = 25 * time.Second // the longest GetRequest wait - MaxGrant = 4096 // bytes of grant the relay stores - MaxNewBody = 1024 // bytes of a create body - maxName = 96 // bytes of name before sealing - sealedExtra = 24 + 16 // secretbox nonce and tag + RequestTTL = 10 * time.Minute // a request lives this long, counted by the relay's clock + DecidedKeep = 2 * time.Minute // a decided request stays readable this long + MaxWait = 25 * time.Second // the longest GetRequest wait + MaxGrant = clusterstate.MaxJoinGrant // bytes of grant the relay stores + MaxNewBody = 1024 // bytes of a create body + maxName = 96 // bytes of name before sealing + sealedExtra = 24 + 16 // secretbox nonce and tag keySize = 32 ) diff --git a/internal/manual/chat/add-or-remove-a-computer.md b/internal/manual/chat/add-or-remove-a-computer.md index 492145ca7b..239aafcff0 100644 --- a/internal/manual/chat/add-or-remove-a-computer.md +++ b/internal/manual/chat/add-or-remove-a-computer.md @@ -200,3 +200,11 @@ Too many requests from one network are refused too: at most 10 new links in an h After a clean quit and final durable save, bare `codeaf` on another paired computer within 24 hours opens the released chat automatically. A sleeping or disconnected laptop still needs `alt+c` on Home. Local unsaved edits, another open local window, a named chat, `--host`, `--at` or `--session` prevent automatic continuation. A running chat never moves by itself. The arrival retains its `Moved from` banner and `not brought along:` facts; `esc` goes Home. Personal memory and stored model, display, approval and limit defaults follow the paired computers too. Limits are shared as values; spending counted against them stays per computer. Automations and machine-specific paths and settings stay local. + +## An approval was interrupted — retrying the same pairing decision + +A new computer receives its sealed pairing grant only after its approval has +been saved. If the reply is interrupted, retrying the same approval keeps the +same decision. A different answer cannot replace a decision already made. If +the link has run out, ask the new computer for a fresh one. Removing a device +blocks its later requests; it cannot take back bytes the device already received. diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index 508db79105..4f94e27aef 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -240,3 +240,11 @@ A running chat on the other computer never moves automatically. Neither does a n ## Why an older chat could not be copied An older chat whose folder is a symbolic link, or whose saved conversation state contains symbolic links, is left untouched. codeaf refuses to copy those links because writing through one could change the original. The original still opens with an older codeaf. + +## A save has not reached the other computer yet — upload failed or another device continued first + +A saved chat reaches your other computers only after its encrypted objects have +been uploaded and its new saved head has been accepted. If an upload fails, the +other computer keeps seeing the previous saved head. Retry when the connection +returns. Uploaded objects are kept for reuse if another device continued the +chat first; this does not discard the work saved on your computer. diff --git a/internal/raftdirectory/README.md b/internal/raftdirectory/README.md index 2dd7f99cb5..930bd9dd5c 100644 --- a/internal/raftdirectory/README.md +++ b/internal/raftdirectory/README.md @@ -43,9 +43,55 @@ overflow. Pure rules retain their legacy behavior for pre-cutover stores. Pairing commands require an existing committed request dependency, atomically write the device metadata and sealed decision, and retain the first decision. The native -and hosted tests provision genesis/request dependencies explicitly. P10 still owns +and hosted tests provision genesis/request dependencies explicitly. The effects adapter below supplies external request/certificate admission, authorization installation, outbox delivery, -blob receipts and upload-before-head. P11 owns actual genesis/import; P12 owns peer +blob receipts and v2 upload-before-head. P11 owns actual genesis/import; P12 owns peer registry; P14 owns watch presence and committed renewal scheduling. No socket can extend a lease through this evaluator. P17 supplies original v1 authentication proof and production client wiring. These boundaries are internal, not new manual claims. + +P10 adds `Effects` and the equivalent hosted `RaftEffectsFacade`. Provision a +separate `Authority{Effects:true}` protocol key; it must not be personal. It can +only record upload evidence, reserve a join dependency, or complete a delivery. +These commands are idempotent by immutable effect keys and do not consume client +sessions. All effect rows count against the identity's application ceiling; +capacity refuses further growth without removing receipts, decisions or blobs. + +Before issuing a receipt, call `Effects.Upload` with the local head ID, its +**different encrypted remote object ID**, and the existing incremental frame +hint. Disk confirmation checks retrieval and flushes directory ancestry; hosted +confirmation awaits R2, its complete index and the supplied storage durability +boundary. A client still uploads its entire encrypted closure before requesting +this receipt: a relay cannot decrypt or traverse it, and an empty frame hint +never proves an empty closure. The receipt binds actor, both head IDs and the +exact frame hint. V2 Create/Publish require the committed receipt, and still report +stale fences/heads before missing upload evidence. The existing Client facade explicitly identifies v1 Create/Publish, as it +already identifies v1 Acquire. Old clients cannot supply the encrypted head RID; +they retain the existing upload-before-publish client ordering and their existing +safe API semantics. This is protocol compatibility in the same evaluator, never +a second writer. P17 must select this marker only from original authenticated v1 +ingress and reject compatibility markers in v2 requests. A v2 body cannot become +v1 merely by requesting it. P17 owns that ingress/client assembly; the production +writer is unchanged until P11 migrates it. Never derive an encrypted remote +object ID from a local turn ID. Receipts admit at most 512 declared frame hints; +a frame plan remains optional and partial. + +Admit calls `LinkEffects.Bind` before logging the request. A reservation excludes +other identities and legacy decision paths. Failed admission retains it until +the ordinary public request expires. Approval verifies the original existing +certificate under `IdentityRec.SigningKey`, checks that its device key matches +the bound request, and installs personal authorization, device metadata and the +sealed decision atomically. P11 must populate that **public** root key from +authenticated genesis/import. An upsert never installs authorization. Peer/node +membership is still P12, and delegated workers remain P15. + +Replay scans at most 64 retained outbox rows per call, from a quorum-authorized +view. Deliver reads the decision through another fresh barrier, sends its hash +as the destination's idempotency key, then commits `CompleteJoin`. A crash at +any boundary repeats only that same decision. A public request that expired +records `gone`; a lost destination reply retries without changing its grant. +Restarting a scan at the beginning is safe. P11/P14 must schedule these bounded +pages on activation/wake; no periodic network loop starts here. Delivery of a +grant approved before revocation cannot restore authority after it: every +subsequent operation still checks the committed device record. Already delivered +sealed bytes cannot be recalled, as with existing personal pairing. diff --git a/internal/raftdirectory/apply.go b/internal/raftdirectory/apply.go index 30a0876bcf..9dcb997001 100644 --- a/internal/raftdirectory/apply.go +++ b/internal/raftdirectory/apply.go @@ -5,6 +5,7 @@ package raftdirectory import ( "cmp" "context" + "crypto/ed25519" "errors" "slices" @@ -83,6 +84,15 @@ func Apply(ctx context.Context, store *raftstore.SQLite, b raftcore.ApplyBatch, // AuthorizedRead runs only within the actor after its current-term quorum // barrier. Enrollment is not inferred from a device metadata upsert. func AuthorizedRead(ctx context.Context, store *raftstore.SQLite, cluster, actor string) (raftstore.Application, error) { + return authorizedRead(ctx, store, cluster, actor, false) +} + +// AuthorizedEffectRead uses the same current-term barrier contract for the +// restricted delivery service. Rotation does not discard a committed outbox. +func AuthorizedEffectRead(ctx context.Context, store *raftstore.SQLite, cluster, actor string) (raftstore.Application, error) { + return authorizedRead(ctx, store, cluster, actor, true) +} +func authorizedRead(ctx context.Context, store *raftstore.SQLite, cluster, actor string, effects bool) (raftstore.Application, error) { app, _, err := store.ApplicationView(ctx) if err != nil { return app, err @@ -94,7 +104,7 @@ func AuthorizedRead(ctx context.Context, store *raftstore.SQLite, cluster, actor } var auth clusterstate.Authority var device clusterstate.Device - if clusterstate.Decode(rows[keys[0]], &auth) != nil || auth.Cluster != cluster || !auth.Personal { + if clusterstate.Decode(rows[keys[0]], &auth) != nil || auth.Cluster != cluster || (auth.Personal == effects) || auth.Effects != effects || len(auth.Key) != ed25519.PublicKeySize { return app, errors.New("raftdirectory: unauthorized") } if raw := rows[keys[1]]; raw != nil { diff --git a/internal/raftdirectory/directory_test.go b/internal/raftdirectory/directory_test.go index ab414fdb3a..e8011edc10 100644 --- a/internal/raftdirectory/directory_test.go +++ b/internal/raftdirectory/directory_test.go @@ -73,11 +73,12 @@ func newGroup(t *testing.T, clock func() time.Time, extra ...raftstore.Write) *g for i := range 50 { actors = append(actors, fmt.Sprintf("dev_%032x", i+1)) } - g.genesis = append(g.genesis, raftstore.Write{Key: raftstore.Key{Namespace: raftstore.State, Key: clusterstate.IdentityKey().Key}, Value: encoded(t, clusterstate.IdentityRec{V: 1})}) + g.genesis = append(g.genesis, raftstore.Write{Key: raftstore.Key{Namespace: raftstore.State, Key: clusterstate.IdentityKey().Key}, Value: encoded(t, clusterstate.IdentityRec{V: 1, Identity: clusterstate.IdentityID(key("root").Public().(ed25519.PublicKey)), SigningKey: key("root").Public().(ed25519.PublicKey)})}) for _, actor := range actors { k := clusterstate.AuthorityKey(actor) g.genesis = append(g.genesis, raftstore.Write{Key: raftstore.Key{Namespace: raftstore.Enrollment, Key: k.Key}, Value: encoded(t, clusterstate.Authority{Cluster: cluster, Key: key(actor).Public().(ed25519.PublicKey), Personal: true})}) } + g.genesis = append(g.genesis, raftstore.Write{Key: raftstore.Key{Namespace: raftstore.Enrollment, Key: clusterstate.AuthorityKey("effects").Key}, Value: encoded(t, clusterstate.Authority{Cluster: cluster, Key: key("effects").Public().(ed25519.PublicKey), Effects: true})}) g.genesis = append(g.genesis, extra...) for i := range 3 { g.paths[i] = filepath.Join(t.TempDir(), "member.db") @@ -390,13 +391,21 @@ func TestR16R30ApproveAndDenyCommitBeforeAnswer(t *testing.T) { writes := []raftstore.Write{} for _, code := range []string{"approve", "deny"} { k := clusterstate.JoinKey(code) - writes = append(writes, raftstore.Write{Key: raftstore.Key{Namespace: raftstore.Outbox, Key: k.Key}, Value: encoded(t, clusterstate.JoinRequest{Device: device, Expires: clock.Now().UnixMilli() + 600000})}) + writes = append(writes, raftstore.Write{Key: raftstore.Key{Namespace: raftstore.Outbox, Key: k.Key}, Value: encoded(t, clusterstate.JoinRequest{Device: device, PublicKey: pub, Expires: clock.Now().UnixMilli() + 600000})}) } g := newGroup(t, clock.Now, writes...) cl := directory.NewRaft(g).For("a") - raw, err := json.Marshal(map[string]any{"V": 1, "device": hex.EncodeToString(pub), "created": 1, "sig": "00"}) + cert := struct { + V uint16 `json:"V"` + Device string `json:"device"` + Created int64 `json:"created"` + Sig string `json:"sig"` + }{1, hex.EncodeToString(pub), 1, ""} + unsigned, _ := json.Marshal(cert) + cert.Sig = hex.EncodeToString(ed25519.Sign(key("root"), append([]byte("codeaf/device-cert/v1\n"), unsigned...))) + raw, err := json.Marshal(cert) must(t, err) - approval := directory.Approval{Device: directory.Device{V: 1, Name: "sealed"}, Cert: base64.RawURLEncoding.EncodeToString(raw), Grant: "sealed-grant"} + approval := directory.Approval{Device: directory.Device{V: 1, Name: "sealed"}, Cert: base64.RawURLEncoding.EncodeToString(raw), Grant: "c2VhbGVkLWdyYW50"} must(t, cl.ApproveRequest(ctx, "approve", approval)) must(t, cl.ApproveRequest(ctx, "approve", approval)) if err := cl.DenyRequest(ctx, "approve"); !errors.Is(err, directory.ErrAlreadyDecided) { @@ -417,7 +426,7 @@ func TestR16R30ApproveAndDenyCommitBeforeAnswer(t *testing.T) { must(t, err) var request clusterstate.JoinRequest must(t, clusterstate.Decode(rows[k], &request)) - if request.Decision == nil || request.Decision.Grant != "sealed-grant" { + if request.Decision == nil || request.Decision.Grant != "c2VhbGVkLWdyYW50" { t.Fatal("answer preceded committed sealed decision") } } diff --git a/internal/raftdirectory/effects.go b/internal/raftdirectory/effects.go new file mode 100644 index 0000000000..ec1fefbcac --- /dev/null +++ b/internal/raftdirectory/effects.go @@ -0,0 +1,160 @@ +package raftdirectory + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "errors" + + "github.com/Agent-Field/codeaf/internal/blobstore" + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" +) + +// EffectHost owns original request persistence, commit and fresh quorum reads. +// Read must recheck service enrollment after its barrier. Local store reads or +// proposal admission cannot satisfy either method's contract. +type EffectHost interface { + CommitEffect(context.Context, clusterstate.Command) (clusterstate.Result, error) + ReadEffect(context.Context, clusterstate.RecordKey) ([]byte, error) +} + +// LinkEffects reserves one public request for one cluster before admission. +// Delivery accepts an exact idempotency key only after the decision committed. +type LinkEffects interface { + Bind(code, cluster string) (directory.Request, error) + Deliver(code, cluster, decision string, d directory.Decision) error +} + +// Effects has only a restricted protocol key, never the identity or content key. +// Operations have immutable content keys, so crash retries need no local counter. +type Effects struct { + Host EffectHost + Cluster, Actor string + Key ed25519.PrivateKey +} + +func (e Effects) commit(ctx context.Context, now int64, m clusterstate.Mutation) error { + c, err := clusterstate.Sign(clusterstate.Command{Schema: clusterstate.SchemaVersion, Cluster: e.Cluster, Actor: e.Actor, Session: "effects", Sequence: 1, Now: now, Change: m}, e.Key) + if err != nil { + return err + } + r, err := e.Host.CommitEffect(ctx, c) + if err != nil { + return err + } + switch r.Code { + case clusterstate.OK: + return nil + case clusterstate.Rotated: + return directory.ErrRotated + case clusterstate.Revoked: + return directory.ErrRevoked + case clusterstate.RequestGone: + return directory.ErrRequestGone + case clusterstate.Capacity: + return directory.ErrFull + default: + return directory.ErrBadRequest + } +} + +// Upload records no success before both store durability and application commit. +// A refused CAS or later revocation retains every uploaded object for reuse. +func (e Effects) Upload(ctx context.Context, now int64, store blobstore.Store, receipt clusterstate.UploadReceipt) (string, error) { + if len(receipt.Frames) > clusterstate.MaxUploadFrames { + return "", directory.ErrTooBig + } + receipt.Frames = append([]string(nil), receipt.Frames...) + if err := blobstore.CheckReferences(ctx, store, receipt.HeadRID, receipt.Frames); err != nil { + return "", err + } + if err := e.commit(ctx, now, clusterstate.Mutation{RecordUpload: &receipt}); err != nil { + return "", err + } + return clusterstate.UploadID(receipt), nil +} +func (e Effects) Admit(ctx context.Context, now int64, links LinkEffects, code string) error { + code = directory.NormalizeCode(code) + r, err := links.Bind(code, e.Cluster) + if err != nil { + return err + } + public, err := base64.RawURLEncoding.DecodeString(r.Pubkey) + if err != nil || clusterstate.DeviceIDOfKey(public) != r.Device { + return directory.ErrBadRequest + } + return e.commit(ctx, now, clusterstate.Mutation{AdmitJoin: &clusterstate.AdmitJoin{Code: code, PublicKey: public, Expires: r.ExpiresAt, Platform: r.Platform}}) +} + +// Deliver replays an outbox row after a fresh barrier. A crash after public +// delivery but before CompleteJoin repeats the same grant, never a new decision. +// Expired public requests record a terminal gone outcome instead of retrying forever. +func (e Effects) Deliver(ctx context.Context, now int64, links LinkEffects, code string) error { + code = directory.NormalizeCode(code) + raw, err := e.Host.ReadEffect(ctx, clusterstate.JoinKey(code)) + if err != nil { + return err + } + var r clusterstate.JoinRequest + if clusterstate.Decode(raw, &r) != nil || r.Decision == nil { + return directory.ErrRequestGone + } + if r.Outcome != "" { + return nil + } + v := r.Decision + d := directory.Decision{State: directory.RequestApproved, Device: v.Device, Grant: v.Grant} + if v.Deny { + d = directory.Decision{State: directory.RequestDenied, Device: r.Device} + } + id := clusterstate.DecisionID(*v) + outcome := "delivered" + if err = links.Deliver(code, e.Cluster, id, d); err != nil { + if !errors.Is(err, directory.ErrRequestGone) { + return err + } + outcome = "gone" + } + return e.commit(ctx, now, clusterstate.Mutation{CompleteJoin: &clusterstate.CompleteJoin{Code: code, Decision: id, Outcome: outcome}}) +} + +// OutboxHost scans a bounded page only after a fresh service authority barrier. +// The cursor is an optimization; restarting at the beginning is always safe. +type OutboxHost interface { + ScanEffects(context.Context, string, int) ([]clusterstate.RecordWrite, error) +} + +const MaxEffectReplay = 64 + +func (e Effects) Replay(ctx context.Context, now int64, links LinkEffects, after string) (string, error) { + host, ok := e.Host.(OutboxHost) + if !ok { + return after, directory.ErrUnreachable + } + rows, err := host.ScanEffects(ctx, after, MaxEffectReplay) + if err != nil { + return after, err + } + if len(rows) > MaxEffectReplay { + return after, directory.ErrFull + } + for _, row := range rows { + if row.Namespace != "outbox" || row.Key <= after { + return after, directory.ErrBadRequest + } + if len(row.Key) >= 5 && row.Key[:5] == "join/" { + var r clusterstate.JoinRequest + if err = clusterstate.Decode(row.Data, &r); err != nil { + return after, err + } + if r.Decision != nil && r.Outcome == "" { + if err = e.Deliver(ctx, now, links, r.Decision.Code); err != nil { + return after, err + } + } + } + after = row.Key + } + return after, nil +} diff --git a/internal/raftdirectory/effects_test.go b/internal/raftdirectory/effects_test.go new file mode 100644 index 0000000000..f83f73bfb4 --- /dev/null +++ b/internal/raftdirectory/effects_test.go @@ -0,0 +1,238 @@ +package raftdirectory_test + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/blobstore" + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/directory/directorytest" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/raftstore" +) + +type effectHost struct { + g *group + failComplete bool +} + +func (h *effectHost) CommitEffect(_ context.Context, c clusterstate.Command) (clusterstate.Result, error) { + h.g.mu.Lock() + defer h.g.mu.Unlock() + if h.failComplete && c.Change.CompleteJoin != nil { + return clusterstate.Result{}, directory.ErrUnreachable + } + return h.g.submit(c) +} +func (h *effectHost) ReadEffect(_ context.Context, k clusterstate.RecordKey) ([]byte, error) { + g := h.g + g.mu.Lock() + defer g.mu.Unlock() + i := g.leader() + if i < 0 { + return nil, directory.ErrUnreachable + } + g.readID++ + id := "effect-" + strings.Repeat("x", int(g.readID)) + g.drive(i, raftcore.Read{RequestID: id}) + g.drain() + if !g.reads[id] { + return nil, directory.ErrUnreachable + } + app, e := raftdirectory.AuthorizedEffectRead(ctx, g.stores[i], cluster, "effects") + if e != nil { + return nil, e + } + sk := raftstore.Key{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key} + rows, _, e := g.stores[i].Read(ctx, app.Applied, []raftstore.Key{sk}) + return rows[sk], e +} +func (h *effectHost) ScanEffects(_ context.Context, after string, limit int) ([]clusterstate.RecordWrite, error) { + if _, e := h.ReadEffect(ctx, clusterstate.IdentityKey()); e != nil { + return nil, e + } + g := h.g + g.mu.Lock() + defer g.mu.Unlock() + i := g.leader() + app, _, e := g.stores[i].ApplicationView(ctx) + if e != nil { + return nil, e + } + rows, _, e := g.stores[i].ScanPage(ctx, app.Applied, raftstore.Outbox, after) + if e != nil { + return nil, e + } + if len(rows) > limit { + rows = rows[:limit] + } + out := []clusterstate.RecordWrite{} + for _, r := range rows { + out = append(out, clusterstate.RecordWrite{RecordKey: clusterstate.RecordKey{Namespace: string(r.Namespace), Key: r.Key.Key}, Data: r.Value}) + } + return out, nil +} +func restartEffects(t *testing.T, g *group) { + t.Helper() + g.queue = nil + for i := range 3 { + p, e := g.stores[i].CreateSnapshot(ctx) + must(t, e) + must(t, g.stores[i].Compact(ctx, p.GetMetadata().GetIndex())) + must(t, g.stores[i].Close()) + s, e := raftstore.Open(g.paths[i], raftstore.Identity{ClusterUUID: cluster, NodeID: uint64(i + 1), Protocol: 1}) + must(t, e) + g.stores[i] = s + recovered, e := s.Recover(ctx) + must(t, e) + n, e := raftcore.New(raftcore.Config{ClusterUUID: cluster, NodeID: uint64(i + 1), Protocol: 1, Seed: uint64(i + 13)}, recovered.Core) + must(t, e) + g.nodes[i] = n + g.drive(i, raftcore.Poll{}) + } + g.drain() + for n := 0; n < 40 && g.leader() < 0; n++ { + for i := range 3 { + g.drive(i, raftcore.Tick{}) + } + g.drain() + } + if g.leader() < 0 { + t.Fatal("no recovered leader") + } +} +func TestR3R16R17R21CommittedBlobAndPairOutbox(t *testing.T) { + clock := directorytest.NewFakeClock() + g := newGroup(t, clock.Now) + host := &effectHost{g: g} + effects := raftdirectory.Effects{Host: host, Cluster: cluster, Actor: "effects", Key: key("effects")} + disk, e := blobstore.NewDisk(t.TempDir()) + must(t, e) + rid := strings.Repeat("1", 64) + frame, e := blobstore.Encode(strings.Repeat("0", 32), []blobstore.Object{{RID: rid, Bytes: []byte("AGEO\x01sealed-object")}}) + must(t, e) + upload := clusterstate.UploadReceipt{Device: "a", Head: "local-head", HeadRID: rid, Frames: []string{blobstore.IDOf(frame)}} + if _, e = effects.Upload(ctx, clock.Now().UnixMilli(), disk, upload); !errors.Is(e, blobstore.ErrNotFound) { + t.Fatal(e) + } + if _, e = disk.PutFrame(ctx, frame); e != nil { + t.Fatal(e) + } + receipt, e := effects.Upload(ctx, clock.Now().UnixMilli(), disk, upload) + must(t, e) + command, e := clusterstate.Sign(clusterstate.Command{Schema: 1, Cluster: cluster, Actor: "a", Session: "v2", Sequence: 1, Now: clock.Now().UnixMilli(), Change: clusterstate.Mutation{Create: &clusterstate.CreateCell{Cell: "chat", Init: clusterstate.CellInit{Head: upload.Head, Frames: upload.Frames}, Receipt: receipt}}}, key("a")) + must(t, e) + result, e := g.submit(command) + must(t, e) + if result.Code != clusterstate.OK { + t.Fatal(result) + } + restartEffects(t, g) + result, e = g.submit(command) + must(t, e) + if result.Code != clusterstate.OK || result.View.Cell.Head != upload.Head { + t.Fatal(result) + } + taken, e := directory.NewRaft(g).For("b").Acquire(ctx, "chat", directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: "a", Fence: 1}}) + must(t, e) + if taken.Cell.Lease.Fence != 2 { + t.Fatal(taken) + } + stale, e := directory.NewRaft(g).For("a").Publish(ctx, "chat", directory.Publish{Fence: 1, OldHead: upload.Head, Head: "different"}) + if !errors.Is(e, directory.ErrFenceStale) { + t.Fatal(stale, e) + } + if _, e = disk.GetFrame(ctx, upload.Frames[0]); e != nil { + t.Fatal("failed publication deleted uploaded frame", e) + } + links := directory.NewLinks(clock.Now, directory.DefaultLinkLimits) + public := key("new-joiner").Public().(ed25519.PublicKey) + b64 := base64.RawURLEncoding.EncodeToString + opened, e := links.Open("local-fixture", directory.NewRequest{Pubkey: b64(public), X25519: b64(public), NameSealed: b64(make([]byte, 40)), Platform: "linux"}) + must(t, e) + must(t, effects.Admit(ctx, clock.Now().UnixMilli(), links, opened.Code)) + if e = effects.Deliver(ctx, clock.Now().UnixMilli(), links, opened.Code); !errors.Is(e, directory.ErrRequestGone) { + t.Fatal("uncommitted grant exposed", e) + } + cert := struct { + V uint16 `json:"V"` + Device string `json:"device"` + Created int64 `json:"created"` + Sig string `json:"sig"` + }{1, hex.EncodeToString(public), 1, ""} + unsigned, _ := json.Marshal(cert) + cert.Sig = hex.EncodeToString(ed25519.Sign(key("root"), append([]byte("codeaf/device-cert/v1\n"), unsigned...))) + raw, _ := json.Marshal(cert) + approval := directory.Approval{Device: directory.Device{V: 1, Name: "c2VhbGVk"}, Cert: b64(raw), Grant: "c2VhbGVkLWdyYW50"} + must(t, directory.NewRaft(g).For("a").ApproveRequest(ctx, opened.Code, approval)) + // Simulate losing the completion after the external destination stored its answer. + host.failComplete = true + if e = effects.Deliver(ctx, clock.Now().UnixMilli(), links, opened.Code); !errors.Is(e, directory.ErrUnreachable) { + t.Fatal(e) + } + request, e := links.Read(ctx, "local-fixture", opened.Code, 0) + must(t, e) + if request.State != directory.RequestApproved || request.Grant == nil || *request.Grant != approval.Grant { + t.Fatal(request) + } + restartEffects(t, g) + host.failComplete = false + _, e = effects.Replay(ctx, clock.Now().UnixMilli(), links, "") + must(t, e) + _, e = effects.Replay(ctx, clock.Now().UnixMilli(), links, "") + must(t, e) + data, e := host.ReadEffect(ctx, clusterstate.JoinKey(opened.Code)) + must(t, e) + var row clusterstate.JoinRequest + must(t, clusterstate.Decode(data, &row)) + if row.Outcome != "delivered" { + t.Fatal(row) + } + auth, e := host.ReadEffect(ctx, clusterstate.AuthorityKey(opened.Device)) + must(t, e) + var enrolled clusterstate.Authority + must(t, clusterstate.Decode(auth, &enrolled)) + if !enrolled.Personal || !ed25519.PublicKey(enrolled.Key).Equal(public) { + t.Fatal(enrolled) + } + // Denials use the same committed outbox; expiry is a terminal delivery + // outcome, rather than an instruction to invent a replacement grant. + for _, expired := range []bool{false, true} { + pending, err := links.Open("local-fixture", directory.NewRequest{Pubkey: b64(public), X25519: b64(public), NameSealed: b64(make([]byte, 40)), Platform: "linux"}) + must(t, err) + must(t, effects.Admit(ctx, clock.Now().UnixMilli(), links, pending.Code)) + must(t, directory.NewRaft(g).For("b").DenyRequest(ctx, pending.Code)) + if expired { + clock.Advance(directory.RequestTTL) + } + must(t, effects.Deliver(ctx, clock.Now().UnixMilli(), links, pending.Code)) + data, err := host.ReadEffect(ctx, clusterstate.JoinKey(pending.Code)) + must(t, err) + var denied clusterstate.JoinRequest + must(t, clusterstate.Decode(data, &denied)) + expected := "delivered" + if expired { + expected = "gone" + } + if denied.Outcome != expected || !denied.Decision.Deny { + t.Fatal(denied) + } + } + // Revocation after ingress still refuses a delayed upload receipt in the log. + must(t, directory.NewRaft(g).For("b").PutDevice(ctx, "a", directory.Device{V: 1})) + must(t, directory.NewRaft(g).For("b").Revoke(ctx, "a")) + if _, e = effects.Upload(ctx, clock.Now().UnixMilli(), disk, upload); !errors.Is(e, directory.ErrRevoked) { + t.Fatal(e) + } + if _, e = disk.GetFrame(ctx, upload.Frames[0]); e != nil { + t.Fatal("revocation deleted reusable bytes", e) + } +} diff --git a/internal/relayserve/blob_authority_test.go b/internal/relayserve/blob_authority_test.go new file mode 100644 index 0000000000..73b3134211 --- /dev/null +++ b/internal/relayserve/blob_authority_test.go @@ -0,0 +1,39 @@ +package relayserve_test + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/blobstore" + "github.com/Agent-Field/codeaf/internal/relayserve" + "github.com/Agent-Field/codeaf/internal/wireauth" +) + +// R16/R21 require a delayed blob completion to recheck committed authority. +func TestR16R21RevocationOrFreezeDuringBlobIOCannotReturnUploadSuccess(t *testing.T) { + for _, refusal := range []error{wireauth.ErrRevoked, wireauth.ErrRotated} { + t.Run(refusal.Error(), func(t *testing.T) { + store := blobstore.NewMemory() + checks := 0 + guarded := relayserve.CommittedBlobStore{Store: store, Authorize: func(context.Context) error { + checks++ + if checks == 2 { + return refusal + } + return nil + }} + frame, e := blobstore.Encode(strings.Repeat("0", 32), []blobstore.Object{{RID: strings.Repeat("1", 64), Bytes: []byte("AGEO\x01sealed")}}) + if e != nil { + t.Fatal(e) + } + if id, e := guarded.PutFrame(context.Background(), frame); id != "" || !errors.Is(e, refusal) { + t.Fatal(id, e) + } + if _, e := store.GetFrame(context.Background(), blobstore.IDOf(frame)); e != nil { + t.Fatal("uploaded frame was deleted", e) + } + }) + } +} diff --git a/internal/relayserve/namespaces.go b/internal/relayserve/namespaces.go index 9271bbdd2f..c48d38d504 100644 --- a/internal/relayserve/namespaces.go +++ b/internal/relayserve/namespaces.go @@ -141,3 +141,57 @@ func (n *namespaces) revoked(identity, device string) (bool, error) { } return d.Revoked(device), nil } + +// CommittedBlobStore checks current committed authority on both sides of an +// external put. A revocation or freeze during disk/R2 I/O leaves the bytes for +// reuse but returns no successful upload to that caller. The receipt command +// rechecks authority again at its own log position before any head may move. +// P11 assembles this only after the namespace's old writer is frozen. +type CommittedBlobStore struct { + blobstore.Store + Authorize func(context.Context) error +} + +func (s CommittedBlobStore) PutFrame(ctx context.Context, frame []byte) (blobstore.FrameID, error) { + if s.Authorize == nil { + return "", wireauth.ErrUnauthorized + } + if err := s.Authorize(ctx); err != nil { + return "", err + } + id, err := s.Store.PutFrame(ctx, frame) + if err != nil { + return "", err + } + if err = s.Authorize(ctx); err != nil { + return "", err + } + return id, nil +} + +// ConfirmReferences preserves the disk store's extra durability boundary through +// this authorization wrapper; embedding Store alone would hide that method. +func (s CommittedBlobStore) ConfirmReferences(ctx context.Context, head string, frames []string) error { + if s.Authorize == nil { + return wireauth.ErrUnauthorized + } + if err := s.Authorize(ctx); err != nil { + return err + } + if err := blobstore.CheckReferences(ctx, s.Store, head, frames); err != nil { + return err + } + return s.Authorize(ctx) +} +func (r readOnlyOnceReplaced) ConfirmReferences(ctx context.Context, head string, frames []string) error { + if r.rotation() != nil { + return wireauth.ErrRotated + } + if err := blobstore.CheckReferences(ctx, r.Store, head, frames); err != nil { + return err + } + if r.rotation() != nil { + return wireauth.ErrRotated + } + return nil +} From 6e3c3e9b8123012516e451ff814c542712c2b071 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 01:12:29 -0400 Subject: [PATCH 023/103] Make hosted blob and pairing effects durable and safe to replay. Reserve each public request for one cluster, commit its device authorization before exposing the sealed grant, and repair interrupted uploads and delivery acknowledgements without deleting content. Co-Authored-By: Claude Opus 5.5 (1M context) --- relay/hosted/src/link/effects.js | 24 ++++++ relay/hosted/src/link/request.js | 60 +++++++++++---- relay/hosted/src/raft/effects.js | 74 +++++++++++++++++++ relay/hosted/src/store.js | 23 +++++- relay/hosted/src/tenant.js | 1 + relay/hosted/test/raft/directory.test.js | 41 ++++++++-- relay/hosted/test/raft/effects.test.js | 62 ++++++++++++++++ relay/hosted/test/raft/link-effects-worker.js | 13 ++++ relay/hosted/test/raft/link-effects.test.js | 38 ++++++++++ relay/hosted/test/store.test.js | 22 ++++++ 10 files changed, 338 insertions(+), 20 deletions(-) create mode 100644 relay/hosted/src/link/effects.js create mode 100644 relay/hosted/src/raft/effects.js create mode 100644 relay/hosted/test/raft/effects.test.js create mode 100644 relay/hosted/test/raft/link-effects-worker.js create mode 100644 relay/hosted/test/raft/link-effects.test.js diff --git a/relay/hosted/src/link/effects.js b/relay/hosted/src/link/effects.js new file mode 100644 index 0000000000..21a021b26d --- /dev/null +++ b/relay/hosted/src/link/effects.js @@ -0,0 +1,24 @@ +import { alreadyDecided, gone } from './refusals.js'; + +// A public code belongs to one committed namespace once reserved. Reservations +// survive a failed proposal until the ordinary request expiry; another identity +// cannot create a second successful decision while the first is uncertain. +export function boundLink(record, cluster) { + if (!record) throw gone(); + if (!cluster || (record.cluster && record.cluster !== cluster) || (!record.cluster && record.state !== 'pending')) throw alreadyDecided(); + return record.cluster ? record : { ...record, cluster }; +} + +// This is effect delivery only. Go already decided and installed authority. +// The exact id and body are checked so a lost RPC reply is safe to replay. +export function deliveredLink(record, cluster, id, state, grant, now, device) { + if (!record) throw gone(); + if (!cluster || record.cluster !== cluster || record.device !== device || !id || !['approved', 'denied'].includes(state)) throw alreadyDecided(); + grant = state === 'approved' ? grant : null; + if (record.delivery) { + if (record.delivery === id && record.state === state && record.grant === grant) return record; + throw alreadyDecided(); + } + if (record.state !== 'pending' || (state === 'approved' && !grant)) throw alreadyDecided(); + return { ...record, state, grant, delivery: id, decided_at: now }; +} diff --git a/relay/hosted/src/link/request.js b/relay/hosted/src/link/request.js index 84a49afa2d..f5b1142318 100644 --- a/relay/hosted/src/link/request.js +++ b/relay/hosted/src/link/request.js @@ -6,10 +6,14 @@ import { DurableObject } from 'cloudflare:workers'; import { limitsOf } from '../limits.js'; import { guarded } from '../wire.js'; import { settled } from './state.js'; +import { Serial } from '../serial.js'; +import { boundLink, deliveredLink } from './effects.js'; +import { alreadyDecided } from './refusals.js'; import { gone } from './refusals.js'; export class LinkRequest extends DurableObject { #record = null; + #changes = new Serial(); #waiting = new Set(); // wake functions of the long polls now waiting constructor(ctx, env) { @@ -22,11 +26,11 @@ export class LinkRequest extends DurableObject { /** open stores the new record, pending, and arms its deletion at the expiry. */ open(record) { - return guarded(async () => { - this.#record = { ...record, state: 'pending', grant: null, decided_at: 0 }; - await this.ctx.storage.put('record', this.#record); - await this.ctx.storage.setAlarm(record.expires_at); - }); + return guarded(() => this.#changes.run(async () => { + if (this.#record) throw alreadyDecided(); + const next = { ...record, state: 'pending', grant: null, decided_at: 0 }; + await this.#save(next, record.expires_at); + })); } /** read answers the record, waiting up to waitMs for it to be decided; null when the wait ends first. */ @@ -40,21 +44,49 @@ export class LinkRequest extends DurableObject { /** decide moves a pending request to `state` once; see state.js for what a repeat does. */ decide(state, grant) { - return guarded(async () => { + return guarded(() => this.#changes.run(async () => { const record = this.#live(); + if (record.cluster) throw alreadyDecided(); const next = settled(record, state, grant, Date.now()); - if (next === record) return; - this.#record = next; - await this.ctx.storage.put('record', next); - await this.ctx.storage.setAlarm(next.decided_at + this.limits.linkDecidedKeepMs); + await this.#save(next, next.decided_at + this.limits.linkDecidedKeepMs); + this.#wake(); + })); + } + + /** bind reserves a request before its immutable dependency enters the log. */ + bind(cluster) { + return guarded(() => this.#changes.run(async () => { + const record = this.#live(), next = boundLink(record, cluster); + if (next !== record) { await this.ctx.storage.put('record', next); this.#record = next; } + return viewOf(next); + })); + } + + /** deliver accepts only an internal committed outbox response. */ + deliver(cluster, id, state, grant, device) { + return guarded(() => this.#changes.run(async () => { + const record = this.#live(), next = deliveredLink(record, cluster, id, state, grant, Date.now(), device); + await this.#save(next, next.decided_at + this.limits.linkDecidedKeepMs); this.#wake(); + })); + } + + // The record and its expiry share a transaction so a crash cannot retain + // an undeliverable grant forever or delete one before its promised window. + async #save(record, deadline) { + await this.ctx.storage.transaction(async tx => { + await tx.put('record', record); + await tx.setAlarm(deadline); }); + this.#record = record; } async alarm() { - this.#record = null; - await this.ctx.storage.deleteAll(); - this.#wake(); + return this.#changes.run(async () => { + await this.ctx.storage.deleteAll(); + this.#record = null; + this.#wake(); + }); } // A pending record is dead at its expiry; a decided one lives until its alarm, so the new device can read it. @@ -78,4 +110,4 @@ export class LinkRequest extends DurableObject { } /** viewOf is the record as the wire shows it: the stored fields except the relay's own bookkeeping. */ -const viewOf = ({ decided_at: _, ...record }) => record; +const viewOf = ({ decided_at: _, cluster: _cluster, delivery: _delivery, ...record }) => record; diff --git a/relay/hosted/src/raft/effects.js b/relay/hosted/src/raft/effects.js new file mode 100644 index 0000000000..a4f01fa472 --- /dev/null +++ b/relay/hosted/src/raft/effects.js @@ -0,0 +1,74 @@ +import { encodeCanonical, decodeCanonical } from './codec.js'; +import { hex, sha256, fromBase64Url } from '../codec.js'; +import { unwrap, Wire } from '../wire.js'; + +const enc = new TextEncoder(), dec = new TextDecoder(); +const segment = text => hex(enc.encode(text)); +const digest = async value => hex(await sha256(enc.encode(encodeCanonical(value)))); + +// The host signs with a committed, restricted effect key and waits for actual +// commit/application. Neither this adapter nor a LinkRequest decides policy. +export class RaftEffectsFacade { + constructor(host, cluster) { this.host = host; this.cluster = cluster; } + async commit(change, now) { + const result = await this.host.commitEffect(change, now); + if (result.code !== 'ok') throw new Wire(result.code, result.code === 'rotated' ? 410 : 409); + } + async upload(store, receipt, durable, now) { + receipt = structuredClone(receipt); + if ((receipt.frames?.length ?? 0) > 512) throw new Wire('bad_request', 400); + if (!receipt.frames?.length) delete receipt.frames; + await store.checkReferences(receipt.head_rid, receipt.frames ?? [], durable); + await this.commit({ record_upload: receipt }, now); + return digest(receipt); + } + async admit(box, code, now) { + code = String(code).trim().toLowerCase(); + const r = unwrap(await box.bind(this.cluster)); + if (r.code !== code) throw new Wire('bad_request',400); + const key = fromBase64Url(r.pubkey); + if (!key || key.length !== 32) throw new Wire('bad_request', 400); + // Canonical byte slices use standard padded base64, not base64url. + const public_key = btoa(String.fromCharCode(...key)); + await this.commit({ admit_join: { code, public_key, expires: r.expires_at, platform:r.platform } }, now); + } + async replay(boxOf, after, now) { + const rows = await this.host.barrierOutbox(after, 64); + if (rows.length > 64) throw new Wire('full', 503); + for (const row of rows) { + if (row.key <= after) throw new Wire('bad_request', 400); + if (row.key.startsWith('join/')) { + const request = decodeCanonical(dec.decode(row.data)); + if (request.decision && !request.outcome) await this.deliver(boxOf(request.decision.code), request.decision.code, now); + } + after = row.key; + } + return after; + } + async deliver(box, code, now) { + code = String(code).trim().toLowerCase(); + const raw = await this.host.barrierRecord({ namespace: 'outbox', key: 'join/' + segment(code) }); + if (!raw) throw new Wire('request_gone', 404); + const row = decodeCanonical(dec.decode(raw)); + if (!row.decision) throw new Wire('request_gone', 404); + if (row.outcome) return; + const id = await digest(row.decision), d = row.decision; + let outcome = 'delivered'; + const reply = await box.deliver(this.cluster, id, d.deny ? 'denied' : 'approved', d.deny ? null : d.grant, row.device); + try { unwrap(reply); } catch (e) { if (e.code !== 'gone') throw e; outcome = 'gone'; } + await this.commit({ complete_join: { code, decision: id, outcome } }, now); + } +} + +// External writes recheck committed authority after I/O. Losing authorization +// never deletes an uploaded frame, and never supplies a successful receipt. +export class RaftBlobTenant { + constructor(store, authorize, durable) { this.store = store; this.authorize = authorize; this.durable = durable; } + async putFrame(frame, admit) { + await this.authorize(); + const stored = await this.store.put(frame, admit); + await this.durable(); + await this.authorize(); + return stored; + } +} diff --git a/relay/hosted/src/store.js b/relay/hosted/src/store.js index ea066ad32b..614cf5b050 100644 --- a/relay/hosted/src/store.js +++ b/relay/hosted/src/store.js @@ -68,6 +68,25 @@ export class R2Store { }); } + /** checkReferences waits for indexing and durability before an upload receipt is proposed. */ + checkReferences(headRID, frames, durable) { + return this.#writes.run(async () => { + if (typeof durable !== 'function' || !/^[0-9a-f]{64}$/.test(headRID)) throw new Damaged(); + const head = await this.get(headRID); + if (!head || head.length !== this.#locate(headRID).len) throw new Damaged(); + for (const id of new Set([this.#locate(headRID).frame, ...frames])) { + if (!/^[0-9a-f]{64}$/.test(id) || !this.#indexed(id)) throw new Damaged(); + const object = await this.bucket.get(this.prefix + id); + if (!object) throw new Damaged(); + const bytes = new Uint8Array(await object.arrayBuffer()); + if (await frameIdOf(bytes) !== id) throw new Damaged(); + const frame = decode(bytes); + for (const ref of frame.objects) if (!this.#locate(ref.rid)) throw new Damaged(); + } + await durable(); + }); + } + /** get answers one object's bytes, or null. */ async get(rid) { const at = this.#locate(rid); @@ -170,7 +189,9 @@ export class R2Store { // #index is one synchronous turn, so no request sees a frame half indexed. An object already held // (the same rid in an earlier frame, with the same bytes) keeps its first place. #index({ id, base, objects }) { - this.sql.exec('INSERT OR IGNORE INTO frames VALUES (?)', id); for (const { rid, off, len } of objects) this.sql.exec('INSERT OR IGNORE INTO objs VALUES (?,?,?,?)', rid, id, base + off, len); + // The completion marker comes last so an interrupted index is repaired + // by the next immutable put, rather than being mistaken for completion. + this.sql.exec('INSERT OR IGNORE INTO frames VALUES (?)', id); } } diff --git a/relay/hosted/src/tenant.js b/relay/hosted/src/tenant.js index ad14dcd51f..5f69efc3e6 100644 --- a/relay/hosted/src/tenant.js +++ b/relay/hosted/src/tenant.js @@ -155,3 +155,4 @@ export class Tenant { // Keeping it separate prevents constructing or invoking the legacy SQL writer // while the integration harness exercises the authoritative application path. export { RaftDirectoryFacade } from './raft/directory.js'; +export { RaftEffectsFacade, RaftBlobTenant } from './raft/effects.js'; diff --git a/relay/hosted/test/raft/directory.test.js b/relay/hosted/test/raft/directory.test.js index 9f5ef55666..cf78488a60 100644 --- a/relay/hosted/test/raft/directory.test.js +++ b/relay/hosted/test/raft/directory.test.js @@ -7,6 +7,10 @@ import { createHash, generateKeyPairSync, sign } from 'node:crypto'; import { Reactor } from '../../src/raft/abi.js'; import { encodeCanonical, decodeCanonical } from '../../src/raft/codec.js'; import { MemberActor } from '../../src/raft/actor.js'; +import { RaftEffectsFacade } from '../../src/raft/effects.js'; +import { R2Store, sealFrame } from '../../src/store.js'; +import { boundLink, deliveredLink } from '../../src/link/effects.js'; +import { encode as encodeFrame } from '../../src/frame.js'; import { directoryEvaluator, RaftDirectoryFacade } from '../../src/raft/directory.js'; const out = process.env.CODEAF_RAFT_PROOF_DIR ?? '/tmp/codeaf-p09'; @@ -30,12 +34,13 @@ class Native { async close() { this.child.stdin.end(); if (this.child.exitCode === null) await new Promise(resolve => this.child.once('exit', resolve)); } } -test('R1 R2 R4 R14 R16 R17 hosted facade uses committed Go policy and native-identical application bytes', { timeout: 30000 }, async () => { +test('R1 R2 R4 R14 R16 R17 R21 hosted facade uses committed Go policy and native-identical application bytes', { timeout: 30000 }, async () => { const reactor = new Reactor(new WebAssembly.Module(readFileSync(out + '/core.wasm'))), native = new Native(); const commands = [], calls = [], hostStores = [], actors = [], queue = [], reads = []; - const cluster = 'directory-p09', keys = Object.fromEntries(['a','b'].map(id => [id, generateKeyPairSync('ed25519')])), sequences = { a: 0, b: 0 }; - const initial = [{ namespace: 'state', key: 'identity', data: bytes({ V: 1, identity: '' }) }]; - for (const id of ['a','b']) initial.push({ namespace: 'enrollment', key: 'actor/' + segment(id), data: bytes({ cluster, key: keys[id].publicKey.export({ format: 'der', type: 'spki' }).subarray(-32).toString('base64'), personal: true }) }); + const cluster = 'directory-p09', keys = Object.fromEntries(['a','b','effects','root','joiner'].map(id => [id, generateKeyPairSync('ed25519')])), sequences = { a: 0, b: 0, effects:0, joiner:0 }; + const root = keys.root.publicKey.export({format:'der',type:'spki'}).subarray(-32); + const initial = [{ namespace:'state',key:'identity',data:bytes({V:1,identity:'id_'+sha(root).slice(0,32),signing_key:root.toString('base64')}) }]; + for (const id of ['a','b','effects']) initial.push({ namespace: 'enrollment', key: 'actor/' + segment(id), data: bytes({ cluster, key: keys[id].publicKey.export({ format: 'der', type: 'spki' }).subarray(-32).toString('base64'), personal: id !== 'effects', ...(id === 'effects' ? {effects:true} : {}) }) }); const size = writes => writes.reduce((n, r) => n + 12 + enc.encode(r.namespace).length + enc.encode(r.key).length + r.data.length, 0); const evaluate = directoryEvaluator(reactor, cluster, 8 * 1024 * 1024); const results = [[],[],[]]; @@ -73,7 +78,20 @@ test('R1 R2 R4 R14 R16 R17 hosted facade uses committed Go policy and native-ide async commit(raw) { const id = leader(), before = results[id - 1].length; commands.push(raw); accept(id, await actors[id - 1].input(3, raw)); await pump(blocked); if (results[id - 1].length === before) throw new Error('unavailable'); return results[id - 1].at(-1); }, async barrierRead(id) { const l = leader(); accept(l, await actors[l - 1].input(4, id)); await pump(blocked); if (!reads.find(r => r.id === id)) throw new Error('unavailable'); return actors[l - 1].store.read('state', 'cell/' + segment('chat')); }, }); - let r = await facade.command(command('a', { create: { cell: 'chat', init: { head: 'sealed-one', class: 'chat', size: '1', keys: {} }, receipt: '' } })); assert.equal(r.code, 'ok'); assert.equal(r.view.cell.lease.fence, '1'); + const effects = new RaftEffectsFacade({ + commitEffect: (change,now) => facade.command(command('effects',change,now)), + async barrierRecord(key) { await facade.read('effect-read-'+commands.length); return actors[leader()-1].store.read(key.namespace,key.key); }, + }, cluster); + const stored=new Map(); let failPut=true; + const bucket={async put(id,b) {if(failPut) throw new Error('blob write failed');stored.set(id,b.slice());},async get(id,opts) {const b=stored.get(id);if(!b)return null;const offset=opts?.range?.offset??0,length=opts?.range?.length??b.length;return {arrayBuffer:async()=>b.slice(offset,offset+length).buffer};}}; + const sqlHost=new Host();hostStores.push(sqlHost);const store=new R2Store(bucket,sqlHost.sql,cluster); + const rid='1'.repeat(64), f=await sealFrame(encodeFrame('0'.repeat(32),[{rid,bytes:enc.encode('AGEO\x01sealed-object')}])); + const receipt={device:'a',head:'sealed-one',head_rid:rid};let durable=0; + await assert.rejects(store.put(f),/blob write failed/); + await assert.rejects(effects.upload(store,receipt,async()=>{durable++;},1000)); + assert.equal(durable,0);failPut=false;await store.put(f); + const receiptID=await effects.upload(store,receipt,async()=>{durable++;},1000);assert.equal(durable,1); + let r = await facade.command(command('a', { create: { cell: 'chat', init: { head: 'sealed-one', class: 'chat', size: '1', keys: {} }, receipt: receiptID } })); assert.equal(r.code, 'ok'); assert.equal(r.view.cell.lease.fence, '1'); const take = command('b', { acquire: { cell: 'chat', force: true, expect: { device: 'a', fence: '1' } } }); r = await facade.command(take); assert.equal(r.code, 'ok'); assert.equal(r.view.cell.lease.fence, '2'); assert.deepEqual(await facade.command(take), r, 'exact retry'); @@ -87,6 +105,19 @@ test('R1 R2 R4 R14 R16 R17 hosted facade uses committed Go policy and native-ide r = await facade.command(command('a', { put_device: { id: 'b', device: { V: 1, name: 'c2VhbGVk', added_by: '', revoked: false, caps: { os: '', arch: '', sandbox: null, container: null, gpu: null, cow: '' } } } })); assert.equal(r.code, 'ok'); r = await facade.command(command('a', { revoke: { device: 'b' } })); assert.equal(r.code, 'ok'); r = await facade.command(delayed); assert.equal(r.code, 'revoked'); + const publicKey=keys.joiner.publicKey.export({format:'der',type:'spki'}).subarray(-32), device='dev_'+sha(publicKey).slice(0,32); + let pending={code:'k7m2q9xd',device,pubkey:publicKey.toString('base64url'),expires_at:10000,platform:'linux',state:'pending',grant:null,decided_at:0}; + let loseDelivery=true; + const box={async bind(c){pending=boundLink(pending,c);return {ok:pending};},async deliver(c,id,state,grant,target){pending=deliveredLink(pending,c,id,state,grant,1000,target);if(loseDelivery){loseDelivery=false;throw new Error('lost delivery reply');}return {ok:null};}}; + await effects.admit(box,'k7m2q9xd',1000); + await assert.rejects(effects.deliver(box,'k7m2q9xd',1000),/request_gone/);assert.equal(pending.state,'pending'); + const cert={V:1,device:publicKey.toString('hex'),created:1,sig:''};cert.sig=sign(null,Buffer.from('codeaf/device-cert/v1\n'+JSON.stringify(cert)),keys.root.privateKey).toString('hex'); + const decision={code:'k7m2q9xd',device,record:{V:1,name:'c2VhbGVk',added_by:'',revoked:false,caps:{os:'',arch:'',sandbox:null,container:null,gpu:null,cow:''}},grant:'c2VhbGVk',cert:Buffer.from(JSON.stringify(cert)).toString('base64url'),deny:false}; + r=await facade.command(command('a',{decide_join:decision}));assert.equal(r.code,'ok'); + assert(actors[leader()-1].store.read('enrollment','actor/'+segment(device))); + await assert.rejects(effects.deliver(box,'k7m2q9xd',1000),/lost delivery reply/);assert.equal(pending.grant,decision.grant); + await effects.deliver(box,'k7m2q9xd',1000);await effects.deliver(box,'k7m2q9xd',1000); + const outbox=decodeCanonical(dec.decode(actors[leader()-1].store.read('outbox','join/'+segment(decision.code))));assert.equal(outbox.outcome,'delivered'); const row = await facade.read('fresh'); assert.equal(decodeCanonical(dec.decode(row)).head, 'sealed-one'); for (const actor of actors) assert.deepEqual(actor.store.read('state', 'cell/' + segment('chat')), row); blocked = true; await assert.rejects(facade.read('minority'), /unavailable/); diff --git a/relay/hosted/test/raft/effects.test.js b/relay/hosted/test/raft/effects.test.js new file mode 100644 index 0000000000..50f019ea26 --- /dev/null +++ b/relay/hosted/test/raft/effects.test.js @@ -0,0 +1,62 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { R2Store, sealFrame } from '../../src/store.js'; +import { encode } from '../../src/frame.js'; +import { memorySql } from '../sql.js'; +import { boundLink, deliveredLink } from '../../src/link/effects.js'; +import { RaftBlobTenant, RaftEffectsFacade } from '../../src/raft/effects.js'; + +const rid='1'.repeat(64), enc=new TextEncoder(); +const frame=()=>sealFrame(encode('0'.repeat(32),[{rid,bytes:enc.encode('AGEO\x01sealed')} ])); +function bucket() { + const data=new Map(); + return {data,async put(key,bytes){data.set(key,bytes.slice());},async get(key,opts){const b=data.get(key);if(!b)return null;const off=opts?.range?.offset??0,n=opts?.range?.length??b.length;return {arrayBuffer:async()=>b.slice(off,off+n).buffer};}}; +} + +test('R21 an upload receipt waits for blob completion, indexing and the storage durability boundary',async()=>{ + const b=bucket(),store=new R2Store(b,memorySql(),'one'),f=await frame();let release,started; + const wait=new Promise(r=>{release=r;}),entered=new Promise(r=>{started=r;});const put=b.put; + b.put=async(...args)=>{started();await wait;return put(...args);}; + const writing=store.put(f);await entered;let ready=false,synced=false; + const checking=store.checkReferences(rid,[f.id],async()=>{synced=true;}).then(()=>{ready=true;}); + await Promise.resolve();assert.equal(ready,false);assert.equal(synced,false); + release();await writing;await checking;assert(synced&&ready); + await assert.rejects(store.checkReferences(rid,[f.id],async()=>{throw Error('sync failed');}),/sync failed/); + b.data.delete('one/f/'+f.id);await assert.rejects(store.checkReferences(rid,[],async()=>{})); +}); + +test('R16 R21 a freeze or revoke during external I/O leaves bytes and returns no upload success',async()=>{ + const b=bucket(),store=new R2Store(b,memorySql(),'one'),f=await frame();let calls=0; + const guarded=new RaftBlobTenant(store,async()=>{if(++calls===2)throw Error('revoked');},async()=>{}); + await assert.rejects(guarded.putFrame(f),/revoked/); + assert(b.data.has('one/f/'+f.id));assert.deepEqual(await store.get(rid),enc.encode('AGEO\x01sealed')); +}); + +test('R16 R17 one public request belongs to one namespace and one exact committed delivery',()=>{ + const pending={device:'device',state:'pending',grant:null,expires_at:10000};const r=boundLink(pending,'one'); + assert.equal(boundLink(r,'one'),r);assert.throws(()=>boundLink(r,'two')); + assert.throws(()=>deliveredLink({...r,device:'replacement'},'one','id','approved','c2VhbGVk',1,'device')); + assert.throws(()=>deliveredLink(r,'two','id','approved','c2VhbGVk',1,'device')); + const done=deliveredLink(r,'one','id','approved','c2VhbGVk',1,'device'); + assert.equal(deliveredLink(done,'one','id','approved','c2VhbGVk',2,'device'),done); + assert.throws(()=>deliveredLink(done,'one','other','approved','c2VhbGVk',2,'device')); + assert.throws(()=>deliveredLink(done,'one','id','approved','b3RoZXI',2,'device')); + assert.throws(()=>deliveredLink(done,'one','id','denied',null,2,'device')); +}); + +test('R17 expired public delivery records a terminal outcome and never invents a new grant',async()=>{ + const {encodeCanonical}=await import('../../src/raft/codec.js'); + const decision={code:'k7m2q9xd',device:'',record:{V:0,name:'',added_by:'',revoked:false,caps:{os:'',arch:'',sandbox:null,container:null,gpu:null,cow:''}},grant:'',cert:'',deny:true}; + const row=enc.encode(encodeCanonical({device:'pending',expires:1,decision}));let outcome; + const effects=new RaftEffectsFacade({async barrierRecord(){return row;},async commitEffect(change){outcome=change.complete_join;return {code:'ok'};}},'one'); + await effects.deliver({async deliver(){return {err:'gone',status:404};}},decision.code,2); + assert.equal(outcome.outcome,'gone');assert.match(outcome.decision,/^[0-9a-f]{64}$/); +}); + +test('R21 an interrupted R2 index has no completion marker and a repeated immutable upload repairs it',async()=>{ + const sql=memorySql(),original=sql.exec.bind(sql);let fail=true; + sql.exec=(q,...args)=>{if(fail&&q.startsWith('INSERT OR IGNORE INTO objs')){fail=false;throw Error('index failed');}return original(q,...args);}; + const store=new R2Store(bucket(),sql,'one'),f=await frame(); + await assert.rejects(store.put(f),/index failed/);assert.equal(await store.put(f),true); + await store.checkReferences(rid,[f.id],async()=>{});assert.equal(await store.put(f),false); +}); diff --git a/relay/hosted/test/raft/link-effects-worker.js b/relay/hosted/test/raft/link-effects-worker.js new file mode 100644 index 0000000000..95b2566115 --- /dev/null +++ b/relay/hosted/test/raft/link-effects-worker.js @@ -0,0 +1,13 @@ +// Only this local fixture exposes RPC controls; the public Worker never does. +export { LinkRequest } from '../../src/link/request.js'; +export default { + async fetch(request, env) { + const url = new URL(request.url); + if (url.pathname === '/ready') return new Response(null, {status:204}); + const {code,op,args=[]} = await request.json(); + if (!['open','bind','read','decide','deliver'].includes(op)) return new Response(null,{status:404}); + const box=env.LINKS.get(env.LINKS.idFromName(code)); + const answer=await box[op](...args); + return Response.json('err' in answer ? answer : {ok:answer.ok??null}); + }, +}; diff --git a/relay/hosted/test/raft/link-effects.test.js b/relay/hosted/test/raft/link-effects.test.js new file mode 100644 index 0000000000..a0203f2f27 --- /dev/null +++ b/relay/hosted/test/raft/link-effects.test.js @@ -0,0 +1,38 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { build } from 'esbuild'; +import { spawn } from 'node:child_process'; +import { createServer } from 'node:net'; +import { mkdtemp,writeFile,readFile,rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; + +// Real local Durable Object persistence is the delivery boundary; killing its +// process retains only the on-disk object storage, never a test-side cache. +test('R3 R16 R17 a real LinkRequest durably reserves and replays its committed delivery after restart',{timeout:30000},async()=>{ + const out=process.env.CODEAF_RAFT_PROOF_DIR??'/tmp/codeaf-p10',dir=await mkdtemp(join(tmpdir(),'p10-link-')); + const [binary,schema]=await Promise.all(['workerd-path','workerd-schema-path'].map(p=>readFile(join(out,p),'utf8'))); + const bundle=await build({entryPoints:['test/raft/link-effects-worker.js'],bundle:true,format:'esm',target:'es2022',external:['cloudflare:*'],write:false}); + await writeFile(join(dir,'worker.js'),bundle.outputFiles[0].contents); + const server=createServer();await new Promise(r=>server.listen(0,'127.0.0.1',r));const port=server.address().port;await new Promise(r=>server.close(r)); + const q=JSON.stringify,config=join(dir,'config.capnp'); + await writeFile(config,`using Workerd=import ${q(schema.trim())}; const config:Workerd.Config=(services=[(name="proof",worker=(compatibilityDate="2026-09-01",modules=[(name="worker.js",esModule=embed ${q(join(dir,'worker.js'))})],durableObjectNamespaces=[(className="LinkRequest",uniqueKey="p10-link")],durableObjectStorage=(localDisk="storage"),bindings=[(name="LINKS",durableObjectNamespace=(className="LinkRequest"))],globalOutbound="blocked")),(name="storage",disk=(path=${q(dir)},writable=true)),(name="blocked",worker=(compatibilityDate="2026-09-01",modules=[(name="blocked.js",esModule="export default {fetch(){throw Error('outbound forbidden')}}")],globalOutbound="blocked"))],sockets=[(name="http",address="127.0.0.1:${port}",http=(),service="proof")]);`); + let child,ended,stderr='';const origin=`http://127.0.0.1:${port}`; + const start=async()=>{child=spawn(binary.trim(),['serve','-I/',config],{stdio:['ignore','ignore','pipe']});child.stderr.on('data',b=>{stderr+=b.toString();});ended=new Promise(r=>child.once('exit',r));for(let n=0;n<100;n++){if(child.exitCode!==null)throw Error(stderr);try{if((await fetch(origin+'/ready',{signal:AbortSignal.timeout(200)})).status===204)return;}catch{}await delay(25);}throw Error('local startup timed out: '+stderr);}; + const stop=async()=>{if(child?.exitCode===null){child.kill('SIGKILL');await ended;}}; + const call=async(code,op,...args)=>{const r=await fetch(origin,{method:'POST',body:JSON.stringify({code,op,args}),signal:AbortSignal.timeout(5000)});assert.equal(r.status,200);return r.json();}; + try { + await start();const code='k7m2q9xd'; + const pending={code,device:'opaque-device',expires_at:Date.now()+60000}; + assert('ok' in await call(code,'open',pending));assert('ok' in await call(code,'bind','one')); + assert.equal((await call(code,'bind','two')).err,'already_decided'); + assert.equal((await call(code,'decide','approved','c2VhbGVk','opaque-device')).err,'already_decided'); + assert.equal((await call(code,'read',0)).ok.state,'pending'); + await stop();await start();assert('ok' in await call(code,'bind','one')); + assert('ok' in await call(code,'deliver','one','decision','approved','c2VhbGVk','opaque-device')); + await stop();await start();assert('ok' in await call(code,'deliver','one','decision','approved','c2VhbGVk','opaque-device')); + assert.equal((await call(code,'deliver','one','decision','approved','b3RoZXI','opaque-device')).err,'already_decided'); + const record=(await call(code,'read',0)).ok;assert.equal(record.state,'approved');assert.equal(record.grant,'c2VhbGVk');assert.equal(record.cluster,undefined);assert.equal(record.delivery,undefined); + } finally {await stop();await rm(dir,{recursive:true,force:true});} +}); diff --git a/relay/hosted/test/store.test.js b/relay/hosted/test/store.test.js index a9cd2fb5b1..4e8b974e2e 100644 --- a/relay/hosted/test/store.test.js +++ b/relay/hosted/test/store.test.js @@ -211,3 +211,25 @@ test('locate over a thousand rids is too_many', async () => { (e) => e instanceof Wire && e.code === 'too_many' && e.status === 400, ); }); + +// R21 exercises the actual local workerd R2 backend: storage failure cannot +// produce upload evidence, and retry repairs the immutable bytes and index. +test('R21 upload references become confirmable only after local R2 and index completion', async () => { + let fail = true; + const bucket = { + async put(...args) { if (fail) throw new Error('injected R2 failure'); return env.bucket.put(...args); }, + get(...args) { return env.bucket.get(...args); }, + }; + const sql = memorySql(), store = new R2Store(bucket, sql, 'id_r21'); + const f = frame([rid(21), rid(22)]), sealed = await sealFrame(f.bytes); + await assert.rejects(store.put(sealed), /injected R2 failure/); + await assert.rejects(store.checkReferences(rid(21), [sealed.id], async () => {})); + fail = false; + assert.equal(await store.put(sealed), true); + let completed = false; + await store.checkReferences(rid(21), [sealed.id], async () => { completed = true; }); + assert(completed); + const cold = new R2Store(bucket, sql, 'id_r21'); + await cold.checkReferences(rid(21), [], async () => {}); + assert.deepEqual(await cold.get(rid(22)), f.objs[1]); +}); From 737326b057024e9d9eff78ce8a16607adc8377c5 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 01:37:41 -0400 Subject: [PATCH 024/103] Freeze legacy namespaces before importing one durable coordination history. Keep the source irrevocably gated, preserve its complete rows and watch evidence, and resume the same three-voter genesis after interruption so old and new writers cannot overlap. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterstate/imports_test.go | 2 +- internal/clusterstate/legacy.go | 194 ++++++++++ internal/directory/export.go | 343 ++++++++++++++++++ internal/directory/export_crash_test.go | 112 ++++++ internal/directory/export_test.go | 316 ++++++++++++++++ internal/directory/feed.go | 5 + internal/directory/links.go | 4 + internal/directory/sqlite.go | 35 +- internal/directory/wire.go | 3 + .../continuing-a-chat-on-another-computer.md | 4 + internal/raftabi/README.md | 7 + internal/raftabi/abi.go | 10 + internal/raftabi/legacy.go | 75 ++++ internal/raftdirectory/apply.go | 5 + internal/raftdirectory/directory_test.go | 11 +- internal/raftdirectory/migration_test.go | 227 ++++++++++++ internal/raftstore/README.md | 33 ++ internal/raftstore/import.go | 158 ++++++++ internal/raftstore/import_test.go | 163 +++++++++ 19 files changed, 1702 insertions(+), 5 deletions(-) create mode 100644 internal/clusterstate/legacy.go create mode 100644 internal/directory/export.go create mode 100644 internal/directory/export_crash_test.go create mode 100644 internal/directory/export_test.go create mode 100644 internal/raftabi/legacy.go create mode 100644 internal/raftdirectory/migration_test.go create mode 100644 internal/raftstore/import.go create mode 100644 internal/raftstore/import_test.go diff --git a/internal/clusterstate/imports_test.go b/internal/clusterstate/imports_test.go index e1115b04af..5b1a089599 100644 --- a/internal/clusterstate/imports_test.go +++ b/internal/clusterstate/imports_test.go @@ -14,7 +14,7 @@ import ( // enter the pure graph when another packet adds a record or rule. func TestR14PureImportBoundary(t *testing.T) { allowed := map[string]bool{ - "bytes": true, "encoding/json": true, "errors": true, "fmt": true, + "io": true, "bytes": true, "encoding/json": true, "errors": true, "fmt": true, // Signature verification, hashing, key encoding and ordering are pure. "encoding/base64": true, "crypto/ed25519": true, "crypto/sha256": true, "encoding/hex": true, "sort": true, "reflect": true, "strconv": true, "strings": true, "unicode/utf8": true, diff --git a/internal/clusterstate/legacy.go b/internal/clusterstate/legacy.go new file mode 100644 index 0000000000..e346e580c2 --- /dev/null +++ b/internal/clusterstate/legacy.go @@ -0,0 +1,194 @@ +package clusterstate + +import ( + "bytes" + "crypto/ed25519" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "io" + "sort" +) + +var ErrMigration = errors.New("codeaf: directory upgrade is incomplete; retry shortly; saved chats are still here") + +// LegacyConfig is allocated once at the source gate. Only public protocol keys +// belong here; content keys and private signing keys never enter an import. +type LegacyConfig struct { + Cluster string `json:"cluster"` + Generation uint64 `json:"generation"` + Identity string `json:"identity"` + SigningKey []byte `json:"signing_key"` + Anchors []uint64 `json:"anchors"` + EffectsKey []byte `json:"effects_key"` + Now int64 `json:"now"` + Certificates []string `json:"certificates,omitempty"` +} + +func (c LegacyConfig) Validate() error { + if c.Cluster == "" || c.Generation != 1 || len(c.SigningKey) != ed25519.PublicKeySize || IdentityID(c.SigningKey) != c.Identity || len(c.EffectsKey) != ed25519.PublicKeySize || bytes.Equal(c.EffectsKey, c.SigningKey) || len(c.Anchors) != 3 || c.Now < 0 || c.Now > int64(MaxSafeInteger) { + return ErrMigration + } + for i, id := range c.Anchors { + if id == 0 || id >= ^uint64(0)-1 || (i > 0 && id <= c.Anchors[i-1]) { + return ErrMigration + } + } + if len(c.Certificates) > MaxLiveSessions { + return ErrMigration + } + seen := map[string]bool{} + for _, text := range c.Certificates { + public, e := legacyCertificate(c, text) + if e != nil || seen[string(public)] || bytes.Equal(public, c.EffectsKey) { + return ErrMigration + } + seen[string(public)] = true + } + return nil +} + +func legacyCertificate(c LegacyConfig, text string) ([]byte, error) { + var cert struct { + Device string `json:"device"` + } + raw, e := base64.RawURLEncoding.DecodeString(text) + if e != nil || len(raw) > 1024 || json.Unmarshal(raw, &cert) != nil { + return nil, ErrMigration + } + public, e := hex.DecodeString(cert.Device) + if e != nil { + return nil, ErrMigration + } + return JoinCertificate(IdentityRec{Identity: c.Identity, SigningKey: c.SigningKey}, JoinRequest{Device: DeviceIDOfKey(public), PublicKey: public}, text) +} + +// LegacyManifest inventories converted rows at a fixed source version. The +// digest covers canonical RecordWrite lines in namespace/key order, excluding +// this manifest. Snapshot row framing is applied only after genesis commits. +type LegacyManifest struct { + Config LegacyConfig `json:"config"` + Version uint64 `json:"version"` + Rows uint64 `json:"rows"` + Bytes uint64 `json:"bytes"` + Digest string `json:"digest"` + Ceiling uint64 `json:"ceiling"` +} + +func LegacyImportKey() RecordKey { return RecordKey{"membership", "legacy-import"} } +func (m LegacyManifest) Validate() error { + h, e := hex.DecodeString(m.Digest) + b, err := Encode(m) + if m.Config.Validate() != nil || m.Rows < 2 || e != nil || len(h) != 32 || hex.EncodeToString(h) != m.Digest || err != nil || uint64(len(b))+128 > LegacyManifestReserve || m.Ceiling != max(uint64(DefaultApplicationBytes), m.Bytes+RecordBytes(LegacyImportKey(), b)) { + return ErrMigration + } + return nil +} + +// Bound the retained provenance independently of the number of source rows. +// Its exact encoded size is charged to the grandfathered inventory. +const LegacyManifestReserve uint64 = 4096 + +// LegacyRow converts the ORIGINAL v1 JSON bytes, never a JavaScript number +// roundtrip. Unknown fields refuse migration instead of silently dropping data. +// HoldUntil is sampled once while the source is frozen and recorded durably. +func LegacyRow(c LegacyConfig, kind, id string, raw []byte, holdUntil int64) (RecordWrite, error) { + if c.Validate() != nil || len(raw) > MaxEvaluationBytes { + return RecordWrite{}, ErrMigration + } + var key RecordKey + var value any + switch kind { + case "identity": + var v IdentityRec + if legacyDecode(raw, &v) != nil || v.V != 1 || (v.Identity != "" && v.Identity != c.Identity) || (len(v.SigningKey) > 0 && !bytes.Equal(v.SigningKey, c.SigningKey)) { + return RecordWrite{}, ErrMigration + } + v.Identity = c.Identity + v.SigningKey = c.SigningKey + key = IdentityKey() + value = v + case "devices": + var v Device + if id == "" || legacyDecode(raw, &v) != nil || v.V != 1 { + return RecordWrite{}, ErrMigration + } + key = DeviceKey(id) + value = v + case "cells": + var v Cell + if id == "" || legacyDecode(raw, &v) != nil || v.V != 1 || v.Lease.Fence > MaxSafeInteger { + return RecordWrite{}, ErrMigration + } + if v.Lease.Expires != 0 { + v.Lease.Expires = max(v.Lease.Expires, holdUntil) + } + key = CellKey(id) + value = v + default: + return RecordWrite{}, ErrMigration + } + data, e := Encode(value) + if e != nil || len(data)+len(key.Key) > MaxEvaluationBytes { + return RecordWrite{}, ErrMigration + } + return RecordWrite{key, data}, nil +} +func legacyDecode(raw []byte, v any) error { + d := json.NewDecoder(bytes.NewReader(raw)) + d.DisallowUnknownFields() + if d.Decode(v) != nil { + return ErrMigration + } + // A second value or malformed trailing bytes must never be ignored. + var extra any + if d.Decode(&extra) != io.EOF { + return ErrMigration + } + return nil +} +func LegacyEffects(c LegacyConfig) RecordWrite { + b, _ := Encode(Authority{Cluster: c.Cluster, Key: c.EffectsKey, Effects: true}) + return RecordWrite{AuthorityKey(DeviceIDOfKey(c.EffectsKey)), b} +} + +// SealLegacyManifest counts its own retained provenance in the imported state. +// Decimal ceiling width converges before the fixed bounded manifest limit. +func SealLegacyManifest(m LegacyManifest) (LegacyManifest, error) { + for range 8 { + b, e := Encode(m) + if e != nil { + return m, e + } + n := max(uint64(DefaultApplicationBytes), m.Bytes+RecordBytes(LegacyImportKey(), b)) + if n == m.Ceiling { + return m, m.Validate() + } + m.Ceiling = n + } + return m, ErrMigration +} + +// LegacyAuthorities carries only certificates already authenticated by ingress. +// Unseen devices later enroll with their original root-signed certificate; an +// ID or a metadata upsert by itself never grants signing authority. +func LegacyAuthorities(c LegacyConfig) ([]RecordWrite, error) { + if c.Validate() != nil { + return nil, ErrMigration + } + out := []RecordWrite{LegacyEffects(c)} + for _, cert := range c.Certificates { + public, e := legacyCertificate(c, cert) + if e != nil { + return nil, e + } + b, e := Encode(Authority{Cluster: c.Cluster, Key: public, Personal: true}) + if e != nil { + return nil, e + } + out = append(out, RecordWrite{AuthorityKey(DeviceIDOfKey(public)), b}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) + return out, nil +} diff --git a/internal/directory/export.go b/internal/directory/export.go new file mode 100644 index 0000000000..62c92beae2 --- /dev/null +++ b/internal/directory/export.go @@ -0,0 +1,343 @@ +package directory + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + + "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +// LegacyGate is irrevocable. A failed inventory keeps the source frozen, and +// ownership publication never makes the backup writable again. +type LegacyGate struct { + Stage string `json:"stage"` + Config clusterstate.LegacyConfig `json:"config"` + Manifest *clusterstate.LegacyManifest `json:"manifest,omitempty"` +} +type LegacyReceipt struct { + Node uint64 + Manifest clusterstate.LegacyManifest + Applied uint64 +} + +func gateIn(tx *sql.Tx) (*LegacyGate, error) { + var raw string + e := tx.QueryRow(`SELECT rec FROM legacy_gate WHERE id=1`).Scan(&raw) + if errors.Is(e, sql.ErrNoRows) { + return nil, nil + } + if e != nil { + return nil, e + } + var g LegacyGate + if json.Unmarshal([]byte(raw), &g) != nil || g.Config.Validate() != nil || (g.Stage != "frozen" && g.Stage != "inventoried" && g.Stage != "raft-owned") { + return nil, clusterstate.ErrMigration + } + return &g, nil +} +func keepGate(tx *sql.Tx, g LegacyGate) error { + b, e := json.Marshal(g) + if e != nil { + return e + } + _, e = tx.Exec(`INSERT OR REPLACE INTO legacy_gate VALUES(1,?)`, string(b)) + return e +} +func (s *SQLite) migrationTx(ctx context.Context, fn func(*sql.Tx) error) error { + tx, e := s.db.BeginTx(ctx, nil) + if e != nil { + return e + } + defer tx.Rollback() + if e = fn(tx); e != nil { + return e + } + s.migrationBoundary("before-commit") + if e = tx.Commit(); e != nil { + return e + } + s.migrationBoundary("after-commit") + return nil +} +func (s *SQLite) LegacyGate(ctx context.Context) (g *LegacyGate, e error) { + e = s.migrationTx(ctx, func(tx *sql.Tx) error { var err error; g, err = gateIn(tx); return err }) + return +} + +// FreezeLegacy waits behind admitted SQL operations, captures exact watch +// evidence, and commits the chosen UUID before any export or member bootstrap. +// Closing old watches forces a fresh authoritative listing after cutover. +func (s *SQLite) FreezeLegacy(ctx context.Context, c clusterstate.LegacyConfig) error { + s.migrationMu.Lock() + defer s.migrationMu.Unlock() + if s.pairing.links != nil { + drain, ok := s.pairing.links.(interface{ WithLegacyGate(func() error) error }) + if !ok { + return clusterstate.ErrMigration + } + return drain.WithLegacyGate(func() error { return s.freezeLegacy(ctx, c) }) + } + return s.freezeLegacy(ctx, c) +} +func (s *SQLite) freezeLegacy(ctx context.Context, c clusterstate.LegacyConfig) error { + if c.Validate() != nil { + return clusterstate.ErrMigration + } + e := s.migrationTx(ctx, func(tx *sql.Tx) error { + g, e := gateIn(tx) + if e != nil { + return e + } + if g != nil { + a, _ := clusterstate.Encode(g.Config) + b, _ := clusterstate.Encode(c) + if !bytes.Equal(a, b) { + return clusterstate.ErrMigration + } + return nil + } + // The source rows are retained byte-for-byte. Only sampled renewal evidence + // enters this separate migration table and the future committed genesis. + rows, e := tx.Query(`SELECT id,rec FROM cells ORDER BY id`) + if e != nil { + return e + } + for rows.Next() { + var id, raw string + if e = rows.Scan(&id, &raw); e != nil { + break + } + var cell Cell + if json.Unmarshal([]byte(raw), &cell) != nil { + // The gate still freezes a malformed source. Inventory reports + // its unsupported row without ever reopening the old writer. + continue + } + until := s.feed.Lifted(id, cell).Lease.Expires + if until > cell.Lease.Expires { + if _, e = tx.Exec(`INSERT INTO legacy_holds VALUES(?,?)`, id, until); e != nil { + break + } + s.migrationBoundary("after-freeze-hold") + } + } + re := rows.Err() + rows.Close() + if e != nil { + return e + } + if re != nil { + return re + } + if e := keepGate(tx, LegacyGate{Stage: "frozen", Config: c}); e != nil { + return e + } + s.migrationBoundary("after-freeze-gate") + return nil + }) + if e == nil { + s.feed.Close() + } + return e +} + +// InventoryLegacy converts one row at a time under the frozen source version. +// Unknown or oversized rows leave the frozen original intact for diagnosis. +func (s *SQLite) InventoryLegacy(ctx context.Context) (m clusterstate.LegacyManifest, e error) { + e = s.migrationTx(ctx, func(tx *sql.Tx) error { + g, e := gateIn(tx) + if e != nil { + return e + } + if g == nil { + return clusterstate.ErrMigration + } + if g.Manifest != nil { + m = *g.Manifest + return nil + } + insert := func(w clusterstate.RecordWrite) error { + _, e := tx.Exec(`INSERT INTO legacy_rows VALUES(?,?,?)`, w.Namespace, w.Key, w.Data) + s.migrationBoundary("after-inventory-row") + return e + } + authorities, e := clusterstate.LegacyAuthorities(g.Config) + if e != nil { + return e + } + for _, w := range authorities { + if e = insert(w); e != nil { + return e + } + } + for _, table := range []string{"identity", "devices", "cells"} { + rows, e := tx.Query(`SELECT id,rec FROM ` + table + ` ORDER BY id`) + if e != nil { + return e + } + for rows.Next() { + var id, raw string + if e = rows.Scan(&id, &raw); e != nil { + break + } + var until int64 + if table == "cells" { + err := tx.QueryRow(`SELECT until_ms FROM legacy_holds WHERE id=?`, id).Scan(&until) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + e = err + break + } + } + w, err := clusterstate.LegacyRow(g.Config, table, id, []byte(raw), until) + if err != nil { + e = fmt.Errorf("directory: cannot import %s record %q: unsupported fields or row exceeds %d bytes: %w", table, id, clusterstate.MaxEvaluationBytes, err) + break + } + if e = insert(w); e != nil { + break + } + } + re := rows.Err() + rows.Close() + if e != nil { + return e + } + if re != nil { + return re + } + } + m = clusterstate.LegacyManifest{Config: g.Config} + m.Version, e = versionIn(tx) + if e != nil { + return e + } + h := sha256.New() + if e = exportRows(tx, func(w clusterstate.RecordWrite) error { + b, e := clusterstate.Encode(w) + if e != nil { + return e + } + h.Write(b) + h.Write([]byte{'\n'}) + m.Rows++ + m.Bytes += clusterstate.RecordBytes(w.RecordKey, w.Data) + return nil + }); e != nil { + return e + } + m.Digest = hex.EncodeToString(h.Sum(nil)) + m, e = clusterstate.SealLegacyManifest(m) + if e != nil { + return e + } + if m.Validate() != nil { + return clusterstate.ErrMigration + } + g.Stage = "inventoried" + g.Manifest = &m + if e := keepGate(tx, *g); e != nil { + return e + } + s.migrationBoundary("after-inventory-gate") + return nil + }) + return +} +func exportRows(tx *sql.Tx, visit func(clusterstate.RecordWrite) error) error { + rows, e := tx.Query(`SELECT namespace,key,data FROM legacy_rows ORDER BY namespace,key`) + if e != nil { + return e + } + defer rows.Close() + for rows.Next() { + var w clusterstate.RecordWrite + if e = rows.Scan(&w.Namespace, &w.Key, &w.Data); e != nil { + return e + } + if e = visit(w); e != nil { + return e + } + } + return rows.Err() +} +func (s *SQLite) ExportLegacy(ctx context.Context, visit func(clusterstate.RecordWrite) error) (m clusterstate.LegacyManifest, e error) { + e = s.migrationTx(ctx, func(tx *sql.Tx) error { + g, err := gateIn(tx) + if err != nil { + return err + } + if g == nil || g.Manifest == nil { + return clusterstate.ErrMigration + } + m = *g.Manifest + return exportRows(tx, visit) + }) + return +} + +// OwnLegacy publishes only after the host observes the SAME durable import +// through a fresh quorum barrier. Receipts are host-internal, never client input. +func (s *SQLite) OwnLegacy(ctx context.Context, receipts []LegacyReceipt) error { + return s.migrationTx(ctx, func(tx *sql.Tx) error { + g, e := gateIn(tx) + if e != nil { + return e + } + if g == nil || g.Manifest == nil { + return clusterstate.ErrMigration + } + want, _ := clusterstate.Encode(*g.Manifest) + seen := map[uint64]bool{} + for _, r := range receipts { + b, _ := clusterstate.Encode(r.Manifest) + for _, id := range g.Config.Anchors { + if r.Node == id && r.Applied >= 1 && bytes.Equal(want, b) { + seen[id] = true + } + } + } + if len(seen) < 2 { + return clusterstate.ErrMigration + } + g.Stage = "raft-owned" + if e := keepGate(tx, *g); e != nil { + return e + } + s.migrationBoundary("after-owned-gate") + return nil + }) +} + +// Tests kill the process at actual durable source transitions. Production has +// no callback and no alternate gate or weaker transaction mode. +func (s *SQLite) migrationBoundary(phase string) { + if s.migrationCut != nil { + s.migrationCut(phase) + } +} + +// MigrationRoute resolves only a durably published owner. The callback must +// return the consensus backend for that UUID, never a cached legacy directory. +func (s *SQLite) MigrationRoute(ctx context.Context, backend func(clusterstate.LegacyManifest) (RaftBackend, error)) (Directory, error) { + g, e := s.LegacyGate(ctx) + if e != nil { + return nil, e + } + if g == nil || g.Stage != "raft-owned" || g.Manifest == nil { + return nil, clusterstate.ErrMigration + } + b, e := backend(*g.Manifest) + if e != nil { + return nil, e + } + if b == nil { + return nil, clusterstate.ErrMigration + } + return NewRaft(b), nil +} diff --git a/internal/directory/export_crash_test.go b/internal/directory/export_crash_test.go new file mode 100644 index 0000000000..e057f04bc7 --- /dev/null +++ b/internal/directory/export_crash_test.go @@ -0,0 +1,112 @@ +package directory + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +func TestR24SourceCrashHelper(t *testing.T) { + path := os.Getenv("P11_SOURCE") + if path == "" { + return + } + s, e := OpenSQLite(path, func() time.Time { return time.UnixMilli(1000) }) + legacyMust(t, e) + var initial *LegacyGate + if os.Getenv("P11_STAGE") == "own" { + initial, e = s.LegacyGate(context.Background()) + legacyMust(t, e) + } + if os.Getenv("P11_STAGE") == "freeze" { + s.clock = func() time.Time { return time.UnixMilli(200000) } + s.feed.clock = s.clock + sub, e := s.Feed().Subscribe("a", []Hold{{Cell: "chat", Fence: 1}}) + legacyMust(t, e) + sub.Ping() + } + phase := os.Getenv("P11_PHASE") + s.migrationCut = func(p string) { + if p == phase { + os.Exit(81) + } + } + switch os.Getenv("P11_STAGE") { + case "freeze": + e = s.FreezeLegacy(context.Background(), legacyConfig()) + case "inventory": + _, e = s.InventoryLegacy(context.Background()) + case "own": + e = s.OwnLegacy(context.Background(), []LegacyReceipt{{Node: 1, Manifest: *initial.Manifest, Applied: 2}, {Node: 2, Manifest: *initial.Manifest, Applied: 2}}) + } + legacyMust(t, e) + t.Fatal("source crash cut was not reached") +} + +// R24 kills between source SQL stages and either side of each durable commit. +// Every replay uses the same UUID; a committed freeze never reopens the writer. +func TestR24SourceStageCrashesAndResume(t *testing.T) { + for _, stage := range []string{"freeze", "inventory", "own"} { + internal := map[string]string{"freeze": "after-freeze-gate", "inventory": "after-inventory-gate", "own": "after-owned-gate"}[stage] + phases := []string{internal, "before-commit", "after-commit"} + if stage == "freeze" { + phases = append(phases, "after-freeze-hold") + } + if stage == "inventory" { + phases = append(phases, "after-inventory-row") + } + for _, phase := range phases { + t.Run(stage+"/"+phase, func(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "source.db") + s, _ := legacySeed(t, path) + if stage != "freeze" { + legacyMust(t, s.FreezeLegacy(ctx, legacyConfig())) + } + if stage == "own" { + _, e := s.InventoryLegacy(ctx) + legacyMust(t, e) + } + legacyMust(t, s.Close()) + cmd := exec.Command(os.Args[0], "-test.run=^TestR24SourceCrashHelper$", "-test.count=1") + cmd.Env = append(os.Environ(), "P11_SOURCE="+path, "P11_STAGE="+stage, "P11_PHASE="+phase) + out, e := cmd.CombinedOutput() + var exit *exec.ExitError + if !errors.As(e, &exit) || exit.ExitCode() != 81 { + t.Fatalf("cut: %v %s", e, out) + } + s, e = OpenSQLite(path, func() time.Time { return time.UnixMilli(1000) }) + legacyMust(t, e) + defer s.Close() + g, e := s.LegacyGate(ctx) + legacyMust(t, e) + // A cut before the first freeze COMMIT is still the old writer; every + // later cut must leave it permanently blocked, even before ownership. + frozen := stage != "freeze" || phase == "after-commit" + if frozen && g == nil { + t.Fatal("lost freeze") + } + if frozen { + if _, e = s.For("a").Cell(ctx, "chat"); !errors.Is(e, clusterstate.ErrMigration) { + t.Fatal("old reader reopened", e) + } + } + legacyMust(t, s.FreezeLegacy(ctx, legacyConfig())) + m, e := s.InventoryLegacy(ctx) + legacyMust(t, e) + legacyMust(t, s.OwnLegacy(ctx, []LegacyReceipt{{Node: 1, Manifest: m, Applied: 2}, {Node: 2, Manifest: m, Applied: 2}})) + g, e = s.LegacyGate(ctx) + legacyMust(t, e) + if g.Stage != "raft-owned" || g.Config.Cluster != legacyConfig().Cluster { + t.Fatal(g) + } + }) + } + } +} diff --git a/internal/directory/export_test.go b/internal/directory/export_test.go new file mode 100644 index 0000000000..a3556cb6c3 --- /dev/null +++ b/internal/directory/export_test.go @@ -0,0 +1,316 @@ +package directory + +import ( + "bytes" + "context" + "crypto/ed25519" + "database/sql" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftstore" +) + +func legacyConfig() clusterstate.LegacyConfig { + root := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{7}, 32)).Public().(ed25519.PublicKey) + effect := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{8}, 32)).Public().(ed25519.PublicKey) + return clusterstate.LegacyConfig{Cluster: "r24-one-uuid", Generation: 1, Identity: clusterstate.IdentityID(root), SigningKey: root, EffectsKey: effect, Anchors: []uint64{1, 2, 3}, Now: 200000} +} +func legacyMust(t *testing.T, e error) { + t.Helper() + if e != nil { + t.Fatal(e) + } +} +func legacySeed(t *testing.T, path string) (*SQLite, Cell) { + t.Helper() + s, e := OpenSQLite(path, func() time.Time { return time.UnixMilli(1000) }) + legacyMust(t, e) + legacyMust(t, s.For("a").PutDevice(context.Background(), "b", Device{V: 1, Name: "c2VhbGVk", Revoked: true, Caps: Caps{OS: "linux"}})) + legacyMust(t, s.For("a").SetVault(context.Background(), "", "opaque-vault")) + v, e := s.For("a").Create(context.Background(), "chat", CellInit{Head: "opaque-head", Frames: []string{"opaque-frame"}, Title: "c2VhbGVk", ParentCell: "parent", Keys: map[string]map[string]string{"key": {"identity": "wrapped"}}, Size: 71}) + legacyMust(t, e) + return s, v.Cell +} + +// R24/R28 compares the full retained application, not List's stripped frame +// hints. Both source handles and every future opener remain fenced forever. +func TestR24R28FrozenExportSameGenesisAndSourceRetention(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "source.db") + s, original := legacySeed(t, path) + defer s.Close() + other, e := OpenSQLite(path, s.clock) + legacyMust(t, e) + defer other.Close() + sub, e := s.Feed().Subscribe("a", []Hold{{Cell: "chat", Fence: original.Lease.Fence}}) + legacyMust(t, e) + defer sub.Close() + // Sampling current evidence preserves a held lease after its SQL expiry. + s.clock = func() time.Time { return time.UnixMilli(200000) } + s.feed.clock = s.clock + sub.Ping() + legacyMust(t, s.FreezeLegacy(ctx, legacyConfig())) + if _, e = s.feed.Subscribe("a", nil); e == nil { + t.Fatal("frozen feed accepted a watch") + } + if _, e = other.For("a").Acquire(ctx, "chat", AcquireOpts{Force: true}); !errors.Is(e, clusterstate.ErrMigration) { + t.Fatal(e) + } + if _, e = s.For("a").List(ctx); !errors.Is(e, clusterstate.ErrMigration) { + t.Fatal(e) + } + m, e := s.InventoryLegacy(ctx) + legacyMust(t, e) + if e = s.OwnLegacy(ctx, []LegacyReceipt{{Node: 1, Manifest: m, Applied: 1}}); e == nil { + t.Fatal("one anchor published ownership") + } + var genesis []byte + for i := uint64(1); i <= 3; i++ { + target, e := raftstore.Open(filepath.Join(t.TempDir(), "anchor.db"), raftstore.Identity{ClusterUUID: m.Config.Cluster, NodeID: i, Protocol: 1}) + legacyMust(t, e) + stream := func(visit func(clusterstate.RecordWrite) error) error { + got, e := s.ExportLegacy(ctx, visit) + if e == nil && got.Digest != m.Digest { + return fmt.Errorf("source changed") + } + return e + } + legacyMust(t, target.ImportLegacy(ctx, m, stream)) + legacyMust(t, target.ImportLegacy(ctx, m, func(func(clusterstate.RecordWrite) error) error { t.Fatal("retry reread source"); return nil })) + r, e := target.Recover(ctx) + legacyMust(t, e) + if !bytes.Equal(genesis, r.Core.Snapshot.Data) && genesis != nil { + t.Fatal("anchors have different genesis") + } + genesis = bytes.Clone(r.Core.Snapshot.Data) + if len(r.Application.ConfState.Voters) != 3 || r.Application.VisibleVersion != m.Version { + t.Fatal("lost voters/version") + } + k := clusterstate.CellKey("chat") + rows, _, e := target.Read(ctx, r.Application.Applied, []raftstore.Key{{Namespace: raftstore.State, Key: k.Key}}) + legacyMust(t, e) + var got Cell + legacyMust(t, clusterstate.Decode(rows[raftstore.Key{Namespace: raftstore.State, Key: k.Key}], &got)) + want := original + want.Lease.Expires = 290000 + a, _ := json.Marshal(got) + b, _ := json.Marshal(want) + if !bytes.Equal(a, b) { + t.Fatalf("lost cell fields: %s", a) + } + legacyMust(t, target.Close()) + } + legacyMust(t, s.OwnLegacy(ctx, []LegacyReceipt{{Node: 1, Manifest: m, Applied: 2}, {Node: 2, Manifest: m, Applied: 2}})) + var raw string + legacyMust(t, s.db.QueryRow(`SELECT rec FROM cells WHERE id='chat'`).Scan(&raw)) + var retained Cell + legacyMust(t, json.Unmarshal([]byte(raw), &retained)) + if retained.Lease.Expires != original.Lease.Expires || len(retained.Frames) != 1 { + t.Fatal("original changed") + } + g, e := s.LegacyGate(ctx) + legacyMust(t, e) + if g.Stage != "raft-owned" { + t.Fatal(g) + } + wrong := legacyConfig() + wrong.Cluster = "different" + if s.FreezeLegacy(ctx, wrong) == nil { + t.Fatal("allocated a second UUID") + } +} + +// R24 inventories an identity above the default ceiling without holding the +// whole export in memory, and refuses an unsupported row without truncation. +func TestR24OversizeRowsRotationAndConcurrentOldWrites(t *testing.T) { + ctx := context.Background() + s, _ := legacySeed(t, filepath.Join(t.TempDir(), "source.db")) + defer s.Close() + payload := strings.Repeat("s", 300000) + for i := 0; i < 30; i++ { + _, e := s.For("a").Create(ctx, fmt.Sprint(i), CellInit{Head: "head", Title: payload, Frames: []string{"frame"}}) + legacyMust(t, e) + } + _, e := s.For("a").Rotate(ctx, RotationReq{Op: OpFreeze}) + legacyMust(t, e) + var wg sync.WaitGroup + wg.Add(1) + done := make(chan struct{}) + go func() { + defer wg.Done() + defer close(done) + for range 20 { + _ = s.seenForTest("b") + } + }() + legacyMust(t, s.FreezeLegacy(ctx, legacyConfig())) + wg.Wait() + <-done + m, e := s.InventoryLegacy(ctx) + legacyMust(t, e) + if m.Ceiling <= clusterstate.DefaultApplicationBytes || m.Bytes < 9000000 { + t.Fatal("oversized inventory stranded", m.Bytes) + } + target, e := raftstore.Open(filepath.Join(t.TempDir(), "anchor.db"), raftstore.Identity{ClusterUUID: m.Config.Cluster, NodeID: 1, Protocol: 1}) + legacyMust(t, e) + defer target.Close() + legacyMust(t, target.ImportLegacy(ctx, m, func(fn func(clusterstate.RecordWrite) error) error { _, e := s.ExportLegacy(ctx, fn); return e })) + r, e := target.Recover(ctx) + legacyMust(t, e) + if r.Application.RecordCount != m.Rows+1 { + t.Fatal("rows lost") + } + var id IdentityRec + key := raftstore.Key{Namespace: raftstore.State, Key: "identity"} + rows, _, e := target.Read(ctx, 1, []raftstore.Key{key}) + legacyMust(t, e) + legacyMust(t, clusterstate.Decode(rows[key], &id)) + if id.Rotation == nil || id.Vault != "opaque-vault" || !bytes.Equal(id.SigningKey, m.Config.SigningKey) { + t.Fatal("lost identity state") + } + bad, _ := legacySeed(t, filepath.Join(t.TempDir(), "bad.db")) + defer bad.Close() + raw, _ := json.Marshal(Cell{V: 1, Title: strings.Repeat("x", clusterstate.MaxEvaluationBytes+1)}) + _, e = bad.db.Exec(`UPDATE cells SET rec=? WHERE id='chat'`, string(raw)) + legacyMust(t, e) + legacyMust(t, bad.FreezeLegacy(ctx, legacyConfig())) + if _, e = bad.InventoryLegacy(ctx); e == nil || !strings.Contains(e.Error(), `cells record "chat"`) { + t.Fatal("oversized row was accepted or its location was not reported", e) + } + var kept string + legacyMust(t, bad.db.QueryRow(`SELECT rec FROM cells WHERE id='chat'`).Scan(&kept)) + if kept != string(raw) { + t.Fatal("oversize source truncated") + } + if _, e = bad.For("a").List(ctx); !errors.Is(e, clusterstate.ErrMigration) { + t.Fatal("failed import reopened source", e) + } +} +func (s *SQLite) seenForTest(id string) error { + return s.For("a").PutDevice(context.Background(), id, Device{V: 1}) +} + +// R24 permits a write admitted before the gate to finish, includes it exactly, +// and refuses a request that was authenticated earlier but entered SQL later. +func TestR24FreezeWaitsForAnAdmittedLegacyWrite(t *testing.T) { + ctx := context.Background() + s, _ := legacySeed(t, filepath.Join(t.TempDir(), "source.db")) + defer s.Close() + entered, release := make(chan struct{}), make(chan struct{}) + writeDone := make(chan error, 1) + go func() { + writeDone <- s.inTx(ctx, func(tx *sql.Tx) error { + close(entered) + <-release + var id IdentityRec + if e := get(tx, "identity", "1", &id); e != nil { + return e + } + id.Vault = "last-admitted-vault" + return put(tx, s.feed, "identity", "1", id, 1000) + }) + }() + <-entered + started := make(chan struct{}) + frozen := make(chan error, 1) + go func() { close(started); frozen <- s.FreezeLegacy(ctx, legacyConfig()) }() + <-started + select { + case e := <-frozen: + t.Fatalf("freeze passed an open write: %v", e) + default: + } + close(release) + legacyMust(t, <-writeDone) + legacyMust(t, <-frozen) + if e := s.For("a").SetVault(ctx, "last-admitted-vault", "too-late"); !errors.Is(e, clusterstate.ErrMigration) { + t.Fatal("delayed old request changed source", e) + } + _, e := s.InventoryLegacy(ctx) + legacyMust(t, e) + found := false + _, e = s.ExportLegacy(ctx, func(w clusterstate.RecordWrite) error { + if w.RecordKey == clusterstate.IdentityKey() { + var id IdentityRec + if e := clusterstate.Decode(w.Data, &id); e != nil { + return e + } + found = id.Vault == "last-admitted-vault" + } + return nil + }) + legacyMust(t, e) + if !found { + t.Fatal("admitted write disappeared") + } +} + +// R24/C1 drains a legacy approval through its public decision before taking +// ownership. The sealed grant stays readable at its original destination. +func TestR24FreezeDrainsAnAdmittedPairingDecision(t *testing.T) { + ctx := context.Background() + s, _ := legacySeed(t, filepath.Join(t.TempDir(), "source.db")) + defer s.Close() + links := NewLinks(s.clock, DefaultLinkLimits) + s.SetLinks(links) + public := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{9}, 32)).Public().(ed25519.PublicKey) + id := clusterstate.DeviceIDOfKey(public) + code := "k7m2q9xd" + links.reqs[code] = &entry{rec: Request{Code: code, Device: id, State: RequestPending, ExpiresAt: 1000000}, until: 1000000, done: make(chan struct{})} + cert := struct { + V uint16 `json:"V"` + Device string `json:"device"` + Made int64 `json:"created"` + Sig string `json:"sig"` + }{V: 1, Device: hex.EncodeToString(public), Made: 1000} + body, e := json.Marshal(cert) + legacyMust(t, e) + cert.Sig = hex.EncodeToString(ed25519.Sign(ed25519.NewKeyFromSeed(bytes.Repeat([]byte{7}, 32)), append([]byte("codeaf/device-cert/v1\n"), body...))) + body, e = json.Marshal(cert) + legacyMust(t, e) + approval := Approval{Device: Device{V: 1, Name: "c2VhbGVk"}, Grant: "c2VhbGVk", Cert: base64.RawURLEncoding.EncodeToString(body)} + entered, release := make(chan struct{}), make(chan struct{}) + s.SetOnJoin(func(Joined) { close(entered); <-release }) + approved := make(chan error, 1) + go func() { approved <- s.For("a").ApproveRequest(ctx, code, approval) }() + <-entered + started := make(chan struct{}) + frozen := make(chan error, 1) + go func() { close(started); frozen <- s.FreezeLegacy(ctx, legacyConfig()) }() + <-started + select { + case e := <-frozen: + t.Fatalf("freeze passed an undecided grant: %v", e) + default: + } + close(release) + legacyMust(t, <-approved) + legacyMust(t, <-frozen) + if links.reqs[code].rec.State != RequestApproved || links.reqs[code].rec.Grant == nil || *links.reqs[code].rec.Grant != approval.Grant { + t.Fatal("admitted sealed grant lost") + } + _, e = s.InventoryLegacy(ctx) + legacyMust(t, e) + found := false + _, e = s.ExportLegacy(ctx, func(w clusterstate.RecordWrite) error { + found = found || w.RecordKey == clusterstate.DeviceKey(id) + return nil + }) + legacyMust(t, e) + if !found { + t.Fatal("granted device missing from import") + } + if e = s.For("a").ApproveRequest(ctx, code, approval); !errors.Is(e, clusterstate.ErrMigration) { + t.Fatal("old duplicate decision bypassed source gate", e) + } +} diff --git a/internal/directory/feed.go b/internal/directory/feed.go index e5f1f40afb..b469b680ac 100644 --- a/internal/directory/feed.go +++ b/internal/directory/feed.go @@ -161,6 +161,11 @@ func (f *Feed) SubscribeBeat(device string, holds []Hold, beat time.Duration) (* func (f *Feed) subscribe(device string, holds []Hold, beat time.Duration) (*Sub, error) { f.mu.Lock() defer f.mu.Unlock() + select { + case <-f.closed: + return nil, errFeedClosed + default: + } if len(f.subs) >= f.cap { return nil, ErrTooManyWatchers } diff --git a/internal/directory/links.go b/internal/directory/links.go index 3cb7823ed7..de6e927306 100644 --- a/internal/directory/links.go +++ b/internal/directory/links.go @@ -304,3 +304,7 @@ func slow(d time.Duration) error { secs := int((d + time.Second - 1) / time.Second) return wireauth.Wait(ErrRateLimited, http.Header{"Retry-After": {strconv.Itoa(max(secs, 1))}}) } + +// WithLegacyGate finishes any admitted pairing decision before the source +// freezes. A later legacy callback must recheck the durable source gate. +func (l *Links) WithLegacyGate(fn func() error) error { l.mu.Lock(); defer l.mu.Unlock(); return fn() } diff --git a/internal/directory/sqlite.go b/internal/directory/sqlite.go index fe14bdf171..9311f87ada 100644 --- a/internal/directory/sqlite.go +++ b/internal/directory/sqlite.go @@ -8,6 +8,8 @@ import ( "sync" "time" + "github.com/Agent-Field/codeaf/internal/clusterstate" + _ "modernc.org/sqlite" // the one SQLite driver the repo uses ) @@ -16,6 +18,9 @@ import ( // migration. The identity table has at most one row: a SQLite file is one // identity's directory. const schema = ` +CREATE TABLE IF NOT EXISTS legacy_gate (id INTEGER PRIMARY KEY CHECK (id=1), rec TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS legacy_rows (namespace TEXT NOT NULL, key TEXT NOT NULL, data BLOB NOT NULL, PRIMARY KEY(namespace,key)); +CREATE TABLE IF NOT EXISTS legacy_holds (id TEXT PRIMARY KEY, until_ms INTEGER NOT NULL); CREATE TABLE IF NOT EXISTS identity (id INTEGER PRIMARY KEY CHECK (id = 1), rec TEXT NOT NULL); CREATE TABLE IF NOT EXISTS devices (id TEXT PRIMARY KEY, rec TEXT NOT NULL); CREATE TABLE IF NOT EXISTS cells (id TEXT PRIMARY KEY, rec TEXT NOT NULL); @@ -28,8 +33,10 @@ CREATE TABLE IF NOT EXISTS version (id INTEGER PRIMARY KEY CHECK (id = 1), n IN // connections to the same file agree on a single winner. type SQLite struct { pairing - db *sql.DB - clock func() time.Time + migrationCut func(string) + migrationMu sync.RWMutex + db *sql.DB + clock func() time.Time // feed is the Status as last committed: the revoked devices a relay turns // away and the rotation the blob wire asks on every frame put, both without @@ -56,6 +63,12 @@ func OpenSQLite(path string, clock func() time.Time) (*SQLite, error) { db.Close() return nil, err } + if g, e := s.LegacyGate(context.Background()); e != nil { + db.Close() + return nil, e + } else if g != nil { + s.feed.Close() + } wireFeed(s.feed, &s.pairing, s.seen) return s, nil } @@ -210,6 +223,14 @@ func (s *SQLite) inTx(ctx context.Context, fn func(*sql.Tx) error) error { if err != nil { return err } + g, err := gateIn(tx) + if err != nil || g != nil { + tx.Rollback() + if err != nil { + return err + } + return clusterstate.ErrMigration + } st, err := s.run(tx, fn) if err != nil { tx.Rollback() @@ -415,6 +436,11 @@ func (s *SQLite) store(tx *sql.Tx, id string, d Device) (Device, error) { } func (c *sqliteClient) ApproveRequest(ctx context.Context, code string, a Approval) error { + c.s.migrationMu.RLock() + defer c.s.migrationMu.RUnlock() + if err := c.txw(ctx, func(*sql.Tx) error { return nil }); err != nil { + return err + } return c.approve(code, a, func(r Request) (d Device, err error) { err = c.txw(ctx, func(tx *sql.Tx) (err error) { d, err = c.s.store(tx, r.Device, a.Device) @@ -425,6 +451,11 @@ func (c *sqliteClient) ApproveRequest(ctx context.Context, code string, a Approv } func (c *sqliteClient) DenyRequest(ctx context.Context, code string) error { + c.s.migrationMu.RLock() + defer c.s.migrationMu.RUnlock() + if err := c.txw(ctx, func(*sql.Tx) error { return nil }); err != nil { + return err + } return c.deny(code, func() error { return c.txw(ctx, func(*sql.Tx) error { return nil }) }) } diff --git a/internal/directory/wire.go b/internal/directory/wire.go index 00d319067f..d195b08eeb 100644 --- a/internal/directory/wire.go +++ b/internal/directory/wire.go @@ -4,6 +4,7 @@ import ( "errors" "net/http" + "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/wireauth" ) @@ -18,6 +19,8 @@ type wireError struct { // wireErrors is the single place a sentinel and its code meet, so the handler // and the client cannot drift apart. Order matters only for errors.Is chains. var wireErrors = []wireError{ + {clusterstate.ErrMigration, "unreachable", http.StatusServiceUnavailable}, + {errFeedClosed, "unreachable", http.StatusServiceUnavailable}, {ErrNotFound, "not_found", http.StatusNotFound}, {ErrExists, "exists", http.StatusConflict}, {ErrLeaseHeld, "lease_held", http.StatusConflict}, diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index 4f94e27aef..50f27db63f 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -248,3 +248,7 @@ been uploaded and its new saved head has been accepted. If an upload fails, the other computer keeps seeing the previous saved head. Retry when the connection returns. Uploaded objects are kept for reuse if another device continued the chat first; this does not discard the work saved on your computer. + +## Relay directory upgrade pauses and saved chats + +A relay directory upgrade can briefly pause pairing and chat moves. Retry after the relay is ready. The upgrade refusal is `codeaf: directory upgrade is incomplete; retry shortly; saved chats are still here`. It does not delete saved chats, files, branches or the encrypted vault. The relay keeps the old directory as a backup and never makes it a second writer after the new directory takes over. A row that cannot be copied safely leaves the upgrade paused for the operator; it is never truncated to make it fit. diff --git a/internal/raftabi/README.md b/internal/raftabi/README.md index 8930fe30ed..696dee7f80 100644 --- a/internal/raftabi/README.md +++ b/internal/raftabi/README.md @@ -151,3 +151,10 @@ protobuf snapshot pointer. Step tag 12 carries peer ID plus snapshot/success flags for transport completion. These helpers provide framing and integrity, never authorization or application policy. The same codecs run in native proof peers and the bundled reactor. + +P11 adds op 12 (canonical LegacyConfig, kind, ID, original v1 JSON and sampled +hold-until; returns canonical RecordWrite), op 13 (validated LegacyManifest; +returns the shared genesis Entry/HardState/ConfState protobuf bytes), and op 14 +(validated public certificates/effects authority rows). These calls convert +bounded bytes and establish no request authentication or successful coordination +receipt. Source/target hosts own the irrevocable migration gate and durability. diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index 582a1ec0cd..11281d8d33 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -164,6 +164,12 @@ func (r *Runtime) Call(b []byte) (out []byte) { func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { e := encoder{} switch op { + case 14: + return legacyAuthorities(d) + case 13: + return legacyGenesis(d) + case 12: + return legacyCodec(d) case 10, 11: return applicationCodec(op, d) case 1: @@ -241,6 +247,10 @@ func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { in = raftcore.StorageFailed{BatchID: d.u64()} case 10: in = raftcore.CancelRead{RequestID: d.str()} + case 14: + return legacyAuthorities(d) + case 13: + return legacyGenesis(d) case 12: in = raftcore.TransportResult{Peer: d.u64(), Snapshot: d.u32() != 0, Success: d.u32() != 0} case 11: diff --git a/internal/raftabi/legacy.go b/internal/raftabi/legacy.go new file mode 100644 index 0000000000..7e5c28e916 --- /dev/null +++ b/internal/raftabi/legacy.go @@ -0,0 +1,75 @@ +package raftabi + +import ( + "github.com/Agent-Field/codeaf/internal/clusterstate" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// The hosted source feeds original SQL JSON to this bounded conversion call. +// It never reimplements directory policy or coerces a v1 fence through Number. +func legacyCodec(d *decoder) ([]byte, error) { + var c clusterstate.LegacyConfig + config := d.blob() + kind, id := d.str(), d.str() + raw := d.blob() + until := int64(d.u64()) + if e := d.end(); e != nil { + return nil, e + } + if clusterstate.Decode(config, &c) != nil { + return nil, clusterstate.ErrMigration + } + w, e := clusterstate.LegacyRow(c, kind, id, raw, until) + if e != nil { + return nil, e + } + b, e := clusterstate.Encode(w) + if e != nil { + return nil, e + } + out := encoder{} + out.blob(b) + return out.b, nil +} + +func legacyGenesis(d *decoder) ([]byte, error) { + var m clusterstate.LegacyManifest + raw := d.blob() + if e := d.end(); e != nil { + return nil, e + } + if clusterstate.Decode(raw, &m) != nil || m.Validate() != nil { + return nil, clusterstate.ErrMigration + } + e := encoder{} + e.message(&pb.Entry{Index: proto.Uint64(1), Term: proto.Uint64(1)}) + e.message(&pb.HardState{Term: proto.Uint64(1), Commit: proto.Uint64(1)}) + e.message(&pb.ConfState{Voters: m.Config.Anchors}) + return e.b, nil +} + +func legacyAuthorities(d *decoder) ([]byte, error) { + var c clusterstate.LegacyConfig + raw := d.blob() + if e := d.end(); e != nil { + return nil, e + } + if clusterstate.Decode(raw, &c) != nil { + return nil, clusterstate.ErrMigration + } + rows, e := clusterstate.LegacyAuthorities(c) + if e != nil { + return nil, e + } + out := encoder{} + out.u32(uint32(len(rows))) + for _, row := range rows { + b, e := clusterstate.Encode(row) + if e != nil { + return nil, e + } + out.blob(b) + } + return out.b, nil +} diff --git a/internal/raftdirectory/apply.go b/internal/raftdirectory/apply.go index 9dcb997001..623f05bc0a 100644 --- a/internal/raftdirectory/apply.go +++ b/internal/raftdirectory/apply.go @@ -20,6 +20,11 @@ import ( // overlay, all exact results and the applied header in one FULL transaction. // Returned results may escape only after this function returns successfully. func Apply(ctx context.Context, store *raftstore.SQLite, b raftcore.ApplyBatch, cluster string, ceiling uint64) ([]clusterstate.Result, error) { + if imported, err := store.ImportedLegacy(ctx); err != nil { + return nil, err + } else if imported != nil { + ceiling = imported.Ceiling + } app, size, err := store.ApplicationView(ctx) if err != nil { return nil, err diff --git a/internal/raftdirectory/directory_test.go b/internal/raftdirectory/directory_test.go index e8011edc10..3fb354c398 100644 --- a/internal/raftdirectory/directory_test.go +++ b/internal/raftdirectory/directory_test.go @@ -61,6 +61,7 @@ type group struct { queue []raftcore.Outbound results [3][]clusterstate.Result reads map[string]bool + keys map[string]ed25519.PrivateKey seq map[string]uint64 genesis []raftstore.Write blocked bool @@ -179,12 +180,18 @@ func (g *group) submit(c clusterstate.Command) (clusterstate.Result, error) { } return g.results[i][before], nil } +func (g *group) commandKey(actor string) ed25519.PrivateKey { + if k := g.keys[actor]; k != nil { + return k + } + return key(actor) +} func (g *group) Change(_ context.Context, actor string, m clusterstate.Mutation) (clusterstate.Result, error) { g.mu.Lock() defer g.mu.Unlock() g.seq[actor]++ seq := g.seq[actor] - c, e := clusterstate.Sign(clusterstate.Command{Schema: 1, Cluster: cluster, Actor: actor, Session: "api", Sequence: seq, Ack: seq - 1, Now: g.clock().UnixMilli(), Change: m}, key(actor)) + c, e := clusterstate.Sign(clusterstate.Command{Schema: 1, Cluster: cluster, Actor: actor, Session: "api", Sequence: seq, Ack: seq - 1, Now: g.clock().UnixMilli(), Change: m}, g.commandKey(actor)) if e != nil { return clusterstate.Result{}, e } @@ -198,7 +205,7 @@ func (g *group) barrier(actor string) (int, raftstore.Application, error) { // Public expiry reads commit the ingress clock before their fresh barrier. g.seq[actor]++ seq := g.seq[actor] - c, e := clusterstate.Sign(clusterstate.Command{Schema: 1, Cluster: cluster, Actor: actor, Session: "api", Sequence: seq, Ack: seq - 1, Now: g.clock().UnixMilli(), Change: clusterstate.Mutation{AdvanceTime: &clusterstate.AdvanceTime{}}}, key(actor)) + c, e := clusterstate.Sign(clusterstate.Command{Schema: 1, Cluster: cluster, Actor: actor, Session: "api", Sequence: seq, Ack: seq - 1, Now: g.clock().UnixMilli(), Change: clusterstate.Mutation{AdvanceTime: &clusterstate.AdvanceTime{}}}, g.commandKey(actor)) if e != nil { return i, raftstore.Application{}, e } diff --git a/internal/raftdirectory/migration_test.go b/internal/raftdirectory/migration_test.go new file mode 100644 index 0000000000..c63be61476 --- /dev/null +++ b/internal/raftdirectory/migration_test.go @@ -0,0 +1,227 @@ +package raftdirectory_test + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/reqsign" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftstore" +) + +func migrationCertificate(t *testing.T, public ed25519.PublicKey) string { + t.Helper() + c := struct { + V uint16 `json:"V"` + Device string `json:"device"` + Created int64 `json:"created"` + Sig string `json:"sig"` + }{V: 1, Device: hex.EncodeToString(public), Created: 1000} + b, e := json.Marshal(c) + must(t, e) + c.Sig = hex.EncodeToString(ed25519.Sign(key("root"), append([]byte("codeaf/device-cert/v1\n"), b...))) + b, e = json.Marshal(c) + must(t, e) + return base64.RawURLEncoding.EncodeToString(b) +} + +type migrationHome struct { + id identity.Identity + dev identity.Dev +} + +func (h migrationHome) IdentityKey() ed25519.PublicKey { return h.id.PublicKey() } +func (h migrationHome) Cert() identity.Cert { return h.dev.Cert } +func (h migrationHome) Sign(b []byte) []byte { return h.dev.Sign(b) } +func savedMigrationHome(t *testing.T, name string) migrationHome { + t.Helper() + root := hex.EncodeToString(key("root").Seed()) + id, e := identity.Unmarshal([]byte(`{"V":1,"signing_seed":"` + root + `","dedup_secret":"` + root + `","cell_key":"` + root + `"}`)) + must(t, e) + home := t.TempDir() + must(t, identity.Save(home, id)) + var cert identity.Cert + raw, e := base64.RawURLEncoding.DecodeString(migrationCertificate(t, key(name).Public().(ed25519.PublicKey))) + must(t, e) + must(t, json.Unmarshal(raw, &cert)) + doc := struct { + V uint16 `json:"V"` + Seed string `json:"device_seed"` + Cert identity.Cert `json:"cert"` + }{1, hex.EncodeToString(key(name).Seed()), cert} + body, e := json.Marshal(doc) + must(t, e) + must(t, os.WriteFile(filepath.Join(home, identity.DeviceFile), body, 0600)) + dev, e := identity.Device(home) + must(t, e) + if dev.ID() != clusterstate.DeviceIDOfKey(key(name).Public().(ed25519.PublicKey)) { + t.Fatal("saved fixture device changed") + } + if !ed25519.Verify(key(name).Public().(ed25519.PublicKey), []byte("fixture"), dev.Sign([]byte("fixture"))) { + t.Fatal("saved device key differs from certificate") + } + return migrationHome{id, dev} +} + +// R24 exercises the old Client and the committed Client at their real doors. +// Both homes keep their certified authority; the source routes only after a +// fresh current-term quorum barrier, and cannot return to legacy writes. +func TestR24TwoCertifiedHomesMigrateThroughQuorumAndRestart(t *testing.T) { + clock := func() time.Time { return time.UnixMilli(1000) } + akey, bkey := key("paired-a").Public().(ed25519.PublicKey), key("paired-b").Public().(ed25519.PublicKey) + a, b := clusterstate.DeviceIDOfKey(akey), clusterstate.DeviceIDOfKey(bkey) + source, e := directory.OpenSQLite(filepath.Join(t.TempDir(), "old.db"), clock) + must(t, e) + defer source.Close() + homes := []migrationHome{savedMigrationHome(t, "paired-a"), savedMigrationHome(t, "paired-b")} + var routeMu sync.RWMutex + var routed directory.Directory = source + server := httptest.NewServer(directory.Handler(func(r *http.Request, body []byte) (string, string, error) { + id, dev, e := reqsign.Verify(r, body, clock()) + if e != nil { + t.Logf("real ingress refusal: %v", e) + } + return id, dev, e + }, func(id string) (directory.Directory, error) { + routeMu.RLock() + defer routeMu.RUnlock() + if id != homes[0].id.ID() { + return nil, directory.ErrUnauthorized + } + return routed, nil + })) + defer server.Close() + clients := []*directory.HTTP{directory.NewHTTP(server.URL, reqsign.SignFor(homes[0], clock), server.Client()), directory.NewHTTP(server.URL, reqsign.SignFor(homes[1], clock), server.Client())} + for i, id := range []string{a, b} { + must(t, clients[i].PutDevice(ctx, id, directory.Device{V: 1, Name: "c2VhbGVk"})) + } + view, e := clients[0].Create(ctx, "chat", directory.CellInit{Head: "saved", Class: "host-bound", Frames: []string{"sealed-frame"}}) + must(t, e) + root := key("root").Public().(ed25519.PublicKey) + effects := key("effects").Public().(ed25519.PublicKey) + cfg := clusterstate.LegacyConfig{Cluster: cluster, Generation: 1, Identity: clusterstate.IdentityID(root), SigningKey: root, EffectsKey: effects, Anchors: []uint64{1, 2, 3}, Now: 1000, Certificates: []string{migrationCertificate(t, akey), migrationCertificate(t, bkey)}} + must(t, source.FreezeLegacy(ctx, cfg)) + manifest, e := source.InventoryLegacy(ctx) + must(t, e) + g := &group{t: t, clock: clock, reads: map[string]bool{}, seq: map[string]uint64{}, keys: map[string]ed25519.PrivateKey{a: key("paired-a"), b: key("paired-b")}} + defer func() { + for _, store := range g.stores { + if store != nil { + must(t, store.Close()) + } + } + }() + for i := range 3 { + g.paths[i] = filepath.Join(t.TempDir(), "member.db") + g.stores[i], e = raftstore.Open(g.paths[i], raftstore.Identity{ClusterUUID: cluster, NodeID: uint64(i + 1), Protocol: 1}) + must(t, e) + must(t, g.stores[i].ImportLegacy(ctx, manifest, func(fn func(clusterstate.RecordWrite) error) error { _, e := source.ExportLegacy(ctx, fn); return e })) + recovered, e := g.stores[i].Recover(ctx) + must(t, e) + g.nodes[i], e = raftcore.New(raftcore.Config{ClusterUUID: cluster, NodeID: uint64(i + 1), Protocol: 1, Seed: uint64(i + 51)}, recovered.Core) + must(t, e) + } + for n := 0; n < 40 && g.leader() < 0; n++ { + for i := range 3 { + g.drive(i, raftcore.Tick{}) + } + g.drain() + } + if g.leader() < 0 { + t.Fatal("imported group cannot elect") + } + // The fixture keys match real certified IDs rather than its older short IDs. + for _, actor := range []string{a, b} { + g.seq[actor] = 0 + } + propose := func(actor string, private ed25519.PrivateKey, m clusterstate.Mutation) clusterstate.Result { + g.seq[actor]++ + seq := g.seq[actor] + c, e := clusterstate.Sign(clusterstate.Command{Schema: 1, Cluster: cluster, Actor: actor, Session: "paired", Sequence: seq, Ack: seq - 1, Now: 1000, Change: m}, private) + must(t, e) + r, e := g.submit(c) + must(t, e) + return r + } + for _, actor := range []struct{ id, name string }{{a, "paired-a"}, {b, "paired-b"}} { + r := propose(actor.id, key(actor.name), clusterstate.Mutation{AdvanceTime: &clusterstate.AdvanceTime{}}) + if r.Code != clusterstate.OK { + t.Fatal("paired home stranded", r.Code) + } + } + leader := g.leader() + g.drive(leader, raftcore.Read{RequestID: "migration"}) + g.drain() + if !g.reads["migration"] { + t.Fatal("no fresh quorum barrier") + } + receipts := []directory.LegacyReceipt{} + for i, s := range g.stores { + m, e := s.ImportedLegacy(context.Background()) + must(t, e) + app, _, e := s.ApplicationView(ctx) + must(t, e) + receipts = append(receipts, directory.LegacyReceipt{Node: uint64(i + 1), Manifest: *m, Applied: app.Applied}) + } + must(t, source.OwnLegacy(ctx, receipts)) + next, e := source.MigrationRoute(ctx, func(clusterstate.LegacyManifest) (directory.RaftBackend, error) { return g, nil }) + must(t, e) + routeMu.Lock() + routed = next + routeMu.Unlock() + g.seq[a] = 0 + g.seq[b] = 0 + for _, client := range clients { + listing, e := client.List(ctx) + must(t, e) + if listing.Cells["chat"].Head != "saved" { + t.Fatal("paired HTTP home lost saved chat") + } + } + + taken, e := clients[1].Acquire(ctx, "chat", directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: a, Fence: view.Cell.Lease.Fence}}) + must(t, e) + if taken.Cell.Lease.Fence != 2 { + t.Fatal("HTTP continuation failed") + } + if _, e = source.For(a).Acquire(ctx, "chat", directory.AcquireOpts{Force: true}); !errors.Is(e, clusterstate.ErrMigration) { + t.Fatal("old writer reopened", e) + } + for i, s := range g.stores { + must(t, s.Close()) + g.stores[i], e = raftstore.Open(g.paths[i], raftstore.Identity{ClusterUUID: cluster, NodeID: uint64(i + 1), Protocol: 1}) + must(t, e) + recovered, e := g.stores[i].Recover(ctx) + must(t, e) + g.nodes[i], e = raftcore.New(raftcore.Config{ClusterUUID: cluster, NodeID: uint64(i + 1), Protocol: 1, Seed: uint64(i + 99)}, recovered.Core) + must(t, e) + g.status[i] = raftcore.Status{} + } + for n := 0; n < 40 && g.leader() < 0; n++ { + for i := range 3 { + g.drive(i, raftcore.Tick{}) + } + g.drain() + } + if g.leader() < 0 { + t.Fatal("restart did not elect") + } + _, e = clients[0].Acquire(ctx, "chat", directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: a, Fence: 1}}) + if !errors.Is(e, directory.ErrLeaseHeld) { + t.Fatal("restart lost winning fence", e) + } +} diff --git a/internal/raftstore/README.md b/internal/raftstore/README.md index 68786b729a..3c04e07d93 100644 --- a/internal/raftstore/README.md +++ b/internal/raftstore/README.md @@ -155,3 +155,36 @@ it, commits more than the 4 MiB trigger through the majority, compacts both stor and caches, and then installs/replays/restarts the learner. The pure format tests pin full-width integer and binary framing vectors independently of the encoder. These are the P05 mechanism gates; P06/P08 own simulator/cross-runtime campaigns. + +## Legacy directory import + +P11 freezes the source with `directory.SQLite.FreezeLegacy`, waits for admitted +legacy writes and pairing decisions, samples hold evidence, then inventories the +original rows through the shared Go converter. `InventoryLegacy` is resumable; +`ExportLegacy` streams canonical rows in namespace/key order. Unknown fields or +rows above the existing one-row envelope refuse the import and retain the frozen +source. It never truncates records or rewrites original JSON. Erased namespaces +remain refused by the existing namespace tombstone before opening the source. + +The source allocates ONE UUID/generation and ordered three-anchor configuration. +Each target calls `ImportLegacy` on its own fresh store, streams the same export, +and commits genesis index/term 1, all rows, version, logical time and complete +three-voter configuration atomically. A portable checkpoint follows. A crash +before checkpoint completion resumes it without replaying imported rows over +later cluster work. Conflicting manifests or nonempty unimported stores refuse. +The `membership/legacy-import` row retains provenance and the byte ceiling +`max(8 MiB, imported rows plus exact provenance bytes)` through compaction. +`raftdirectory.Apply` reads that persisted ceiling rather than a caller override. + +`OwnLegacy` accepts host-internal readback receipts for the same manifest from +at least two distinct configured anchors; the host obtains them through a fresh +current-term quorum barrier. Only then does `MigrationRoute` return the Raft +facade. Freeze, inventory and ownership never reopen the old writer, including +stale handles and restarts. The full production host/client routing is assembled +in P17. No API caller may submit import receipts or allocate replacement UUIDs. +Public root and original authenticated device certificates install authority; +metadata alone cannot enroll a device. Previously unseen personal devices must +present their root-signed certificate at the later enrollment/ingress door. +Legacy public link decisions drain before freeze and remain in their original +request store through its existing TTL; pending codes enter P10's reserved +outbox after cutover. No grant is regenerated or automatically replayed as work. diff --git a/internal/raftstore/import.go b/internal/raftstore/import.go new file mode 100644 index 0000000000..13e81cd92d --- /dev/null +++ b/internal/raftstore/import.go @@ -0,0 +1,158 @@ +package raftstore + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// ImportLegacy installs one source-authorized genesis, never three independent +// singleton bootstraps. The host keeps the source frozen until quorum readback. +// A crash rolls back all rows or retains the entire genesis; a retry can only +// finish its checkpoint, never replace later committed cluster work. +func (s *SQLite) ImportLegacy(ctx context.Context, m clusterstate.LegacyManifest, stream func(func(clusterstate.RecordWrite) error) error) error { + if m.Validate() != nil || m.Config.Cluster != s.identity.ClusterUUID { + return ErrBatch + } + member := false + for _, id := range m.Config.Anchors { + member = member || id == s.identity.NodeID + } + if !member { + return ErrIdentity + } + encoded, e := clusterstate.Encode(m) + if e != nil || uint64(len(encoded))+128 > clusterstate.LegacyManifestReserve { + return ErrBatch + } + s.mu.Lock() + e = s.transaction(ctx, func(tx *sql.Tx) error { + a, e := getApplication(ctx, tx) + if e != nil { + return e + } + k := clusterstate.LegacyImportKey() + var old []byte + err := tx.QueryRowContext(ctx, `SELECT data FROM records WHERE namespace=? AND key=?`, k.Namespace, k.Key).Scan(&old) + if err == nil { + if !bytes.Equal(old, encoded) || a.Applied == 0 { + return ErrBatch + } + return nil + } + if !errors.Is(err, sql.ErrNoRows) { + return err + } + tip, e := getMeta(ctx, tx, "log_tip") + if e != nil { + return e + } + h := &pb.HardState{} + if e = getProto(ctx, tx, "hardstate", h); e != nil { + return e + } + if a.Applied != 0 || a.RecordCount != 0 || !bytes.Equal(tip, indexKey(0)) || h.GetTerm() != 0 || h.GetVote() != 0 || h.GetCommit() != 0 { + return ErrBatch + } + count, total := uint64(0), uint64(0) + last := clusterstate.RecordKey{} + digest := sha256.New() + add := func(w clusterstate.RecordWrite) error { + key := Key{Namespace: Namespace(w.Namespace), Key: w.Key} + if !validKey(key) || len(w.Data) > MaxRecordBytes { + return ErrBatch + } + sum := checksum([]byte("records"), []byte(w.Namespace), []byte(w.Key), w.Data) + if _, e := tx.ExecContext(ctx, `INSERT INTO records VALUES(?,?,?,?)`, w.Namespace, w.Key, w.Data, sum); e != nil { + return e + } + a.RecordCount++ + xorHash(&a.RecordsHash, sum) + s.boundary("after-import-row") + return nil + } + if e = stream(func(w clusterstate.RecordWrite) error { + if w.RecordKey == k || count > 0 && (w.Namespace < last.Namespace || w.Namespace == last.Namespace && w.Key <= last.Key) { + return ErrBatch + } + b, e := clusterstate.Encode(w) + if e != nil { + return ErrBatch + } + digest.Write(b) + digest.Write([]byte{'\n'}) + total += clusterstate.RecordBytes(w.RecordKey, w.Data) + count++ + if count > m.Rows || total > m.Bytes { + return ErrBatch + } + last = w.RecordKey + return add(w) + }); e != nil { + return e + } + if count != m.Rows || total != m.Bytes || hex.EncodeToString(digest.Sum(nil)) != m.Digest { + return ErrBatch + } + if e = add(clusterstate.RecordWrite{RecordKey: k, Data: encoded}); e != nil { + return e + } + a.Applied = 1 + a.VisibleVersion = m.Version + a.LogicalTime = m.Config.Now + a.ConfState = &pb.ConfState{Voters: append([]uint64(nil), m.Config.Anchors...)} + entry := &pb.Entry{Index: proto.Uint64(1), Term: proto.Uint64(1)} + data, _ := wire.Marshal(entry) + idx := indexKey(1) + if _, e = tx.ExecContext(ctx, `INSERT INTO raft_log VALUES(?,?,?)`, idx, data, checksum([]byte("raft_log"), idx, data)); e != nil { + return e + } + if e = putMeta(ctx, tx, "log_tip", idx); e != nil { + return e + } + if e = putProto(ctx, tx, "hardstate", &pb.HardState{Term: proto.Uint64(1), Commit: proto.Uint64(1)}); e != nil { + return e + } + if e = putApplication(ctx, tx, a); e != nil { + return e + } + s.boundary("after-import-genesis") + return nil + }) + s.mu.Unlock() + if e != nil { + return e + } + _, e = s.CreateSnapshot(ctx) + return e +} + +// ImportedLegacy returns persisted provenance, including the capacity granted +// to the original rows. Hosts consult it rather than accepting a request limit. +func (s *SQLite) ImportedLegacy(ctx context.Context) (*clusterstate.LegacyManifest, error) { + app, _, e := s.ApplicationView(ctx) + if e != nil { + return nil, e + } + k := clusterstate.LegacyImportKey() + key := Key{Namespace: Membership, Key: k.Key} + rows, _, e := s.Read(ctx, app.Applied, []Key{key}) + if e != nil { + return nil, e + } + if rows[key] == nil { + return nil, nil + } + var m clusterstate.LegacyManifest + if clusterstate.Decode(rows[key], &m) != nil || m.Validate() != nil || m.Config.Cluster != s.identity.ClusterUUID { + return nil, ErrCorrupt + } + return &m, nil +} diff --git a/internal/raftstore/import_test.go b/internal/raftstore/import_test.go new file mode 100644 index 0000000000..0c8f3e7a57 --- /dev/null +++ b/internal/raftstore/import_test.go @@ -0,0 +1,163 @@ +package raftstore + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "testing" + + "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +func importFixture(t *testing.T) (clusterstate.LegacyManifest, []clusterstate.RecordWrite) { + t.Helper() + root := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{1}, 32)).Public().(ed25519.PublicKey) + effect := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{2}, 32)).Public().(ed25519.PublicKey) + c := clusterstate.LegacyConfig{Cluster: "r24-import", Generation: 1, Identity: clusterstate.IdentityID(root), SigningKey: root, EffectsKey: effect, Anchors: []uint64{1, 2, 3}, Now: 12345} + id, e := clusterstate.LegacyRow(c, "identity", "", []byte(`{"V":1,"vault":"sealed-vault","rotation":{"state":"retired","by":"device","at":100,"retire_at":999999}}`), 0) + must(t, e) + cell, e := clusterstate.LegacyRow(c, "cells", "chat", []byte(`{"V":1,"head":"head","class":"host-bound","size":91,"keys":{},"lease":{"device":"device","fence":9007199254740991,"expires":0,"pending":3,"released_at":44},"frames":["frame"]}`), 0) + must(t, e) + rows := []clusterstate.RecordWrite{clusterstate.LegacyEffects(c), id, cell} + sort.Slice(rows, func(i, j int) bool { + a, b := rows[i], rows[j] + return a.Namespace < b.Namespace || a.Namespace == b.Namespace && a.Key < b.Key + }) + m := clusterstate.LegacyManifest{Config: c, Version: 71, Rows: uint64(len(rows))} + h := sha256.New() + for _, r := range rows { + b, e := clusterstate.Encode(r) + must(t, e) + h.Write(b) + h.Write([]byte{'\n'}) + m.Bytes += clusterstate.RecordBytes(r.RecordKey, r.Data) + } + m.Digest = hex.EncodeToString(h.Sum(nil)) + m, e = clusterstate.SealLegacyManifest(m) + must(t, e) + return m, rows +} +func fixtureStream(rows []clusterstate.RecordWrite) func(func(clusterstate.RecordWrite) error) error { + return func(visit func(clusterstate.RecordWrite) error) error { + for _, r := range rows { + if e := visit(r); e != nil { + return e + } + } + return nil + } +} +func TestR24ImportCrashHelper(t *testing.T) { + path := os.Getenv("P11_TARGET") + if path == "" { + return + } + m, rows := importFixture(t) + s, e := Open(path, Identity{ClusterUUID: m.Config.Cluster, NodeID: 1, Protocol: 1}) + must(t, e) + n, e := strconv.Atoi(os.Getenv("P11_NUMBER")) + must(t, e) + count := 0 + s.cut = func(phase string) { + if phase == os.Getenv("P11_PHASE") { + count++ + if count == n { + os.Exit(82) + } + } + } + must(t, s.ImportLegacy(context.Background(), m, fixtureStream(rows))) + t.Fatal("import crash cut not reached") +} + +// R24/R3 cuts every atomic genesis row and both sides of its FULL commit, +// followed by the streamed checkpoint's source/chunk/pointer boundaries. +func TestR24ImportCrashesNeverExposePartialGenesis(t *testing.T) { + m, rows := importFixture(t) + cuts := map[string]int{"after-begin": 2, "after-import-row": 4, "after-import-genesis": 1, "before-commit": 2, "after-commit": 2, "after-snapshot-source-row": 4, "after-snapshot-created-chunk": 1, "after-snapshot-pointer": 1} + phases := make([]string, 0, len(cuts)) + for p := range cuts { + phases = append(phases, p) + } + sort.Strings(phases) + for _, phase := range phases { + for number := 1; number <= cuts[phase]; number++ { + t.Run(fmt.Sprintf("%s/%d", phase, number), func(t *testing.T) { + path := filepath.Join(t.TempDir(), "member.db") + s, e := Open(path, Identity{ClusterUUID: m.Config.Cluster, NodeID: 1, Protocol: 1}) + must(t, e) + must(t, s.Close()) + cmd := exec.Command(os.Args[0], "-test.run=^TestR24ImportCrashHelper$", "-test.count=1") + cmd.Env = append(os.Environ(), "P11_TARGET="+path, "P11_PHASE="+phase, "P11_NUMBER="+strconv.Itoa(number)) + out, e := cmd.CombinedOutput() + var exit *exec.ExitError + if !errors.As(e, &exit) || exit.ExitCode() != 82 { + t.Fatalf("cut: %v %s", e, out) + } + s, e = Open(path, Identity{ClusterUUID: m.Config.Cluster, NodeID: 1, Protocol: 1}) + must(t, e) + defer s.Close() + r, e := s.Recover(context.Background()) + must(t, e) + if r.Application.RecordCount != 0 && r.Application.RecordCount != 4 { + t.Fatal("partial import became authoritative") + } + must(t, s.ImportLegacy(context.Background(), m, fixtureStream(rows))) + r, e = s.Recover(context.Background()) + must(t, e) + if r.Application.RecordCount != 4 || r.Application.VisibleVersion != 71 || r.Core.Snapshot.GetMetadata().GetIndex() != 1 || len(r.Application.ConfState.Voters) != 3 { + t.Fatal("resume lost genesis") + } + changed := m + changed.Version++ + changed, e = clusterstate.SealLegacyManifest(changed) + must(t, e) + if e = s.ImportLegacy(context.Background(), changed, fixtureStream(rows)); !errors.Is(e, ErrBatch) { + t.Fatal("changed import accepted", e) + } + }) + } + } +} + +// R24/R28 refuses missing, reordered and altered streams atomically. A failed +// import cannot become an excuse to truncate source data or bootstrap a voter. +func TestR24ImportCorruptionAndConflictingStreams(t *testing.T) { + m, rows := importFixture(t) + for _, kind := range []string{"missing", "reordered", "changed", "identity"} { + t.Run(kind, func(t *testing.T) { + s, e := Open(filepath.Join(t.TempDir(), "member.db"), Identity{ClusterUUID: m.Config.Cluster, NodeID: 1, Protocol: 1}) + must(t, e) + defer s.Close() + bad := append([]clusterstate.RecordWrite(nil), rows...) + candidate := m + switch kind { + case "missing": + bad = bad[:2] + case "reordered": + bad[0], bad[1] = bad[1], bad[0] + case "changed": + bad[1].Data = []byte("changed") + case "identity": + candidate.Config.Cluster = "another" + } + if e = s.ImportLegacy(context.Background(), candidate, fixtureStream(bad)); e == nil { + t.Fatal("invalid source imported") + } + r, e := s.Recover(context.Background()) + must(t, e) + if !r.Fresh || r.Application.RecordCount != 0 { + t.Fatal("failed import changed authority") + } + }) + } +} From 8ac4df09b8ab201444799f899624c7c487ff1666 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 01:37:41 -0400 Subject: [PATCH 025/103] Resume hosted imports through bounded durable row staging. Drain admitted legacy effects before freezing, fence incomplete member activation, and compare the hosted genesis with native storage so eviction and retries retain the original authority and content. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/codeaf-raft-probe/actor.go | 53 +++++- relay/hosted/src/directory.js | 9 + relay/hosted/src/identity.js | 5 +- relay/hosted/src/raft/README.md | 20 +++ relay/hosted/src/raft/abi.js | 1 + relay/hosted/src/raft/legacy.js | 167 ++++++++++++++++++ relay/hosted/src/raft/member.js | 28 ++- relay/hosted/src/tenant.js | 9 +- relay/hosted/test/raft/legacy-worker.js | 49 +++++ relay/hosted/test/raft/legacy-workerd.test.js | 54 ++++++ relay/hosted/test/raft/legacy.test.js | 101 +++++++++++ 11 files changed, 484 insertions(+), 12 deletions(-) create mode 100644 relay/hosted/src/raft/legacy.js create mode 100644 relay/hosted/test/raft/legacy-worker.js create mode 100644 relay/hosted/test/raft/legacy-workerd.test.js create mode 100644 relay/hosted/test/raft/legacy.test.js diff --git a/cmd/codeaf-raft-probe/actor.go b/cmd/codeaf-raft-probe/actor.go index ff8c1998df..c07df2026e 100644 --- a/cmd/codeaf-raft-probe/actor.go +++ b/cmd/codeaf-raft-probe/actor.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strconv" + "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/raftabi" "github.com/Agent-Field/codeaf/internal/raftcore" "github.com/Agent-Field/codeaf/internal/raftsnapshot" @@ -19,15 +20,16 @@ import ( ) type actorRequest struct { - Op string `json:"op"` - Cluster string `json:"cluster"` - ID uint64 `json:"id,string"` - Tag uint32 `json:"tag"` - Value json.RawMessage `json:"value"` - Data []byte `json:"data"` - Generation string `json:"generation"` - Offset uint64 `json:"offset"` - Hash string `json:"hash"` + Rows []clusterstate.RecordWrite `json:"rows"` + Op string `json:"op"` + Cluster string `json:"cluster"` + ID uint64 `json:"id,string"` + Tag uint32 `json:"tag"` + Value json.RawMessage `json:"value"` + Data []byte `json:"data"` + Generation string `json:"generation"` + Offset uint64 `json:"offset"` + Hash string `json:"hash"` } type probeActor struct { core *raftcore.Core @@ -121,6 +123,39 @@ func (a *probeActor) open() error { func (a *probeActor) call(r actorRequest, dir string) (any, error) { ctx := context.Background() switch r.Op { + case "legacy-import": + var m clusterstate.LegacyManifest + if err := clusterstate.Decode(r.Value, &m); err != nil { + return nil, err + } + if a.store == nil { + a.cfg = raftcore.Config{ClusterUUID: r.Cluster, NodeID: r.ID, Seed: 100 + r.ID, Protocol: 1} + a.path = filepath.Join(dir, fmt.Sprintf("%s-%d.db", r.Cluster, r.ID)) + s, err := raftstore.Open(a.path, raftstore.Identity{ClusterUUID: r.Cluster, NodeID: r.ID, Protocol: 1}) + if err != nil { + return nil, err + } + a.store = s + } + if err := a.store.ImportLegacy(ctx, m, func(visit func(clusterstate.RecordWrite) error) error { + for _, row := range r.Rows { + if e := visit(row); e != nil { + return e + } + } + return nil + }); err != nil { + return nil, err + } + recovered, err := a.store.Recover(ctx) + if err != nil { + return nil, err + } + snapshot, err := actorWire.Marshal(recovered.Core.Snapshot) + if err != nil { + return nil, err + } + return map[string]any{"snapshot": snapshot, "applied": fmt.Sprint(recovered.Application.Applied)}, nil case "init": if a.store != nil { if err := a.store.Close(); err != nil { diff --git a/relay/hosted/src/directory.js b/relay/hosted/src/directory.js index 4e8b381896..5f1182f378 100644 --- a/relay/hosted/src/directory.js +++ b/relay/hosted/src/directory.js @@ -2,6 +2,7 @@ // and every method here is one synchronous turn (no await between a read and the write that // depends on it), so nothing can interleave inside a swap: that is the compare-and-swap the // lease rules need. The rules themselves are pure and live in rules.js. +import { Wire } from './wire.js'; import { makeRules, RuleError } from './rules.js'; import { lifted } from './vouch.js'; import { CLOSE_REVOKED, CLOSE_ROTATED, NOBODY } from './watch.js'; @@ -24,13 +25,19 @@ export class Directory { this.clock = clock; this.rules = makeRules(policy); this.watchers = watchers; + sql.exec('CREATE TABLE IF NOT EXISTS legacy_gate (one INTEGER PRIMARY KEY CHECK(one=1), doc TEXT NOT NULL)'); sql.exec('CREATE TABLE IF NOT EXISTS dir (kind TEXT NOT NULL, id TEXT NOT NULL, doc TEXT NOT NULL, PRIMARY KEY (kind, id)) WITHOUT ROWID'); sql.exec('CREATE TABLE IF NOT EXISTS dirver (one INTEGER PRIMARY KEY CHECK (one = 1), n INTEGER NOT NULL)'); sql.exec('INSERT OR IGNORE INTO dirver VALUES (1, 0)'); this.version = sql.exec('SELECT n FROM dirver').one().n; } + assertLegacy() { + if(this.sql.exec('SELECT doc FROM legacy_gate WHERE one=1').toArray().length) throw new Wire('unreachable',503); + } + read(kind, id) { + this.assertLegacy(); const row = this.sql.exec('SELECT doc FROM dir WHERE kind = ? AND id = ?', kind, id).toArray()[0]; return row ? JSON.parse(row.doc) : null; } @@ -40,6 +47,7 @@ export class Directory { * version. This is the one place a version is counted, so no verb, rotation included, decides alone. */ write(kind, id, doc) { + this.assertLegacy(); const before = this.#stored(kind, id); this.sql.exec('INSERT OR REPLACE INTO dir VALUES (?,?,?)', kind, id, JSON.stringify(doc)); if (!before || this.#visible(kind, id, before) !== this.#visible(kind, id, doc)) this.#bump(); @@ -182,6 +190,7 @@ export class Directory { /** list answers the whole Listing: the identity record, every device and every cell. */ list() { + this.assertLegacy(); const out = { now: this.clock(), identity: { V: 1, identity: this.identity }, devices: {}, cells: {} }; for (const { kind, id, doc } of this.sql.exec('SELECT kind, id, doc FROM dir').toArray()) { if (kind === 'identity') out.identity = JSON.parse(doc); diff --git a/relay/hosted/src/identity.js b/relay/hosted/src/identity.js index 367a31e812..1fa4b3d300 100644 --- a/relay/hosted/src/identity.js +++ b/relay/hosted/src/identity.js @@ -7,6 +7,7 @@ // (needs no body), count a put as in flight, read the body under its cap, verify the request // signature over it, and only then spend the caller's rate budget and serve. import { DurableObject } from 'cloudflare:workers'; +import { LegacyAdmission } from './raft/legacy.js'; import { signedUri } from './base.js'; import { readBody } from './body.js'; import { dropTables, erasePage } from './erasure.js'; @@ -32,6 +33,7 @@ export class IdentityDO extends DurableObject { constructor(ctx, env) { super(ctx, env); + this.legacyAdmission = new LegacyAdmission(ctx.storage); this.limits = limitsOf(env); this.policy = policyOf(env.LEASE_POLICY); this.watchers = new Watchers( @@ -47,7 +49,7 @@ export class IdentityDO extends DurableObject { } fetch(request) { - return respond(() => this.#serve(request)); + return respond(() => this.legacyAdmission.run(() => this.#serve(request))); } /** webSocketMessage runs only for a client frame that is not `ping` (the platform answers that without waking us); such a frame is ignored (contract 21.3). */ @@ -103,6 +105,7 @@ export class IdentityDO extends DurableObject { if (!this.#tenant) this.ctx.storage.put('identity', identity); // the alarm wakes with no request to name it this.#tenant ??= new Tenant({ identity, + legacyAdmission: this.legacyAdmission, sql: this.ctx.storage.sql, bucket: this.env.FRAMES, clock: Date.now, diff --git a/relay/hosted/src/raft/README.md b/relay/hosted/src/raft/README.md index 55e06b8ae4..800fbf5b94 100644 --- a/relay/hosted/src/raft/README.md +++ b/relay/hosted/src/raft/README.md @@ -66,3 +66,23 @@ process death, full joint configuration restart and co-resident identities. Unit tests cover durability waits, queue bounds, all snapshot namespaces, partial/corrupt generations, revocation during an await and native wire bytes. This is internal host proof, not the final client journeys or a deployment. + +P11 adds `LegacyAdmission` around full legacy IdentityDO requests, so freeze +waits for admitted pairing/blob awaits. `Tenant.migration` supplies a +`LegacySource`: freeze once, inventory original SQL rows through Go ABI op 12, +then stream one converted row at a time. Sampled watch renewals are stored apart +from original JSON; old watchers close and reread. A durable source gate blocks +old methods even through previously constructed Directory objects. An erased +source is refused, never enrolled as a fresh group. + +`RaftMemberDO.legacyImport(config, manifest, row, finish)` stages bounded rows +and records `legacy-pending` before provisioning. Pending imports cannot +initialize/load a core after eviction. Exact retries survive interruption; +finish checks the complete digest, installs a three-voter genesis and portable +checkpoint, then removes only redundant staging rows and clears the pending +marker durably. The stored ceiling includes all grandfathered rows. One import +request per member can run at once, preventing page buffers stacking in memory. +After fresh quorum readback, `LegacySource.own` publishes `raft-owned`; `route` +then resolves only that UUID. There is no rollback to a legacy writer. P17 +assembles the complete facade, receipt ingress and existing-client doors; P14 +owns continuing watch renewal/scheduling. Source backup cleanup is explicit. diff --git a/relay/hosted/src/raft/abi.js b/relay/hosted/src/raft/abi.js index 80d14e540c..652d84273a 100644 --- a/relay/hosted/src/raft/abi.js +++ b/relay/hosted/src/raft/abi.js @@ -119,5 +119,6 @@ export class HostCodec { const r = this.call(frame(8).blob(state).blob(block)); const v = { state: r.blob(), digest: r.blob() }; r.end(); return v; } + genesis(data) {const r=this.call(frame(13).blob(data));const v={entry:r.blob(),hard:r.blob(),conf:r.blob()};r.end();return v;} snapshot(data) { const r = this.call(frame(9).blob(data)); const b = r.blob(); r.end(); return b; } } diff --git a/relay/hosted/src/raft/legacy.js b/relay/hosted/src/raft/legacy.js new file mode 100644 index 0000000000..b436aaa9be --- /dev/null +++ b/relay/hosted/src/raft/legacy.js @@ -0,0 +1,167 @@ +import { frame, response, HostCodec } from './abi.js'; +import { encodeCanonical, decodeCanonical } from './codec.js'; +import { ROW_BYTES, DEFAULT_ROWS_BYTES, bytes, b64, unb64, hex } from './storage.js'; +import { Wire } from '../wire.js'; + +const enc = new TextEncoder(), dec = new TextDecoder('utf-8', {fatal:true}); +const data = v => enc.encode(encodeCanonical(v)); +const check = v => { if (!v) throw new Error('codeaf: directory upgrade is incomplete; retry shortly; saved chats are still here'); }; +const size = r => 12 + enc.encode(r.namespace).length + enc.encode(r.key).length + r.data.length; +const xor = (a,b) => { for(let i=0;i<32;i++) a[i]^=b[i]; }; +const importKey = 'legacy-import'; +const reserve = 4096; + +// Original JSON bytes go to the shared Go conversion function, including every +// numeric fence and sealed field. JavaScript never evaluates a lease rule. +export function convertLegacy(runtime, config, kind, id, raw, until=0) { + const r = response(runtime.call(frame(12).blob(data(config)).str(kind).str(id).blob(enc.encode(raw)).u64(until).finish())); + const row = decodeCanonical(dec.decode(r.blob())); r.end(); + return { namespace:row.namespace, key:row.key, data:unb64(row.data) }; +} + +// Admission spans the entire legacy request, including pairing/blob awaits. +// Freeze first stops new admissions, then waits for admitted effects to finish. +// A failed freeze stays unavailable until recovery; it never opens two writers. +export class LegacyAdmission { + constructor(storage) { this.storage=storage;this.pending=new Set();this.frozen=false; } + run(fn) { + if(this.frozen)return Promise.reject(new Wire('unreachable',503)); + const p=Promise.resolve().then(fn);this.pending.add(p); + return p.finally(()=>this.pending.delete(p)); + } + async drain(fn) { this.frozen=true;await Promise.allSettled([...this.pending]);return fn(); } +} + +// The source object serializes migration with admitted request handlers. Its +// durable gate is checked again by the legacy writer, including stale handles. +// Awaiting sync precedes bootstrap, ownership publication and every reply. +export class LegacySource { + constructor(storage, directory, runtime, admission) { + this.storage=storage;this.admission=admission;this.dir=directory;this.sql=storage.sql;this.runtime=runtime;this.codec=new HostCodec(runtime); + this.sql.exec('CREATE TABLE IF NOT EXISTS legacy_gate (one INTEGER PRIMARY KEY CHECK(one=1), doc TEXT NOT NULL)'); + this.sql.exec('CREATE TABLE IF NOT EXISTS legacy_rows (namespace TEXT NOT NULL, key TEXT NOT NULL, data BLOB NOT NULL, PRIMARY KEY(namespace,key))'); + this.sql.exec('CREATE TABLE IF NOT EXISTS legacy_holds (id TEXT PRIMARY KEY, until_ms INTEGER NOT NULL)'); + } + gate() { const r=this.sql.exec('SELECT doc FROM legacy_gate WHERE one=1').toArray()[0];return r?decodeCanonical(r.doc):null; } + keep(g) { this.sql.exec('INSERT OR REPLACE INTO legacy_gate VALUES(1,?)',encodeCanonical(g)); } + async freeze(config) { + if(this.admission)return this.admission.drain(()=>this.freezeSource(config)); + return this.freezeSource(config); + } + async freezeSource(config) { + if(await this.storage.get?.('gone'))throw new Wire('gone',410); + // Go validates the binding before the source acquires an irreversible gate. + convertLegacy(this.runtime,config,'identity','',JSON.stringify({V:1,identity:config.identity})); + this.storage.transactionSync(()=>{ + const old=this.gate();if(old){check(encodeCanonical(old.config)===encodeCanonical(config));return;} + for(const {id,doc} of this.sql.exec("SELECT id,doc FROM dir WHERE kind='cells' ORDER BY id")) { + let c;try{c=JSON.parse(doc);}catch{continue;} + if(!c.lease)continue; + const until=c.lease.expires===0?0:this.dir.watchers.vouchedUntil(c.lease.device,id,c.lease.fence); + if(until>c.lease.expires)this.sql.exec('INSERT INTO legacy_holds VALUES(?,?)',id,until); + } + this.keep({stage:'frozen',config}); + }); + await this.storage.sync(); + this.dir.watchers.closeAll(1012,'reconnecting'); + } + async inventory() { + let manifest; + this.storage.transactionSync(()=>{ + const g=this.gate();check(g);if(g.manifest){manifest=g.manifest;return;} + const put=r=>this.sql.exec('INSERT INTO legacy_rows VALUES(?,?,?)',r.namespace,r.key,r.data); + const authorities=response(this.runtime.call(frame(14).blob(data(g.config)).finish())); + const countAuthorities=authorities.u32(); + for(let i=0;i{ + const g=this.gate();check(g?.manifest);const nodes=new Set(); + for(const r of receipts)if(g.config.anchors.includes(String(r.node))&&BigInt(r.applied)>=1n&&encodeCanonical(r.manifest)===encodeCanonical(g.manifest))nodes.add(String(r.node)); + check(nodes.size>=2);g.stage='raft-owned';this.keep(g); + });await this.storage.sync(); + } + // A stale routing hint can never resurrect the old writer. The host assembles + // the committed facade only for this exact durable UUID/generation. + route(factory) {const g=this.gate();if(g?.stage!=='raft-owned')throw new Wire('unreachable',503);return factory(g.manifest);} +} + +// Member storage imports a bounded row stream in one durable genesis. Existing +// imported stores return their provenance without overwriting later cluster work. +export async function importLegacy(store, manifest, source) { + const m=manifest, encoded=data(m), k={namespace:'membership',key:importKey}; + check(m.config.cluster===store.identity.cluster&&m.config.anchors.includes(store.identity.id)&&encoded.length+128<=reserve); + check(String(store.identity.ceiling)===m.ceiling); + await store.transaction(()=>{ + const app=store.application(),old=store.read(k.namespace,k.key); + if(old){check(hex(old)===hex(encoded)&&BigInt(app.applied)>=1n);return;} + check(app.applied==='0'&&app.count===0&&store.get('tip')==='0'&&store.get('hard')===''); + const aggregate=new Uint8Array(32);let count=0,total=0,last='',state=new Uint8Array(),digest=store.codec.hash(state).digest; + const add=r=>{ + check(r.data.length<=ROW_BYTES&&r.key.length>0); + const sum=store.checksum('records',r.namespace,r.key,r.data);xor(aggregate,sum); + store.rows('INSERT INTO records VALUES(?,?,?,?)',r.namespace,r.key,r.data,hex(sum)); + }; + for(const r of source()){ + const order=r.namespace+'\0'+r.key;check(order>last&&!(r.namespace===k.namespace&&r.key===k.key));last=order; + const encoded=data({namespace:r.namespace,key:r.key,data:b64(r.data)}),line=new Uint8Array(encoded.length+1);line.set(encoded);line[encoded.length]=10; + ({state,digest}=store.codec.hash(line,state));count++;total+=size(r);check(BigInt(count)<=BigInt(m.rows)&&BigInt(total)<=BigInt(m.bytes));add(r); + } + check(String(count)===m.rows&&String(total)===m.bytes&&hex(digest)===m.digest);add({...k,data:encoded}); + // These protobuf messages contain only the common genesis term/index and + // the source-authorized complete voter set, encoded by the Go ABI below. + const genesis=store.codec.genesis(data(m)); + store.rows('INSERT INTO journal VALUES(?,?,?)','0000000000000001',genesis.entry,store.hash(genesis.entry)); + store.put('tip','1');store.put('hard',b64(genesis.hard)); + store.put('application',{applied:'1',conf:b64(genesis.conf),version:m.version,now:String(m.config.now),count:count+1,size:total+size({...k,data:encoded}),hash:hex(aggregate)}); + }); + return store.checkpoint(); +} + +// Cross-object import stages one bounded row per request. Incomplete stages +// never start a core and never become application authority. Retries compare +// exact bytes; final installation rechecks the source's complete stream digest. +export async function stageLegacyImport(store, manifest, row) { + store.codec.genesis(data(manifest)); + check(manifest.config.cluster===store.identity.cluster&&manifest.config.anchors.includes(store.identity.id)&&String(store.identity.ceiling)===manifest.ceiling); + await store.transaction(()=>{ + store.rows('CREATE TABLE IF NOT EXISTS legacy_stage (one INTEGER PRIMARY KEY CHECK(one=1), doc TEXT NOT NULL)'); + store.rows('CREATE TABLE IF NOT EXISTS legacy_stage_rows (namespace TEXT NOT NULL, key TEXT NOT NULL, data BLOB NOT NULL, PRIMARY KEY(namespace,key))'); + const old=store.one('SELECT doc FROM legacy_stage WHERE one=1'),encoded=encodeCanonical(manifest); + if(old)check(old.doc===encoded);else store.rows('INSERT INTO legacy_stage VALUES(1,?)',encoded); + if(!row)return; + check(['state','enrollment','outbox','tombstones','sessions','membership'].includes(row.namespace)&&row.key.length>0&&row.data.length<=ROW_BYTES&&size(row)<=ROW_BYTES+28); + const before=store.one('SELECT data FROM legacy_stage_rows WHERE namespace=? AND key=?',row.namespace,row.key); + if(before){check(hex(bytes(before.data))===hex(row.data));return;} + const use=store.one('SELECT count(*) AS n,coalesce(sum(12+length(namespace)+length(key)+length(data)),0) AS bytes FROM legacy_stage_rows'); + check(BigInt(use.n+1)<=BigInt(manifest.rows)&&BigInt(use.bytes+size(row))<=BigInt(manifest.bytes)); + store.rows('INSERT INTO legacy_stage_rows VALUES(?,?,?)',row.namespace,row.key,row.data); + }); +} +export async function finishLegacyImport(store, manifest) { + const snapshot=await importLegacy(store,manifest,function*(){for(const r of store.rows('SELECT namespace,key,data FROM legacy_stage_rows ORDER BY namespace,key'))yield {...r,data:bytes(r.data)};}); + // These are redundant staging bytes. The immutable legacy ORIGINAL remains + // at its source; imported provenance and rows remain in every snapshot. + await store.transaction(()=>{store.rows('DROP TABLE IF EXISTS legacy_stage_rows');store.rows('DROP TABLE IF EXISTS legacy_stage');}); + return snapshot; +} diff --git a/relay/hosted/src/raft/member.js b/relay/hosted/src/raft/member.js index c55bd9eea9..739b76c144 100644 --- a/relay/hosted/src/raft/member.js +++ b/relay/hosted/src/raft/member.js @@ -2,7 +2,9 @@ import { DurableObject } from 'cloudflare:workers'; import { MemberActor, sharedRuntime } from './actor.js'; import { MemberInbox } from './inbox.js'; import { directoryEvaluator } from './directory.js'; -import { DEFAULT_ROWS_BYTES } from './storage.js'; +import { DEFAULT_ROWS_BYTES, MemberStorage } from './storage.js'; +import { HostCodec } from './abi.js'; +import { stageLegacyImport, finishLegacyImport } from './legacy.js'; export { installRaftModule, runtimeMetrics } from './actor.js'; // Only internal namespace calls can initialize a provisioned member. The @@ -11,7 +13,30 @@ export { installRaftModule, runtimeMetrics } from './actor.js'; export class RaftMemberDO extends DurableObject { constructor(ctx, env) { super(ctx, env); this.actor = null; this.loading = null; this.ctx = ctx; this.env = env; } evaluator(config) { return directoryEvaluator(sharedRuntime(), config.cluster, config.ceiling ?? DEFAULT_ROWS_BYTES); } + // Source migration feeds bounded immutable rows before a core can activate. + // The durable pending marker fences initialize/load across process death. + legacyImport(config, manifest, row = null, finish = false) { + if(this.actor||this.loading||this.importWork)return Promise.reject(new Error('raft member: active or import busy; retry')); + config=structuredClone(config);manifest=structuredClone(manifest);row=row&&structuredClone(row); + this.importPending=true; + const run=Promise.resolve().then(async()=>{ + const store=new MemberStorage(this.ctx.storage,new HostCodec(sharedRuntime())); + const canonical={...config,id:String(config.id),seed:String(config.seed),bootstrap:manifest.config.anchors.map(String),ceiling:Number(manifest.ceiling)}; + const previous=await this.ctx.storage.get('raft-config'); + if(previous&&JSON.stringify(previous)!==JSON.stringify(canonical))throw new Error('raft member: provisioned identity cannot change'); + await this.ctx.storage.put('legacy-pending',true);await this.ctx.storage.sync(); + if(!previous){await this.ctx.storage.put('raft-config',canonical);await this.ctx.storage.sync();} + await store.open({cluster:canonical.cluster,id:canonical.id,protocol:1},canonical.ceiling); + await stageLegacyImport(store,manifest,row); + if(!finish)return; + const snapshot=await finishLegacyImport(store,manifest); + await this.ctx.storage.delete('legacy-pending');await this.ctx.storage.sync();this.importPending=false; + return snapshot; + }); + this.importWork=run;return run.finally(()=>{this.importWork=null;}); + } async initialize(config) { + if(await this.ctx.storage.get('legacy-pending')||this.importPending)throw new Error('raft member: import incomplete; retry'); config = { ...config, id: String(config.id), seed: String(config.seed), bootstrap: config.bootstrap.map(String) }; if (this.loading) { if (this.loadingConfig !== JSON.stringify(config)) throw new Error('raft member: different activation is in progress'); @@ -38,6 +63,7 @@ export class RaftMemberDO extends DurableObject { async load() { if (this.actor) return; + if(await this.ctx.storage.get('legacy-pending')||this.importPending)throw new Error('raft member: import incomplete; retry'); if (this.loading) return this.loading; this.loading = this.restore(); try { return await this.loading; } finally { this.loading = null; } diff --git a/relay/hosted/src/tenant.js b/relay/hosted/src/tenant.js index 5f69efc3e6..d464bb2d95 100644 --- a/relay/hosted/src/tenant.js +++ b/relay/hosted/src/tenant.js @@ -1,5 +1,6 @@ // One identity's namespace: its directory, its frames, its quota, and the relay's own count of // what the identity asked of it. Route handlers talk to this and to nothing beneath it. +import { LegacySource } from './raft/legacy.js'; import { Directory } from './directory.js'; import { Quota } from './limits.js'; import { Meta } from './meta.js'; @@ -22,8 +23,9 @@ const MANY_KEY_ID = '0'.repeat(32); const servedOf = (range, size) => (range ? Math.min(range.length ?? Infinity, size - range.offset) : size); export class Tenant { - constructor({ identity, sql, bucket, clock, policy, limits, flight, arm, watchers, reserve = async () => {} }) { + constructor({ identity, legacyAdmission, sql, bucket, clock, policy, limits, flight, arm, watchers, reserve = async () => {} }) { this.identity = identity; + this.legacyAdmission = legacyAdmission; this.reserve = reserve; this.clock = clock; this.limits = limits; @@ -37,6 +39,10 @@ export class Tenant { this.stats = new Stats(this.meta, () => arm(clock() + limits.statsFlushMs)); } + // The host serializes this handoff with full request handlers, including + // pairing delivery awaits, before invoking any source migration stage. + migration(storage, runtime) { return new LegacySource(storage, this.dir, runtime, this.legacyAdmission); } + /** * putFrame validates the frame, stores it unless it is over quota, and counts it once. A frame * the store already holds costs the identity nothing, so a client that resends is never refused for it. @@ -105,6 +111,7 @@ export class Tenant { /** watch opens a socket for `device` that is told the directory's version now and on every visible change, and that vouches for the leases it names in `holds`. `beat` is the period it declared its pings at (contract 21.12.3). */ watch(device, { holds, beat, events = false }) { + this.dir.assertLegacy(); const now = this.clock(); this.dir.seen(device, now); return this.watchers.accept(device, this.dir.version, now, { holds, beat, events }); diff --git a/relay/hosted/test/raft/legacy-worker.js b/relay/hosted/test/raft/legacy-worker.js new file mode 100644 index 0000000000..14c7f060d8 --- /dev/null +++ b/relay/hosted/test/raft/legacy-worker.js @@ -0,0 +1,49 @@ +// Only this loopback fixture exposes migration controls. Production invokes +// the same source/member adapters after authenticating its original ingress. +import core from 'core.wasm'; +import { RaftMemberDO } from '../../src/raft/member.js'; +import { installRaftModule, sharedRuntime } from '../../src/raft/actor.js'; +installRaftModule(core); +import { Reactor, HostCodec } from '../../src/raft/abi.js'; +import { Directory } from '../../src/directory.js'; +import { policyOf } from '../../src/rules.js'; +import { LegacySource, LegacyAdmission, importLegacy } from '../../src/raft/legacy.js'; +import { MemberStorage, b64, unb64 } from '../../src/raft/storage.js'; +let runtime,codec; +export class ActorFixture extends RaftMemberDO{ + constructor(ctx,env){super(ctx,env);this.tail=Promise.resolve();this.admission=new LegacyAdmission(ctx.storage);} + fetch(request){const p=this.tail.then(()=>this.control(request));this.tail=p.catch(()=>{});return p;} + async control(request){ + try{ + runtime??=sharedRuntime();codec??=new HostCodec(runtime); + const {op,...v}=await request.json();let result; + if(op==='stage'||op==='finish') { + const snapshot=await this.legacyImport(v.memberConfig,v.manifest,v.row?{...v.row,data:unb64(v.row.data)}:null,op==='finish');result=snapshot?b64(snapshot):true; + }else if(op==='activate') {result=await this.initialize(v.memberConfig); + }else if(op==='import'||op==='member'){ + const store=new MemberStorage(this.ctx.storage,codec);await store.open({cluster:v.manifest.config.cluster,id:v.node,protocol:1},Number(v.manifest.ceiling)); + if(op==='import')await importLegacy(store,v.manifest,function*(){for(const r of v.rows)yield {...r,data:unb64(r.data)};}); + const recovered=store.recover();result={applied:String(recovered.applied),snapshot:b64(recovered.snapshot),provenance:b64(store.read('membership','legacy-import'))}; + }else{ + const dir=new Directory(this.ctx.storage.sql,v.config.identity,()=>v.config.now,policyOf()); + const source=new LegacySource(this.ctx.storage,dir,runtime,this.admission); + switch(op){ + case 'erase-marker':await this.ctx.storage.put('gone',1000);await this.ctx.storage.sync();result=true;break; + case 'seed':for(const r of v.records)dir.write(r.kind,r.id,r.doc);await this.ctx.storage.sync();result=true;break; + case 'freeze':await source.freeze(v.config);result=true;break; + case 'inventory':result=await source.inventory();break; + case 'rows':result=[...source.rows()].map(r=>({...r,data:b64(r.data)}));break; + case 'own':await source.own(v.receipts);result=true;break; + case 'gate':result=source.gate();break; + case 'legacy':dir.release('chat','a',1);result=true;break; + case 'original':result=this.ctx.storage.sql.exec("SELECT doc FROM dir WHERE kind='cells' AND id='chat'").toArray()[0].doc;break; + default:return new Response(null,{status:404}); + } + } + return new Response(JSON.stringify({ok:result},(_,v)=>typeof v==='bigint'?String(v):v),{headers:{'content-type':'application/json'}}); + }catch(e){return Response.json({error:e.message,status:e.status??503});} + } +} +export default{ + async fetch(request,env){if(new URL(request.url).pathname==='/ready')return new Response(null,{status:204});const name=new URL(request.url).pathname.slice(1);return env.ACTORS.get(env.ACTORS.idFromName(name)).fetch(request);} +}; diff --git a/relay/hosted/test/raft/legacy-workerd.test.js b/relay/hosted/test/raft/legacy-workerd.test.js new file mode 100644 index 0000000000..47ff87cc4a --- /dev/null +++ b/relay/hosted/test/raft/legacy-workerd.test.js @@ -0,0 +1,54 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { encodeCanonical } from '../../src/raft/codec.js'; +import { build } from 'esbuild'; +import { writeFile, mkdtemp, copyFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { generateKeyPairSync, createHash } from 'node:crypto'; +import { openWorkerd } from './workerd.js'; + +test('R24 R3 R28 actual hosted source and three imports survive process death at every migration stage',{timeout:60000},async()=>{ + const proof=process.env.CODEAF_RAFT_PROOF_DIR??'/tmp/codeaf-p11',out=await mkdtemp(join(tmpdir(),'p11-wasm-')),persist=await mkdtemp(join(tmpdir(),'p11-disk-')); + const bundled=await build({entryPoints:['test/raft/legacy-worker.js'],bundle:true,format:'esm',target:'es2022',external:['core.wasm','cloudflare:*'],write:false}); + await writeFile(join(out,'worker.js'),bundled.outputFiles[0].contents);await copyFile(join(proof,'core.wasm'),join(out,'core.wasm'));await writeFile(join(out,'storage-worker.js'),'export class ProofStore{}'); + const pub=()=>generateKeyPairSync('ed25519').publicKey.export({format:'der',type:'spki'}).subarray(-32); + const root=pub(),cfg={cluster:'one-hosted-uuid',generation:'1',identity:'id_'+createHash('sha256').update(root).digest('hex').slice(0,32),signing_key:root.toString('base64'),effects_key:pub().toString('base64'),anchors:['1','2','3'],now:1000}; + const records=[{kind:'identity',id:'',doc:{V:1,identity:cfg.identity,vault:'sealed-vault',rotation:{state:'frozen',by:'a',at:900}}},{kind:'devices',id:'a',doc:{V:1,name:'c2VhbGVk',added_by:cfg.identity,revoked:false,caps:{os:'linux',arch:'amd64',sandbox:null,container:null,gpu:null,cow:'none'}}},{kind:'cells',id:'chat',doc:{V:1,head:'saved',durable_at:1000,class:'host-bound',size:7,keys:{key:{identity:'wrapped'}},lease:{device:'a',fence:1,expires:0,pending:0,released_at:900},frames:['sealed-frame'],title:'c2VhbGVk'}}]; + let native; + let server;const restart=async()=>{if(server)await server.dispose();server=await openWorkerd(out,persist,true);}; + const call=async(name,op,extra={})=>{const r=await fetch(server.origin+'/'+name,{method:'POST',body:JSON.stringify({op,config:cfg,...extra}),signal:AbortSignal.timeout(5000)});assert.equal(r.status,200);return r.json();}; + const ok=async(...args)=>{const r=await call(...args);assert('ok'in r,JSON.stringify(r));return r.ok;}; + try{ + await restart();await ok('source','seed',{records});const original=await ok('source','original'); + await ok('source','freeze');await restart();assert.equal((await ok('source','gate')).stage,'frozen');assert.equal((await call('source','legacy')).status,503); + const manifest=await ok('source','inventory');await restart();assert.deepEqual(await ok('source','inventory'),manifest);const rows=await ok('source','rows'); + const snapshots=[]; + for(const node of ['1','2','3']){ + const memberConfig={cluster:cfg.cluster,id:node,seed:String(100+Number(node)),bootstrap:cfg.anchors,ceiling:Number(manifest.ceiling)}; + await ok('anchor-'+node,'stage',{memberConfig,manifest,row:rows[0]});await restart(); + assert('error'in await call('anchor-'+node,'activate',{memberConfig}),'incomplete import activated'); + for(const row of rows)await ok('anchor-'+node,'stage',{memberConfig,manifest,row}); + const snapshot=await ok('anchor-'+node,'finish',{memberConfig,manifest});snapshots.push(snapshot);await restart(); + const result=await ok('anchor-'+node,'member',{node,manifest});assert.equal(result.applied,'1');assert.equal(result.snapshot,snapshot); + assert.equal(await ok('anchor-'+node,'finish',{memberConfig,manifest}),snapshot); + const active=await ok('anchor-'+node,'activate',{memberConfig});assert.equal(active.status.applied,'1');assert.equal(active.status.role,0); + assert.equal((await call('source','legacy')).status,503); + } + assert.equal(snapshots[0],snapshots[1]);assert.equal(snapshots[1],snapshots[2]); + // A real native FULL-synchronous store imports exactly the hosted export. + // This compares the complete portable genesis bytes, not a JS summary. + native=spawn(join(proof,'native-peer'),['actors',join(persist,'native')],{stdio:['pipe','pipe','inherit'],env:{...process.env,GOMEMLIMIT:'32MiB'}}); + let buffer=Buffer.alloc(0),waiter; + native.stdout.on('data',chunk=>{buffer=Buffer.concat([buffer,chunk]);if(waiter&&buffer.length>=4){const n=buffer.readUInt32LE();assert(n<2*1024*1024);if(buffer.length>=n+4){const response=JSON.parse(buffer.subarray(4,n+4));buffer=buffer.subarray(n+4);clearTimeout(waiter.timer);const done=waiter;waiter=null;done.resolve(response);}}}); + const nativeCall=value=>new Promise((resolve,reject)=>{waiter={resolve,reject,timer:setTimeout(()=>reject(Error('native import timeout')),5000)};const body=Buffer.from(JSON.stringify(value)),header=Buffer.alloc(4);header.writeUInt32LE(body.length);native.stdin.write(header);native.stdin.write(body);}); + const nativeImport=await nativeCall({op:'legacy-import',cluster:cfg.cluster,id:'1',value:JSON.parse(encodeCanonical(manifest)),rows});assert.equal(nativeImport.applied,'1',JSON.stringify(nativeImport));assert.equal(nativeImport.snapshot,snapshots[0]); + assert.deepEqual(await nativeCall({op:'legacy-import',cluster:cfg.cluster,id:'1',value:JSON.parse(encodeCanonical(manifest)),rows:[]}),nativeImport); + native.stdin.end();await new Promise(resolve=>native.once('exit',resolve));native=null; + assert('error'in await call('source','own',{receipts:[{node:'1',applied:'1',manifest}]})); + await ok('source','own',{receipts:[{node:'1',applied:'2',manifest},{node:'2',applied:'2',manifest}]});await restart();const gate=await ok('source','gate');assert.equal(gate.stage,'raft-owned');assert.deepEqual(gate.manifest,manifest);assert.equal(await ok('source','original'),original); + await ok('erased','erase-marker');assert.equal((await call('erased','freeze')).status,410);assert.equal(await ok('erased','gate'),null); + const conflict=await call('source','freeze',{config:{...cfg,cluster:'second-uuid'}});assert('error'in conflict);assert.equal((await ok('source','gate')).config.cluster,cfg.cluster);assert.equal((await call('source','legacy')).status,503); + }finally{if(native?.exitCode===null){native.kill('SIGKILL');await new Promise(resolve=>native.once('exit',resolve));}if(server)await server.dispose();await rm(out,{recursive:true,force:true});await rm(persist,{recursive:true,force:true});} +}); diff --git a/relay/hosted/test/raft/legacy.test.js b/relay/hosted/test/raft/legacy.test.js new file mode 100644 index 0000000000..ef374d6d2a --- /dev/null +++ b/relay/hosted/test/raft/legacy.test.js @@ -0,0 +1,101 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { DatabaseSync } from 'node:sqlite'; +import { spawn } from 'node:child_process'; +import { createHash, generateKeyPairSync, sign } from 'node:crypto'; +import { Reactor, HostCodec } from '../../src/raft/abi.js'; +import { Directory } from '../../src/directory.js'; +import { LegacySource, importLegacy, convertLegacy } from '../../src/raft/legacy.js'; +import { MemberStorage, bytes, b64 } from '../../src/raft/storage.js'; +import { encodeCanonical, decodeCanonical } from '../../src/raft/codec.js'; +import { policyOf } from '../../src/rules.js'; +const out=process.env.CODEAF_RAFT_PROOF_DIR??'/tmp/codeaf-p11'; +const enc=new TextEncoder(),dec=new TextDecoder(); +const sha=b=>createHash('sha256').update(b).digest('hex'); +const config=()=>{ + const key=generateKeyPairSync('ed25519'), effects=generateKeyPairSync('ed25519'), device=generateKeyPairSync('ed25519'); + const publicKey=key.publicKey.export({format:'der',type:'spki'}).subarray(-32),pub=device.publicKey.export({format:'der',type:'spki'}).subarray(-32); + const cert={V:1,device:pub.toString('hex'),created:1000,sig:''};cert.sig=sign(null,Buffer.concat([Buffer.from('codeaf/device-cert/v1\n'),Buffer.from(JSON.stringify(cert))]),key.privateKey).toString('hex'); + return {cluster:'r24-hosted',generation:'1',identity:'id_'+sha(publicKey).slice(0,32),signing_key:publicKey.toString('base64'),effects_key:effects.publicKey.export({format:'der',type:'spki'}).subarray(-32).toString('base64'),anchors:['1','2','3'],now:200000,certificates:[Buffer.from(JSON.stringify(cert)).toString('base64url')]}; +}; +class Host{ + constructor(){this.db=new DatabaseSync(':memory:');this.sql={exec:(q,...p)=>{const rows=this.db.prepare(q).all(...p);return {toArray:()=>rows,one:()=>{assert.equal(rows.length,1);return rows[0];},[Symbol.iterator]:()=>rows[Symbol.iterator]()};}};this.gate=null;} + transactionSync(fn){this.db.exec('BEGIN');try{const v=fn();this.db.exec('COMMIT');return v;}catch(e){this.db.exec('ROLLBACK');throw e;}} + async sync(){if(this.gate)await this.gate;} +} +class Native{ + constructor(){this.child=spawn(out+'/native-peer',[],{stdio:['pipe','pipe','inherit'],env:{...process.env,GOMEMLIMIT:'32MiB'}});this.buffer=Buffer.alloc(0);this.child.stdout.on('data',b=>{this.buffer=Buffer.concat([this.buffer,b]);this.pump();});} + pump(){if(!this.waiter||this.buffer.length<4)return;const n=this.buffer.readUInt32LE();assert(n<=2*1024*1024);if(this.buffer.length{assert(!this.waiter);this.waiter={resolve,reject,timer:setTimeout(()=>reject(Error('native migration ABI timeout')),5000)};const h=Buffer.alloc(4);h.writeUInt32LE(b.length);this.child.stdin.write(h);this.child.stdin.write(b);this.pump();});} + async close(){this.child.stdin.end();if(this.child.exitCode===null)await new Promise(r=>this.child.once('exit',r));} +} + +// R24 compares every original SQL row through wasm and native conversion, +// preserves watch evidence, and checks actual portable genesis/checkpoint bytes. +test('R24 R14 R28 hosted source freezes stale handles and imports identical three-voter genesis',{timeout:30000},async()=>{ + const wasm=new Reactor(new WebAssembly.Module(readFileSync(out+'/core.wasm'))),calls=[],runtime={call(b){const r=wasm.call(b);calls.push([b.slice(),r.slice()]);return r;}},codec=new HostCodec(runtime),host=new Host(),cfg=config(); + let native; + const watchers={broadcast(){},announce(){},closeDevice(){},closeAll(){},vouchedUntil:()=>290000}; + const directory=new Directory(host.sql,cfg.identity,()=>200000,policyOf(),watchers); + directory.write('identity','',{V:1,identity:cfg.identity,vault:'opaque-vault',rotation:{state:'retired',by:'a',at:1000,retire_at:900000}}); + directory.write('devices','a',{V:1,name:'c2VhbGVk',added_by:cfg.identity,revoked:true,caps:{os:'linux',arch:'amd64',sandbox:null,container:null,gpu:null,cow:'none'},created:100,last_seen:200}); + const raw='{"V":1,"head":"head","durable_at":1000,"class":"host-bound","size":71,"keys":{"key":{"identity":"wrapped"}},"lease":{"device":"a","fence":9007199254740991,"expires":91000,"pending":3},"title":"c2VhbGVk","frames":["sealed-frame"],"parent_cell":"parent","orphan_turns":2,"archived":true}'; + host.sql.exec('INSERT INTO dir VALUES(?,?,?)','cells','chat',raw); + const stale=new Directory(host.sql,cfg.identity,()=>200000,policyOf(),watchers),source=new LegacySource(host,directory,runtime),stores=[]; + try{ + native=new Native(); + let release;host.gate=new Promise(r=>release=r);let complete=false;const freezing=source.freeze(cfg).then(()=>complete=true);await new Promise(r=>setImmediate(r));assert.equal(complete,false);assert.throws(()=>stale.release('chat','a',9007199254740991),e=>e.status===503);release();host.gate=null;await freezing; + const m=await source.inventory(),again=await source.inventory();assert.deepEqual(again,m);assert.equal(m.version,'2'); + await assert.rejects(source.own([{node:'1',applied:'1',manifest:m}]));assert.equal(source.gate().stage,'inventoried'); + const snapshots=[]; + for(const id of ['1','2','3']){ + const h=new Host();stores.push(h);const store=new MemberStorage(h,codec);await store.open({cluster:cfg.cluster,id,protocol:1},Number(m.ceiling)); + const snapshot=await importLegacy(store,m,()=>source.rows());snapshots.push(snapshot); + assert.deepEqual(await importLegacy(store,m,()=>{throw Error('retry reread source');}),snapshot); + const recovered=store.recover();assert.equal(recovered.applied,1n);assert.equal(store.application().version,m.version); + const cell=decodeCanonical(dec.decode(store.read('state','cell/'+Buffer.from('chat').toString('hex'))));assert.equal(cell.lease.fence,'9007199254740991');assert.equal(cell.lease.expires,290000);assert.deepEqual(cell.frames,['sealed-frame']);assert.equal(cell.archived,true); + } + assert.deepEqual(snapshots[0],snapshots[1]);assert.deepEqual(snapshots[1],snapshots[2]); + await source.own([{node:'1',applied:'2',manifest:m},{node:'2',applied:'2',manifest:m}]);assert.equal(source.gate().stage,'raft-owned');assert.equal(source.route(m=>m.config.cluster),cfg.cluster); + const reopened=new LegacySource(host,stale,runtime);await reopened.freeze(cfg);assert.deepEqual(await reopened.inventory(),m);await assert.rejects(reopened.freeze({...cfg,cluster:'new'})); + assert.equal(host.sql.exec("SELECT doc FROM dir WHERE kind='cells'").toArray()[0].doc,raw); + for(const [input,expected]of calls)assert.deepEqual(new Uint8Array(await native.call(input)),expected,'native/wasm migration ABI parity'); + }finally{if(native)await native.close();for(const h of stores)h.db.close();host.db.close();} +}); + +test('R24 R28 oversized and unknown legacy fields refuse without truncation; old releases stay released',async()=>{ + const runtime=new Reactor(new WebAssembly.Module(readFileSync(out+'/core.wasm'))),cfg=config(); + const released='{"V":1,"head":"head","size":1,"class":"chat","keys":{},"lease":{"device":"a","fence":7,"expires":0,"pending":0,"released_at":80}}'; + const row=convertLegacy(runtime,cfg,'cells','released',released,900000);const cell=decodeCanonical(dec.decode(row.data));assert.equal(cell.lease.expires,0);assert.equal(cell.lease.released_at,80); + assert.throws(()=>convertLegacy(runtime,cfg,'cells','new',released.slice(0,-1)+',"future":"keep-me"}')); + const host=new Host(),dir=new Directory(host.sql,cfg.identity,()=>200000,policyOf()),source=new LegacySource(host,dir,runtime); + const original=JSON.stringify({V:1,title:'s'.repeat(1024*1024+1)});host.sql.exec('INSERT INTO dir VALUES(?,?,?)','cells','large',original); + await source.freeze(cfg);await assert.rejects(source.inventory(),/cells record "large".*1048576/);assert.equal(source.gate().stage,'frozen');assert.equal(host.sql.exec("SELECT doc FROM dir WHERE kind='cells'").toArray()[0].doc,original);assert.throws(()=>dir.list(),e=>e.status===503);host.db.close(); +}); + +test('R24 freeze drains admitted legacy effects before inventory and refuses subsequent admissions',async()=>{ + const {LegacyAdmission}=await import('../../src/raft/legacy.js'); + const runtime=new Reactor(new WebAssembly.Module(readFileSync(out+'/core.wasm'))),cfg=config(),host=new Host(),admission=new LegacyAdmission(host),dir=new Directory(host.sql,cfg.identity,()=>200000,policyOf()),source=new LegacySource(host,dir,runtime,admission); + let release;const blocked=new Promise(r=>release=r),admitted=admission.run(async()=>{await blocked;dir.setVault('','last-admitted-vault');}); + const freezing=source.freeze(cfg);await assert.rejects(admission.run(()=>dir.setVault('','late')),e=>e.status===503); + assert.equal(source.gate(),null);release();await admitted;await freezing;await source.inventory(); + const identity=[...source.rows()].find(r=>r.key==='identity');assert.equal(decodeCanonical(dec.decode(identity.data)).vault,'last-admitted-vault');host.db.close(); +}); + +test('R24 R28 staged cross-object rows resume after reopen and preserve a grandfathered identity',async()=>{ + const {stageLegacyImport,finishLegacyImport}=await import('../../src/raft/legacy.js'); + const runtime=new Reactor(new WebAssembly.Module(readFileSync(out+'/core.wasm'))),codec=new HostCodec(runtime),cfg=config(),host=new Host(),directory=new Directory(host.sql,cfg.identity,()=>200000,policyOf()),source=new LegacySource(host,directory,runtime); + const title='s'.repeat(300000);for(let i=0;i<30;i++)directory.write('cells',String(i),{V:1,head:'head',class:'host-bound',size:1,keys:{},lease:{device:'a',fence:1,expires:0,pending:0},title}); + await source.freeze(cfg);const m=await source.inventory();assert(BigInt(m.ceiling)>8n*1024n*1024n); + const target=new Host();let store=new MemberStorage(target,codec);await store.open({cluster:cfg.cluster,id:'1',protocol:1},Number(m.ceiling)); + let n=0; + for(const row of source.rows()){ + await stageLegacyImport(store,m,row);await stageLegacyImport(store,m,row); + if(++n===3){store=new MemberStorage(target,codec);assert.equal((await store.open({cluster:cfg.cluster,id:'1',protocol:1})).applied,0n);assert.equal(store.application().count,0);} + } + const snapshot=await finishLegacyImport(store,m);assert(snapshot.length>0);assert.equal(store.application().count,Number(m.rows)+1);assert(store.application().size<=Number(m.ceiling));assert.equal(store.one("SELECT count(*) AS n FROM sqlite_master WHERE name LIKE 'legacy_stage%'").n,0); + // A cold opener uses the persisted imported ceiling even with the default + // argument, so same-sized or shrinking updates remain possible. + store=new MemberStorage(target,codec);assert.equal((await store.open({cluster:cfg.cluster,id:'1',protocol:1})).applied,1n);assert.equal(store.identity.ceiling,Number(m.ceiling));assert.equal([...source.rows()].length,Number(m.rows));target.db.close();host.db.close(); +}); From 0ee83f3bf55cf2c186f817f5f9dfab2327f13237 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 02:15:06 -0400 Subject: [PATCH 026/103] Record durable learner intents so pairing cannot enlarge the voting quorum. Keep admission separate from applied configuration, retain every retired node ID, and complete authorized joins after the requester sleeps. Replay configuration at each log position so follower batching cannot change policy. Prove bounded churn, snapshot catch-up, exact retries and restart with the production core and native stores. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterstate/apply.go | 34 +- internal/clusterstate/codec.go | 2 +- internal/clusterstate/membership.go | 413 ++++++++++++++++++++++ internal/clusterstate/membership_test.go | 305 ++++++++++++++++ internal/clusterstate/types.go | 54 +-- internal/raftabi/abi.go | 12 +- internal/raftabi/application.go | 50 ++- internal/raftabi/membership.go | 89 +++++ internal/raftcore/README.md | 33 ++ internal/raftcore/core.go | 6 +- internal/raftcore/membership.go | 157 ++++++++ internal/raftcore/membership_test.go | 91 +++++ internal/raftcore/types.go | 3 +- internal/raftdirectory/apply.go | 9 +- internal/raftdirectory/directory_test.go | 19 +- internal/raftdirectory/membership_test.go | 232 ++++++++++++ internal/raftdirectory/migration_test.go | 2 +- internal/raftsim/README.md | 9 + internal/raftsim/membership_test.go | 85 +++++ internal/raftsim/simulator.go | 15 +- internal/rafttransport/README.md | 11 + internal/rafttransport/envelope.go | 3 +- internal/rafttransport/registry.go | 22 ++ 23 files changed, 1606 insertions(+), 50 deletions(-) create mode 100644 internal/clusterstate/membership.go create mode 100644 internal/clusterstate/membership_test.go create mode 100644 internal/raftabi/membership.go create mode 100644 internal/raftcore/membership.go create mode 100644 internal/raftcore/membership_test.go create mode 100644 internal/raftdirectory/membership_test.go create mode 100644 internal/raftsim/membership_test.go create mode 100644 internal/rafttransport/registry.go diff --git a/internal/clusterstate/apply.go b/internal/clusterstate/apply.go index 873010be55..d4f6a2e30a 100644 --- a/internal/clusterstate/apply.go +++ b/internal/clusterstate/apply.go @@ -54,6 +54,7 @@ type Authority struct { Key []byte `json:"key"` Personal bool `json:"personal"` Effects bool `json:"effects,omitempty"` + Anchor bool `json:"anchor,omitempty"` } type session struct { Sequence uint64 `json:"sequence"` @@ -80,11 +81,13 @@ type JoinRequest struct { // ApplicationMeta is an immutable view at one applied index. Byte accounting // includes record framing and keys; capacity is a refusal, never an eviction. type ApplicationMeta struct { - Cluster string `json:"cluster"` - Now int64 `json:"now"` - Version uint64 `json:"version"` - Bytes uint64 `json:"bytes"` - Ceiling uint64 `json:"ceiling"` + Index uint64 `json:"index"` + Configuration Configuration `json:"configuration"` + Cluster string `json:"cluster"` + Now int64 `json:"now"` + Version uint64 `json:"version"` + Bytes uint64 `json:"bytes"` + Ceiling uint64 `json:"ceiling"` } type Evaluation struct { Now int64 `json:"now"` @@ -135,6 +138,7 @@ func Sign(c Command, key ed25519.PrivateKey) (Command, error) { func Dependencies(c Command) []RecordKey { keys := []RecordKey{IdentityKey(), DeviceKey(c.Actor), AuthorityKey(c.Actor), SessionKey(c.Actor, c.Session), SessionsKey()} m := c.Change + keys = append(keys, membershipDependencies(m)...) if m.PutDevice != nil { keys = append(keys, DeviceKey(m.PutDevice.ID)) } @@ -246,6 +250,16 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er target = new(Device) case k.Namespace == "state" && len(k.Key) >= 5 && k.Key[:5] == "cell/": target = new(Cell) + case k == MembershipKey(): + var h Membership + if Decode(b, &h) != nil || h.Validate() != nil { + return out, ErrDependency + } + target = new(Membership) + case k.Namespace == "membership": + target = new(DeviceMember) + case k.Namespace == "tombstones" || k.Namespace == "enrollment" && len(k.Key) > 5 && k.Key[:5] == "node/": + target = new(MemberRecord) case k.Namespace == "enrollment": target = new(Authority) case k == SessionsKey(): @@ -284,7 +298,7 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er } digest, _ := Digest(c) signed, e := SigningBytes(c) - if e != nil || c.Cluster != meta.Cluster || auth.Cluster != c.Cluster || (!auth.Personal && !auth.Effects) || len(auth.Key) != ed25519.PublicKeySize || !ed25519.Verify(auth.Key, signed, c.Signature) || digest != c.RequestDigest { + if e != nil || c.Cluster != meta.Cluster || auth.Cluster != c.Cluster || (!auth.Personal && !auth.Effects && !auth.Anchor) || len(auth.Key) != ed25519.PublicKeySize || !ed25519.Verify(auth.Key, signed, c.Signature) || digest != c.RequestDigest { return out, nil } // Revocation precedes dedup: a saved old response cannot authorize a revoked @@ -294,7 +308,7 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er return out, nil } service := c.Change.RecordUpload != nil || c.Change.AdmitJoin != nil || c.Change.CompleteJoin != nil - if service != auth.Effects || auth.Personal && auth.Effects { + if service != auth.Effects || auth.Personal && auth.Effects || auth.Anchor && (auth.Personal || auth.Effects || c.Change.SettleMembership == nil && c.Change.CloseSession == nil) { return out, nil } var last session @@ -338,7 +352,11 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er if service { code = applyEffect(c, out.Now, rows, put) } else { - code = applyMutation(c, out.Now, identity, rows, put, &out.Result) + if membershipMutation(c.Change) { + code = applyMembership(c, meta, rows, put, &out.Result) + } else { + code = applyMutation(c, out.Now, identity, rows, put, &out.Result) + } } } if err != nil { diff --git a/internal/clusterstate/codec.go b/internal/clusterstate/codec.go index 525a0408a4..ac6043e6f6 100644 --- a/internal/clusterstate/codec.go +++ b/internal/clusterstate/codec.go @@ -336,7 +336,7 @@ func (r Result) Validate() error { func (c Code) valid() bool { switch c { case OK, NotFound, Exists, LeaseHeld, FenceStale, HeadMoved, CAS, Revoked, - SelfRevoke, Unauthorized, Rotated, Conflict, Capacity, RotationStep, BadGrace, RequestGone, AlreadyDecided, BadRequest: + SelfRevoke, Unauthorized, Rotated, Conflict, Capacity, UpdateRequired, RotationStep, BadGrace, RequestGone, AlreadyDecided, BadRequest: return true } return false diff --git a/internal/clusterstate/membership.go b/internal/clusterstate/membership.go new file mode 100644 index 0000000000..1127c5149c --- /dev/null +++ b/internal/clusterstate/membership.go @@ -0,0 +1,413 @@ +package clusterstate + +import ( + "bytes" + "crypto/ed25519" + "strconv" +) + +const ( + MaxLearners = 32 + MaxMemberRecords = 4096 + ClusterProtocol uint16 = 1 +) + +// Configuration is supplied by the host from DURABLY APPLIED Raft state, never +// from a request. Joint fields are retained so admission cannot cross a swap. +type Configuration struct { + Voters []uint64 `json:"voters"` + Learners []uint64 `json:"learners"` + Outgoing []uint64 `json:"outgoing"` + LearnersNext []uint64 `json:"learners_next"` + AutoLeave bool `json:"auto_leave"` +} + +func (c Configuration) Joint() bool { + return len(c.Outgoing) > 0 || len(c.LearnersNext) > 0 || c.AutoLeave +} +func hasID(ids []uint64, id uint64) bool { + for _, n := range ids { + if n == id { + return true + } + } + return false +} + +// MemberRecord contains protocol authority only. Retired IDs are never deleted; +// a device that rejoins receives a fresh ID and a fresh transport incarnation. +type MemberRecord struct { + Node uint64 `json:"node"` + Device string `json:"device"` + Key []byte `json:"key"` + Incarnation uint64 `json:"incarnation"` + Protocol uint16 `json:"protocol"` + Anchor bool `json:"anchor"` + Retired bool `json:"retired"` + ReadyIndex uint64 `json:"ready_index"` +} +type MembershipIntent struct { + Node uint64 `json:"node"` + Remove bool `json:"remove"` + Revision uint64 `json:"revision"` +} +type Membership struct { + Revision uint64 `json:"revision"` + Index uint64 `json:"index"` + NextNode uint64 `json:"next_node"` + Anchors []uint64 `json:"anchors"` + Nodes []uint64 `json:"nodes"` + Live uint32 `json:"live"` + Limit uint32 `json:"limit"` + Pending *MembershipIntent `json:"pending,omitempty"` +} +type InitMembership struct { + Anchors []MemberRecord `json:"anchors"` + Limit uint32 `json:"limit"` +} +type EnrollMember struct { + Node uint64 `json:"node"` + Revision uint64 `json:"revision"` + Device string `json:"device"` + Protocol uint16 `json:"protocol"` +} +type RemoveMember struct { + Revision uint64 `json:"revision"` + Node uint64 `json:"node"` + Device string `json:"device"` +} +type SettleMembership struct { + Revision uint64 `json:"revision"` + Node uint64 `json:"node"` +} +type MembershipResult struct { + Node uint64 `json:"node"` + Revision uint64 `json:"revision"` + Barrier uint64 `json:"barrier"` +} +type DeviceMember struct { + Node uint64 `json:"node"` +} + +func AnchorActor(node uint64) string { return "anchor/" + strconv.FormatUint(node, 10) } + +func MembershipKey() RecordKey { return RecordKey{"membership", "registry"} } +func MemberKey(node uint64) RecordKey { + return RecordKey{"enrollment", "node/" + strconv.FormatUint(node, 10)} +} +func MemberDeviceKey(device string) RecordKey { + return RecordKey{"membership", "device/" + segment(device)} +} +func MemberTombstoneKey(node uint64) RecordKey { + return RecordKey{"tombstones", "node/" + strconv.FormatUint(node, 10)} +} + +func membershipDependencies(m Mutation) []RecordKey { + keys := []RecordKey{} + switch { + case m.InitMembership != nil: + keys = append(keys, MembershipKey()) + for _, a := range m.InitMembership.Anchors { + keys = append(keys, MemberKey(a.Node), AuthorityKey(AnchorActor(a.Node))) + } + case m.EnrollMember != nil: + keys = append(keys, MembershipKey(), DeviceKey(m.EnrollMember.Device), AuthorityKey(m.EnrollMember.Device), MemberDeviceKey(m.EnrollMember.Device), MemberKey(m.EnrollMember.Node), MemberTombstoneKey(m.EnrollMember.Node)) + case m.RemoveMember != nil: + v := m.RemoveMember + keys = append(keys, MembershipKey(), MemberKey(v.Node), MemberTombstoneKey(v.Node), MemberDeviceKey(v.Device)) + case m.SettleMembership != nil: + v := m.SettleMembership + keys = append(keys, MembershipKey(), MemberKey(v.Node)) + } + return keys +} +func membershipMutation(m Mutation) bool { + return m.InitMembership != nil || m.EnrollMember != nil || m.RemoveMember != nil || m.SettleMembership != nil +} +func validNode(id uint64) bool { return id > 0 && id < ^uint64(0)-1 } +func applyMembership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, put func(RecordKey, any), result *Result) Code { + m := c.Change + var h Membership + raw := rows[MembershipKey()] + if raw != nil && Decode(raw, &h) != nil { + return BadRequest + } + if m.InitMembership != nil { + v := m.InitMembership + if raw != nil { + return Conflict + } + if len(v.Anchors) != 3 || meta.Configuration.Joint() || len(meta.Configuration.Voters) != 3 || len(meta.Configuration.Learners) != 0 || v.Limit < 3 || v.Limit > MaxMemberRecords || meta.Index == 0 { + return BadRequest + } + seen := map[string]bool{} + h = Membership{Revision: 1, Index: meta.Index, Limit: v.Limit} + for i, a := range v.Anchors { + if !validNode(a.Node) || !validNode(a.Node+1) || i > 0 && a.Node <= v.Anchors[i-1].Node || !hasID(meta.Configuration.Voters, a.Node) || len(a.Key) != ed25519.PublicKeySize || seen[string(a.Key)] || a.Device != "" || !a.Anchor || a.Retired || a.Incarnation != 1 || a.Protocol != ClusterProtocol || a.ReadyIndex != 0 || rows[MemberKey(a.Node)] != nil { + return BadRequest + } + var old Authority + if b := rows[AuthorityKey(AnchorActor(a.Node))]; b != nil { + if Decode(b, &old) != nil || !old.Anchor || old.Personal || old.Effects || old.Cluster != c.Cluster || !bytes.Equal(old.Key, a.Key) { + return Conflict + } + } + seen[string(a.Key)] = true + h.Anchors = append(h.Anchors, a.Node) + h.Nodes = append(h.Nodes, a.Node) + h.NextNode = a.Node + 1 + } + for _, a := range v.Anchors { + put(MemberKey(a.Node), a) + put(AuthorityKey(AnchorActor(a.Node)), Authority{Cluster: c.Cluster, Key: a.Key, Anchor: true}) + } + put(MembershipKey(), h) + result.Membership = &MembershipResult{Revision: h.Revision, Barrier: meta.Index} + return OK + } + if raw == nil || h.Revision == ^uint64(0) || meta.Index == 0 { + return Conflict + } + if meta.Configuration.Joint() { + return Conflict + } + // P12 only edits learners. A voter swap is a separate authorized P13 intent. + for _, id := range h.Anchors { + if !hasID(meta.Configuration.Voters, id) { + return Conflict + } + } + if len(meta.Configuration.Voters) != len(h.Anchors) { + return Conflict + } + switch { + case m.EnrollMember != nil: + v := m.EnrollMember + if v.Revision != h.Revision || h.Pending != nil { + return Conflict + } + if v.Protocol != ClusterProtocol { + return UpdateRequired + } + var authority Authority + var device Device + var prior DeviceMember + if Decode(rows[AuthorityKey(v.Device)], &authority) != nil || authority.Cluster != c.Cluster || !authority.Personal || authority.Effects || len(authority.Key) != ed25519.PublicKeySize { + return Unauthorized + } + if b := rows[DeviceKey(v.Device)]; b != nil { + if Decode(b, &device) != nil { + return BadRequest + } + if device.Revoked { + return Revoked + } + } + if b := rows[MemberDeviceKey(v.Device)]; b != nil { + if Decode(b, &prior) != nil { + return BadRequest + } + if prior.Node != 0 { + return Conflict + } + } + if h.Live >= MaxLearners || len(h.Nodes) >= int(h.Limit) || !validNode(h.NextNode+1) { + return Capacity + } + node := h.NextNode + if v.Node != node || rows[MemberKey(node)] != nil || rows[MemberTombstoneKey(node)] != nil { + return Conflict + } + for _, n := range h.Nodes { + if n >= node { + return Conflict + } + } + h.NextNode++ + h.Revision++ + h.Index = meta.Index + h.Live++ + h.Nodes = append(h.Nodes, node) + h.Pending = &MembershipIntent{Node: node, Revision: h.Revision} + put(MemberKey(node), MemberRecord{Node: node, Device: v.Device, Key: bytes.Clone(authority.Key), Incarnation: 1, Protocol: ClusterProtocol}) + put(MemberDeviceKey(v.Device), DeviceMember{Node: node}) + put(MembershipKey(), h) + result.Membership = &MembershipResult{Node: node, Revision: h.Revision} + return OK + case m.RemoveMember != nil: + v := m.RemoveMember + if v.Revision != h.Revision || h.Pending != nil { + return Conflict + } + var member MemberRecord + if Decode(rows[MemberKey(v.Node)], &member) != nil { + return NotFound + } + if member.Anchor || member.Retired || member.Node != v.Node || member.Device != v.Device { + return Conflict + } + var mapping DeviceMember + if Decode(rows[MemberDeviceKey(v.Device)], &mapping) != nil || mapping.Node != v.Node { + return Conflict + } + member.Retired = true + member.ReadyIndex = 0 + h.Revision++ + h.Index = meta.Index + h.Live-- + h.Pending = &MembershipIntent{Node: v.Node, Remove: true, Revision: h.Revision} + put(MemberKey(v.Node), member) + put(MemberTombstoneKey(v.Node), member) + put(MemberDeviceKey(v.Device), DeviceMember{}) + put(MembershipKey(), h) + result.Membership = &MembershipResult{Node: v.Node, Revision: h.Revision} + return OK + case m.SettleMembership != nil: + v := m.SettleMembership + if v.Revision != h.Revision || h.Pending == nil || h.Pending.Node != v.Node { + return Conflict + } + present := hasID(meta.Configuration.Learners, v.Node) + if present == h.Pending.Remove { + return Conflict + } + var member MemberRecord + if Decode(rows[MemberKey(v.Node)], &member) != nil || member.Node != v.Node || member.Retired != h.Pending.Remove { + return Conflict + } + if !member.Retired { + member.ReadyIndex = meta.Index + put(MemberKey(v.Node), member) + } + h.Pending = nil + h.Index = meta.Index + put(MembershipKey(), h) + result.Membership = &MembershipResult{Node: v.Node, Revision: h.Revision, Barrier: meta.Index} + return OK + } + return BadRequest +} + +// PeerEnrollment is a derived view, not a second registry. Hosts publish it +// under their dispatch lock before acknowledging any committed revocation. +type PeerEnrollment struct { + Node uint64 `json:"node"` + Incarnation uint64 `json:"incarnation"` + Key []byte `json:"key"` + Voter bool `json:"voter"` + Revoked bool `json:"revoked"` +} + +func MembershipView(read ReadRecord, conf Configuration, applied uint64) (Membership, []PeerEnrollment, error) { + var h Membership + raw, err := read(MembershipKey()) + if err != nil { + return h, nil, err + } + if raw == nil { + return h, nil, nil + } + if Decode(raw, &h) != nil || h.Validate() != nil || h.Index > applied { + return h, nil, ErrDependency + } + peers := make([]PeerEnrollment, 0, len(h.Nodes)) + for i, id := range h.Nodes { + b, e := read(MemberKey(id)) + if e != nil { + return h, nil, e + } + var m MemberRecord + if Decode(b, &m) != nil || m.Node != id || i > 0 && h.Nodes[i-1] >= id || m.Protocol != ClusterProtocol || len(m.Key) != ed25519.PublicKeySize || m.Incarnation == 0 { + return h, nil, ErrDependency + } + var device []byte + if m.Device != "" { + device, e = read(DeviceKey(m.Device)) + if e != nil { + return h, nil, e + } + } + peer, e := PeerFromRecord(m, device, conf) + if e != nil { + return h, nil, e + } + peers = append(peers, peer) + } + return h, peers, nil +} + +// LearnerReady is evaluated on the learner's own durable applied state. A +// leader's replication Match, an uploaded snapshot or a registry grant alone +// cannot make the member ready to serve its advertised enrollment barrier. +func LearnerReady(m MemberRecord, conf Configuration, applied, advertised uint64) bool { + return !m.Anchor && !m.Retired && m.ReadyIndex > 0 && advertised >= m.ReadyIndex && applied >= advertised && hasID(conf.Learners, m.Node) +} + +// AnchorEnrollments validates the source-provisioned transport manifest used +// before the first registry command can commit. Only the SAME three bootstrap +// anchors may exchange protocol traffic; a personal device needs committed +// enrollment. This manifest contains no application or content authority. +func AnchorEnrollments(anchors []MemberRecord, voters []uint64) ([]PeerEnrollment, error) { + if len(anchors) != 3 || len(voters) != 3 { + return nil, ErrDependency + } + seen := map[string]bool{} + peers := []PeerEnrollment{} + for i, m := range anchors { + if !validNode(m.Node) || i > 0 && anchors[i-1].Node >= m.Node || !hasID(voters, m.Node) || !m.Anchor || m.Device != "" || m.Retired || m.ReadyIndex != 0 || m.Protocol != ClusterProtocol || m.Incarnation != 1 || len(m.Key) != ed25519.PublicKeySize || seen[string(m.Key)] { + return nil, ErrDependency + } + seen[string(m.Key)] = true + peers = append(peers, PeerEnrollment{Node: m.Node, Incarnation: m.Incarnation, Key: bytes.Clone(m.Key), Voter: true}) + } + return peers, nil +} + +func (h Membership) Validate() error { + if h.Revision == 0 || h.Index == 0 || len(h.Anchors) != 3 || len(h.Nodes) < 3 || h.Limit < 3 || h.Limit > MaxMemberRecords || len(h.Nodes) > int(h.Limit) || h.Live > MaxLearners || !validNode(h.NextNode) { + return ErrDependency + } + for i, id := range h.Nodes { + if !validNode(id) || id >= h.NextNode || i > 0 && h.Nodes[i-1] >= id { + return ErrDependency + } + } + for i, id := range h.Anchors { + if !hasID(h.Nodes, id) || i > 0 && h.Anchors[i-1] >= id { + return ErrDependency + } + } + if h.Pending != nil && (h.Pending.Revision != h.Revision || !hasID(h.Nodes, h.Pending.Node) || hasID(h.Anchors, h.Pending.Node)) { + return ErrDependency + } + return nil +} + +// EnrollmentMessage is shared by later ingress adapters and the manual. The +// protocol refusal distinguishes an old build from missing saved content. +func EnrollmentMessage(code Code) string { + switch code { + case UpdateRequired: + return "this codeaf build must be updated before it can join this cluster; your saved chats are still here" + case Capacity: + return "this cluster has reached its enrollment limit; remove a joined computer or ask the relay operator for capacity" + } + return "" +} + +// PeerFromRecord is shared by native registry scans and paged hosted reads. +// Device metadata only narrows existing enrollment; it never creates a peer. +func PeerFromRecord(m MemberRecord, device []byte, conf Configuration) (PeerEnrollment, error) { + if !validNode(m.Node) || m.Protocol != ClusterProtocol || len(m.Key) != ed25519.PublicKeySize || m.Incarnation == 0 { + return PeerEnrollment{}, ErrDependency + } + revoked := m.Retired + if m.Device != "" && device != nil { + var d Device + if Decode(device, &d) != nil { + return PeerEnrollment{}, ErrDependency + } + revoked = revoked || d.Revoked + } + return PeerEnrollment{Node: m.Node, Incarnation: m.Incarnation, Key: bytes.Clone(m.Key), Voter: hasID(conf.Voters, m.Node) || hasID(conf.Outgoing, m.Node), Revoked: revoked}, nil +} diff --git a/internal/clusterstate/membership_test.go b/internal/clusterstate/membership_test.go new file mode 100644 index 0000000000..62144c711a --- /dev/null +++ b/internal/clusterstate/membership_test.go @@ -0,0 +1,305 @@ +package clusterstate_test + +import ( + "crypto/ed25519" + "fmt" + "math/rand" + "reflect" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +func membershipApp(t *testing.T, limit uint32) *application { + a := newApplication(t) + a.meta.Configuration = cs.Configuration{Voters: []uint64{1, 2, 3}} + anchors := []cs.MemberRecord{} + for id := uint64(1); id <= 3; id++ { + anchors = append(anchors, cs.MemberRecord{Node: id, Key: actorKey(fmt.Sprint("anchor-", id)).Public().(ed25519.PublicKey), Incarnation: 1, Protocol: 1, Anchor: true}) + } + if r := memberRun(a, cs.Mutation{InitMembership: &cs.InitMembership{Anchors: anchors, Limit: limit}}); r.Code != cs.OK { + t.Fatal(r) + } + return a +} +func memberRun(a *application, m cs.Mutation) cs.Result { a.meta.Index++; return a.run("a", 1000, m) } +func memberHeader(a *application) cs.Membership { + var h cs.Membership + if e := cs.Decode(a.rows[cs.MembershipKey()], &h); e != nil { + a.t.Fatal(e) + } + return h +} +func enroll(a *application, device string) cs.Result { + h := memberHeader(a) + return memberRun(a, cs.Mutation{EnrollMember: &cs.EnrollMember{Node: h.NextNode, Revision: h.Revision, Device: device, Protocol: 1}}) +} +func settle(a *application) cs.Result { + h := memberHeader(a) + id := h.Pending.Node + if h.Pending.Remove { + ids := []uint64{} + for _, n := range a.meta.Configuration.Learners { + if n != id { + ids = append(ids, n) + } + } + a.meta.Configuration.Learners = ids + } else { + a.meta.Configuration.Learners = append(a.meta.Configuration.Learners, id) + } + return memberRun(a, cs.Mutation{SettleMembership: &cs.SettleMembership{Revision: h.Revision, Node: id}}) +} +func addAuthority(a *application, id string) { + a.put(cs.AuthorityKey(id), cs.Authority{Cluster: "test", Key: actorKey(id).Public().(ed25519.PublicKey), Personal: true}) + a.put(cs.DeviceKey(id), cs.Device{V: 1}) +} + +// R9/R11/R12/R16/R17 require bounded learner-only admission, durable allocation, +// one resumable intent, exact retry, and no revival of a retired node ID. +func TestR9R11R12R16R17LearnerAdmissionAndRetainedTombstones(t *testing.T) { + a := membershipApp(t, cs.MaxMemberRecords) + h := memberHeader(a) + command := a.prepareBare("a", 1000, cs.Mutation{EnrollMember: &cs.EnrollMember{Node: 4, Revision: h.Revision, Device: "b", Protocol: 1}}) + a.meta.Index++ + r := a.evaluate(command).Result + if r.Code != cs.OK || r.Membership.Node != 4 || r.Membership.Barrier != 0 { + t.Fatal(r) + } + if got := a.evaluate(command).Result; !reflect.DeepEqual(got, r) { + t.Fatal("R17 retry changed allocation", got, r) + } + if got := memberRun(a, cs.Mutation{SettleMembership: &cs.SettleMembership{Revision: 2, Node: 4}}); got.Code != cs.Conflict { + t.Fatal("ready before configuration", got) + } + if got := enroll(a, "a"); got.Code != cs.Conflict { + t.Fatal("second concurrent intent", got) + } + done := settle(a) + if done.Code != cs.OK || done.Membership.Barrier == 0 { + t.Fatal(done) + } + var m cs.MemberRecord + _ = cs.Decode(a.rows[cs.MemberKey(4)], &m) + if cs.LearnerReady(m, a.meta.Configuration, done.Membership.Barrier-1, done.Membership.Barrier) || !cs.LearnerReady(m, a.meta.Configuration, done.Membership.Barrier, done.Membership.Barrier) { + t.Fatal("R11 ready ignored applied barrier") + } + h = memberHeader(a) + if got := memberRun(a, cs.Mutation{RemoveMember: &cs.RemoveMember{Revision: h.Revision, Node: 4, Device: "b"}}); got.Code != cs.OK { + t.Fatal(got) + } + if a.rows[cs.MemberTombstoneKey(4)] == nil { + t.Fatal("R12 missing tombstone") + } + _, peers, e := cs.MembershipView(func(k cs.RecordKey) ([]byte, error) { return a.rows[k], nil }, a.meta.Configuration, a.meta.Index) + if e != nil || !peers[3].Revoked { + t.Fatal("R16 removed transport remains admitted", e, peers) + } + if got := settle(a); got.Code != cs.OK { + t.Fatal(got) + } + if got := enroll(a, "b"); got.Code != cs.OK || got.Membership.Node != 5 { + t.Fatal("R12 reused removed node", got) + } + if got := settle(a); got.Code != cs.OK { + t.Fatal(got) + } + if got := memberRun(a, cs.Mutation{Revoke: &cs.DeviceID{Device: "b"}}); got.Code != cs.OK { + t.Fatal(got) + } + _, peers, e = cs.MembershipView(func(k cs.RecordKey) ([]byte, error) { return a.rows[k], nil }, a.meta.Configuration, a.meta.Index) + if e != nil || !peers[4].Revoked { + t.Fatal("R16 committed device revoke missed routing", e) + } + if got := enroll(a, "b"); got.Code != cs.Revoked { + t.Fatal(got) + } + if len(a.meta.Configuration.Voters) != 3 { + t.Fatal("R9 pairing changed quorum") + } +} + +// R11/R27 cap live learners separately from permanently retained records. +// Capacity pressure must retain every prior enrollment and node tombstone. +func TestR11R27LearnerAndRetainedRecordLimits(t *testing.T) { + a := membershipApp(t, cs.MaxMemberRecords) + for i := 0; i < cs.MaxLearners; i++ { + id := fmt.Sprint("personal-", i) + addAuthority(a, id) + if r := enroll(a, id); r.Code != cs.OK { + t.Fatal(i, r) + } + if r := settle(a); r.Code != cs.OK { + t.Fatal(r) + } + } + addAuthority(a, "overflow") + before := append([]byte(nil), a.rows[cs.MembershipKey()]...) + if r := enroll(a, "overflow"); r.Code != cs.Capacity { + t.Fatal(r) + } + if !reflect.DeepEqual(before, a.rows[cs.MembershipKey()]) { + t.Fatal("capacity altered records") + } + b := membershipApp(t, 4) + if r := enroll(b, "b"); r.Code != cs.OK { + t.Fatal(r) + } + settle(b) + h := memberHeader(b) + memberRun(b, cs.Mutation{RemoveMember: &cs.RemoveMember{Revision: h.Revision, Node: 4, Device: "b"}}) + settle(b) + if r := enroll(b, "b"); r.Code != cs.Capacity { + t.Fatal("retained ceiling silently erased history", r) + } + if b.rows[cs.MemberTombstoneKey(4)] == nil { + t.Fatal("lost tombstone") + } +} + +// R9/R12/R16 reject unsupported protocols, forged metadata-only enrollment, +// stale revisions, voter removal and configuration changes during joint state. +func TestR9R12R16MembershipRefusals(t *testing.T) { + a := membershipApp(t, 100) + a.put(cs.DeviceKey("metadata-only"), cs.Device{V: 1}) + if r := enroll(a, "metadata-only"); r.Code != cs.Unauthorized { + t.Fatal(r) + } + h := memberHeader(a) + if r := memberRun(a, cs.Mutation{EnrollMember: &cs.EnrollMember{Node: h.NextNode, Revision: h.Revision, Device: "b", Protocol: 2}}); r.Code != cs.UpdateRequired { + t.Fatal(r) + } + if r := memberRun(a, cs.Mutation{RemoveMember: &cs.RemoveMember{Revision: h.Revision, Node: 1}}); r.Code != cs.Conflict { + t.Fatal("anchor removed", r) + } + a.meta.Configuration.Outgoing = []uint64{1, 2, 3} + if r := enroll(a, "b"); r.Code != cs.Conflict { + t.Fatal("joined during joint state", r) + } + a.meta.Configuration.Outgoing = nil + if r := memberRun(a, cs.Mutation{EnrollMember: &cs.EnrollMember{Node: 3, Revision: h.Revision, Device: "b", Protocol: 1}}); r.Code != cs.Conflict { + t.Fatal("allocated reused id", r) + } +} + +// R11/R12/R16/R17 compare random request/configuration orderings with a small +// independent admission model, including stale observations and revocation. +func TestR11R12R17SeededMembershipCampaign(t *testing.T) { + for seed := 0; seed < seedCount(t); seed++ { + t.Run(fmt.Sprintf("seed-%04d", seed), func(t *testing.T) { + a := membershipApp(t, 100) + rng := rand.New(rand.NewSource(int64(seed))) + revision, next := uint64(1), uint64(4) + active, target := uint64(0), uint64(4) + pending := "" + present, revoked := false, false + retired := map[uint64]bool{} + for step := 0; step < 16; step++ { + var mutation cs.Mutation + want := cs.OK + switch rng.Intn(7) { + case 0, 1: + mutation.EnrollMember = &cs.EnrollMember{Node: next, Revision: revision, Device: "b", Protocol: 1} + switch { + case pending != "": + want = cs.Conflict + case revoked: + want = cs.Revoked + case active != 0: + want = cs.Conflict + default: + active = next + target = next + next++ + revision++ + pending = "add" + } + case 2: + mutation.RemoveMember = &cs.RemoveMember{Revision: revision, Node: target, Device: "b"} + switch { + case pending != "": + want = cs.Conflict + case target >= next: + want = cs.NotFound + case retired[target]: + want = cs.Conflict + default: + retired[target] = true + active = 0 + revision++ + pending = "remove" + } + case 3: + mutation.SettleMembership = &cs.SettleMembership{Revision: revision, Node: target} + if pending == "" || pending == "add" && !present || pending == "remove" && present { + want = cs.Conflict + } else { + pending = "" + } + case 4: + // Explicit host configuration delivery is independent of request order. + if pending == "add" { + present = true + a.meta.Configuration.Learners = []uint64{target} + } else if pending == "remove" { + present = false + a.meta.Configuration.Learners = nil + } + continue + case 5: + mutation.EnrollMember = &cs.EnrollMember{Node: next, Revision: revision - 1, Device: "b", Protocol: 1} + want = cs.Conflict + case 6: + mutation.Revoke = &cs.DeviceID{Device: "b"} + revoked = true + } + got := memberRun(a, mutation) + if got.Code != want { + t.Fatalf("step %d got %s want %s mutation=%+v", step, got.Code, want, mutation) + } + h := memberHeader(a) + if h.NextNode != next || h.Revision != revision { + t.Fatalf("allocation differs: %+v next=%d revision=%d", h, next, revision) + } + for id := range retired { + var m cs.MemberRecord + if cs.Decode(a.rows[cs.MemberTombstoneKey(id)], &m) != nil || !m.Retired { + t.Fatal("retirement lost", id) + } + } + } + }) + } +} + +// R9/R12/R16 require an anchor to finish an already authorized join while its +// requester sleeps, without granting that protocol key personal authority. +func TestR9R12R16AnchorCanOnlyCompleteCommittedIntent(t *testing.T) { + a := membershipApp(t, 100) + if r := enroll(a, "b"); r.Code != cs.OK { + t.Fatal(r) + } + h := memberHeader(a) + actor := cs.AnchorActor(1) + run := func(sequence uint64, m cs.Mutation) cs.Result { + a.meta.Index++ + c, e := cs.Sign(cs.Command{Schema: 1, Cluster: "test", Actor: actor, Session: "completion", Sequence: sequence, Ack: sequence - 1, Now: 1000, Change: m}, actorKey("anchor-1")) + if e != nil { + t.Fatal(e) + } + return a.evaluate(c).Result + } + if r := run(1, cs.Mutation{SetVault: &cs.VaultCAS{New: "forbidden"}}); r.Code != cs.Unauthorized { + t.Fatal("anchor gained content authority", r) + } + if r := run(1, cs.Mutation{EnrollMember: &cs.EnrollMember{Node: 5, Revision: h.Revision, Device: "a", Protocol: 1}}); r.Code != cs.Unauthorized { + t.Fatal("anchor invented enrollment", r) + } + if r := run(1, cs.Mutation{SettleMembership: &cs.SettleMembership{Node: 4, Revision: h.Revision}}); r.Code != cs.Conflict { + t.Fatal("anchor skipped applied configuration", r) + } + a.meta.Configuration.Learners = []uint64{4} + if r := run(2, cs.Mutation{SettleMembership: &cs.SettleMembership{Node: 4, Revision: h.Revision}}); r.Code != cs.OK || r.Membership.Barrier == 0 { + t.Fatal("anchor could not resume authorized enrollment", r) + } +} diff --git a/internal/clusterstate/types.go b/internal/clusterstate/types.go index a48c868724..40728af1a2 100644 --- a/internal/clusterstate/types.go +++ b/internal/clusterstate/types.go @@ -44,24 +44,28 @@ type Command struct { // their typed operations here; unknown fields fail decoding on older members. // There is no arbitrary JSON payload in which task text or secrets can hide. type Mutation struct { - RecordUpload *UploadReceipt `json:"record_upload,omitempty"` - AdmitJoin *AdmitJoin `json:"admit_join,omitempty"` - CompleteJoin *CompleteJoin `json:"complete_join,omitempty"` - Rotate *RotateRequest `json:"rotate,omitempty"` - CloseSession *CloseSession `json:"close_session,omitempty"` - DecideJoin *JoinDecision `json:"decide_join,omitempty"` - Create *CreateCell `json:"create,omitempty"` - Acquire *AcquireCell `json:"acquire,omitempty"` - Beat *BeatCell `json:"beat,omitempty"` - BeatBatch *BeatBatch `json:"beat_batch,omitempty"` - Publish *PublishCell `json:"publish,omitempty"` - Release *ReleaseCell `json:"release,omitempty"` - Archive *CellID `json:"archive,omitempty"` - PutDevice *PutDevice `json:"put_device,omitempty"` - Revoke *DeviceID `json:"revoke,omitempty"` - SetVault *VaultCAS `json:"set_vault,omitempty"` - AdvanceTime *AdvanceTime `json:"advance_time,omitempty"` - DeviceSeen *DeviceSeen `json:"device_seen,omitempty"` + InitMembership *InitMembership `json:"init_membership,omitempty"` + EnrollMember *EnrollMember `json:"enroll_member,omitempty"` + RemoveMember *RemoveMember `json:"remove_member,omitempty"` + SettleMembership *SettleMembership `json:"settle_membership,omitempty"` + RecordUpload *UploadReceipt `json:"record_upload,omitempty"` + AdmitJoin *AdmitJoin `json:"admit_join,omitempty"` + CompleteJoin *CompleteJoin `json:"complete_join,omitempty"` + Rotate *RotateRequest `json:"rotate,omitempty"` + CloseSession *CloseSession `json:"close_session,omitempty"` + DecideJoin *JoinDecision `json:"decide_join,omitempty"` + Create *CreateCell `json:"create,omitempty"` + Acquire *AcquireCell `json:"acquire,omitempty"` + Beat *BeatCell `json:"beat,omitempty"` + BeatBatch *BeatBatch `json:"beat_batch,omitempty"` + Publish *PublishCell `json:"publish,omitempty"` + Release *ReleaseCell `json:"release,omitempty"` + Archive *CellID `json:"archive,omitempty"` + PutDevice *PutDevice `json:"put_device,omitempty"` + Revoke *DeviceID `json:"revoke,omitempty"` + SetVault *VaultCAS `json:"set_vault,omitempty"` + AdvanceTime *AdvanceTime `json:"advance_time,omitempty"` + DeviceSeen *DeviceSeen `json:"device_seen,omitempty"` } type CellID struct { @@ -136,12 +140,13 @@ type DeviceSeen struct { // no free-form diagnostic text, so logging a refusal cannot disclose payloads. // Now and Version are application time/version, never a local clock/Raft index. type Result struct { - Schema uint16 `json:"schema"` - Code Code `json:"code"` - Now int64 `json:"now"` - Version uint64 `json:"version"` - View *CellView `json:"view,omitempty"` - Items []ItemResult `json:"items,omitempty"` + Membership *MembershipResult `json:"membership,omitempty"` + Schema uint16 `json:"schema"` + Code Code `json:"code"` + Now int64 `json:"now"` + Version uint64 `json:"version"` + View *CellView `json:"view,omitempty"` + Items []ItemResult `json:"items,omitempty"` } type ItemResult struct { @@ -171,6 +176,7 @@ const ( AlreadyDecided Code = "already_decided" BadRequest Code = "bad_request" Capacity Code = "capacity" + UpdateRequired Code = "update_required" ) // Row encodes one application record for paged reads and streamed snapshots. diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index 11281d8d33..a54dc5fceb 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -164,13 +164,23 @@ func (r *Runtime) Call(b []byte) (out []byte) { func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { e := encoder{} switch op { + case 20: + return configurationAtEntry(d) + case 19: + return anchorRegistry(d) + case 17: + return r.membershipPlan(d) + case 18: + return learnerReady(d) + case 16: + return registryCodec(d) case 14: return legacyAuthorities(d) case 13: return legacyGenesis(d) case 12: return legacyCodec(d) - case 10, 11: + case 10, 11, 15: return applicationCodec(op, d) case 1: cfg := raftcore.Config{ClusterUUID: d.str(), NodeID: d.u64(), Seed: d.u64()} diff --git a/internal/raftabi/application.go b/internal/raftabi/application.go index 3786a121da..3be24c7d9f 100644 --- a/internal/raftabi/application.go +++ b/internal/raftabi/application.go @@ -1,6 +1,10 @@ package raftabi -import "github.com/Agent-Field/codeaf/internal/clusterstate" +import ( + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + pb "go.etcd.io/raft/v3/raftpb" +) func applicationCodec(op uint32, d *decoder) ([]byte, error) { raw := d.blob() @@ -41,6 +45,12 @@ func applicationCodec(op uint32, d *decoder) ([]byte, error) { rows[k] = b size += len(b) + len(k.Key) + len(k.Namespace) } + if op == 15 { + meta.Index = d.u64() + conf := new(pb.ConfState) + d.message(conf) + meta.Configuration = raftcore.Configuration(conf) + } if err := d.end(); err != nil { return nil, err } @@ -72,3 +82,41 @@ func applicationCodec(op uint32, d *decoder) ([]byte, error) { } return e.b, nil } + +// registryCodec pages one peer at a time so large retained device metadata +// cannot require materializing an identity or its full registry in the reactor. +func registryCodec(d *decoder) ([]byte, error) { + node, applied := d.u64(), d.u64() + conf := new(pb.ConfState) + d.message(conf) + header, member, device := d.blob(), d.blob(), d.blob() + if err := d.end(); err != nil { + return nil, err + } + var h clusterstate.Membership + var m clusterstate.MemberRecord + if clusterstate.Decode(header, &h) != nil || clusterstate.Decode(member, &m) != nil || h.Index > applied || m.Node != node { + return nil, clusterstate.ErrDependency + } + found := false + for _, id := range h.Nodes { + found = found || id == node + } + if !found { + return nil, clusterstate.ErrDependency + } + if h.Validate() != nil { + return nil, clusterstate.ErrDependency + } + peer, err := clusterstate.PeerFromRecord(m, device, raftcore.Configuration(conf)) + if err != nil { + return nil, err + } + b, err := clusterstate.Encode(peer) + if err != nil { + return nil, err + } + e := encoder{} + e.blob(b) + return e.b, nil +} diff --git a/internal/raftabi/membership.go b/internal/raftabi/membership.go new file mode 100644 index 0000000000..4307be1177 --- /dev/null +++ b/internal/raftabi/membership.go @@ -0,0 +1,89 @@ +package raftabi + +import ( + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + pb "go.etcd.io/raft/v3/raftpb" +) + +func (r *Runtime) membershipPlan(d *decoder) ([]byte, error) { + handle, raw := d.u32(), d.blob() + if err := d.end(); err != nil { + return nil, err + } + node := r.nodes[handle] + if node == nil { + return nil, errFrame + } + var h clusterstate.Membership + if clusterstate.Decode(raw, &h) != nil { + return nil, clusterstate.ErrCodec + } + p, err := node.PlanMembership(h) + if err != nil { + return nil, err + } + e := encoder{} + e.blob(p.Change) + e.flag(p.Checkpoint) + e.flag(p.Settle) + return e.b, nil +} +func learnerReady(d *decoder) ([]byte, error) { + raw := d.blob() + conf := new(pb.ConfState) + d.message(conf) + applied, advertised := d.u64(), d.u64() + if err := d.end(); err != nil { + return nil, err + } + var m clusterstate.MemberRecord + if clusterstate.Decode(raw, &m) != nil { + return nil, clusterstate.ErrCodec + } + e := encoder{} + e.flag(clusterstate.LearnerReady(m, raftcore.Configuration(conf), applied, advertised)) + return e.b, nil +} + +func anchorRegistry(d *decoder) ([]byte, error) { + raw := d.blob() + if err := d.end(); err != nil { + return nil, err + } + var input struct { + Anchors []clusterstate.MemberRecord `json:"anchors"` + Voters []uint64 `json:"voters"` + } + if err := clusterstate.Decode(raw, &input); err != nil { + return nil, err + } + peers, err := clusterstate.AnchorEnrollments(input.Anchors, input.Voters) + if err != nil { + return nil, err + } + b, err := clusterstate.Encode(peers) + if err != nil { + return nil, err + } + e := encoder{} + e.blob(b) + return e.b, nil +} + +func configurationAtEntry(d *decoder) ([]byte, error) { + conf := new(pb.ConfState) + d.message(conf) + entry := new(pb.Entry) + d.message(entry) + if err := d.end(); err != nil { + return nil, err + } + after, err := raftcore.AdvanceConfiguration(conf, entry) + if err != nil { + return nil, err + } + e := encoder{} + e.message(after) + return e.b, nil +} diff --git a/internal/raftcore/README.md b/internal/raftcore/README.md index 73459c71be..a532c1ad71 100644 --- a/internal/raftcore/README.md +++ b/internal/raftcore/README.md @@ -135,3 +135,36 @@ conservative persist-before-send protocol without sending an acknowledgment for an uninstalled snapshot. The later Apply effect confirms that installed index and applies any committed suffix before `Applied`. Recover the application before constructing a core; no application rows or chunks belong in its log cache. + + +## Committed learner intents + +P12's `PlanMembership` accepts only the durable `clusterstate.Membership` header +under the member serialization lock. `InitMembership` records exactly the three +already-applied anchors and their protocol public keys; source provisioning must +supply that SAME manifest to the bootstrap transport. Personal enrollment must +already have committed certificate authority. Metadata upserts grant none. + +The header retains a single add/remove intent and configuration revision. Plan, +submit its deterministic ConfChangeV2, wait for durable application, then plan +again. A durable uncommitted configuration returns ErrBusy. Checkpoint when +requested before allowing compacted catch-up: a pre-admission snapshot cannot +restore a learner it excludes. Submit authorized SettleMembership only after the +configuration is applied. The evaluator checks the host's applied ConfState; +a request cannot supply it. `MembershipResult.Barrier` is zero on admission, +and the settlement entry index afterward. The learner must durably apply that +barrier before reporting ready. Current committed authorization must also hold. + +A removal immediately retires transport authority, but its configuration change +still completes through Raft. IDs and tombstones survive snapshots and are never +reused. New IDs increase monotonically. Live learners are limited separately +from retained records; exhaustion refuses enrollment without deleting anything. +P13 owns promotion, joint swaps, transfer completion and anchor fallback. No +P12 policy command removes/promotes a voter. Configuration/provisioning APIs are +internal host seams, never unauthenticated client endpoints. + +Provisioned anchor keys receive a restricted committed authority that can only +settle an existing intent or close their request session. They cannot enroll, +remove, read directory content, mutate chats, pair or rotate. Thus a crashed +requester need not return for an already authorized admission to finish. The +settlement still requires the exact applied configuration and pending revision. diff --git a/internal/raftcore/core.go b/internal/raftcore/core.go index c18acc19e9..3aa170e8ff 100644 --- a/internal/raftcore/core.go +++ b/internal/raftcore/core.go @@ -3,6 +3,7 @@ package raftcore import ( "bytes" "encoding/binary" + "errors" "slices" "github.com/Agent-Field/codeaf/internal/clusterstate" @@ -269,7 +270,10 @@ func (c *Core) Step(input Input) (effects Effects, err error) { if len(in.Bytes) > MaxUncommittedBytes || proto.Unmarshal(in.Bytes, m) != nil || !c.validPeer(m) { return effects, ErrPeer } - if err = c.node.Step(m); err != nil { + // A delayed response can outlive committed learner removal. RawNode + // refuses it before changing consensus; discard that stale delivery rather + // than treating ordinary membership churn as a failed member. + if err = c.node.Step(m); err != nil && !errors.Is(err, raft.ErrStepPeerNotFound) { return effects, err } case Propose: diff --git a/internal/raftcore/membership.go b/internal/raftcore/membership.go new file mode 100644 index 0000000000..8fcc6ac7ef --- /dev/null +++ b/internal/raftcore/membership.go @@ -0,0 +1,157 @@ +package raftcore + +import ( + "github.com/Agent-Field/codeaf/internal/clusterstate" + "go.etcd.io/raft/v3/confchange" + pb "go.etcd.io/raft/v3/raftpb" + "go.etcd.io/raft/v3/tracker" + "google.golang.org/protobuf/proto" +) + +// Configuration copies complete configuration state across the pure policy +// boundary without introducing a protobuf dependency into clusterstate. +func Configuration(c *pb.ConfState) clusterstate.Configuration { + return clusterstate.Configuration{Voters: append([]uint64(nil), c.GetVoters()...), Learners: append([]uint64(nil), c.GetLearners()...), Outgoing: append([]uint64(nil), c.GetVotersOutgoing()...), LearnersNext: append([]uint64(nil), c.GetLearnersNext()...), AutoLeave: c.GetAutoLeave()} +} + +type MembershipPlan struct { + Change []byte + Checkpoint bool + Settle bool +} + +// PlanMembership reconciles a DURABLY COMMITTED intent after any restart or +// leader change. Registry admission and Raft membership are separate stages; +// neither an uncommitted configuration nor a lagging learner pins compaction. +func (c *Core) PlanMembership(h clusterstate.Membership) (MembershipPlan, error) { + var p MembershipPlan + if c.stopped { + return p, ErrStopped + } + if !c.leader() { + return p, ErrLeader + } + if c.pending != nil { + return p, ErrBusy + } + if h.Index > c.applied || h.Revision == 0 { + return p, ErrConfig + } + intent := h.Pending + if intent == nil { + return p, nil + } + if intent.Revision != h.Revision || intent.Node == 0 { + return p, ErrConfig + } + conf := c.store.conf + if len(conf.GetVotersOutgoing()) != 0 || len(conf.GetLearnersNext()) != 0 || conf.GetAutoLeave() { + return p, ErrBusy + } + for _, id := range conf.GetVoters() { + if id == intent.Node { + return p, ErrConfig + } + } + // A previously proposed configuration may be durable but uncommitted. Wait + // for Raft to decide it instead of queuing another membership mutation. + last, err := c.store.LastIndex() + if err != nil { + return p, err + } + if last > c.applied { + entries, err := c.store.Entries(c.applied+1, last+1, ^uint64(0)) + if err != nil { + return p, err + } + for _, entry := range entries { + if entry.GetType() != pb.EntryNormal { + return p, ErrBusy + } + } + } + present := false + for _, id := range conf.GetLearners() { + present = present || id == intent.Node + } + if present == !intent.Remove { + p.Settle = true + if present { + snap, err := c.store.Snapshot() + if err != nil { + return p, err + } + included := false + for _, id := range snap.GetMetadata().GetConfState().GetLearners() { + included = included || id == intent.Node + } + // etcd refuses a snapshot that excludes its recipient. A newly admitted + // learner catching up across compaction therefore needs a fresh checkpoint. + p.Checkpoint = !included + } + return p, nil + } + kind := pb.ConfChangeAddLearnerNode + if intent.Remove { + kind = pb.ConfChangeRemoveNode + } + context, err := clusterstate.Encode(*intent) + if err != nil { + return p, err + } + cc := &pb.ConfChangeV2{Changes: []*pb.ConfChangeSingle{{Type: kind.Enum(), NodeId: proto.Uint64(intent.Node)}}, Context: context} + p.Change, err = wire.Marshal(cc) + return p, err +} + +// AdvanceConfiguration reconstructs the configuration at each committed log +// position for paged application evaluation. Followers may receive a config +// entry and its settlement command in ONE Ready; using either the batch's old +// or final ConfState for every entry would make policy depend on batching. +// This invokes the same pinned library changer used by RawNode, not host rules. +func AdvanceConfiguration(before *pb.ConfState, entry *pb.Entry) (after *pb.ConfState, err error) { + defer func() { + if recover() != nil { + after = nil + err = ErrConfig + } + }() + cc := new(pb.ConfChangeV2) + switch entry.GetType() { + case pb.EntryNormal: + return clone(before), nil + case pb.EntryConfChange: + v := new(pb.ConfChange) + if proto.Unmarshal(entry.Data, v) != nil { + return nil, ErrCommand + } + cc = v.AsV2() + case pb.EntryConfChangeV2: + if proto.Unmarshal(entry.Data, cc) != nil { + return nil, ErrCommand + } + default: + return nil, ErrCommand + } + tracker := tracker.MakeProgressTracker(MaxInflightBatches, MaxMessageBytes) + changer := confchange.Changer{Tracker: tracker, LastIndex: entry.GetIndex()} + cfg, progress, err := confchange.Restore(changer, before) + if err != nil { + return nil, err + } + changer.Tracker.Config = cfg + changer.Tracker.Progress = progress + if cc.LeaveJoint() { + cfg, progress, err = changer.LeaveJoint() + } else if auto, joint := cc.EnterJoint(); joint { + cfg, progress, err = changer.EnterJoint(auto, cc.Changes...) + } else { + cfg, progress, err = changer.Simple(cc.Changes...) + } + if err != nil { + return nil, err + } + changer.Tracker.Config = cfg + changer.Tracker.Progress = progress + return changer.Tracker.ConfState(), nil +} diff --git a/internal/raftcore/membership_test.go b/internal/raftcore/membership_test.go new file mode 100644 index 0000000000..c49a2987a2 --- /dev/null +++ b/internal/raftcore/membership_test.go @@ -0,0 +1,91 @@ +package raftcore + +import ( + "errors" + "strings" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// R11/R12 serialize actual configuration proposals, reconcile them after +// restart and refresh a compacted snapshot to include the admitted learner. +func TestR11R12MembershipIntentReconciliationAndStaleResponse(t *testing.T) { + g := newGroup(t) + leader := g.elect() + c := g.nodes[leader-1] + h := cs.Membership{Revision: 2, Index: c.applied, Pending: &cs.MembershipIntent{Node: 4, Revision: 2}} + p, e := c.PlanMembership(h) + if e != nil || len(p.Change) == 0 || p.Settle { + t.Fatal(p, e) + } + // Withhold replication responses. A durable-but-uncommitted configuration + // is not permission to issue another learner change or mark the first ready. + g.drop = func(from, to uint64) bool { return to == leader || to == 4 } + g.call(leader, Configure{Change: p.Change}) + g.pump() + if _, e = c.PlanMembership(h); !errors.Is(e, ErrBusy) { + t.Fatal("uncommitted configuration accepted another intent", e) + } + g.drop = func(from, to uint64) bool { return to == 4 } + g.call(leader, Tick{}) + g.pump() + p, e = c.PlanMembership(h) + if e != nil || !p.Settle || !p.Checkpoint { + t.Fatal("admission did not request fresh snapshot", p, e) + } + // Restart from the actual flushed configuration, not the caller's intent. + cfg := c.cfg + cfg.Bootstrap = nil + cfg.Seed++ + restored, e := New(cfg, g.durable[leader-1]) + if e != nil { + t.Fatal(e) + } + if _, e = restored.PlanMembership(h); !errors.Is(e, ErrLeader) { + t.Fatal("restart restored leader authority", e) + } + confBytes, e := raftsnapshot.Configuration(c.store.conf) + if e != nil { + t.Fatal(e) + } + term, e := c.store.Term(c.applied) + if e != nil { + t.Fatal(e) + } + manifest := raftsnapshot.Manifest{Version: 1, Cluster: c.cfg.ClusterUUID, Protocol: 1, Index: c.applied, Term: term, Conf: confBytes, RecordsHash: strings.Repeat("0", 64), Digest: raftsnapshot.ID(nil)} + data, e := manifest.Encode() + if e != nil { + t.Fatal(e) + } + snapshot := &pb.Snapshot{Data: data, Metadata: &pb.SnapshotMetadata{Index: proto.Uint64(c.applied), Term: proto.Uint64(term), ConfState: clone(c.store.conf)}} + g.call(leader, CompactSnapshot{Snapshot: snapshot}) + g.pump() + p, e = c.PlanMembership(h) + if e != nil || !p.Settle || p.Checkpoint { + t.Fatal("already matching snapshot not recognized", p, e) + } + h.Revision++ + h.Index = c.applied + h.Pending = &cs.MembershipIntent{Node: 4, Revision: h.Revision, Remove: true} + p, e = c.PlanMembership(h) + if e != nil { + t.Fatal(e) + } + g.call(leader, Configure{Change: p.Change}) + g.pump() + p, e = c.PlanMembership(h) + if e != nil || !p.Settle || p.Checkpoint { + t.Fatal("remove did not reconcile", p, e) + } + before := c.status() + raw, _ := wire.Marshal(&pb.Message{Type: pb.MsgAppResp.Enum(), From: proto.Uint64(4), To: proto.Uint64(leader), Term: proto.Uint64(before.Term), Index: proto.Uint64(before.Commit)}) + g.call(leader, PeerMessage{Bytes: raw}) + g.pump() + if c.status().Term != before.Term || len(c.store.conf.GetVoters()) != 3 || len(c.store.conf.GetLearners()) != 0 { + t.Fatal("retired response changed authority") + } +} diff --git a/internal/raftcore/types.go b/internal/raftcore/types.go index 981bbb4be2..6c2e03f731 100644 --- a/internal/raftcore/types.go +++ b/internal/raftcore/types.go @@ -4,13 +4,14 @@ package raftcore import ( "errors" + "github.com/Agent-Field/codeaf/internal/clusterstate" "go.etcd.io/raft/v3" pb "go.etcd.io/raft/v3/raftpb" ) const ( - ProtocolVersion uint16 = 1 + ProtocolVersion uint16 = clusterstate.ClusterProtocol ElectionTicks = 6 HeartbeatTicks = 1 MaxMessageBytes = 64 << 10 diff --git a/internal/raftdirectory/apply.go b/internal/raftdirectory/apply.go index 623f05bc0a..bde8c0248d 100644 --- a/internal/raftdirectory/apply.go +++ b/internal/raftdirectory/apply.go @@ -48,11 +48,16 @@ func Apply(ctx context.Context, store *raftstore.SQLite, b raftcore.ApplyBatch, return v, nil } results := []clusterstate.Result{} + conf := app.ConfState for _, entry := range b.Entries { + conf, err = raftcore.AdvanceConfiguration(conf, entry) + if err != nil { + return nil, err + } if entry.GetType() != pb.EntryNormal || len(entry.GetData()) == 0 { continue } - e, err := clusterstate.Evaluate(entry.GetData(), clusterstate.ApplicationMeta{Cluster: cluster, Now: update.LogicalTime, Version: update.VisibleVersion, Bytes: size, Ceiling: ceiling}, read) + e, err := clusterstate.Evaluate(entry.GetData(), clusterstate.ApplicationMeta{Index: entry.GetIndex(), Configuration: raftcore.Configuration(conf), Cluster: cluster, Now: update.LogicalTime, Version: update.VisibleVersion, Bytes: size, Ceiling: ceiling}, read) if err != nil { return nil, err } @@ -109,7 +114,7 @@ func authorizedRead(ctx context.Context, store *raftstore.SQLite, cluster, actor } var auth clusterstate.Authority var device clusterstate.Device - if clusterstate.Decode(rows[keys[0]], &auth) != nil || auth.Cluster != cluster || (auth.Personal == effects) || auth.Effects != effects || len(auth.Key) != ed25519.PublicKeySize { + if clusterstate.Decode(rows[keys[0]], &auth) != nil || auth.Cluster != cluster || (auth.Personal == effects) || auth.Effects != effects || auth.Anchor || len(auth.Key) != ed25519.PublicKeySize { return app, errors.New("raftdirectory: unauthorized") } if raw := rows[keys[1]]; raw != nil { diff --git a/internal/raftdirectory/directory_test.go b/internal/raftdirectory/directory_test.go index 3fb354c398..5acebb56d6 100644 --- a/internal/raftdirectory/directory_test.go +++ b/internal/raftdirectory/directory_test.go @@ -54,22 +54,23 @@ type group struct { t *testing.T mu sync.Mutex clock func() time.Time - paths [3]string - stores [3]*raftstore.SQLite - nodes [3]*raftcore.Core - status [3]raftcore.Status + paths []string + stores []*raftstore.SQLite + nodes []*raftcore.Core + status []raftcore.Status queue []raftcore.Outbound - results [3][]clusterstate.Result + results [][]clusterstate.Result reads map[string]bool keys map[string]ed25519.PrivateKey seq map[string]uint64 genesis []raftstore.Write + absent map[uint64]bool blocked bool readID uint64 } func newGroup(t *testing.T, clock func() time.Time, extra ...raftstore.Write) *group { - g := &group{t: t, clock: clock, reads: map[string]bool{}, seq: map[string]uint64{}} + g := &group{paths: make([]string, 3), stores: make([]*raftstore.SQLite, 3), nodes: make([]*raftcore.Core, 3), status: make([]raftcore.Status, 3), results: make([][]clusterstate.Result, 3), absent: map[uint64]bool{}, t: t, clock: clock, reads: map[string]bool{}, seq: map[string]uint64{}} actors := []string{devA, devB, "a", "b"} for i := range 50 { actors = append(actors, fmt.Sprintf("dev_%032x", i+1)) @@ -110,7 +111,9 @@ func newGroup(t *testing.T, clock func() time.Time, extra ...raftstore.Write) *g } func (g *group) drive(i int, in raftcore.Input) { e, err := g.nodes[i].Step(in) - must(g.t, err) + if err != nil { + g.t.Fatalf("node %d input %T: %v", i+1, in, err) + } for n := 0; n < 100; n++ { g.status[i] = e.Status g.queue = append(g.queue, e.Send...) @@ -150,7 +153,7 @@ func (g *group) drain() { } m := g.queue[0] g.queue = g.queue[1:] - if !g.blocked { + if !g.blocked && !g.absent[m.To] && int(m.To) <= len(g.nodes) { g.drive(int(m.To)-1, raftcore.PeerMessage{Bytes: m.Bytes}) } } diff --git a/internal/raftdirectory/membership_test.go b/internal/raftdirectory/membership_test.go new file mode 100644 index 0000000000..3487eb8128 --- /dev/null +++ b/internal/raftdirectory/membership_test.go @@ -0,0 +1,232 @@ +package raftdirectory_test + +import ( + "crypto/ed25519" + "fmt" + "path/filepath" + "reflect" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/rafttransport" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +func membershipChange(t *testing.T, g *group, m cs.Mutation) cs.Result { + t.Helper() + r, e := g.Change(ctx, "a", m) + must(t, e) + if r.Code != cs.OK { + t.Fatalf("membership %T refused: %+v", m, r) + } + return r +} +func registry(t *testing.T, g *group, i int) cs.Membership { + t.Helper() + app, _, e := g.stores[i].ApplicationView(ctx) + must(t, e) + rows, _, e := g.stores[i].Read(ctx, app.Applied, []raftstore.Key{{Namespace: raftstore.Membership, Key: cs.MembershipKey().Key}}) + must(t, e) + var h cs.Membership + must(t, cs.Decode(rows[raftstore.Key{Namespace: raftstore.Membership, Key: cs.MembershipKey().Key}], &h)) + return h +} +func recoverMembership(t *testing.T, g *group) { + t.Helper() + g.queue = nil + for i := range g.nodes { + must(t, g.stores[i].Close()) + s, e := raftstore.Open(g.paths[i], raftstore.Identity{ClusterUUID: cluster, NodeID: uint64(i + 1), Protocol: 1}) + must(t, e) + g.stores[i] = s + r, e := s.Recover(ctx) + must(t, e) + g.nodes[i], e = raftcore.New(raftcore.Config{ClusterUUID: cluster, NodeID: uint64(i + 1), Protocol: 1, Seed: uint64(i + 123)}, r.Core) + must(t, e) + g.drive(i, raftcore.Poll{}) + } + g.drain() + for n := 0; n < 40 && g.leader() < 0; n++ { + for i := range g.nodes { + if !g.absent[uint64(i+1)] { + g.drive(i, raftcore.Tick{}) + } + } + g.drain() + } + if g.leader() < 0 { + t.Fatal("restart failed election") + } +} +func reconcileMembership(t *testing.T, g *group) { + t.Helper() + i := g.leader() + h := registry(t, g, i) + p, e := g.nodes[i].PlanMembership(h) + must(t, e) + if len(p.Change) > 0 { + g.drive(i, raftcore.Configure{Change: p.Change}) + g.drain() + p, e = g.nodes[i].PlanMembership(h) + must(t, e) + } + if !p.Settle { + t.Fatal("configuration did not reach intent") + } + if p.Checkpoint { + snapshot, e := g.stores[i].CreateSnapshot(ctx) + must(t, e) + must(t, g.stores[i].Compact(ctx, snapshot.GetMetadata().GetIndex())) + g.drive(i, raftcore.CompactSnapshot{Snapshot: snapshot}) + } + membershipChange(t, g, cs.Mutation{SettleMembership: &cs.SettleMembership{Revision: h.Revision, Node: h.Pending.Node}}) +} + +// R9/R10/R11/R12/R13/R16/R17 cross actual native FULL transactions, restart at +// each enrollment stage, compact with a sleeping learner, and rebuild routing +// from durable revocations/tombstones rather than trusting a socket or cache. +func TestR9R10R11R12R13R16R17NativeEnrollmentRestartAndSnapshotCatchUp(t *testing.T) { + g := newGroup(t, func() time.Time { return time.UnixMilli(1000) }) + anchors := []cs.MemberRecord{} + for id := uint64(1); id <= 3; id++ { + anchors = append(anchors, cs.MemberRecord{Node: id, Key: key(fmt.Sprint("anchor-", id)).Public().(ed25519.PublicKey), Anchor: true, Incarnation: 1, Protocol: 1}) + } + membershipChange(t, g, cs.Mutation{InitMembership: &cs.InitMembership{Anchors: anchors, Limit: cs.MaxMemberRecords}}) + recoverMembership(t, g) + h := registry(t, g, g.leader()) + seq := uint64(1) + enrolled, e := cs.Sign(cs.Command{Schema: 1, Cluster: cluster, Actor: "a", Session: "enrollment", Sequence: seq, Ack: seq - 1, Now: 1000, Change: cs.Mutation{EnrollMember: &cs.EnrollMember{Node: 4, Revision: h.Revision, Device: "b", Protocol: 1}}}, key("a")) + must(t, e) + r, e := g.submit(enrolled) + must(t, e) + if r.Code != cs.OK { + t.Fatal(r) + } + if r.Membership.Barrier != 0 { + t.Fatal("enrollment claimed ready") + } + recoverMembership(t, g) + // Learner transport is provisioned without a listening port or a singleton + // bootstrap. It stays asleep while admission and writes compact past it. + retry, e := g.submit(enrolled) + must(t, e) + if !reflect.DeepEqual(retry, r) { + t.Fatal("R17 restart changed enrollment result") + } + path := filepath.Join(t.TempDir(), "learner.db") + s, e := raftstore.Open(path, raftstore.Identity{ClusterUUID: cluster, NodeID: 4, Protocol: 1}) + must(t, e) + n, e := raftcore.New(raftcore.Config{ClusterUUID: cluster, NodeID: 4, Seed: 4, Protocol: 1}, raftcore.RecoveredState{}) + must(t, e) + g.paths = append(g.paths, path) + g.stores = append(g.stores, s) + g.nodes = append(g.nodes, n) + g.status = append(g.status, raftcore.Status{}) + g.results = append(g.results, nil) + g.absent[4] = true + g.drive(3, raftcore.Poll{}) + lagged := (g.leader() + 1) % 3 + g.absent[uint64(lagged+1)] = true + leader := g.leader() + pending := registry(t, g, leader) + plan, e := g.nodes[leader].PlanMembership(pending) + must(t, e) + g.drive(leader, raftcore.Configure{Change: plan.Change}) + g.drain() + // Keep the uncompacted suffix for this deliberately lagged anchor. The + // fresh checkpoint is made before the new learner's snapshot catch-up. + anchorNode := uint64(leader + 1) + settlement, e := cs.Sign(cs.Command{Schema: 1, Cluster: cluster, Actor: cs.AnchorActor(anchorNode), Session: "completion", Sequence: 1, Now: 1000, Change: cs.Mutation{SettleMembership: &cs.SettleMembership{Revision: pending.Revision, Node: pending.Pending.Node}}}, key(fmt.Sprint("anchor-", anchorNode))) + must(t, e) + completed, e := g.submit(settlement) + must(t, e) + if completed.Code != cs.OK { + t.Fatal("anchor could not resume authorized admission", completed) + } + g.absent[uint64(lagged+1)] = false + for step := 0; step < 3; step++ { + g.drive(g.leader(), raftcore.Tick{}) + g.drain() + } + for replica := 0; replica < 3; replica++ { + if !reflect.DeepEqual(registry(t, g, replica), registry(t, g, g.leader())) { + t.Fatal("R12 mixed configuration/settlement batch diverged") + } + } + recoverMembership(t, g) + for step := 0; step < 3; step++ { + _, e := g.Change(ctx, "a", cs.Mutation{AdvanceTime: &cs.AdvanceTime{}}) + must(t, e) + } + i := g.leader() + snapshot, e := g.stores[i].CreateSnapshot(ctx) + must(t, e) + must(t, g.stores[i].Compact(ctx, snapshot.GetMetadata().GetIndex())) + g.drive(i, raftcore.CompactSnapshot{Snapshot: snapshot}) + // Stage streamed chunks in the actual recipient store before RawNode sees + // MsgSnap. Upload completion is not reported as an append acknowledgment. + installNativeSnapshot(t, g, i, 3, snapshot) + retry, e = g.submit(enrolled) + must(t, e) + if !reflect.DeepEqual(retry, r) { + t.Fatal("R17 compaction changed enrollment result") + } + g.absent[4] = false + for step := 0; step < 12 && g.status[3].Applied < g.status[i].Applied; step++ { + g.drive(i, raftcore.Tick{}) + g.drain() + } + app, _, e := g.stores[3].ApplicationView(ctx) + must(t, e) + if app.Applied < snapshot.GetMetadata().GetIndex() { + t.Fatal("learner did not apply advertised barrier") + } + h = registry(t, g, i) + g.absent[4] = true + membershipChange(t, g, cs.Mutation{RemoveMember: &cs.RemoveMember{Revision: h.Revision, Node: 4, Device: "b"}}) + reconcileMembership(t, g) + recoverMembership(t, g) + app, _, e = g.stores[g.leader()].ApplicationView(ctx) + must(t, e) + read := func(k cs.RecordKey) ([]byte, error) { + rows, _, e := g.stores[g.leader()].Read(ctx, app.Applied, []raftstore.Key{{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key}}) + return rows[raftstore.Key{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key}], e + } + view, e := rafttransport.CommittedView(rafttransport.Scope{Identity: "test", Cluster: cluster, Epoch: 1, Protocol: 1}, 1, app.Applied, raftcore.Configuration(app.ConfState), read) + must(t, e) + if len(view.Members) != 4 || !view.Members[3].Revoked { + t.Fatal("retired node admitted after restart") + } + router, e := rafttransport.NewRouter(rafttransport.DefaultLimits()) + must(t, e) + must(t, router.Install(view)) + view.Revision++ + view.Members[3].Revoked = false + if router.Install(view) == nil { + t.Fatal("routing accepted retired authority") + } +} + +func installNativeSnapshot(t *testing.T, g *group, from, to int, snapshot *pb.Snapshot) { + t.Helper() + id, e := g.stores[to].BeginSnapshot(ctx, snapshot.Data) + must(t, e) + manifest, e := raftsnapshot.Decode(snapshot.Data) + must(t, e) + for offset := uint64(0); offset < manifest.Chunks; offset++ { + b, e := g.stores[from].ReadSnapshotChunk(ctx, id, offset) + must(t, e) + must(t, g.stores[to].PutSnapshotChunk(ctx, id, offset, b, raftsnapshot.ID(b))) + } + _, e = g.stores[to].CompleteSnapshot(ctx, id) + must(t, e) + raw, e := proto.Marshal(&pb.Message{Type: pb.MsgSnap.Enum(), From: proto.Uint64(uint64(from + 1)), To: proto.Uint64(uint64(to + 1)), Term: proto.Uint64(g.status[from].Term), Snapshot: snapshot}) + must(t, e) + g.drive(to, raftcore.PeerMessage{Bytes: raw}) + g.drain() +} diff --git a/internal/raftdirectory/migration_test.go b/internal/raftdirectory/migration_test.go index c63be61476..7745ab0765 100644 --- a/internal/raftdirectory/migration_test.go +++ b/internal/raftdirectory/migration_test.go @@ -117,7 +117,7 @@ func TestR24TwoCertifiedHomesMigrateThroughQuorumAndRestart(t *testing.T) { must(t, source.FreezeLegacy(ctx, cfg)) manifest, e := source.InventoryLegacy(ctx) must(t, e) - g := &group{t: t, clock: clock, reads: map[string]bool{}, seq: map[string]uint64{}, keys: map[string]ed25519.PrivateKey{a: key("paired-a"), b: key("paired-b")}} + g := &group{paths: make([]string, 3), stores: make([]*raftstore.SQLite, 3), nodes: make([]*raftcore.Core, 3), status: make([]raftcore.Status, 3), results: make([][]clusterstate.Result, 3), absent: map[uint64]bool{}, t: t, clock: clock, reads: map[string]bool{}, seq: map[string]uint64{}, keys: map[string]ed25519.PrivateKey{a: key("paired-a"), b: key("paired-b")}} defer func() { for _, store := range g.stores { if store != nil { diff --git a/internal/raftsim/README.md b/internal/raftsim/README.md index a6ee1e6354..4b4030bd5a 100644 --- a/internal/raftsim/README.md +++ b/internal/raftsim/README.md @@ -124,3 +124,12 @@ other X/J requirements are not claimed by a metadata simulator. Porcupine v1.1.0 is MIT licensed and only imported by the simulator checker. Neither this dependency nor the simulator is linked into `bin/codeaf`. + + +P12 extends the bounded topology to three voters plus at most 32 learner cores. +Its full-fanout case admits the learners in one internal configuration batch, +sleeps all of them, commits writes, compacts, restores a lagging learner through +a fresh snapshot and removes it with delayed responses. The unchanged 2,000-event +ceiling still applies. Separate signed-policy tests prove sequential admission, +its 33rd-learner refusal, revision races and retained ID tombstones; the raw +simulator configuration action is not a public admission policy. diff --git a/internal/raftsim/membership_test.go b/internal/raftsim/membership_test.go new file mode 100644 index 0000000000..f2c4c47733 --- /dev/null +++ b/internal/raftsim/membership_test.go @@ -0,0 +1,85 @@ +package raftsim + +import ( + "testing" + + "github.com/Agent-Field/codeaf/internal/raftsim/model" +) + +// R9/R11/R12/R13 keep three voters while 32 learners join and depart. Sleeping +// learners are real cores with durable images, not a second consensus model. +// A lagging learner installs a post-admission snapshot instead of pinning logs. +func TestR9R11R12R13ThirtyTwoDeadLearnersDoNotPinCommitOrCompaction(t *testing.T) { + s := group(t, Config{Seed: 121211, Nodes: 35, Duplicate: 50, Delay: 2}) + leader := s.Leader() + for node := 3; node < 35; node++ { + s.Do(Action{Kind: "crash", Node: node, Value: 1}) + } + // One bounded catch-up batch exercises the full 32-member fanout; signed + // policy tests independently prove sequential admission and its live cap. + s.Do(Action{Kind: "configure", Node: leader, Count: 5}) + s.Drain() + if s.Err != nil { + t.Fatal(s.Err) + } + conf := s.Nodes[leader].Stable.Conf + if len(conf.GetVoters()) != 3 || len(conf.GetLearners()) != 32 { + t.Fatal("R9 learner admission changed voting topology", conf) + } + create(t, s, "a") + s.Do(Action{Kind: "snapshot", Node: leader}) + s.Drain() + before := s.Nodes[leader].Stable.Applied + for n := 0; n < 2; n++ { + submit(t, s, leader, model.Call{Kind: "advance_time", Actor: "owner", At: int64(n + 1000)}) + } + if s.Nodes[leader].Stable.Applied <= before { + t.Fatal("R11 dead learners blocked commitment") + } + s.Do(Action{Kind: "snapshot", Node: leader}) + s.Drain() + if len(s.Nodes[leader].Stable.Entries) != 0 { + t.Fatal("R11 dead learners pinned retained log") + } + s.Do(Action{Kind: "restart", Node: 3}) + for n := 0; n < 16 && s.Nodes[3].Stable.Applied < s.Nodes[leader].Stable.Applied; n++ { + s.Ticks(1) + } + if s.Nodes[3].Stable.Applied != s.Nodes[leader].Stable.Applied || len(s.Nodes[3].Stable.Generations) == 0 { + t.Fatalf("R13 learner failed snapshot catch-up: %v, events=%d", s.Err, s.Events) + } + s.Do(Action{Kind: "configure", Node: leader, Other: 3, Count: 4}) + s.Drain() + for _, id := range s.Nodes[leader].Stable.Conf.GetLearners() { + if id == 4 { + t.Fatal("R12 removal did not apply") + } + } + s.Do(Action{Kind: "configure", Node: leader, Count: 6}) + s.Drain() + if len(s.Nodes[leader].Stable.Conf.GetVoters()) != 3 || len(s.Nodes[leader].Stable.Conf.GetLearners()) != 0 { + t.Fatal("R11 churn did not retire all learners") + } + t.Logf("R11 full fanout/churn used %d/%d events", s.Events, MaxEvents) + finish(t, s) +} + +// R9 checks every subset of sleeping paired laptops without changing quorum. +func TestR9EverySleepingSubsetOfTwoLearnerLaptops(t *testing.T) { + for mask := 0; mask < 4; mask++ { + s := group(t, Config{Seed: uint64(1290 + mask), Nodes: 5}) + leader := s.Leader() + s.Do(Action{Kind: "configure", Node: leader, Count: 5}) + s.Drain() + for node := 3; node < 5; node++ { + if mask&(1<<(node-3)) != 0 { + s.Do(Action{Kind: "crash", Node: node, Value: 1}) + } + } + create(t, s, "a") + if len(s.Nodes[leader].Stable.Conf.GetVoters()) != 3 { + t.Fatal("R9 pairing requires a laptop for quorum") + } + finish(t, s) + } +} diff --git a/internal/raftsim/simulator.go b/internal/raftsim/simulator.go index 3410fbd4ae..d5a95f3e17 100644 --- a/internal/raftsim/simulator.go +++ b/internal/raftsim/simulator.go @@ -9,6 +9,7 @@ import ( "fmt" "slices" + "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/raftcore" "github.com/Agent-Field/codeaf/internal/raftsim/model" "go.etcd.io/raft/v3" @@ -157,7 +158,7 @@ func New(config Config) (*Simulator, error) { if config.MaxEvents == 0 { config.MaxEvents = MaxEvents } - if config.PaddingBytes < 0 || config.PaddingBytes > 640<<10 || config.Nodes < 3 || config.Nodes > 5 || config.MaxEvents > MaxEvents || config.MaxEvents < 1 || config.Delay < 0 || config.Delay > 20 || config.Drop < 0 || config.Drop > 1000 || config.Duplicate < 0 || config.Duplicate > 1000 { + if config.PaddingBytes < 0 || config.PaddingBytes > 640<<10 || config.Nodes < 3 || config.Nodes > 3+clusterstate.MaxLearners || config.MaxEvents > MaxEvents || config.MaxEvents < 1 || config.Delay < 0 || config.Delay > 20 || config.Drop < 0 || config.Drop > 1000 || config.Duplicate < 0 || config.Duplicate > 1000 { return nil, errors.New("raftsim: invalid bounded configuration") } s := &Simulator{initial: config, Config: config, ReadAckDelay: map[int]int64{}, rng: config.Seed + 1, FailNext: map[int]bool{}, Blocked: map[[2]int]bool{}, Stalled: map[int]int64{}, catalog: map[string]model.Call{}, leaders: map[uint64]int{}, commits: map[uint64]string{}, applications: map[uint64]string{}, maxFence: map[string]uint64{}, maxCommit: make([]uint64, config.Nodes), maxApplied: make([]uint64, config.Nodes), appGeneration: make([]uint64, config.Nodes)} @@ -664,6 +665,18 @@ func (s *Simulator) Do(a Action) int { change = &pb.ConfChangeV2{Changes: []*pb.ConfChangeSingle{{Type: pb.ConfChangeAddLearnerNode.Enum(), NodeId: proto.Uint64(uint64(a.Other + 1))}}} case 2: change = &pb.ConfChangeV2{Transition: pb.ConfChangeTransitionJointExplicit.Enum(), Changes: []*pb.ConfChangeSingle{{Type: pb.ConfChangeAddLearnerNode.Enum(), NodeId: proto.Uint64(3)}, {Type: pb.ConfChangeAddNode.Enum(), NodeId: proto.Uint64(4)}}} + case 6: + change = &pb.ConfChangeV2{} + for node := 3; node < len(s.Nodes); node++ { + change.Changes = append(change.Changes, &pb.ConfChangeSingle{Type: pb.ConfChangeRemoveNode.Enum(), NodeId: proto.Uint64(uint64(node + 1))}) + } + case 5: + change = &pb.ConfChangeV2{} + for node := 3; node < len(s.Nodes); node++ { + change.Changes = append(change.Changes, &pb.ConfChangeSingle{Type: pb.ConfChangeAddLearnerNode.Enum(), NodeId: proto.Uint64(uint64(node + 1))}) + } + case 4: + change = &pb.ConfChangeV2{Changes: []*pb.ConfChangeSingle{{Type: pb.ConfChangeRemoveNode.Enum(), NodeId: proto.Uint64(uint64(a.Other + 1))}}} case 3: change = &pb.ConfChangeV2{} default: diff --git a/internal/rafttransport/README.md b/internal/rafttransport/README.md index 8619407654..5b11e4e6d1 100644 --- a/internal/rafttransport/README.md +++ b/internal/rafttransport/README.md @@ -141,3 +141,14 @@ unit tests pin auth, generation, socket revocation and all queue reservations. P06/P06b's 2,000-seed simulator soak remains a required regression gate. Full R10 enrollment/worker journeys, R11 churn/snapshot interoperability, application revocation and the R15 full-state sentinel campaign remain later packet gates. + + +P12's `CommittedView` derives protocol authority from the applied registry and +current device revocations. Hold the actor lock while reading and installing +that view, before acknowledging the command. Use the applied index as its +revision, so a device revoke updates routing even without a membership edit. +Retain retired entries; Router.Install refuses their resurrection. Increment +and persist router generation before admitting exchanges after inbox loss. +An authenticated source's three provisioned anchor keys may bootstrap protocol +traffic before registry commitment. A personal key requires committed enrollment; +never accept a caller's arbitrary view as an enrollment grant. diff --git a/internal/rafttransport/envelope.go b/internal/rafttransport/envelope.go index 3046153cc0..e1c6ec7690 100644 --- a/internal/rafttransport/envelope.go +++ b/internal/rafttransport/envelope.go @@ -12,6 +12,7 @@ import ( "io" "strings" + "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/raftcore" "github.com/Agent-Field/codeaf/internal/raftsnapshot" "go.etcd.io/raft/v3" @@ -24,7 +25,7 @@ const ( MaxBatchBytes = raftcore.MaxMessageBytes MaxChunkBytes = raftsnapshot.ChunkBytes MaxInflight = raftcore.MaxInflightBatches - MaxLearners = 32 + MaxLearners = clusterstate.MaxLearners MaxMessages = 64 MaxQueuedBytes = 1 << 20 // Encoded envelopes include base64 payloads, signatures and bounded IDs. diff --git a/internal/rafttransport/registry.go b/internal/rafttransport/registry.go new file mode 100644 index 0000000000..876117ffdc --- /dev/null +++ b/internal/rafttransport/registry.go @@ -0,0 +1,22 @@ +package rafttransport + +import "github.com/Agent-Field/codeaf/internal/clusterstate" + +// CommittedView derives routing authority from the serialized applied store. +// The caller holds its actor lock through Install and before replying to any +// registry/revocation command. Applied index is the view revision, so a device +// revoke fences sockets even when no membership intent changed revision. +func CommittedView(scope Scope, generation, applied uint64, conf clusterstate.Configuration, read clusterstate.ReadRecord) (View, error) { + _, peers, err := clusterstate.MembershipView(read, conf, applied) + if err != nil { + return View{}, err + } + if len(peers) == 0 || applied == 0 { + return View{}, ErrAuth + } + v := View{Scope: scope, Generation: generation, Revision: applied} + for _, p := range peers { + v.Members = append(v.Members, Member{Node: p.Node, Incarnation: p.Incarnation, Key: p.Key, Voter: p.Voter, Revoked: p.Revoked}) + } + return v, nil +} From 9858a28394de1071c704adc22aa0ec452ec09c70 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 02:15:06 -0400 Subject: [PATCH 027/103] Derive hosted peer authority from committed membership so stale sockets cannot restore retired nodes. Page registry policy through the shared Go reactor and use the applied join barrier before reporting readiness. Preserve the source-provisioned three-anchor bootstrap, isolate the old mechanism seam in its test fixture, and exercise real workerd restarts with a dead learner. Document enrollment limits and refusals with the implementation. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../continuing-a-chat-on-another-computer.md | 16 +++ relay/hosted/src/raft/README.md | 29 ++++++ relay/hosted/src/raft/abi.js | 1 + relay/hosted/src/raft/actor.js | 24 ++++- relay/hosted/src/raft/directory.js | 4 +- relay/hosted/src/raft/member.js | 36 ++++--- relay/hosted/src/raft/membership.js | 31 ++++++ relay/hosted/src/raft/storage.js | 3 +- relay/hosted/test/raft/actor-worker.js | 36 +++++++ relay/hosted/test/raft/membership-worker.js | 43 ++++++++ .../test/raft/membership-workerd.test.js | 55 +++++++++++ relay/hosted/test/raft/membership.test.js | 98 +++++++++++++++++++ 12 files changed, 360 insertions(+), 16 deletions(-) create mode 100644 relay/hosted/src/raft/membership.js create mode 100644 relay/hosted/test/raft/membership-worker.js create mode 100644 relay/hosted/test/raft/membership-workerd.test.js create mode 100644 relay/hosted/test/raft/membership.test.js diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index 50f27db63f..6b86a113e6 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -252,3 +252,19 @@ chat first; this does not discard the work saved on your computer. ## Relay directory upgrade pauses and saved chats A relay directory upgrade can briefly pause pairing and chat moves. Retry after the relay is ready. The upgrade refusal is `codeaf: directory upgrade is incomplete; retry shortly; saved chats are still here`. It does not delete saved chats, files, branches or the encrypted vault. The relay keeps the old directory as a backup and never makes it a second writer after the new directory takes over. A row that cannot be copied safely leaves the upgrade paused for the operator; it is never truncated to make it fit. + + +## Joined computers, learner capacity and an outdated build + +Joining the relay's coordination group does not make a computer a voter. Its +learner role does not require other laptops to stay awake, and a slow learner +cannot hold back relay writes or keep old coordination logs forever. A learner +is ready only after it has applied the committed join barrier. Removing one +retires its node ID permanently; joining again allocates a fresh ID. + +The group admits at most 32 live learners. Retired enrollment records are kept, +with a separate operator capacity limit; reaching either limit refuses new +enrollment instead of deleting saved chats or old records. The capacity sentence +is `this cluster has reached its enrollment limit; remove a joined computer or ask the relay operator for capacity`. +An unsupported protocol says `this codeaf build must be updated before it can join this cluster; your saved chats are still here`. +This does not add a leadership command or a worker setup command. diff --git a/relay/hosted/src/raft/README.md b/relay/hosted/src/raft/README.md index 800fbf5b94..3f972e600b 100644 --- a/relay/hosted/src/raft/README.md +++ b/relay/hosted/src/raft/README.md @@ -86,3 +86,32 @@ After fresh quorum readback, `LegacySource.own` publishes `raft-owned`; `route` then resolves only that UUID. There is no rollback to a legacy writer. P17 assembles the complete facade, receipt ingress and existing-client doors; P14 owns continuing watch renewal/scheduling. Source backup cleanup is explicit. + + +P12 adds membership-aware evaluation (ABI op 15), paged peer derivation (16), +core reconciliation (17), local learner catch-up (18) and provisioned-anchor +manifest validation (19). Original ABI operations remain unchanged. Native and +hosted application adapters supply the entry index and its exact configuration +as trusted host metadata. Operation 20 replays each committed configuration +entry with the pinned library changer, so batching cannot alter policy. No request may supply those fields. + +Provision `config.anchors` with the same three public protocol records and IDs +used by the source manifest/InitMembership. Only these anchors can bootstrap +peer traffic before the registry commits. Once present, every peer check derives +from the durable registry and current device revocations, including after awaits. +`enrollment(view, applied)` only publishes an identical committed view; it cannot +introduce a member. The old probe fixture's arbitrary registry is test-only. +`membership()` resumes the recorded intent, refreshes an obsolete checkpoint and +reports whether a signed settlement may be proposed. It never synthesizes that +settlement signature or declares a join successful. `joinStatus(barrier)` additionally +checks current authorization and the learner's own applied index. + +P17 still assembles authenticated client enrollment, registry publication and +quiet-mode guards. P13 owns voter swaps; P15 owns delegated worker certificates. +Do not advertise a public leader/worker command from these internal methods. + +Provisioned anchor keys receive a restricted committed authority that can only +settle an existing intent or close their request session. They cannot enroll, +remove, read directory content, mutate chats, pair or rotate. Thus a crashed +requester need not return for an already authorized admission to finish. The +settlement still requires the exact applied configuration and pending revision. diff --git a/relay/hosted/src/raft/abi.js b/relay/hosted/src/raft/abi.js index 652d84273a..169d3c904e 100644 --- a/relay/hosted/src/raft/abi.js +++ b/relay/hosted/src/raft/abi.js @@ -119,6 +119,7 @@ export class HostCodec { const r = this.call(frame(8).blob(state).blob(block)); const v = { state: r.blob(), digest: r.blob() }; r.end(); return v; } + configuration(conf, entry) { const r = this.call(frame(20).blob(conf).blob(entry)); const next = r.blob(); r.end(); return next; } genesis(data) {const r=this.call(frame(13).blob(data));const v={entry:r.blob(),hard:r.blob(),conf:r.blob()};r.end();return v;} snapshot(data) { const r = this.call(frame(9).blob(data)); const b = r.blob(); r.end(); return b; } } diff --git a/relay/hosted/src/raft/actor.js b/relay/hosted/src/raft/actor.js index a575e110df..b5a7dbe651 100644 --- a/relay/hosted/src/raft/actor.js +++ b/relay/hosted/src/raft/actor.js @@ -1,5 +1,6 @@ import { CORE_MEMORY_BUDGET, Reactor, HostCodec, createFrame, response, effects, frame, stepFrame } from './abi.js'; -import { MemberStorage, DEFAULT_ROWS_BYTES } from './storage.js'; +import { MemberStorage, DEFAULT_ROWS_BYTES, unb64 } from './storage.js'; +import { committedRegistry } from './membership.js'; const RESERVATION = 6 * 1024 * 1024; const CACHE_BYTES = 2 * 1024 * 1024; @@ -86,6 +87,27 @@ export class MemberActor { try { const snapshot = await this.store.checkpoint(); await this.drive(11, snapshot); this.checkpointIndex = this.status.applied; return snapshot; } catch (error) { this.stopped = true; throw error; } }); } + // Reconcile only the committed intent. Core planning prevents a second + // configuration while the first is uncommitted and refreshes snapshots that + // would otherwise exclude the newly admitted learner. + membership() { return this.serial(async () => { + const header = this.store.read('membership', 'registry'); + if (!header) throw new Error('raft member: enrollment is absent'); + let r = response(this.reactor.call(frame(17).u32(this.handle).blob(header).finish())); + const change = r.blob(), checkpoint = r.u32() !== 0, settle = r.u32() !== 0; r.end(); + if (change.length) return { ...(await this.drive(8, change)), settle: false }; + if (checkpoint) { + const snapshot = await this.store.checkpoint(); await this.drive(11, snapshot); this.checkpointIndex = this.status.applied; + } + return { send: [], reads: [], status: this.status, settle }; + }); } + joinStatus(advertised) { return this.serial(() => { + const raw = this.store.read('enrollment', 'node/' + this.config.id); + if (!raw || committedRegistry(this.reactor, this.store)[this.config.id]?.revoked) return { ready: false }; + const app = this.store.application(); + const r = response(this.reactor.call(frame(18).blob(raw).blob(unb64(app.conf)).u64(app.applied).u64(advertised).finish())); + const ready = r.u32() !== 0; r.end(); return { ready }; + }); } begin(data) { data = data.slice(); return this.serial(() => this.store.beginSnapshot(data)); } chunk(id, offset, data, hash) { data = data.slice(); return this.serial(() => this.store.putChunk(id, offset, data, hash)); } complete(id) { return this.serial(() => this.store.completeSnapshot(id)); } diff --git a/relay/hosted/src/raft/directory.js b/relay/hosted/src/raft/directory.js index 2ae57de7ba..5f697abf37 100644 --- a/relay/hosted/src/raft/directory.js +++ b/relay/hosted/src/raft/directory.js @@ -1,4 +1,5 @@ import { frame, response } from './abi.js'; +import { unb64 } from './storage.js'; // JavaScript supplies immutable dependency bytes and persists the Go write-set. // It never interprets leases, clocks, revocations, CAS or request sequences. @@ -6,8 +7,9 @@ export function directoryEvaluator(runtime, cluster, ceiling) { return (command, read, app) => { let r = response(runtime.call(frame(11).blob(command).finish())); const keys = Array.from({ length: r.u32() }, () => ({ namespace: r.str(), key: r.str() })); r.end(); - const w = frame(10).blob(command).str(cluster).u64(app.now).u64(app.version).u64(app.size).u64(ceiling).u32(keys.length); + const w = (app.index === undefined ? frame(10) : frame(15)).blob(command).str(cluster).u64(app.now).u64(app.version).u64(app.size).u64(ceiling).u32(keys.length); for (const k of keys) { const data = read(k.namespace, k.key); w.str(k.namespace).str(k.key).u32(data == null ? 0 : 1).blob(data ?? new Uint8Array()); } + if (app.index !== undefined) w.u64(app.index).blob(unb64(app.conf)); r = response(runtime.call(w.finish())); const now = r.u64(), version = r.u64(), result = r.blob(); const writes = Array.from({ length: r.u32() }, () => ({ namespace: r.str(), key: r.str(), data: r.blob() })); r.end(); diff --git a/relay/hosted/src/raft/member.js b/relay/hosted/src/raft/member.js index 739b76c144..fe454d2aab 100644 --- a/relay/hosted/src/raft/member.js +++ b/relay/hosted/src/raft/member.js @@ -4,12 +4,13 @@ import { MemberInbox } from './inbox.js'; import { directoryEvaluator } from './directory.js'; import { DEFAULT_ROWS_BYTES, MemberStorage } from './storage.js'; import { HostCodec } from './abi.js'; +import { committedRegistry, provisionedRegistry } from './membership.js'; import { stageLegacyImport, finishLegacyImport } from './legacy.js'; export { installRaftModule, runtimeMetrics } from './actor.js'; // Only internal namespace calls can initialize a provisioned member. The // public Worker exports this class but does not route directory traffic here. -// P12 supplies the durable committed enrollment view and P09 the evaluator. +// Enrollment is derived from the durable Go registry; P09 supplies evaluation. export class RaftMemberDO extends DurableObject { constructor(ctx, env) { super(ctx, env); this.actor = null; this.loading = null; this.ctx = ctx; this.env = env; } evaluator(config) { return directoryEvaluator(sharedRuntime(), config.cluster, config.ceiling ?? DEFAULT_ROWS_BYTES); } @@ -57,7 +58,7 @@ export class RaftMemberDO extends DurableObject { if (!previous) { await this.ctx.storage.put('raft-config', canonical); await this.ctx.storage.sync(); } const actor = new MemberActor(this.ctx.storage, sharedRuntime(), this.evaluator(config)); const result = await actor.init(canonical); - this.config = canonical; this.registry = await this.ctx.storage.get('raft-registry'); this.actor = actor; + this.config = canonical; this.registry = actor.store.read('membership', 'registry') ? committedRegistry(sharedRuntime(), actor.store) : await this.ctx.storage.get('raft-registry'); this.actor = actor; return result; } @@ -71,8 +72,10 @@ export class RaftMemberDO extends DurableObject { async restore() { const config = await this.ctx.storage.get('raft-config'); if (!config) throw new Error('raft member: not provisioned'); const actor = new MemberActor(this.ctx.storage, sharedRuntime(), this.evaluator(config)); - this.recoveredEffects = await actor.init(config); this.config = config; this.registry = await this.ctx.storage.get('raft-registry'); this.actor = actor; + this.recoveredEffects = await actor.init(config); this.config = config; this.registry = actor.store.read('membership', 'registry') ? committedRegistry(sharedRuntime(), actor.store) : await this.ctx.storage.get('raft-registry'); this.actor = actor; } + async membership() { await this.load(); return this.actor.membership(); } + async joinStatus(advertised) { await this.load(); return this.actor.joinStatus(advertised); } async drainRecovery() { await this.load(); return this.actor.serial(() => { const effects = this.recoveredEffects; this.recoveredEffects = null; return effects; }); @@ -80,8 +83,8 @@ export class RaftMemberDO extends DurableObject { async peer(raw) { await this.load(); const config = this.config; - if (!this.registry) throw new Error('raft member: enrollment is absent'); - this.inbox ??= new MemberInbox(config.scope, config.id, this.actor.codec, () => this.registry, fn => this.actor.serial(fn)); + if (!this.actor.store.read('membership', 'registry') && !config.anchors) throw new Error('raft member: enrollment is absent'); + this.inbox ??= new MemberInbox(config.scope, config.id, this.actor.codec, () => this.actor.store.read('membership', 'registry') ? committedRegistry(sharedRuntime(), this.actor.store) : provisionedRegistry(sharedRuntime(), config), fn => this.actor.serial(fn)); return this.inbox.receive(raw, async (envelope, payload) => { if (envelope.chunk) { if (!this.actor.store.one('SELECT id FROM snapshots WHERE id=?', envelope.chunk.generation)) return { staged: false }; @@ -104,19 +107,26 @@ export class RaftMemberDO extends DurableObject { return this.actor.drive(2, payload); }); } - // A registry update is trusted assembly input derived from committed state. - // Serialize it with dispatch and persist before acknowledging revocation. + // Refresh under the dispatch queue; never accept a caller-supplied membership + // list as evidence that a device joined or a revoked transport key returned. + async refreshEnrollment() { + await this.load(); + return this.actor.serial(async () => { + const view = committedRegistry(sharedRuntime(), this.actor.store); + await this.ctx.storage.put('raft-registry', view); await this.ctx.storage.sync(); this.registry = view; + return view; + }); + } + // Existing internal callers may request publication, but cannot supply a + // different view or invent enrollment before the committed manifest exists. async enrollment(view, applied) { view = structuredClone(view); await this.load(); return this.actor.serial(async () => { if (BigInt(applied) > this.actor.status.applied) throw new Error('raft member: enrollment barrier not applied'); - const old = await this.ctx.storage.get('raft-registry') ?? {}; - for (const [id, member] of Object.entries(old)) { - const next = view[id]; - if (!next || (member.revoked && !next.revoked) || BigInt(next.incarnation) < BigInt(member.incarnation) || (next.incarnation === member.incarnation && next.key !== member.key)) throw new Error('raft member: enrollment regression'); - } - await this.ctx.storage.put('raft-registry', view); await this.ctx.storage.sync(); this.registry = view; + const committed = committedRegistry(sharedRuntime(), this.actor.store); + if (JSON.stringify(committed) !== JSON.stringify(view)) throw new Error('raft member: enrollment differs from committed registry'); + await this.ctx.storage.put('raft-registry', committed); await this.ctx.storage.sync(); this.registry = committed; }); } } diff --git a/relay/hosted/src/raft/membership.js b/relay/hosted/src/raft/membership.js new file mode 100644 index 0000000000..e931275e2b --- /dev/null +++ b/relay/hosted/src/raft/membership.js @@ -0,0 +1,31 @@ +import { frame, response } from './abi.js'; +import { unb64 } from './storage.js'; +import { encodeCanonical } from './codec.js'; + +const decode = b => JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(b)); +const segment = s => [...new TextEncoder().encode(s)].map(b => b.toString(16).padStart(2, '0')).join(''); + +// The host pages public addressing dependencies; Go decides roles and revoked +// authority. Run under the member queue after durable apply and before replies. +export function committedRegistry(runtime, store) { + const header = store.read('membership', 'registry'); + if (!header) throw new Error('raft member: enrollment is absent'); + const app = store.application(), h = decode(header), view = {}; + for (const node of h.nodes) { + const raw = store.read('enrollment', 'node/' + node); + if (!raw) throw new Error('raft member: enrollment is incomplete'); + const m = decode(raw), device = m.device ? store.read('state', 'device/' + segment(m.device)) : null; + const r = response(runtime.call(frame(16).u64(node).u64(app.applied).blob(unb64(app.conf)).blob(header).blob(raw).blob(device ?? new Uint8Array()).finish())); + const peer = decode(r.blob()); r.end(); + view[peer.node] = { incarnation: peer.incarnation, key: peer.key, voter: peer.voter, revoked: peer.revoked }; + } + return view; +} + +// Only authenticated internal provisioning supplies the three public anchor +// records. Before registry commitment, personal keys cannot submit peer data. +export function provisionedRegistry(runtime, config) { + const raw = new TextEncoder().encode(encodeCanonical({ anchors: config.anchors, voters: config.bootstrap })); + const r = response(runtime.call(frame(19).blob(raw).finish())), peers = decode(r.blob()); r.end(); + return Object.fromEntries(peers.map(p => [p.node, { incarnation: p.incarnation, key: p.key, voter: p.voter, revoked: p.revoked }])); +} diff --git a/relay/hosted/src/raft/storage.js b/relay/hosted/src/raft/storage.js index 2e36ee4656..bbf2470a2d 100644 --- a/relay/hosted/src/raft/storage.js +++ b/relay/hosted/src/raft/storage.js @@ -129,10 +129,11 @@ export class MemberStorage { for (const b of a.entries) { const e = this.codec.inspect(1, b); check(e.index === index + 1n); const persisted = this.one('SELECT data FROM journal WHERE idx=?', key(e.index)); check(persisted && equal(bytes(persisted.data), b)); + if (e.type !== 0) app.conf = b64(this.codec.configuration(unb64(app.conf), b)); if (e.type === 0 && e.data.length) { check(evaluate, 'raft storage: application evaluator is absent'); let readBytes = 0, writeBytes = 0; - const change = evaluate(e.data, (ns, k) => { const data = this.read(ns, k); readBytes += data?.length ?? 0; check(readBytes <= ROW_BYTES, 'raft storage: evaluation read-set capacity'); return data; }, { ...app }); + const change = evaluate(e.data, (ns, k) => { const data = this.read(ns, k); readBytes += data?.length ?? 0; check(readBytes <= ROW_BYTES, 'raft storage: evaluation read-set capacity'); return data; }, { ...app, index: e.index }); check(change && !change.then && Array.isArray(change.writes)); for (const r of change.writes) { check(namespaces.includes(r.namespace) && typeof r.key === 'string' && enc.encode(r.key).length > 0 && enc.encode(r.key).length <= ROW_BYTES && r.data instanceof Uint8Array && r.data.length <= ROW_BYTES); diff --git a/relay/hosted/test/raft/actor-worker.js b/relay/hosted/test/raft/actor-worker.js index 05235d8290..a5de74de17 100644 --- a/relay/hosted/test/raft/actor-worker.js +++ b/relay/hosted/test/raft/actor-worker.js @@ -4,6 +4,7 @@ import core from './core.wasm'; import { installRaftModule, runtimeMetrics, MemberActor, sharedRuntime } from '../../src/raft/actor.js'; import { RaftMemberDO } from '../../src/raft/member.js'; import { frame, response } from '../../src/raft/abi.js'; +import { MemberInbox } from '../../src/raft/inbox.js'; import { b64, unb64 } from '../../src/raft/storage.js'; export { IdentityDO, NewcomerGate, LinkGate, LinkRequest, PairGate, Mailbox } from '../../src/worker.js'; installRaftModule(core); @@ -16,6 +17,41 @@ const evaluate = (command, read, app) => { }; export class ActorFixture extends RaftMemberDO { evaluator() { return evaluate; } + async enrollment(view, applied) { + await this.load(); + return this.actor.serial(async () => { + if (BigInt(applied) > this.actor.status.applied) throw new Error('raft member: enrollment barrier not applied'); + const old = await this.ctx.storage.get('raft-registry') ?? {}; + for (const [id, before] of Object.entries(old)) { + const after = view[id]; + if (!after || before.revoked && !after.revoked || BigInt(after.incarnation) < BigInt(before.incarnation) || after.incarnation === before.incarnation && after.key !== before.key) throw new Error('raft member: enrollment regression'); + } + await this.ctx.storage.put('raft-registry', view); await this.ctx.storage.sync(); this.registry = view; + }); + } + async peer(raw) { + // The mechanism fixture isolates host durability from policy. Its explicit + // local registry is not a production admission or authority path. + await this.load(); + this.inbox ??= new MemberInbox(this.config.scope, this.config.id, this.actor.codec, () => this.registry, fn => this.actor.serial(fn)); + return this.inbox.receive(raw, async (envelope, payload) => { + if (envelope.chunk) { + if (!this.actor.store.one('SELECT id FROM snapshots WHERE id=?', envelope.chunk.generation)) return { staged: false }; + await this.actor.store.putChunk(envelope.chunk.generation, Number(BigInt(envelope.chunk.offset)), payload, envelope.chunk.hash); return { staged: true }; + } + const message = this.actor.codec.inspect(4, payload); + if (message.snapshot.length) { + const snapshot = this.actor.codec.inspect(3, message.snapshot); + if (snapshot.index > this.actor.status.applied) { + const id = await this.actor.store.beginSnapshot(snapshot.data), manifest = this.actor.store.generation(id).manifest; + if (this.actor.store.one('SELECT count(*) AS n FROM chunks WHERE generation=?', id).n !== Number(manifest.chunks)) return { staged: true, installed: false }; + await this.actor.store.completeSnapshot(id); + } + const result = await this.actor.drive(2, payload); return { ...result, installed: this.actor.status.applied >= snapshot.index }; + } + return this.actor.drive(2, payload); + }); + } async fetch(request) { const url = new URL(request.url), b = request.method === 'POST' ? await request.json() : {}; try { diff --git a/relay/hosted/test/raft/membership-worker.js b/relay/hosted/test/raft/membership-worker.js new file mode 100644 index 0000000000..03f1c54577 --- /dev/null +++ b/relay/hosted/test/raft/membership-worker.js @@ -0,0 +1,43 @@ +// Only this local fixture supplies genesis public authorities and driving RPCs. +// All membership evaluation, planning, storage and peer admission are production. +import core from 'core.wasm'; +import { RaftMemberDO } from '../../src/raft/member.js'; +import { installRaftModule, sharedRuntime } from '../../src/raft/actor.js'; +import { directoryEvaluator } from '../../src/raft/directory.js'; +import { committedRegistry } from '../../src/raft/membership.js'; +import { b64, unb64 } from '../../src/raft/storage.js'; +export class ProofStore {} +installRaftModule(core); +const reply = v => new Response(JSON.stringify(v ?? null, (_,x) => typeof x === 'bigint' ? String(x) : x instanceof Uint8Array ? b64(x) : x)); +export class ActorFixture extends RaftMemberDO { + evaluator(config) { + const evaluate = directoryEvaluator(sharedRuntime(),config.cluster,8*1024*1024); + return (raw,read,app) => { + const base = read('state','identity') ? [] : config.genesis.map(r => ({...r,data:unb64(r.data)})); + const size = base.reduce((n,r) => n+12+r.namespace.length+r.key.length+r.data.length,0); + const r = evaluate(raw,(ns,k) => base.find(r => r.namespace===ns && r.key===k)?.data ?? read(ns,k),{...app,size:app.size+size}); + return {...r,writes:[...base,...r.writes]}; + }; + } + async fetch(request) { + const b = await request.json(); + try { + if (b.op==='init') return reply(await this.initialize(b.config)); + await this.load(); + if (b.op==='input') return reply(await this.actor.input(b.tag,[2,3,8].includes(b.tag)?unb64(b.value):b.value)); + if (b.op==='peer') return reply(await this.peer(unb64(b.value))); + if (b.op==='membership') return reply(await this.membership()); + if (b.op==='checkpoint') return reply(await this.actor.checkpoint()); + if (b.op==='registry') return reply(await this.refreshEnrollment()); + if (b.op==='header') return reply(JSON.parse(new TextDecoder().decode(this.actor.store.read('membership','registry')))); + if (b.op==='row') return reply(this.actor.store.read(b.namespace,b.key)); + if (b.op==='status') return reply(await this.actor.input(0)); + if (b.op==='enrollment') return reply(await this.enrollment(b.view,b.applied)); + throw Error('fixture operation'); + } catch (e) { return reply({error:e.message}); } + } +} +export default { async fetch(request,env) { + const u = new URL(request.url); if (u.pathname==='/ready') return new Response(null,{status:204}); + const name = u.pathname.slice(1); return env.ACTORS.get(env.ACTORS.idFromName(name)).fetch(request); +} }; diff --git a/relay/hosted/test/raft/membership-workerd.test.js b/relay/hosted/test/raft/membership-workerd.test.js new file mode 100644 index 0000000000..f14cd8ad85 --- /dev/null +++ b/relay/hosted/test/raft/membership-workerd.test.js @@ -0,0 +1,55 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { build } from 'esbuild'; +import { mkdtemp, writeFile, copyFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { generateKeyPairSync, createHash, sign } from 'node:crypto'; +import { encodeCanonical } from '../../src/raft/codec.js'; +import { openWorkerd } from './workerd.js'; + +// R11/R12/R16 exercise the ACTUAL hosted peer and committed-registry doors. +// A learner remains absent during writes/compaction and both process restarts. +test('R9 R11 R12 R13 R16 actual workerd admits only committed learners and retains retired authority across restarts',{timeout:45000},async()=>{ + const proof=process.env.CODEAF_RAFT_PROOF_DIR??'/tmp/codeaf-p12',out=await mkdtemp(join(tmpdir(),'p12-wasm-')),disk=await mkdtemp(join(tmpdir(),'p12-disk-')); + const bundle=await build({entryPoints:['test/raft/membership-worker.js'],bundle:true,format:'esm',target:'es2022',external:['core.wasm','cloudflare:*'],write:false}); + await writeFile(join(out,'worker.js'),bundle.outputFiles[0].contents);await copyFile(join(proof,'core.wasm'),join(out,'core.wasm'));await writeFile(join(out,'storage-worker.js'),'export class ProofStore{}'); + const keys=Object.fromEntries(['a','b','1','2','3'].map(id=>[id,generateKeyPairSync('ed25519')])),pub=id=>keys[id].publicKey.export({format:'der',type:'spki'}).subarray(-32).toString('base64'); + const cluster='p12-workerd',sha=b=>createHash('sha256').update(b).digest('hex'),bytes=v=>Buffer.from(encodeCanonical(v)),scope={identity:cluster,cluster,epoch:'1',protocol:1}; + const anchors=['1','2','3'].map(node=>({node,device:'',key:pub(node),incarnation:'1',protocol:1,anchor:true,retired:false,ready_index:'0'})); + const genesis=['a','b'].flatMap(id=>[{namespace:'enrollment',key:'actor/'+Buffer.from(id).toString('hex'),data:bytes({cluster,key:pub(id),personal:true}).toString('base64')},{namespace:'state',key:'device/'+Buffer.from(id).toString('hex'),data:bytes({V:1,name:'',added_by:'',revoked:false,caps:{os:'',arch:'',sandbox:null,container:null,gpu:null,cow:''}}).toString('base64')}]); + genesis.push({namespace:'state',key:'identity',data:bytes({V:1,identity:'test'}).toString('base64')}); + let server,transport=0;const sequences={};const status={},queue=[]; + const restart=async()=>{if(server)await server.dispose();server=await openWorkerd(out,disk,true);}; + const call=async(id,op,extra={})=>{const r=await fetch(server.origin+'/'+id,{method:'POST',body:JSON.stringify({op,...extra}),signal:AbortSignal.timeout(5000)});assert.equal(r.status,200);const v=await r.json();if(v?.error)throw Error(v.error);return v;}; + const accept=(id,r)=>{if(r?.status)status[id]=r.status;queue.push(...(r?.send??[]).map(m=>({...m,from:id})));}; + const pump=async()=>{for(let n=0;queue.length;n++){assert(n<2000);const m=queue.shift();if(Number(m.to)>3)continue; + const e={scope,from:String(m.from),to:String(m.to),incarnation:'1',destination:'1',sequence:String(++transport),chunk:null,digest:sha(Buffer.from(m.bytes,'base64')),payload:m.bytes,signature:null}; + e.signature=sign(null,Buffer.from('codeaf/raft/envelope/1\n'+JSON.stringify(e)),keys[e.from].privateKey).toString('base64'); + accept(Number(m.to),await call(m.to,'peer',{value:Buffer.from(JSON.stringify(e)).toString('base64')})); + }}; + const leader=()=>Number(Object.keys(status).find(id=>status[id].role===2)); + const command=(change,signer='a')=>{const actor=signer==='a'?'a':'anchor/'+signer,n=sequences[actor]=(sequences[actor]??0)+1,c={schema:1,cluster,actor,session:'api',sequence:String(n),request_digest:'',now:1000,change,signature:null};if(n>1)c.ack=String(n-1);c.request_digest=sha(bytes({cluster,actor,session:'api',sequence:c.sequence,change}));c.signature=sign(null,Buffer.concat([Buffer.from('codeaf.directory.command.v1\0'),bytes(c)]),keys[signer].privateKey).toString('base64');return bytes(c).toString('base64');}; + const submit=async raw=>{const i=leader(),before=BigInt(status[i].applied);accept(i,await call(i,'input',{tag:3,value:raw}));await pump();assert(BigInt(status[i].applied)>before,'dead learner stalled commitment');}; + const header=()=>call(leader(),'header'); + const settle=async()=>{const i=leader();accept(i,await call(i,'membership'));await pump();const p=await call(i,'membership');accept(i,p);await pump();assert(p.settle);const h=await header();await submit(command({settle_membership:{revision:h.revision,node:h.pending.node}},String(i)));}; + const tick=async()=>{for(const id of [1,2,3])accept(id,await call(id,'input',{tag:1}));await pump();}; + try{ + await restart();for(const id of [1,2,3])accept(id,await call(id,'init',{config:{cluster,id:String(id),seed:String(id+20),bootstrap:['1','2','3'],anchors,scope,genesis}})); + await pump();for(let n=0;n<30&&!leader();n++)await tick();assert(leader()); + await assert.rejects(call(leader(),'enrollment',{view:{},applied:'0'}),/absent/); + await submit(command({init_membership:{anchors,limit:4096}})); + let h=await header();const enrolled=command({enroll_member:{node:h.next_node,revision:h.revision,device:'b',protocol:1}});await submit(enrolled);await settle(); + let view=await call(leader(),'registry');assert.equal(view['4'].voter,false);assert.equal(view['4'].revoked,false); + await submit(command({set_vault:{old:'',new:'sealed-ref'}}));await call(leader(),'checkpoint'); + await restart();for(const id of [1,2,3])accept(id,await call(id,'status'));await pump();for(let n=0;n<30&&!leader();n++)await tick();assert(leader()); + h=await header();assert.equal(h.next_node,'5');view=await call(leader(),'registry');assert.equal(view['4'].voter,false); + await submit(enrolled);assert.equal((await header()).next_node,'5'); + await assert.rejects(call(leader(),'enrollment',{view:{...view,'4':{...view['4'],voter:true}},applied:status[leader()].applied}),/differs/); + h=await header();await submit(command({remove_member:{revision:h.revision,node:'4',device:'b'}}));await settle(); + assert.equal((await call(leader(),'registry'))['4'].revoked,true);await call(leader(),'checkpoint'); + await restart();for(const id of [1,2,3])accept(id,await call(id,'status'));await pump();for(let n=0;n<30&&!leader();n++)await tick(); + assert(await call(leader(),'row',{namespace:'tombstones',key:'node/4'}));assert.equal((await call(leader(),'registry'))['4'].revoked,true); + h=await header();await submit(command({enroll_member:{node:h.next_node,revision:h.revision,device:'b',protocol:1}}));await submit(command({revoke:{device:'b'}}));assert.equal((await call(leader(),'registry'))['5'].revoked,true); + }finally{if(server)await server.dispose();await rm(out,{recursive:true,force:true});await rm(disk,{recursive:true,force:true});} +}); diff --git a/relay/hosted/test/raft/membership.test.js b/relay/hosted/test/raft/membership.test.js new file mode 100644 index 0000000000..19a06a535c --- /dev/null +++ b/relay/hosted/test/raft/membership.test.js @@ -0,0 +1,98 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { spawn } from 'node:child_process'; +import { DatabaseSync } from 'node:sqlite'; +import { generateKeyPairSync, createHash, sign } from 'node:crypto'; +import { Reactor } from '../../src/raft/abi.js'; +import { MemberActor } from '../../src/raft/actor.js'; +import { directoryEvaluator } from '../../src/raft/directory.js'; +import { committedRegistry, provisionedRegistry } from '../../src/raft/membership.js'; +import { encodeCanonical } from '../../src/raft/codec.js'; + +const out = process.env.CODEAF_RAFT_PROOF_DIR ?? '/tmp/codeaf-p12'; +const enc = new TextEncoder(), dec = new TextDecoder(), bytes = v => enc.encode(encodeCanonical(v)); +const sha = b => createHash('sha256').update(b).digest('hex'); +const segment = s => Buffer.from(s).toString('hex'); +class Host { + constructor() { this.db = new DatabaseSync(':memory:'); this.sql = { exec: (q, ...p) => { const rows = this.db.prepare(q).all(...p); return { toArray: () => rows, [Symbol.iterator]: () => rows[Symbol.iterator]() }; } }; } + transactionSync(fn) { this.db.exec('BEGIN'); try { const r = fn(); this.db.exec('COMMIT'); return r; } catch (e) { this.db.exec('ROLLBACK'); throw e; } } + async sync() {} +} +class Native { + constructor() { + this.child = spawn(out + '/native-peer', [], { stdio: ['pipe','pipe','inherit'], env: { ...process.env, GOMEMLIMIT: '32MiB' } }); this.buffer = Buffer.alloc(0); + this.child.stdout.on('data', b => { this.buffer = Buffer.concat([this.buffer,b]); if (!this.waiter || this.buffer.length < 4) return; const n = this.buffer.readUInt32LE(); if (this.buffer.length < n + 4) return; const r = this.buffer.subarray(4,n+4); this.buffer = this.buffer.subarray(n+4); const w = this.waiter; this.waiter = null; clearTimeout(w.timer); w.resolve(new Uint8Array(r)); }); + this.child.on('exit', code => { if (this.waiter) { clearTimeout(this.waiter.timer); this.waiter.reject(Error('native exited ' + code)); } }); + } + call(b) { return new Promise((resolve,reject) => { assert(!this.waiter); this.waiter = { resolve,reject,timer:setTimeout(() => reject(Error('native timeout')),5000) }; const h = Buffer.alloc(4); h.writeUInt32LE(b.length); this.child.stdin.write(h); this.child.stdin.write(b); }); } + async close() { this.child.stdin.end(); if (this.child.exitCode === null) await new Promise(r => this.child.once('exit',r)); } +} + +test('R9 R11 R12 R13 R14 R16 R17 hosted learner intents, exact retries, tombstones and native policy parity', { timeout:30000 }, async () => { + const reactor = new Reactor(new WebAssembly.Module(readFileSync(out + '/core.wasm'))), native = new Native(), calls = []; + const policy = { call(b) { const result = reactor.call(b); calls.push([b.slice(),result.slice()]); return result; } }; + const keys = Object.fromEntries(['a','b','x1','x2','x3'].map(id => [id,generateKeyPairSync('ed25519')])); + const pub = id => keys[id].publicKey.export({ format:'der',type:'spki' }).subarray(-32).toString('base64'); + const cluster = 'p12-hosted', hosts = [], actors = [], queue = [], seq = {a:0,b:0}, results = []; + const anchors = [1,2,3].map(node => ({node:String(node),device:'',key:pub('x'+node),incarnation:'1',protocol:1,anchor:true,retired:false,ready_index:'0'})); + const genesis = ['a','b'].flatMap(id => [ + {namespace:'enrollment',key:'actor/'+segment(id),data:bytes({cluster,key:pub(id),personal:true})}, + {namespace:'state',key:'device/'+segment(id),data:bytes({V:1,name:'',added_by:'',revoked:false,caps:{os:'',arch:'',sandbox:null,container:null,gpu:null,cow:''}})}, + ]); + genesis.push({namespace:'state',key:'identity',data:bytes({V:1,identity:'test'})}); + const size = rows => rows.reduce((n,r) => n+12+r.namespace.length+r.key.length+r.data.length,0); + const configs = [], absent = new Set(); + const accept = e => queue.push(...e.send); + const pump = async () => { for (let n=0;queue.length;n++) { assert(n<2000); const m=queue.shift(), id=Number(m.to); if (!actors[id-1] || absent.has(id)) continue; accept(await actors[id-1].input(2,m.bytes)); } }; + const leader = () => actors.findIndex(a => a.status.role===2); + const add = async id => { + const host = new Host(); hosts.push(host); + const evaluator = (raw,read,app) => { + const base = read('state','identity') ? [] : genesis; + const r = directoryEvaluator(policy,cluster,8*1024*1024)(raw,(ns,k) => base.find(r => r.namespace===ns && r.key===k)?.data ?? read(ns,k),{...app,size:app.size+size(base)}); + if (!results[id-1]) results[id-1]=[]; results[id-1].push(JSON.parse(dec.decode(r.result))); return {...r,writes:[...base,...r.writes]}; + }; + const config = {cluster,id:String(id),seed:String(id+44),bootstrap:id<=3?['1','2','3']:[]}; configs[id-1]=config; + const actor = new MemberActor(host,reactor,evaluator); actors[id-1]=actor; accept(await actor.init(config)); + }; + const command = change => { + const sequence=++seq.a, c={schema:1,cluster,actor:'a',session:'api',sequence:String(sequence),request_digest:'',now:1000,change,signature:null}; + if (sequence>1)c.ack=String(sequence-1); + c.request_digest=sha(bytes({cluster,actor:c.actor,session:c.session,sequence:c.sequence,change})); + c.signature=sign(null,Buffer.concat([Buffer.from('codeaf.directory.command.v1\0'),bytes(c)]),keys.a.privateKey).toString('base64');return bytes(c); + }; + const submit = async raw => {const i=leader(),n=results[i]?.length??0;accept(await actors[i].input(3,raw));await pump();assert.equal(results[i].length,n+1);return results[i].at(-1);}; + const registry = () => committedRegistry(policy,actors[leader()].store); + const header = () => JSON.parse(dec.decode(actors[leader()].store.read('membership','registry'))); + const settle = async () => { + let i=leader(),e=await actors[i].membership();accept(e);await pump();e=await actors[i].membership();accept(e);await pump();assert.equal(e.settle,true); + const h=header();const r=await submit(command({settle_membership:{revision:h.revision,node:h.pending.node}}));assert.equal(r.code,'ok');return r; + }; + try { + for (let id=1;id<=3;id++)await add(id); + await pump();for (let n=0;n<30&&leader()<0;n++){for(const a of actors)accept(await a.input(1));await pump();}assert(leader()>=0); + assert.equal(Object.keys(provisionedRegistry(policy,{anchors,bootstrap:['1','2','3']})).length,3); + assert.throws(()=>provisionedRegistry(policy,{anchors:anchors.slice(0,2),bootstrap:['1','2','3']})); + let r=await submit(command({init_membership:{anchors,limit:4096}}));assert.equal(r.code,'ok');assert.equal(Object.keys(registry()).length,3); + let h=header();const enroll=command({enroll_member:{node:h.next_node,revision:h.revision,device:'b',protocol:1}}); + r=await submit(enroll);assert.equal(r.code,'ok');assert.equal(r.membership.node,'4');assert.equal(r.membership.barrier,'0'); + assert.deepEqual(await submit(enroll),r); + await add(4);absent.add(4);const lagged=(leader()+1)%3;absent.add(lagged+1);accept(await actors[leader()].membership());await pump();h=header();r=await submit(command({settle_membership:{revision:h.revision,node:h.pending.node}}));assert.equal(r.code,'ok');absent.delete(lagged+1);for(let n=0;n<3;n++){accept(await actors[leader()].input(1));await pump();}for(let n=0;n<3;n++)assert.deepEqual(actors[n].store.read('membership','registry'),actors[leader()].store.read('membership','registry'),'mixed configuration/settlement batch diverged');assert.equal(Object.keys(registry()).length,4);assert.equal(registry()['4'].voter,false); + const i=leader(),snapshot=await actors[i].checkpoint(),info=actors[i].codec.inspect(3,snapshot),id=actors[i].store.hash(info.data); + await actors[3].begin(info.data);const m=actors[i].store.generation(id).manifest; + for(let n=0;n30); + } finally {await native.close();for(const h of hosts)h.db.close();} +}); From 925abd46c0bf3338dde6680b52650875c90f49c9 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 03:22:55 -0400 Subject: [PATCH 028/103] Keep leadership policy and ranked election timing durable across fallback. Use committed owner intents and explicit joint consensus for personal leadership, require a fresh quorum read for completion and retain it to prevent automatic regain. Rank randomized election windows from the immutable provisioned anchor manifest while preserving quorum and vote rules and exact versioned replay. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterstate/apply.go | 6 +- internal/clusterstate/leadership.go | 175 ++++++++++++++++++ internal/clusterstate/leadership_test.go | 162 +++++++++++++++++ internal/clusterstate/membership.go | 41 +++-- internal/clusterstate/types.go | 48 ++--- internal/raftcore/README.md | 41 ++++- internal/raftcore/core.go | 66 +++++-- internal/raftcore/core_test.go | 8 +- internal/raftcore/election_test.go | 88 +++++++++ internal/raftcore/leadership.go | 181 +++++++++++++++++++ internal/raftcore/leadership_test.go | 151 ++++++++++++++++ internal/raftcore/membership.go | 3 + internal/raftcore/random.go | 14 ++ internal/raftcore/types.go | 30 ++- third_party/raft-patch/README.md | 7 + third_party/raft-patch/election-random.patch | 55 +++--- third_party/raft/raft.go | 19 +- 17 files changed, 992 insertions(+), 103 deletions(-) create mode 100644 internal/clusterstate/leadership.go create mode 100644 internal/clusterstate/leadership_test.go create mode 100644 internal/raftcore/election_test.go create mode 100644 internal/raftcore/leadership.go create mode 100644 internal/raftcore/leadership_test.go diff --git a/internal/clusterstate/apply.go b/internal/clusterstate/apply.go index d4f6a2e30a..2f2e9ca458 100644 --- a/internal/clusterstate/apply.go +++ b/internal/clusterstate/apply.go @@ -81,6 +81,7 @@ type JoinRequest struct { // ApplicationMeta is an immutable view at one applied index. Byte accounting // includes record framing and keys; capacity is a refusal, never an eviction. type ApplicationMeta struct { + Term uint64 `json:"term,omitempty"` Index uint64 `json:"index"` Configuration Configuration `json:"configuration"` Cluster string `json:"cluster"` @@ -308,7 +309,10 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er return out, nil } service := c.Change.RecordUpload != nil || c.Change.AdmitJoin != nil || c.Change.CompleteJoin != nil - if service != auth.Effects || auth.Personal && auth.Effects || auth.Anchor && (auth.Personal || auth.Effects || c.Change.SettleMembership == nil && c.Change.CloseSession == nil) { + if service != auth.Effects || auth.Personal && auth.Effects || auth.Anchor && (auth.Personal || auth.Effects || c.Change.SettleMembership == nil && c.Change.SettleLeadership == nil && c.Change.CompleteLeadership == nil && c.Change.RestoreAnchors == nil && c.Change.CloseSession == nil) { + return out, nil + } + if (c.Change.RestoreAnchors != nil || c.Change.CompleteLeadership != nil) && !auth.Anchor { return out, nil } var last session diff --git a/internal/clusterstate/leadership.go b/internal/clusterstate/leadership.go new file mode 100644 index 0000000000..db8dbf9661 --- /dev/null +++ b/internal/clusterstate/leadership.go @@ -0,0 +1,175 @@ +package clusterstate + +// LeadershipIntent survives election and restart. Settled means the requested +// configuration is durable, not that the target won an election. Only a fresh +// leader-side quorum barrier can produce a successful leadership receipt. +type LeadershipIntent struct { + Term uint64 `json:"term"` + Node uint64 `json:"node"` + Standby uint64 `json:"standby"` + Revision uint64 `json:"revision"` + Barrier uint64 `json:"barrier"` + Restore bool `json:"restore"` + Completed bool `json:"completed"` + Settled bool `json:"settled"` +} + +type RequestLeadership struct { + Device string `json:"device"` + Revision uint64 `json:"revision"` + Node uint64 `json:"node"` +} +type RestoreAnchors struct { + Revision uint64 `json:"revision"` +} +type CompleteLeadership struct { + Revision uint64 `json:"revision"` +} + +type SettleLeadership struct { + Revision uint64 `json:"revision"` +} + +// LeadershipReceipt contains coordination metadata only. It grants no chat +// ownership or content access and is issued by the elected target after a +// current-term commit and a quorum read, never by proposal admission. +type LeadershipReceipt struct { + Node uint64 `json:"node"` + Term uint64 `json:"term"` + Revision uint64 `json:"revision"` + Barrier uint64 `json:"barrier"` + Voters []uint64 `json:"voters"` +} + +const LeadershipUnavailable = "this cluster cannot coordinate right now; reconnect enough of its voting computers and try again; your saved chats are still here" + +func leadershipMutation(m Mutation) bool { + return m.RequestLeadership != nil || m.RestoreAnchors != nil || m.SettleLeadership != nil || m.CompleteLeadership != nil +} +func leadershipDependencies(m Mutation) []RecordKey { + keys := []RecordKey{MembershipKey()} + if v := m.RequestLeadership; v != nil { + keys = append(keys, MemberKey(v.Node), DeviceKey(v.Device), AuthorityKey(v.Device)) + } + return keys +} + +func sameIDs(a, b []uint64) bool { + if len(a) != len(b) { + return false + } + for _, id := range a { + if !hasID(b, id) { + return false + } + } + return true +} + +// LeadershipVoters describes policy; hosts still use Raft's configuration +// machinery to enact it. Three provisioned anchors remain enrolled throughout. +func (h Membership) LeadershipVoters() []uint64 { + v := append([]uint64(nil), h.Anchors...) + if l := h.Leadership; l != nil && !l.Restore { + for i, id := range v { + if id == l.Standby { + v[i] = l.Node + } + } + } + return v +} +func (h Membership) StableTopology(c Configuration) bool { + if c.Joint() || !sameIDs(c.Voters, h.LeadershipVoters()) { + return false + } + if l := h.Leadership; l != nil { + learner := l.Standby + if l.Restore { + learner = l.Node + } + return hasID(c.Learners, learner) + } + return true +} + +func applyLeadership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, put func(RecordKey, any), result *Result, h Membership) Code { + if h.Pending != nil { + return Conflict + } + m := c.Change + switch { + case m.RequestLeadership != nil: + v := m.RequestLeadership + if meta.Term == 0 { + return BadRequest + } + if v.Revision != h.Revision || meta.Configuration.Joint() || h.Leadership != nil && (!h.Leadership.Settled || h.Leadership.Restore || h.Leadership.Node != v.Node) { + return Conflict + } + if !h.StableTopology(meta.Configuration) { + return Conflict + } + var member MemberRecord + if Decode(rows[MemberKey(v.Node)], &member) != nil { + return NotFound + } + if member.Anchor || member.Retired || member.Protocol != ClusterProtocol || member.ReadyIndex == 0 || member.Node != v.Node || member.Device != v.Device { + return Conflict + } + // The target's own committed personal authority and revocation are + // checked independently of the owner who requests the transfer. + var auth Authority + var device Device + if Decode(rows[AuthorityKey(member.Device)], &auth) != nil || !auth.Personal || auth.Anchor || auth.Effects || auth.Cluster != c.Cluster { + return Unauthorized + } + if b := rows[DeviceKey(member.Device)]; b != nil { + if Decode(b, &device) != nil { + return BadRequest + } + if device.Revoked { + return Revoked + } + } + h.Revision++ + h.Leadership = &LeadershipIntent{Node: v.Node, Standby: h.Anchors[len(h.Anchors)-1], Revision: h.Revision, Barrier: meta.Index, Term: meta.Term} + case m.RestoreAnchors != nil: + v := m.RestoreAnchors + if v.Revision != h.Revision || h.Leadership == nil || h.Leadership.Restore { + return Conflict + } + h.Revision++ + l := *h.Leadership + l.Revision, l.Barrier, l.Restore, l.Settled, l.Completed = h.Revision, meta.Index, true, false, false + h.Leadership = &l + case m.CompleteLeadership != nil: + if m.CompleteLeadership.Revision != h.Revision || h.Leadership == nil || h.Leadership.Restore || !h.Leadership.Settled || h.Leadership.Completed || !h.StableTopology(meta.Configuration) { + return Conflict + } + l := *h.Leadership + l.Completed = true + h.Leadership = &l + case m.SettleLeadership != nil: + v := m.SettleLeadership + if v.Revision != h.Revision || h.Leadership == nil || h.Leadership.Settled || !h.StableTopology(meta.Configuration) { + return Conflict + } + l := *h.Leadership + if l.Restore { + h.Leadership = nil + } else { + l.Settled = true + h.Leadership = &l + } + default: + return BadRequest + } + h.Index = meta.Index + put(MembershipKey(), h) + result.Membership = &MembershipResult{Revision: h.Revision, Barrier: meta.Index} + if h.Leadership != nil { + result.Membership.Node = h.Leadership.Node + } + return OK +} diff --git a/internal/clusterstate/leadership_test.go b/internal/clusterstate/leadership_test.go new file mode 100644 index 0000000000..0b886814ac --- /dev/null +++ b/internal/clusterstate/leadership_test.go @@ -0,0 +1,162 @@ +package clusterstate_test + +import ( + "fmt" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "math/rand" + "testing" +) + +func leadReady(t *testing.T) *application { + a := membershipApp(t, cs.MaxMemberRecords) + a.meta.Term = 1 + if r := enroll(a, "b"); r.Code != cs.OK { + t.Fatal(r) + } + if r := settle(a); r.Code != cs.OK { + t.Fatal(r) + } + return a +} +func leadRequest(a *application) cs.Result { + h := memberHeader(a) + return memberRun(a, cs.Mutation{RequestLeadership: &cs.RequestLeadership{Revision: h.Revision, Node: 4, Device: "b"}}) +} +func anchorChange(a *application, m cs.Mutation) cs.Result { + a.meta.Index++ + c := a.prepareBare(cs.AnchorActor(1), 1000, m) + c, e := cs.Sign(c, actorKey("anchor-1")) + if e != nil { + a.t.Fatal(e) + } + return a.evaluate(c).Result +} +func forwardConfiguration(a *application) { + a.meta.Configuration = cs.Configuration{Voters: []uint64{1, 2, 4}, Learners: []uint64{3}} +} +func settleLead(a *application) cs.Result { + return anchorChange(a, cs.Mutation{SettleLeadership: &cs.SettleLeadership{Revision: memberHeader(a).Revision}}) +} + +// R6/R7/R8/R9/R12 require a durable intent rather than an editable voter list. +// Joint state cannot settle; fallback needs provisioned anchor authority, keeps +// the third anchor, and leaves the returning personal member without a vote. +func TestR6R7R8R9R12LeadershipPolicy(t *testing.T) { + a := leadReady(t) + if r := leadRequest(a); r.Code != cs.OK { + t.Fatal(r) + } + h := memberHeader(a) + if h.Leadership.Settled || len(a.meta.Configuration.Voters) != 3 { + t.Fatal("intent claimed an election") + } + if r := settleLead(a); r.Code != cs.Conflict { + t.Fatal("settled before swap", r) + } + a.meta.Configuration = cs.Configuration{Voters: []uint64{1, 2, 4}, Outgoing: []uint64{1, 2, 3}, LearnersNext: []uint64{3}} + if r := settleLead(a); r.Code != cs.Conflict { + t.Fatal("settled joint state", r) + } + forwardConfiguration(a) + if r := settleLead(a); r.Code != cs.OK { + t.Fatal(r) + } + h = memberHeader(a) + if r := memberRun(a, cs.Mutation{RestoreAnchors: &cs.RestoreAnchors{Revision: h.Revision}}); r.Code != cs.Unauthorized { + t.Fatal("personal caller forced fallback", r) + } + if r := anchorChange(a, cs.Mutation{RestoreAnchors: &cs.RestoreAnchors{Revision: h.Revision}}); r.Code != cs.OK { + t.Fatal(r) + } + if r := anchorChange(a, cs.Mutation{RestoreAnchors: &cs.RestoreAnchors{Revision: h.Revision}}); r.Code != cs.Conflict { + t.Fatal("simultaneous fallback did not serialize", r) + } + if r := settleLead(a); r.Code != cs.Conflict { + t.Fatal("fallback before reverse swap", r) + } + a.meta.Configuration = cs.Configuration{Voters: []uint64{1, 2, 3}, Learners: []uint64{4}} + if r := settleLead(a); r.Code != cs.OK { + t.Fatal(r) + } + if memberHeader(a).Leadership != nil || !memberHeader(a).StableTopology(a.meta.Configuration) { + t.Fatal("P automatically regained a vote") + } + if cs.LeadershipUnavailable != "this cluster cannot coordinate right now; reconnect enough of its voting computers and try again; your saved chats are still here" { + t.Fatal("R8 unavailable sentence changed") + } +} + +// R6/R12/R16 refuse retired, revoked, unready and mismatched targets, including +// requests signed by a different still-authorized personal owner. +func TestR6R12R16LeadershipAdmission(t *testing.T) { + for _, kind := range []string{"revoked", "unready", "wrong-device", "anchor", "stale"} { + t.Run(kind, func(t *testing.T) { + a := leadReady(t) + h := memberHeader(a) + v := cs.RequestLeadership{Revision: h.Revision, Node: 4, Device: "b"} + switch kind { + case "revoked": + memberRun(a, cs.Mutation{Revoke: &cs.DeviceID{Device: "b"}}) + case "unready": + var m cs.MemberRecord + _ = cs.Decode(a.rows[cs.MemberKey(4)], &m) + m.ReadyIndex = 0 + a.put(cs.MemberKey(4), m) + case "wrong-device": + v.Device = "a" + case "anchor": + v.Node = 3 + case "stale": + v.Revision-- + } + if r := memberRun(a, cs.Mutation{RequestLeadership: &v}); r.Code == cs.OK { + t.Fatal("ineligible target admitted", kind) + } + }) + } +} + +// R6/R7/R12 randomize durable stage delivery against the independently stated +// expected topology. Duplicate and stale requests cannot settle an earlier +// stage, nor can returning P create a new promotion without an owner request. +func TestR6R7R12SeededLeadershipIntentOrder(t *testing.T) { + for seed := 0; seed < seedCount(t); seed++ { + t.Run(fmt.Sprintf("seed-%03d", seed), func(t *testing.T) { + a := leadReady(t) + rng := rand.New(rand.NewSource(int64(seed))) + if r := leadRequest(a); r.Code != cs.OK { + t.Fatal(r) + } + for i := 0; i < rng.Intn(4)+1; i++ { + if r := settleLead(a); r.Code != cs.Conflict { + t.Fatal(r) + } + } + // Loss may occur before, during, or after the forward configuration. + switch rng.Intn(3) { + case 1: + a.meta.Configuration = cs.Configuration{Voters: []uint64{1, 2, 4}, Outgoing: []uint64{1, 2, 3}, LearnersNext: []uint64{3}} + case 2: + forwardConfiguration(a) + if r := settleLead(a); r.Code != cs.OK { + t.Fatal(r) + } + } + rev := memberHeader(a).Revision + if r := anchorChange(a, cs.Mutation{RestoreAnchors: &cs.RestoreAnchors{Revision: rev}}); r.Code != cs.OK { + t.Fatal(r) + } + if r := anchorChange(a, cs.Mutation{RestoreAnchors: &cs.RestoreAnchors{Revision: rev}}); r.Code != cs.Conflict { + t.Fatal(r) + } + a.meta.Configuration = cs.Configuration{Voters: []uint64{1, 2, 3}, Learners: []uint64{4}} + if r := settleLead(a); r.Code != cs.OK { + t.Fatal(r) + } + h := memberHeader(a) + if h.Leadership != nil || len(h.LeadershipVoters()) != 3 || h.Validate() != nil { + t.Fatal(h) + } + }) + } +} diff --git a/internal/clusterstate/membership.go b/internal/clusterstate/membership.go index 1127c5149c..ae717a0b34 100644 --- a/internal/clusterstate/membership.go +++ b/internal/clusterstate/membership.go @@ -52,14 +52,15 @@ type MembershipIntent struct { Revision uint64 `json:"revision"` } type Membership struct { - Revision uint64 `json:"revision"` - Index uint64 `json:"index"` - NextNode uint64 `json:"next_node"` - Anchors []uint64 `json:"anchors"` - Nodes []uint64 `json:"nodes"` - Live uint32 `json:"live"` - Limit uint32 `json:"limit"` - Pending *MembershipIntent `json:"pending,omitempty"` + Leadership *LeadershipIntent `json:"leadership,omitempty"` + Revision uint64 `json:"revision"` + Index uint64 `json:"index"` + NextNode uint64 `json:"next_node"` + Anchors []uint64 `json:"anchors"` + Nodes []uint64 `json:"nodes"` + Live uint32 `json:"live"` + Limit uint32 `json:"limit"` + Pending *MembershipIntent `json:"pending,omitempty"` } type InitMembership struct { Anchors []MemberRecord `json:"anchors"` @@ -104,6 +105,9 @@ func MemberTombstoneKey(node uint64) RecordKey { func membershipDependencies(m Mutation) []RecordKey { keys := []RecordKey{} + if leadershipMutation(m) { + return leadershipDependencies(m) + } switch { case m.InitMembership != nil: keys = append(keys, MembershipKey()) @@ -122,7 +126,7 @@ func membershipDependencies(m Mutation) []RecordKey { return keys } func membershipMutation(m Mutation) bool { - return m.InitMembership != nil || m.EnrollMember != nil || m.RemoveMember != nil || m.SettleMembership != nil + return leadershipMutation(m) || m.InitMembership != nil || m.EnrollMember != nil || m.RemoveMember != nil || m.SettleMembership != nil } func validNode(id uint64) bool { return id > 0 && id < ^uint64(0)-1 } func applyMembership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, put func(RecordKey, any), result *Result) Code { @@ -168,16 +172,10 @@ func applyMembership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, if raw == nil || h.Revision == ^uint64(0) || meta.Index == 0 { return Conflict } - if meta.Configuration.Joint() { - return Conflict - } - // P12 only edits learners. A voter swap is a separate authorized P13 intent. - for _, id := range h.Anchors { - if !hasID(meta.Configuration.Voters, id) { - return Conflict - } + if leadershipMutation(m) { + return applyLeadership(c, meta, rows, put, result, h) } - if len(meta.Configuration.Voters) != len(h.Anchors) { + if h.Leadership != nil && !h.Leadership.Settled || !h.StableTopology(meta.Configuration) { return Conflict } switch { @@ -243,7 +241,7 @@ func applyMembership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, if Decode(rows[MemberKey(v.Node)], &member) != nil { return NotFound } - if member.Anchor || member.Retired || member.Node != v.Node || member.Device != v.Device { + if member.Anchor || member.Retired || member.Node != v.Node || member.Device != v.Device || hasID(meta.Configuration.Voters, v.Node) { return Conflict } var mapping DeviceMember @@ -380,6 +378,11 @@ func (h Membership) Validate() error { if h.Pending != nil && (h.Pending.Revision != h.Revision || !hasID(h.Nodes, h.Pending.Node) || hasID(h.Anchors, h.Pending.Node)) { return ErrDependency } + if l := h.Leadership; l != nil { + if !hasID(h.Nodes, l.Node) || hasID(h.Anchors, l.Node) || !hasID(h.Anchors, l.Standby) || l.Revision > h.Revision || l.Term == 0 && !l.Restore || l.Completed && (!l.Settled || l.Restore) || l.Barrier == 0 || l.Barrier > h.Index || h.Pending != nil && !l.Settled { + return ErrDependency + } + } return nil } diff --git a/internal/clusterstate/types.go b/internal/clusterstate/types.go index 40728af1a2..0388b49e25 100644 --- a/internal/clusterstate/types.go +++ b/internal/clusterstate/types.go @@ -44,28 +44,32 @@ type Command struct { // their typed operations here; unknown fields fail decoding on older members. // There is no arbitrary JSON payload in which task text or secrets can hide. type Mutation struct { - InitMembership *InitMembership `json:"init_membership,omitempty"` - EnrollMember *EnrollMember `json:"enroll_member,omitempty"` - RemoveMember *RemoveMember `json:"remove_member,omitempty"` - SettleMembership *SettleMembership `json:"settle_membership,omitempty"` - RecordUpload *UploadReceipt `json:"record_upload,omitempty"` - AdmitJoin *AdmitJoin `json:"admit_join,omitempty"` - CompleteJoin *CompleteJoin `json:"complete_join,omitempty"` - Rotate *RotateRequest `json:"rotate,omitempty"` - CloseSession *CloseSession `json:"close_session,omitempty"` - DecideJoin *JoinDecision `json:"decide_join,omitempty"` - Create *CreateCell `json:"create,omitempty"` - Acquire *AcquireCell `json:"acquire,omitempty"` - Beat *BeatCell `json:"beat,omitempty"` - BeatBatch *BeatBatch `json:"beat_batch,omitempty"` - Publish *PublishCell `json:"publish,omitempty"` - Release *ReleaseCell `json:"release,omitempty"` - Archive *CellID `json:"archive,omitempty"` - PutDevice *PutDevice `json:"put_device,omitempty"` - Revoke *DeviceID `json:"revoke,omitempty"` - SetVault *VaultCAS `json:"set_vault,omitempty"` - AdvanceTime *AdvanceTime `json:"advance_time,omitempty"` - DeviceSeen *DeviceSeen `json:"device_seen,omitempty"` + CompleteLeadership *CompleteLeadership `json:"complete_leadership,omitempty"` + RequestLeadership *RequestLeadership `json:"request_leadership,omitempty"` + RestoreAnchors *RestoreAnchors `json:"restore_anchors,omitempty"` + SettleLeadership *SettleLeadership `json:"settle_leadership,omitempty"` + InitMembership *InitMembership `json:"init_membership,omitempty"` + EnrollMember *EnrollMember `json:"enroll_member,omitempty"` + RemoveMember *RemoveMember `json:"remove_member,omitempty"` + SettleMembership *SettleMembership `json:"settle_membership,omitempty"` + RecordUpload *UploadReceipt `json:"record_upload,omitempty"` + AdmitJoin *AdmitJoin `json:"admit_join,omitempty"` + CompleteJoin *CompleteJoin `json:"complete_join,omitempty"` + Rotate *RotateRequest `json:"rotate,omitempty"` + CloseSession *CloseSession `json:"close_session,omitempty"` + DecideJoin *JoinDecision `json:"decide_join,omitempty"` + Create *CreateCell `json:"create,omitempty"` + Acquire *AcquireCell `json:"acquire,omitempty"` + Beat *BeatCell `json:"beat,omitempty"` + BeatBatch *BeatBatch `json:"beat_batch,omitempty"` + Publish *PublishCell `json:"publish,omitempty"` + Release *ReleaseCell `json:"release,omitempty"` + Archive *CellID `json:"archive,omitempty"` + PutDevice *PutDevice `json:"put_device,omitempty"` + Revoke *DeviceID `json:"revoke,omitempty"` + SetVault *VaultCAS `json:"set_vault,omitempty"` + AdvanceTime *AdvanceTime `json:"advance_time,omitempty"` + DeviceSeen *DeviceSeen `json:"device_seen,omitempty"` } type CellID struct { diff --git a/internal/raftcore/README.md b/internal/raftcore/README.md index a532c1ad71..76ff1e136a 100644 --- a/internal/raftcore/README.md +++ b/internal/raftcore/README.md @@ -94,10 +94,25 @@ needs ticks; the safe quiescence protocol belongs to P14. ## Replay contract -Election timeout is 6–11 ticks, with a heartbeat every tick. Hosts currently +Election timing version 2 gives provisioned anchors A/B/C disjoint randomized +windows of 6–7/8–9/10–11 ticks; a personal voter uses 12–13. Rank is the index in +the immutable, sorted `Membership.Anchors` manifest, not current ConfState voter +order. Before registry initialization the same source-provisioned bootstrap +manifest supplies it. Native recovery supplies `RecoveredState.Anchors` from the +checksummed applied registry, including installed snapshots. Hosted create frames +carry that same durable list. Conflicting host/recovered manifests are refused. +An unprovisioned learner uses the personal window. Heartbeats remain every tick; +CheckQuorum and PreVote leader leases retain their six-tick interval. Hosts currently schedule 5-second ticks (P14); no wall duration is read here. SplitMix64's exact 64-bit constants, overflow and rejection sampling are specified in random.go; -seed zero is valid. Every timeout reset consumes this per-node provider and +seed zero is valid. Every version-2 timeout reset consumes one bounded draw of width two and adds +twice the immutable rank through the existing provider. The library's separate +`ElectionRandomBound=8` admits these offsets without changing `ElectionTick`. +Version 1 remains available only as explicit internal replay compatibility for +old recorded inputs; production defaults and the ABI always use version 2. +The simulator writes its timing version and reads unversioned old recordings as +version 1, preserving their exact event digests. Every timeout reset uses this +per-node provider and never upstream global randomness. RNG state/draws appear only in diagnostics. Wire protobuf marshaling is deterministic. RawNode already orders meaningful peer traversal; preserve its per-peer message order. Completed read contexts @@ -168,3 +183,25 @@ settle an existing intent or close their request session. They cannot enroll, remove, read directory content, mutate chats, pair or rotate. Thus a crashed requester need not return for an already authorized admission to finish. The settlement still requires the exact applied configuration and pending revision. + +P13 leadership hosts reconcile `Membership.Leadership` under their actor queue: +obtain authenticated target `LeadershipEvidence` from its durable store at the +intent barrier; call `PlanLeadership`; durably apply each explicit joint enter +and separate leave; commit `SettleLeadership`; then execute the planned transfer. +A zero-byte protobuf with `Configure=true` is the explicit leave, not no action. +Only the elected target can issue `LeadershipReceipt`, after a new current-term +ReadIndex response and durable apply. The read proof is consumed by that receipt; +starting another read invalidates any previous proof. Commit `CompleteLeadership` +through provisioned anchor authority after that observation, then obtain a new +read/receipt before returning success. A completed intent never drives another +automatic transfer after a later anchor election. + +An elected anchor's `Fallback` plan authorizes only housekeeping intent admission, +not a voter mutation. Commit `RestoreAnchors` at the expected revision, obtain C's +actual durable evidence, reconcile the reverse joint enter/leave, and commit its +settlement. An interrupted joint state is finished before reversing. Concurrent +fallback requests conflict on the same revision. Returning P stays a learner; +only another personal request may promote it. Public handlers accept the typed +signed request and never expose Configure or host-supplied readiness evidence. +P14 schedules ticks/wake/housekeeping, and P17 assembles these host/control seams +with the personal member lifecycle and the one production directory route. diff --git a/internal/raftcore/core.go b/internal/raftcore/core.go index 3aa170e8ff..3930a5ec36 100644 --- a/internal/raftcore/core.go +++ b/internal/raftcore/core.go @@ -46,17 +46,20 @@ type readRequest struct { // queued implicitly. Storage failure or an unknown committed command stops the // handle. Recovery always starts as a follower from supplied durable bytes. type Core struct { - cfg Config - node *raft.RawNode - store *storage - rng splitMix64 - applied uint64 - serial uint64 - readSerial uint64 - reads map[string]readRequest - barriers map[string]uint64 - pending *pending - stopped bool + cfg Config + node *raft.RawNode + store *storage + rng splitMix64 + electionRank int + applied uint64 + serial uint64 + quorumTerm uint64 + quorumIndex uint64 + readSerial uint64 + reads map[string]readRequest + barriers map[string]uint64 + pending *pending + stopped bool } func clone[T proto.Message](v T) T { @@ -91,6 +94,37 @@ func New(cfg Config, recovered RecoveredState) (core *Core, err error) { if cfg.ClusterUUID == "" || cfg.NodeID == 0 || raft.IsLocalMsgTarget(cfg.NodeID) { return nil, ErrConfig } + if cfg.TimingVersion == 0 { + cfg.TimingVersion = ElectionTimingVersion + } + if cfg.TimingVersion != 1 && cfg.TimingVersion != ElectionTimingVersion { + return nil, ErrConfig + } + anchors := cfg.Anchors + if len(recovered.Anchors) != 0 { + if len(anchors) != 0 && !slices.Equal(anchors, recovered.Anchors) { + return nil, ErrConfig + } + anchors = recovered.Anchors + } else if len(anchors) == 0 && len(cfg.Bootstrap) == 3 { + anchors = slices.Clone(cfg.Bootstrap) + slices.Sort(anchors) + } + rank := 3 + if len(anchors) != 0 { + if len(anchors) != 3 { + return nil, ErrConfig + } + for i, id := range anchors { + if id == 0 || raft.IsLocalMsgTarget(id) || i > 0 && id <= anchors[i-1] { + return nil, ErrConfig + } + if id == cfg.NodeID { + rank = i + } + } + } + cfg.Anchors = slices.Clone(anchors) snapshot := clone(recovered.Snapshot) snapIndex := snapshot.GetMetadata().GetIndex() last := snapIndex @@ -116,7 +150,7 @@ func New(cfg Config, recovered RecoveredState) (core *Core, err error) { if conf == nil { conf = &pb.ConfState{} } - c := &Core{cfg: cfg, rng: splitMix64{state: cfg.Seed}, applied: recovered.Applied, reads: make(map[string]readRequest), barriers: make(map[string]uint64)} + c := &Core{cfg: cfg, electionRank: rank, rng: splitMix64{state: cfg.Seed}, applied: recovered.Applied, reads: make(map[string]readRequest), barriers: make(map[string]uint64)} c.store = &storage{MemoryStorage: raft.NewMemoryStorage(), conf: conf} if !raft.IsEmptySnap(snapshot) { if err := c.store.ApplySnapshot(snapshot); err != nil { @@ -131,9 +165,13 @@ func New(cfg Config, recovered RecoveredState) (core *Core, err error) { return nil, ErrConfig } } + randomBound := ElectionRandomBound + if cfg.TimingVersion == 1 { + randomBound = ElectionTicks + } c.node, err = raft.NewRawNode(&raft.Config{ ID: cfg.NodeID, ElectionTick: ElectionTicks, HeartbeatTick: HeartbeatTicks, - Storage: c.store, Applied: recovered.Applied, ElectionRandom: c.rng.bounded, + Storage: c.store, Applied: recovered.Applied, ElectionRandom: c.electionRandom, ElectionRandomBound: randomBound, PreVote: true, CheckQuorum: true, ReadOnlyOption: raft.ReadOnlySafe, DisableProposalForwarding: true, AsyncStorageWrites: false, MaxSizePerMsg: MaxMessageBytes, MaxCommittedSizePerReady: MaxMessageBytes, @@ -298,6 +336,7 @@ func (c *Core) Step(input Input) (effects Effects, err error) { return effects, err } case Read: + c.quorumIndex, c.quorumTerm = 0, 0 if !c.leader() { return effects, ErrLeader } @@ -514,6 +553,7 @@ func (c *Core) finish(e *Effects) { if r.term == s.GetTerm() && c.leader() && c.applied >= index { if !r.cancelled { e.Reads = append(e.Reads, ReadBarrier{RequestID: r.id, Index: index}) + c.quorumTerm, c.quorumIndex = s.GetTerm(), index } delete(c.reads, key) delete(c.barriers, key) diff --git a/internal/raftcore/core_test.go b/internal/raftcore/core_test.go index 87009433e3..225cf5b5b4 100644 --- a/internal/raftcore/core_test.go +++ b/internal/raftcore/core_test.go @@ -31,11 +31,13 @@ type testGroup struct { drop func(from, to uint64) bool } -func newGroup(t *testing.T) *testGroup { +func newGroup(t *testing.T) *testGroup { return newTimingGroup(t, ElectionTimingVersion) } + +func newTimingGroup(t *testing.T, version uint16) *testGroup { t.Helper() g := &testGroup{t: t, applied: make([][]*pb.Entry, 3), durable: make([]RecoveredState, 3)} for id := uint64(1); id <= 3; id++ { - c, err := New(Config{ClusterUUID: "test-cluster", NodeID: id, Seed: 100 + id, Protocol: ProtocolVersion, Bootstrap: []uint64{3, 1, 2}}, RecoveredState{}) + c, err := New(Config{TimingVersion: version, ClusterUUID: "test-cluster", NodeID: id, Seed: 100 + id, Protocol: ProtocolVersion, Bootstrap: []uint64{3, 1, 2}}, RecoveredState{}) if err != nil { t.Fatal(err) } @@ -172,7 +174,7 @@ func command(t *testing.T, sequence uint64) []byte { // pass a NEW read barrier before a host can report a completed transfer. func TestR14SeededReplayAndR6Transfer(t *testing.T) { run := func() []byte { - g := newGroup(t) + g := newTimingGroup(t, 1) leader := g.elect() g.call(leader, Propose{command(t, 1)}) g.pump() diff --git a/internal/raftcore/election_test.go b/internal/raftcore/election_test.go new file mode 100644 index 0000000000..6641e5ecd4 --- /dev/null +++ b/internal/raftcore/election_test.go @@ -0,0 +1,88 @@ +package raftcore + +import ( + "fmt" + "os" + "strconv" + "testing" + + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// R7/R14/T1/T2 prove rank from the immutable committed anchor order, including +// C outside the voters and P promoted. Node IDs deliberately differ from rank. +// Every timeout is observed through emitted protocol messages, not core fields. +func TestR7R14RankedElectionWindows(t *testing.T) { + count := 64 + if value := os.Getenv("RAFTSIM_SEEDS"); value != "" { + var err error + count, err = strconv.Atoi(value) + if err != nil || count < 1 { + t.Fatal("invalid RAFTSIM_SEEDS") + } + } + for seed := 0; seed < count; seed++ { + for rank, id := range []uint64{11, 28, 99, 201} { + for _, recovered := range []bool{false, true} { + cfg := Config{ClusterUUID: "ranked", NodeID: id, Seed: uint64(seed), Protocol: ProtocolVersion} + state := RecoveredState{ConfState: &pb.ConfState{Voters: []uint64{11, 28, 99, 201}}} + if recovered { + state.Anchors = []uint64{11, 28, 99} + } else { + cfg.Anchors = []uint64{11, 28, 99} + } + c, err := New(cfg, state) + if err != nil { + t.Fatal(err) + } + at := 0 + for tick := 1; tick <= 14; tick++ { + effects, err := c.Step(Tick{}) + if err != nil { + t.Fatal(err) + } + for { + for _, send := range effects.Send { + m := new(pb.Message) + if err := proto.Unmarshal(send.Bytes, m); err != nil { + t.Fatal(err) + } + if m.GetType() == pb.MsgPreVote && at == 0 { + at = tick + } + } + if effects.Persist != nil { + effects, err = c.Step(Persisted{BatchID: effects.Persist.ID}) + } else if effects.Apply != nil { + effects, err = c.Step(Applied{BatchID: effects.Apply.ID}) + } else { + break + } + if err != nil { + t.Fatal(err) + } + } + if at > 0 { + break + } + } + min := 6 + 2*rank + if at < min || at > min+1 { + t.Fatalf("R7 seed=%d rank=%d recovered=%v first campaign at %d ticks, want %d–%d", seed, rank, recovered, at, min, min+1) + } + } + } + } +} + +// R7/R14/T2 refuse a host's conflicting manifest instead of choosing a local +// rank after restart. Missing or invalid versions cannot silently change replay. +func TestR7R14RankRecoveryRefusesConflictingProvisioning(t *testing.T) { + for _, anchors := range [][]uint64{{11, 28, 100}, {11, 11, 99}, {99, 28, 11}, {11, 28}, {0, 28, 99}} { + _, err := New(Config{ClusterUUID: "ranked", NodeID: 11, Protocol: ProtocolVersion, Anchors: anchors}, RecoveredState{Anchors: []uint64{11, 28, 99}}) + if err == nil { + t.Fatal("R7 conflicting manifest accepted", fmt.Sprint(anchors)) + } + } +} diff --git a/internal/raftcore/leadership.go b/internal/raftcore/leadership.go new file mode 100644 index 0000000000..af00bad889 --- /dev/null +++ b/internal/raftcore/leadership.go @@ -0,0 +1,181 @@ +package raftcore + +import ( + "github.com/Agent-Field/codeaf/internal/clusterstate" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// LeadershipEvidence is supplied by the authenticated host from the target's +// durable store at the advertised barrier. It is not accepted from a public +// request. RecentActive in the actual Raft progress adds the reverse direction; +// queued transport, an uploaded snapshot or a socket alone cannot qualify. +type LeadershipEvidence struct { + Node uint64 `json:"node"` + Revision uint64 `json:"revision"` + Applied uint64 `json:"applied"` + Protocol uint16 `json:"protocol"` + Durable bool `json:"durable"` +} +type LeadershipPlan struct { + Configure bool `json:"configure"` + Change []byte `json:"change"` + Settle bool `json:"settle"` + Transfer uint64 `json:"transfer"` + Fallback bool `json:"fallback"` +} + +func (c *Core) configurationPending() (bool, error) { + last, err := c.store.LastIndex() + if err != nil || last <= c.applied { + return false, err + } + entries, err := c.store.Entries(c.applied+1, last+1, ^uint64(0)) + for _, entry := range entries { + if entry.GetType() != pb.EntryNormal { + return true, err + } + } + return false, err +} + +// PlanLeadership resumes ONLY a durably committed policy intent. Leaving a +// joint state is always its own durable entry; reversing an interrupted swap +// first finishes that joint state, never edits its voter list in place. +func (c *Core) PlanLeadership(h clusterstate.Membership, evidence LeadershipEvidence) (LeadershipPlan, error) { + var p LeadershipPlan + if c.stopped { + return p, ErrStopped + } + if !c.leader() { + return p, ErrLeader + } + if c.pending != nil { + return p, ErrBusy + } + if h.Validate() != nil || h.Index > c.applied || h.Pending != nil { + return p, ErrConfig + } + pending, err := c.configurationPending() + if err != nil { + return p, err + } + if pending { + return p, ErrBusy + } + l := h.Leadership + if l == nil { + return p, nil + } + conf := Configuration(c.store.conf) + status := c.node.Status() + // Fallback housekeeping is restricted to a provisioned anchor that has + // actually won a new term. Absent quorum cannot elect or commit this work. + if !l.Restore && status.Lead != l.Node && contains(h.Anchors, status.Lead) { + progress, ok := status.Progress[l.Node] + term, e := c.store.Term(status.GetCommit()) + p.Fallback = e == nil && term == status.GetTerm() && (l.Completed || status.GetTerm() > l.Term && (!ok || !progress.RecentActive)) + } + if p.Fallback { + return p, nil + } + if conf.Joint() { + // Both possible joint configurations must be the exact policy swap. + forward := h + copyIntent := *l + copyIntent.Restore = false + forward.Leadership = ©Intent + incoming := conf.Voters + outgoing := conf.Outgoing + if !(equalIDs(incoming, forward.LeadershipVoters()) && equalIDs(outgoing, h.Anchors) || equalIDs(incoming, h.Anchors) && equalIDs(outgoing, forward.LeadershipVoters())) { + return p, ErrConfig + } + p.Configure = true + p.Change, err = wire.Marshal(&pb.ConfChangeV2{}) + return p, err + } + if h.StableTopology(conf) { + if !l.Settled { + p.Settle = true + return p, nil + } + if !l.Restore && !l.Completed && c.cfg.NodeID != l.Node { + if !c.qualified(l.Node, h.Revision, h.Index, evidence) { + return p, ErrTransfer + } + p.Transfer = l.Node + } + return p, nil + } + forward := h + intent := *l + intent.Restore = false + forward.Leadership = &intent + old := h.Anchors + promote, demote := l.Node, l.Standby + if l.Restore { + old = forward.LeadershipVoters() + promote, demote = demote, promote + } + if !equalIDs(conf.Voters, old) || !contains(conf.Learners, promote) { + return p, ErrConfig + } + if !c.qualified(promote, h.Revision, l.Barrier, evidence) { + return p, ErrTransfer + } + cc := &pb.ConfChangeV2{Transition: pb.ConfChangeTransitionJointExplicit.Enum(), Changes: []*pb.ConfChangeSingle{ + {Type: pb.ConfChangeAddLearnerNode.Enum(), NodeId: proto.Uint64(demote)}, + {Type: pb.ConfChangeAddNode.Enum(), NodeId: proto.Uint64(promote)}, + }} + p.Configure = true + p.Change, err = wire.Marshal(cc) + return p, err +} +func (c *Core) qualified(node, revision, barrier uint64, e LeadershipEvidence) bool { + s := c.node.Status() + progress, ok := s.Progress[node] + return ok && progress.RecentActive && progress.Match >= s.GetCommit() && progress.Match >= barrier && e.Node == node && e.Revision == revision && e.Applied >= barrier && e.Durable && e.Protocol == ProtocolVersion +} +func contains(ids []uint64, id uint64) bool { + for _, n := range ids { + if n == id { + return true + } + } + return false +} +func equalIDs(a, b []uint64) bool { + if len(a) != len(b) { + return false + } + for _, id := range a { + if !contains(b, id) { + return false + } + } + return true +} + +// LeadershipReceipt refuses cached leader hints and barriers from an earlier +// term. quorumIndex is recorded only when this core emits a completed read. +func (c *Core) LeadershipReceipt(h clusterstate.Membership) (clusterstate.LeadershipReceipt, error) { + var r clusterstate.LeadershipReceipt + if c.stopped { + return r, ErrStopped + } + if !c.leader() { + return r, ErrLeader + } + l := h.Leadership + s := c.node.BasicStatus() + if h.Validate() != nil || l == nil || l.Restore || !l.Settled || l.Node != c.cfg.NodeID || h.Index > c.applied || !h.StableTopology(Configuration(c.store.conf)) || c.quorumTerm != s.GetTerm() || c.quorumIndex < h.Index || c.applied < c.quorumIndex { + return r, ErrTransfer + } + term, err := c.store.Term(c.quorumIndex) + if err != nil || term != s.GetTerm() { + return r, ErrTransfer + } + r = clusterstate.LeadershipReceipt{Node: c.cfg.NodeID, Term: s.GetTerm(), Revision: h.Revision, Barrier: c.quorumIndex, Voters: append([]uint64(nil), c.store.conf.GetVoters()...)} + c.quorumTerm, c.quorumIndex = 0, 0 + return r, nil +} diff --git a/internal/raftcore/leadership_test.go b/internal/raftcore/leadership_test.go new file mode 100644 index 0000000000..02afad7277 --- /dev/null +++ b/internal/raftcore/leadership_test.go @@ -0,0 +1,151 @@ +package raftcore + +import ( + "errors" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "testing" +) + +func leadershipGroup(t *testing.T) (*testGroup, cs.Membership) { + g := newGroup(t) + g.elect() + n, e := New(Config{ClusterUUID: "test-cluster", NodeID: 4, Seed: 104, Protocol: 1}, RecoveredState{}) + if e != nil { + t.Fatal(e) + } + g.nodes = append(g.nodes, n) + g.applied = append(g.applied, nil) + g.durable = append(g.durable, RecoveredState{}) + leader := g.leader() + h := cs.Membership{Revision: 2, Index: g.nodes[leader-1].applied, NextNode: 5, Anchors: []uint64{1, 2, 3}, Nodes: []uint64{1, 2, 3, 4}, Live: 1, Limit: 4096, Pending: &cs.MembershipIntent{Node: 4, Revision: 2}} + p, e := g.nodes[leader-1].PlanMembership(h) + if e != nil { + t.Fatal(e) + } + g.call(leader, Configure{p.Change}) + g.pump() + h.Pending = nil + h.Revision++ + h.Index = g.nodes[g.leader()-1].applied + h.Leadership = &cs.LeadershipIntent{Node: 4, Standby: 3, Revision: h.Revision, Barrier: h.Index, Term: g.nodes[g.leader()-1].status().Term} + return g, h +} +func leadEvidence(g *testGroup, h cs.Membership, node uint64) LeadershipEvidence { + return LeadershipEvidence{Node: node, Revision: h.Revision, Applied: g.nodes[node-1].applied, Protocol: 1, Durable: true} +} +func reconcileLeadCore(t *testing.T, g *testGroup, h cs.Membership) { + t.Helper() + for i := 0; i < 8; i++ { + leader := g.leader() + if leader == 0 { + leader = g.elect() + } + target := h.Leadership.Node + if h.Leadership.Restore { + target = h.Leadership.Standby + } + p, e := g.nodes[leader-1].PlanLeadership(h, leadEvidence(g, h, target)) + if errors.Is(e, ErrTransfer) { + g.call(leader, Tick{}) + g.pump() + continue + } + if e != nil { + t.Fatal(p, e) + } + if !p.Configure { + if !p.Settle { + t.Fatal("configuration not settled", p) + } + return + } + g.call(leader, Configure{p.Change}) + g.pump() + } + t.Fatal("swap failed to settle") +} + +// R6/R7/R8/R12 gate the real joint swap, target durable barrier and actual +// election. No fabricated read index, cached status or previous-term read can +// be reported as a successful leadership transfer. +func TestR6R7R8R12LeadershipBarrierAndJointPolicy(t *testing.T) { + g, h := leadershipGroup(t) + leader := g.leader() + evidence := leadEvidence(g, h, 4) + for _, bad := range []LeadershipEvidence{{}, {Node: 4, Revision: h.Revision, Applied: evidence.Applied, Protocol: 1}, {Node: 4, Revision: h.Revision, Applied: h.Index - 1, Protocol: 1, Durable: true}, {Node: 4, Revision: h.Revision - 1, Applied: evidence.Applied, Protocol: 1, Durable: true}} { + if _, err := g.nodes[leader-1].PlanLeadership(h, bad); !errors.Is(err, ErrTransfer) { + t.Fatal("unqualified target admitted", err) + } + } + reconcileLeadCore(t, g, h) + h.Leadership.Settled = true + h.Index = g.nodes[g.leader()-1].applied + leader = g.leader() + p, err := g.nodes[leader-1].PlanLeadership(h, leadEvidence(g, h, 4)) + if err != nil { + t.Fatal(err) + } + if p.Transfer != 4 { + t.Fatal(p) + } + g.call(leader, Transfer{p.Transfer}) + g.pump() + if g.leader() != 4 { + t.Fatal("requested node did not win") + } + if _, err := g.nodes[3].LeadershipReceipt(h); !errors.Is(err, ErrTransfer) { + t.Fatal("intent/election mistaken for success", err) + } + g.call(4, Read{RequestID: "leadership"}) + g.pump() + r, err := g.nodes[3].LeadershipReceipt(h) + if err != nil || r.Node != 4 || r.Barrier < h.Index || len(r.Voters) != 3 { + t.Fatal(r, err) + } + if _, err = g.nodes[3].LeadershipReceipt(h); !errors.Is(err, ErrTransfer) { + t.Fatal("consumed quorum proof was reusable", err) + } + // A sleeping P and a lost anchor leave no majority. No false success can + // escape the remaining anchor, including from its earlier term's barrier. + g.drop = func(from, to uint64) bool { return from == 4 || to == 4 || from == 2 || to == 2 } + g.call(4, Read{RequestID: "isolated-leadership"}) + g.pump() + if _, err = g.nodes[3].LeadershipReceipt(h); !errors.Is(err, ErrTransfer) { + t.Fatal("minority reused earlier read", err) + } + for n := 0; n < 13; n++ { + for _, id := range []uint64{1, 3} { + g.call(id, Tick{}) + } + g.pump() + } + if g.nodes[0].leader() { + t.Fatal("minority elected a leader") + } + // Restore the surviving A+B path, then perform the reverse joint swap + // without the absent P. C remains an enrolled warm learner until promotion. + g.drop = func(from, to uint64) bool { return from == 4 || to == 4 } + // The isolated old leader must relinquish before helper leader inspection. + for n := 0; n < 7; n++ { + g.call(4, Tick{}) + g.pump() + } + g.elect() + h.Revision++ + h.Index = g.nodes[g.leader()-1].applied + h.Leadership = &cs.LeadershipIntent{Node: 4, Standby: 3, Revision: h.Revision, Barrier: h.Index, Restore: true} + reconcileLeadCore(t, g, h) + conf := g.nodes[g.leader()-1].store.conf + if !equalIDs(conf.GetVoters(), []uint64{1, 2, 3}) || !equalIDs(conf.GetLearners(), []uint64{4}) || len(conf.GetVotersOutgoing()) != 0 { + t.Fatal(conf) + } + // Returning P is a learner even after restart with its old durable bytes. + g.drop = nil + g.tick(2) + if contains(g.nodes[3].store.conf.GetVoters(), 4) || !contains(g.nodes[3].store.conf.GetLearners(), 4) { + t.Fatal("P regained vote") + } + if _, err := g.nodes[3].LeadershipReceipt(h); err == nil { + t.Fatal("old receipt retained authority") + } +} diff --git a/internal/raftcore/membership.go b/internal/raftcore/membership.go index 8fcc6ac7ef..7ac6cf60be 100644 --- a/internal/raftcore/membership.go +++ b/internal/raftcore/membership.go @@ -37,6 +37,9 @@ func (c *Core) PlanMembership(h clusterstate.Membership) (MembershipPlan, error) if h.Index > c.applied || h.Revision == 0 { return p, ErrConfig } + if h.Leadership != nil && !h.Leadership.Settled { + return p, ErrBusy + } intent := h.Pending if intent == nil { return p, nil diff --git a/internal/raftcore/random.go b/internal/raftcore/random.go index 74e3cf679c..3c61e9bd50 100644 --- a/internal/raftcore/random.go +++ b/internal/raftcore/random.go @@ -28,3 +28,17 @@ func (r *splitMix64) bounded(n int) int { } } } + +// electionRandom keeps anchor ranks in disjoint windows while leaving Raft's +// quorum-check and leader-lease interval unchanged. Rank is immutable provisioned +// configuration, never the current voter order after a personal-voter swap. +// TIMING VERSION 2 consumes one bounded draw per timeout reset in every runtime. +func (c *Core) electionRandom(bound int) int { + if c.cfg.TimingVersion == 1 { + return c.rng.bounded(bound) + } + if bound != ElectionRandomBound { + panic("raftcore: election timing version mismatch") + } + return c.electionRank*ElectionWindowTicks + c.rng.bounded(ElectionWindowTicks) +} diff --git a/internal/raftcore/types.go b/internal/raftcore/types.go index 6c2e03f731..3440590de4 100644 --- a/internal/raftcore/types.go +++ b/internal/raftcore/types.go @@ -11,16 +11,19 @@ import ( ) const ( - ProtocolVersion uint16 = clusterstate.ClusterProtocol - ElectionTicks = 6 - HeartbeatTicks = 1 - MaxMessageBytes = 64 << 10 - MaxUncommittedBytes = 1 << 20 - MaxInflightBatches = 8 - MaxReadRequests = 128 - MaxRequestIDBytes = 128 - SnapshotEntries = 1024 - SnapshotLogBytes = 4 << 20 + ProtocolVersion uint16 = clusterstate.ClusterProtocol + ElectionTicks = 6 + ElectionWindowTicks = 2 + ElectionTimingVersion = 2 + ElectionRandomBound = 4 * ElectionWindowTicks + HeartbeatTicks = 1 + MaxMessageBytes = 64 << 10 + MaxUncommittedBytes = 1 << 20 + MaxInflightBatches = 8 + MaxReadRequests = 128 + MaxRequestIDBytes = 128 + SnapshotEntries = 1024 + SnapshotLogBytes = 4 << 20 ) var ( @@ -47,6 +50,12 @@ type Config struct { Seed uint64 Protocol uint16 Bootstrap []uint64 + // Anchors is the immutable provisioned manifest in committed rank order. + // Recovery supplies it from applied registry rows, never current voters. + Anchors []uint64 + // TimingVersion is explicit replay compatibility, never a deployment switch. + // Zero selects the current version; version 1 reproduces recorded old inputs. + TimingVersion uint16 } // RecoveredState is immutable input copied at construction. ConfState is the @@ -54,6 +63,7 @@ type Config struct { // Entries contains the contiguous suffix after Snapshot, not just committed // entries. Applied belongs to the atomic application transaction. type RecoveredState struct { + Anchors []uint64 HardState *pb.HardState ConfState *pb.ConfState Snapshot *pb.Snapshot diff --git a/third_party/raft-patch/README.md b/third_party/raft-patch/README.md index eadd0d1d8a..d92500b347 100644 --- a/third_party/raft-patch/README.md +++ b/third_party/raft-patch/README.md @@ -7,6 +7,13 @@ Only `raft.go` is modified. `election-random.patch` is the complete upstream diff, including the local-change comments; no election algorithm is rewritten. The added optional `Config.ElectionRandom func(int) int` belongs to one node. +`ElectionRandomBound` optionally changes only that provider's bounded offset; +zero retains upstream's bound. `ElectionTick`, CheckQuorum, PreVote lease checks, +votes and transfer remain unchanged. codeaf uses bound eight and returns a +ranked width-two draw so anchors ask before a promoted personal voter. Rank +comes from the committed provisioned-anchor manifest. This timing extension +also modifies only `raft.go`; the complete recorded patch reverses to the same +independently hashed upstream bytes. It is consulted only by `resetRandomizedElectionTimeout`. Nil preserves the upstream behavior for upstream tests/users; invalid bounds panic. codeaf always sets it to the core-owned SplitMix64 provider, so the upstream global RNG is diff --git a/third_party/raft-patch/election-random.patch b/third_party/raft-patch/election-random.patch index c0b6c642b4..7a6bca5751 100644 --- a/third_party/raft-patch/election-random.patch +++ b/third_party/raft-patch/election-random.patch @@ -1,47 +1,40 @@ --- a/raft.go +++ b/raft.go -@@ -126,6 +126,11 @@ - type Config struct { - // ID is the identity of the local raft. ID cannot be 0. - ID uint64 -+ +@@ -14,0 +15,2 @@ ++// codeaf modifies election randomness and its offset bound for deterministic, ++// anchor-ranked timing; quorum checks and voting rules retain upstream behavior. +@@ -129,0 +132,9 @@ + // ElectionRandom optionally supplies per-node bounded election randomness. + // It must return a value in [0, n) and must not be shared between nodes. + // A nil provider preserves the upstream default for existing callers. + ElectionRandom func(n int) int - - // ElectionTick is the number of Node.Tick invocations that must pass between - // elections. That is, if a follower does not receive any message from the -@@ -419,6 +424,7 @@ - // [electiontimeout, 2 * electiontimeout - 1]. It gets reset - // when raft changes its state to follower or candidate. - randomizedElectionTimeout int ++ ++ // ElectionRandomBound optionally widens the bounded offset without changing ++ // ElectionTick's quorum and leader-lease timing. Zero retains upstream timing. ++ ElectionRandomBound int ++ +@@ -293,0 +305,3 @@ ++ if c.ElectionRandomBound < 0 || c.ElectionRandomBound > int(^uint(0)>>1)-c.ElectionTick { ++ return errors.New("invalid election random bound") ++ } +@@ -421,0 +436,2 @@ + electionRandom func(int) int - disableProposalForwarding bool - stepDownOnRemoval bool - -@@ -448,6 +454,7 @@ - - r := &raft{ - id: c.ID, ++ electionRandomBound int +@@ -450,0 +467,2 @@ + electionRandom: c.ElectionRandom, - lead: None, - isLearner: false, - raftLog: raftlog, -@@ -2051,7 +2058,15 @@ - } - - func (r *raft) resetRandomizedElectionTimeout() { ++ electionRandomBound: c.ElectionRandomBound, +@@ -2054 +2072,13 @@ - r.randomizedElectionTimeout = r.electionTimeout + globalRand.Intn(r.electionTimeout) + random := r.electionRandom + if random == nil { + random = globalRand.Intn + } -+ v := random(r.electionTimeout) -+ if v < 0 || v >= r.electionTimeout { ++ bound := r.electionRandomBound ++ if bound == 0 { ++ bound = r.electionTimeout ++ } ++ v := random(bound) ++ if v < 0 || v >= bound { + panic("raft: election random value outside bound") + } + r.randomizedElectionTimeout = r.electionTimeout + v - } - - func (r *raft) sendTimeoutNow(to uint64) { diff --git a/third_party/raft/raft.go b/third_party/raft/raft.go index 8f94a11d96..c72c646d71 100644 --- a/third_party/raft/raft.go +++ b/third_party/raft/raft.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// codeaf modifies election randomness and its offset bound for deterministic, +// anchor-ranked timing; quorum checks and voting rules retain upstream behavior. package raft import ( @@ -132,6 +134,10 @@ type Config struct { // A nil provider preserves the upstream default for existing callers. ElectionRandom func(n int) int + // ElectionRandomBound optionally widens the bounded offset without changing + // ElectionTick's quorum and leader-lease timing. Zero retains upstream timing. + ElectionRandomBound int + // ElectionTick is the number of Node.Tick invocations that must pass between // elections. That is, if a follower does not receive any message from the // leader of current term before ElectionTick has elapsed, it will become @@ -296,6 +302,9 @@ type Config struct { } func (c *Config) validate() error { + if c.ElectionRandomBound < 0 || c.ElectionRandomBound > int(^uint(0)>>1)-c.ElectionTick { + return errors.New("invalid election random bound") + } if c.ID == None { return errors.New("cannot use none as id") } @@ -425,6 +434,7 @@ type raft struct { // when raft changes its state to follower or candidate. randomizedElectionTimeout int electionRandom func(int) int + electionRandomBound int disableProposalForwarding bool stepDownOnRemoval bool @@ -455,6 +465,7 @@ func newRaft(c *Config) *raft { r := &raft{ id: c.ID, electionRandom: c.ElectionRandom, + electionRandomBound: c.ElectionRandomBound, lead: None, isLearner: false, raftLog: raftlog, @@ -2062,8 +2073,12 @@ func (r *raft) resetRandomizedElectionTimeout() { if random == nil { random = globalRand.Intn } - v := random(r.electionTimeout) - if v < 0 || v >= r.electionTimeout { + bound := r.electionRandomBound + if bound == 0 { + bound = r.electionTimeout + } + v := random(bound) + if v < 0 || v >= bound { panic("raft: election random value outside bound") } r.randomizedElectionTimeout = r.electionTimeout + v From a1ff577898ab233ed3738918a65acab3196c8aa8 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 03:22:55 -0400 Subject: [PATCH 029/103] Carry leadership and anchor rank through native and hosted control doors. Recover the committed anchor order through registry and import snapshots in both runtimes. Serialize readiness and control effects, recheck current authorization, persist original request retries and document the actual commands, refusals and fallback limits. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/codeaf-raft-probe/actor.go | 29 +- cmd/codeaf/cluster.go | 75 ++++ cmd/codeaf/cluster_test.go | 53 +++ cmd/codeaf/main.go | 4 +- cmd/codeaf/usage.go | 6 +- internal/clusterctl/control.go | 173 ++++++++ internal/clusterctl/control_test.go | 44 +++ .../continuing-a-chat-on-another-computer.md | 21 +- internal/manual/chat_test.go | 1 + internal/raftabi/README.md | 25 ++ internal/raftabi/abi.go | 11 + internal/raftabi/application.go | 3 + internal/raftabi/leadership.go | 49 +++ internal/raftdirectory/apply.go | 2 +- internal/raftdirectory/directory_test.go | 2 +- internal/raftdirectory/leadership.go | 113 ++++++ internal/raftdirectory/leadership_test.go | 374 ++++++++++++++++++ internal/raftstore/recover.go | 30 +- internal/raftstore/sqlite.go | 4 + internal/syncsetup/cluster.go | 147 +++++++ internal/syncsetup/cluster_test.go | 107 +++++ relay/hosted/src/raft/abi.js | 6 +- relay/hosted/src/raft/actor.js | 35 +- relay/hosted/src/raft/directory.js | 10 +- relay/hosted/src/raft/member.js | 3 + relay/hosted/src/raft/storage.js | 2 +- 26 files changed, 1317 insertions(+), 12 deletions(-) create mode 100644 cmd/codeaf/cluster.go create mode 100644 cmd/codeaf/cluster_test.go create mode 100644 internal/clusterctl/control.go create mode 100644 internal/clusterctl/control_test.go create mode 100644 internal/raftabi/leadership.go create mode 100644 internal/raftdirectory/leadership.go create mode 100644 internal/raftdirectory/leadership_test.go create mode 100644 internal/syncsetup/cluster.go create mode 100644 internal/syncsetup/cluster_test.go diff --git a/cmd/codeaf-raft-probe/actor.go b/cmd/codeaf-raft-probe/actor.go index c07df2026e..38e406501e 100644 --- a/cmd/codeaf-raft-probe/actor.go +++ b/cmd/codeaf-raft-probe/actor.go @@ -20,6 +20,7 @@ import ( ) type actorRequest struct { + Header []byte `json:"header"` Rows []clusterstate.RecordWrite `json:"rows"` Op string `json:"op"` Cluster string `json:"cluster"` @@ -123,6 +124,32 @@ func (a *probeActor) open() error { func (a *probeActor) call(r actorRequest, dir string) (any, error) { ctx := context.Background() switch r.Op { + case "leadership-plan", "leadership-receipt": + var h clusterstate.Membership + if err := clusterstate.Decode(r.Header, &h); err != nil { + return nil, err + } + var value any + if r.Op == "leadership-receipt" { + v, err := a.core.LeadershipReceipt(h) + if err != nil { + return nil, err + } + value = v + } else { + var evidence raftcore.LeadershipEvidence + if err := clusterstate.Decode(r.Value, &evidence); err != nil { + return nil, err + } + v, err := a.core.PlanLeadership(h, evidence) + if err != nil { + return nil, err + } + value = v + } + b, err := clusterstate.Encode(value) + return json.RawMessage(b), err + case "legacy-import": var m clusterstate.LegacyManifest if err := clusterstate.Decode(r.Value, &m); err != nil { @@ -162,7 +189,7 @@ func (a *probeActor) call(r actorRequest, dir string) (any, error) { return nil, err } } - a.cfg = raftcore.Config{ClusterUUID: r.Cluster, NodeID: r.ID, Seed: 100 + r.ID, Protocol: 1, Bootstrap: []uint64{1, 2, 3}} + a.cfg = raftcore.Config{ClusterUUID: r.Cluster, NodeID: r.ID, Seed: 100 + r.ID, Protocol: 1, Anchors: []uint64{1, 2, 3}, Bootstrap: []uint64{1, 2, 3}} a.path = filepath.Join(dir, fmt.Sprintf("%s-%d.db", r.Cluster, r.ID)) if err := a.open(); err != nil { return nil, err diff --git a/cmd/codeaf/cluster.go b/cmd/codeaf/cluster.go new file mode 100644 index 0000000000..1935e3063d --- /dev/null +++ b/cmd/codeaf/cluster.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + "errors" + "fmt" + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/syncsetup" + "io" + "os" + "time" +) + +const clusterHelp = `codeaf cluster status + Show the voting computers and the current coordinator. +codeaf cluster lead + Ask this joined computer to coordinate. Keep its persistent store available. + Success includes the elected computer, term and quorum read barrier. + Three voters tolerate one unavailable member. A sleeping personal leader is + replaced by the relay anchors; it leads again only after another request. +` + +type clusterDoor interface { + ClusterStatus(context.Context) (clusterctl.Status, error) + LeadCluster(context.Context, uint64) (cs.LeadershipReceipt, error) +} + +func runCluster(args []string) error { + if askedForHelp(args) { + fmt.Fprint(usageOut, clusterHelp) + return exitHelped + } + if len(args) != 1 || args[0] != "status" && args[0] != "lead" { + return errors.New("usage: codeaf cluster status | lead") + } + if syncsetup.Quiet(home.Dir()) { + return syncsetup.ErrQuiet + } + s, ok, err := syncsetup.Open(home.Dir()) + if err != nil { + return err + } + if !ok { + return errors.New("sync is off on this computer") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + return clusterCommand(ctx, s, args[0], os.Stdout) +} +func clusterCommand(ctx context.Context, c clusterDoor, verb string, out io.Writer) error { + if verb == "lead" { + r, err := c.LeadCluster(ctx, 0) + if err != nil { + return err + } + _, err = fmt.Fprintf(out, "computer %d is coordinating · term %d · revision %d · read barrier %d\n", r.Node, r.Term, r.Revision, r.Barrier) + return err + } + s, err := c.ClusterStatus(ctx) + if err != nil { + return err + } + if s.Leader != 0 { + fmt.Fprintf(out, "coordinator %d\n", s.Leader) + } + if len(s.Configuration.Voters) > 0 { + fmt.Fprintf(out, "voting computers %v\n", s.Configuration.Voters) + } + if len(s.Configuration.Outgoing) > 0 { + fmt.Fprintf(out, "finishing the change from %v\n", s.Configuration.Outgoing) + } + return nil +} diff --git a/cmd/codeaf/cluster_test.go b/cmd/codeaf/cluster_test.go new file mode 100644 index 0000000000..060de1645f --- /dev/null +++ b/cmd/codeaf/cluster_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "bytes" + "context" + "errors" + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "strings" + "testing" +) + +type clusterFixture struct { + receipt cs.LeadershipReceipt + err error +} + +func (c clusterFixture) ClusterStatus(context.Context) (clusterctl.Status, error) { + return clusterctl.Status{Leader: 4, Configuration: cs.Configuration{Voters: []uint64{1, 2, 4}, Learners: []uint64{3}}}, c.err +} +func (c clusterFixture) LeadCluster(context.Context, uint64) (cs.LeadershipReceipt, error) { + return c.receipt, c.err +} + +// R6/R8/R30 pin the actual CLI receipt and refusal. A pending transfer never +// prints a successful role, and help remains usable before pairing or setup. +func TestR6R8R30ClusterCommandReceiptAndRefusal(t *testing.T) { + var out bytes.Buffer + c := clusterFixture{receipt: cs.LeadershipReceipt{Node: 4, Term: 8, Revision: 3, Barrier: 19, Voters: []uint64{1, 2, 4}}} + if e := clusterCommand(context.Background(), c, "lead", &out); e != nil { + t.Fatal(e) + } + if got := out.String(); got != "computer 4 is coordinating · term 8 · revision 3 · read barrier 19\n" { + t.Fatal(got) + } + out.Reset() + c.err = clusterctl.ErrUnavailable + if e := clusterCommand(context.Background(), c, "lead", &out); !errors.Is(e, clusterctl.ErrUnavailable) || out.Len() != 0 { + t.Fatal(e, out.String()) + } + c.err = nil + if e := clusterCommand(context.Background(), c, "status", &out); e != nil { + t.Fatal(e) + } + if !strings.Contains(out.String(), "voting computers [1 2 4]") { + t.Fatal(out.String()) + } + for _, arg := range []string{"--help", "-h", "-help"} { + if code := exitCodeOf(runCluster([]string{arg})); code != 0 { + t.Fatal(code) + } + } +} diff --git a/cmd/codeaf/main.go b/cmd/codeaf/main.go index 37ceec6d1e..d3a3f1b4c1 100644 --- a/cmd/codeaf/main.go +++ b/cmd/codeaf/main.go @@ -330,6 +330,8 @@ func run() error { // whoever hosts the relay and not typed in a conversation, so like // `engine` it is absent from the usage text. return relayserve.Main(os.Args[2:]) + case "cluster": + return runCluster(os.Args[2:]) case "devices": // Who is allowed to open a conversation here, and the door for taking // that back. REVOKING IS THIS MACHINE'S DECISION AND ONLY THIS @@ -606,7 +608,7 @@ Housekeeping — changes state on disk or on the network approve a new device; or use or show a six-digit code for your chats codeaf serve [--workspace path] [--via url] be reachable from your other devices without ssh, with a pairing code - codeaf devices + codeaf devices | codeaf cluster status | lead list your devices and the ones that can use this machine codeaf devices revoke [--all] | rename remove one (cannot undo what a device holds), or name this one diff --git a/cmd/codeaf/usage.go b/cmd/codeaf/usage.go index 11d39f1c6a..c578952c85 100644 --- a/cmd/codeaf/usage.go +++ b/cmd/codeaf/usage.go @@ -450,6 +450,10 @@ func askedForHelp(args []string) bool { // commandHelp answers that gesture for a door with no flag set of its own. func commandHelp(name string) error { + if name == "cluster" { + fmt.Fprint(usageOut, clusterHelp) + return exitHelped + } writeCommandUsage(usageOut, commandFlags(name)) return exitHelped } @@ -519,7 +523,7 @@ func commandWords() []string { // programs this build carries are joined to it where it is read // ([commandWords]). var knownCommands = []string{ - "chat", "resume", "pair", "serve", "devices", "do", "plan", "revise", "run", "exec", + "chat", "resume", "pair", "serve", "devices", "cluster", "do", "plan", "revise", "run", "exec", "show", "models", "pool", "notebook", "collections", "competence", "services", "patch", "doc", "web", "image", "doctor", diff --git a/internal/clusterctl/control.go b/internal/clusterctl/control.go new file mode 100644 index 0000000000..2da0f84881 --- /dev/null +++ b/internal/clusterctl/control.go @@ -0,0 +1,173 @@ +// Package clusterctl is the authenticated leadership door. Consensus hosts +// supply committed reads and completed transfers; this wire owns no voter list. +package clusterctl + +import ( + "bytes" + "context" + "errors" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/wireauth" + "io" + "net/http" + "strings" + "time" +) + +const Path = "/v2/cluster/control" + +var ErrUnavailable = errors.New(cs.LeadershipUnavailable) + +type CommittedRefusal struct { + Code cs.Code `json:"code"` + Sequence uint64 `json:"sequence"` +} + +func (e CommittedRefusal) Error() string { + return "the leadership request could not complete; refresh cluster status and try again" +} + +type Status struct { + Cluster string `json:"cluster"` + Self uint64 `json:"self"` + Leader uint64 `json:"leader"` + Membership cs.Membership `json:"membership"` + Configuration cs.Configuration `json:"configuration"` +} + +// Backend must authorize against committed state after a fresh quorum read. +// Lead returns only after the target is elected, commits its term and passes a +// new read barrier. Admission, a pending transfer and cached status are errors. +// A durably refused or superseded intent returns CommittedRefusal with its +// original sequence, allowing the client to acknowledge it before a new request. +type Backend interface { + Status(context.Context, string) (Status, error) + Lead(context.Context, cs.Command) (cs.LeadershipReceipt, error) +} + +func Handler(auth wireauth.Authenticate, open func(string) (Backend, error)) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != Path || r.Method != http.MethodGet && r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, cs.MaxCommandBytes)) + if err != nil { + http.Error(w, "request is too large", http.StatusRequestEntityTooLarge) + return + } + identity, device, err := auth(r, body) + if err != nil { + http.Error(w, "this computer is not allowed to coordinate this cluster", http.StatusForbidden) + return + } + backend, err := open(identity) + if err != nil { + http.Error(w, cs.LeadershipUnavailable, http.StatusServiceUnavailable) + return + } + var result any + if r.Method == http.MethodGet { + if len(body) != 0 { + http.Error(w, "unexpected request body", 400) + return + } + result, err = backend.Status(r.Context(), device) + } else { + var c cs.Command + c, err = cs.DecodeCommand(body) + if err != nil || c.Actor != device || c.Change.RequestLeadership == nil { + http.Error(w, "refresh cluster status before requesting leadership", 400) + return + } + result, err = backend.Lead(r.Context(), c) + } + if err != nil { + var refused CommittedRefusal + if errors.As(err, &refused) { + b, _ := cs.Encode(refused) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusConflict) + _, _ = w.Write(b) + return + } + http.Error(w, cs.LeadershipUnavailable, http.StatusServiceUnavailable) + return + } + b, err := cs.Encode(result) + if err != nil { + http.Error(w, cs.LeadershipUnavailable, 503) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(b) + }) +} + +type Client struct { + Base string + Sign wireauth.Sign + HTTP *http.Client +} + +func (c Client) request(ctx context.Context, method string, body []byte, out any) error { + req, err := http.NewRequestWithContext(ctx, method, strings.TrimRight(c.Base, "/")+Path, bytes.NewReader(body)) + if err != nil { + return err + } + if c.Sign == nil { + return wireauth.ErrUnauthorized + } + c.Sign(req, body) + hc := http.Client{Timeout: 30 * time.Second} + if c.HTTP != nil { + hc = *c.HTTP + } + if hc.Timeout <= 0 || hc.Timeout > 30*time.Second { + hc.Timeout = 30 * time.Second + } + hc.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + res, err := hc.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + b, err := io.ReadAll(io.LimitReader(res.Body, cs.MaxEvaluationBytes+1)) + if err != nil { + return err + } + if len(b) > cs.MaxEvaluationBytes { + return ErrUnavailable + } + if res.StatusCode == http.StatusConflict { + var refused CommittedRefusal + if cs.Decode(b, &refused) == nil { + return refused + } + } + if res.StatusCode != 200 { + if res.StatusCode == 404 { + return errors.New("this relay does not offer cluster leadership yet; your saved chats are still here") + } + return ErrUnavailable + } + return cs.Decode(b, out) +} +func (c Client) Status(ctx context.Context) (s Status, err error) { + err = c.request(ctx, http.MethodGet, nil, &s) + return +} +func (c Client) Lead(ctx context.Context, command cs.Command) (r cs.LeadershipReceipt, err error) { + if command.Change.RequestLeadership == nil { + return r, errors.New("a leadership request is required") + } + b, err := cs.EncodeCommand(command) + if err != nil { + return r, err + } + err = c.request(ctx, http.MethodPost, b, &r) + if err == nil && (r.Node != command.Change.RequestLeadership.Node || r.Term == 0 || r.Barrier == 0 || len(r.Voters) != 3 || r.Revision <= command.Change.RequestLeadership.Revision) { + err = ErrUnavailable + } + return +} diff --git a/internal/clusterctl/control_test.go b/internal/clusterctl/control_test.go new file mode 100644 index 0000000000..34909623b8 --- /dev/null +++ b/internal/clusterctl/control_test.go @@ -0,0 +1,44 @@ +package clusterctl_test + +import ( + "context" + ctl "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "net/http" + "net/http/httptest" + "testing" +) + +type retained struct{ nodes []uint64 } + +func (b retained) Status(context.Context, string) (ctl.Status, error) { + h := cs.Membership{Revision: 2, Index: 1, NextNode: b.nodes[len(b.nodes)-1] + 1, Anchors: b.nodes[:3], Nodes: b.nodes, Live: 1, Limit: cs.MaxMemberRecords} + if err := h.Validate(); err != nil { + return ctl.Status{}, err + } + return ctl.Status{Cluster: "large-node-ids", Self: b.nodes[3], Leader: b.nodes[0], Membership: h, Configuration: cs.Configuration{Voters: b.nodes[:3]}}, nil +} +func (b retained) Lead(context.Context, cs.Command) (cs.LeadershipReceipt, error) { + return cs.LeadershipReceipt{}, ctl.ErrUnavailable +} + +// R6/R27/R30 keep status readable when permanent node tombstones use the full +// uint64 range. The response envelope is distinct from the smaller log command +// budget; neither response truncation nor deleting retained records is valid. +func TestR6R27R30StatusRetainsFullWidthNodeHistory(t *testing.T) { + nodes := make([]uint64, cs.MaxMemberRecords) + for i := range nodes { + nodes[i] = 1<<63 + uint64(i) + } + backend := retained{nodes} + server := httptest.NewServer(ctl.Handler(func(*http.Request, []byte) (string, string, error) { return "identity", "device", nil }, func(string) (ctl.Backend, error) { return backend, nil })) + defer server.Close() + client := ctl.Client{Base: server.URL, Sign: func(*http.Request, []byte) {}, HTTP: server.Client()} + status, err := client.Status(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(status.Membership.Nodes) != len(nodes) || status.Membership.Nodes[len(nodes)-1] != nodes[len(nodes)-1] { + t.Fatal("retained IDs were truncated") + } +} diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index 6b86a113e6..5326202240 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -267,4 +267,23 @@ with a separate operator capacity limit; reaching either limit refuses new enrollment instead of deleting saved chats or old records. The capacity sentence is `this cluster has reached its enrollment limit; remove a joined computer or ask the relay operator for capacity`. An unsupported protocol says `this codeaf build must be updated before it can join this cluster; your saved chats are still here`. -This does not add a leadership command or a worker setup command. +Joining alone does not request leadership. Use `codeaf cluster lead` after this computer has joined with persistent storage. A worker setup command is not available yet. + +## Which computer coordinates: cluster status and cluster lead + +`codeaf cluster status` asks the relay for the current coordinator and voting computers. `codeaf cluster lead` requests leadership for this computer after it has joined with persistent storage. The target must catch up and exchange recent traffic; a pending request is not success. A successful line names the computer, term, revision and read barrier. Leadership moves coordination metadata; it does not move chat ownership or download chat content. Continue a chat through the usual continue door. + +The default has three persistent relay voters. Paired computers join without votes. To lead, one caught-up personal computer replaces one anchor through a majority-approved change, retaining three voters; that anchor stays caught up without voting. When the personal leader disappears, the two remaining anchor voters can elect a replacement and restore the third anchor. The returning computer does not automatically lead again. Request leadership explicitly. + +Three voters tolerate one unavailable member. Losing the personal leader and one of its two voting anchors pauses coordination. Saved chats remain. The refusal is `this cluster cannot coordinate right now; reconnect enough of its voting computers and try again; your saved chats are still here`. Network partitions and scheduling stalls can delay elections beyond the healthy fallback target. A relay without this door says `this relay does not offer cluster leadership yet; your saved chats are still here`. Persistent member/client assembly is required; a paired API client alone cannot lead. It says `this computer must join the cluster with persistent storage before it can lead`. If a reply is lost, running `codeaf cluster lead` again resumes the saved request rather than silently choosing a new configuration. + +## How long does a replacement leader take after my laptop sleeps? + +With the two remaining relay voters reachable, codeaf aims to accept a new +conditional operation within about 65 seconds. Relay anchors ask to lead in +ordered randomized windows: the first at 30–35 seconds, the second at 40–45, +and the third at 50–55. A promoted personal computer uses 60–65 seconds. +Network partitions or scheduling stalls can take longer. If the personal leader +and either remaining relay voter are both unavailable before the third anchor +is restored, writes wait for a majority to return. Returning computers stay +learners; leadership returns only when you request it again. diff --git a/internal/manual/chat_test.go b/internal/manual/chat_test.go index 471f65bbe4..6070d68cce 100644 --- a/internal/manual/chat_test.go +++ b/internal/manual/chat_test.go @@ -3212,6 +3212,7 @@ func TestTheChatManualAnswersTheQuestionsPeopleAsk(t *testing.T) { {"a folder was left out of the move because it could not be read", "home"}, {"how do I take a chat back from my other computer", "continuing-a-chat-on-another-computer"}, {"what happens if two computers continue the same chat at once", "continuing-a-chat-on-another-computer"}, + {"which computer coordinates cluster status cluster lead", "continuing-a-chat-on-another-computer"}, {"why does it say this chat is arriving", "continuing-a-chat-on-another-computer"}, {"does /remember work on my other computer", "what-i-remember"}, {"are my memories shared across computers", "what-i-remember"}, diff --git a/internal/raftabi/README.md b/internal/raftabi/README.md index 696dee7f80..44015d1ad7 100644 --- a/internal/raftabi/README.md +++ b/internal/raftabi/README.md @@ -158,3 +158,28 @@ returns the shared genesis Entry/HardState/ConfState protobuf bytes), and op 14 (validated public certificates/effects authority rows). These calls convert bounded bytes and establish no request authentication or successful coordination receipt. Source/target hosts own the irrevocable migration gate and durability. + +P13 adds op 21 (node handle, canonical Membership header, canonical target +LeadershipEvidence; returns a canonical LeadershipPlan) and op 22 (node handle, +canonical Membership header; returns a consumed current-term LeadershipReceipt). +These are internal host calls, never public configuration or readiness inputs. +Plans distinguish Configure from empty bytes because the explicit leave-joint +protobuf has zero bytes. Operation 15 accepts an optional trailing entry term; +new hosts supply the actual committed entry term when evaluating leadership +intents, and old directory-only frames retain their prior representation. + +The expanded mixed-runtime proof sets its Node coordinator old-generation limit to 16 MiB so +its retained request/report garbage does not consume the native/workerd share +of the unchanged 256 MiB combined RSS gate. The same six hosted handles, native +process death, both personal-leader roles and all snapshot/configuration steps +remain in the workload; production isolate/core ceilings are unchanged. + +## Ranked election timing (R7/R14) + +ABI 1 create optionally appends an anchor count (exactly three) and three uint64 +IDs after the applied index. Old mechanism frames may omit it. Hosts obtain the +ordered IDs from the durably applied membership registry, or the same provisioned +bootstrap manifest before initialization. They must never use the current voter +list, which changes when C becomes a standby. The core uses election timing +version 2 in native and WASI. Rank survives snapshots and restarts as registry +state; absent provisioning identifies a personal learner, never an invented rank. diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index a54dc5fceb..679123f988 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -164,6 +164,8 @@ func (r *Runtime) Call(b []byte) (out []byte) { func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { e := encoder{} switch op { + case 21, 22: + return r.leadership(d, op == 22) case 20: return configurationAtEntry(d) case 19: @@ -209,6 +211,15 @@ func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { } recovered.Entries = d.entries() recovered.Applied = d.u64() + if len(d.b) != 0 { + n := d.u32() + if n != 3 { + return nil, errFrame + } + for range n { + recovered.Anchors = append(recovered.Anchors, d.u64()) + } + } if err := d.end(); err != nil { return nil, err } diff --git a/internal/raftabi/application.go b/internal/raftabi/application.go index 3be24c7d9f..3df4d4caf0 100644 --- a/internal/raftabi/application.go +++ b/internal/raftabi/application.go @@ -50,6 +50,9 @@ func applicationCodec(op uint32, d *decoder) ([]byte, error) { conf := new(pb.ConfState) d.message(conf) meta.Configuration = raftcore.Configuration(conf) + if len(d.b) > 0 { + meta.Term = d.u64() + } } if err := d.end(); err != nil { return nil, err diff --git a/internal/raftabi/leadership.go b/internal/raftabi/leadership.go new file mode 100644 index 0000000000..88c0fe1726 --- /dev/null +++ b/internal/raftabi/leadership.go @@ -0,0 +1,49 @@ +package raftabi + +import ( + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" +) + +func (r *Runtime) leadership(d *decoder, receipt bool) ([]byte, error) { + handle, raw := d.u32(), d.blob() + node := r.nodes[handle] + if node == nil { + return nil, errFrame + } + var h clusterstate.Membership + if err := clusterstate.Decode(raw, &h); err != nil { + return nil, err + } + var value any + if receipt { + if err := d.end(); err != nil { + return nil, err + } + v, err := node.LeadershipReceipt(h) + if err != nil { + return nil, err + } + value = v + } else { + var evidence raftcore.LeadershipEvidence + if err := clusterstate.Decode(d.blob(), &evidence); err != nil { + return nil, err + } + if err := d.end(); err != nil { + return nil, err + } + v, err := node.PlanLeadership(h, evidence) + if err != nil { + return nil, err + } + value = v + } + b, err := clusterstate.Encode(value) + if err != nil { + return nil, err + } + e := encoder{} + e.blob(b) + return e.b, nil +} diff --git a/internal/raftdirectory/apply.go b/internal/raftdirectory/apply.go index bde8c0248d..5945dc969e 100644 --- a/internal/raftdirectory/apply.go +++ b/internal/raftdirectory/apply.go @@ -57,7 +57,7 @@ func Apply(ctx context.Context, store *raftstore.SQLite, b raftcore.ApplyBatch, if entry.GetType() != pb.EntryNormal || len(entry.GetData()) == 0 { continue } - e, err := clusterstate.Evaluate(entry.GetData(), clusterstate.ApplicationMeta{Index: entry.GetIndex(), Configuration: raftcore.Configuration(conf), Cluster: cluster, Now: update.LogicalTime, Version: update.VisibleVersion, Bytes: size, Ceiling: ceiling}, read) + e, err := clusterstate.Evaluate(entry.GetData(), clusterstate.ApplicationMeta{Term: entry.GetTerm(), Index: entry.GetIndex(), Configuration: raftcore.Configuration(conf), Cluster: cluster, Now: update.LogicalTime, Version: update.VisibleVersion, Bytes: size, Ceiling: ceiling}, read) if err != nil { return nil, err } diff --git a/internal/raftdirectory/directory_test.go b/internal/raftdirectory/directory_test.go index 5acebb56d6..a7dcf59ca9 100644 --- a/internal/raftdirectory/directory_test.go +++ b/internal/raftdirectory/directory_test.go @@ -160,7 +160,7 @@ func (g *group) drain() { } func (g *group) leader() int { for i, s := range g.status { - if s.Role == raft.StateLeader { + if s.Role == raft.StateLeader && !g.absent[uint64(i+1)] { return i } } diff --git a/internal/raftdirectory/leadership.go b/internal/raftdirectory/leadership.go new file mode 100644 index 0000000000..5cd618f80f --- /dev/null +++ b/internal/raftdirectory/leadership.go @@ -0,0 +1,113 @@ +package raftdirectory + +import ( + "context" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftstore" +) + +// LeadershipState is called under the native actor lock. A public status read +// first passes its ordinary authorization and current-term quorum barrier; +// this helper never opens content storage or supplies a local-success route. +func LeadershipState(ctx context.Context, store *raftstore.SQLite) (cs.Membership, error) { + app, _, err := store.ApplicationView(ctx) + if err != nil { + return cs.Membership{}, err + } + k := raftstore.Key{Namespace: raftstore.Membership, Key: cs.MembershipKey().Key} + rows, _, err := store.Read(ctx, app.Applied, []raftstore.Key{k}) + if err != nil { + return cs.Membership{}, err + } + var h cs.Membership + if cs.Decode(rows[k], &h) != nil || h.Validate() != nil || h.Index > app.Applied { + return h, cs.ErrDependency + } + return h, nil +} + +// DurableLeadershipEvidence is an authenticated target response. The caller +// binds it to that member's transport generation and the requested revision; +// the leader separately checks actual Raft acknowledgment progress. A diskless +// host cannot call this with a store and therefore cannot qualify as a voter. +func DurableLeadershipEvidence(ctx context.Context, store *raftstore.SQLite, node, revision, barrier uint64) (raftcore.LeadershipEvidence, error) { + if store.MemberIdentity().NodeID != node { + return raftcore.LeadershipEvidence{}, raftcore.ErrTransfer + } + app, _, err := store.ApplicationView(ctx) + if err != nil { + return raftcore.LeadershipEvidence{}, err + } + h, err := LeadershipState(ctx, store) + if err != nil { + return raftcore.LeadershipEvidence{}, err + } + l := h.Leadership + if l == nil || h.Revision != revision || app.Applied < barrier || l.Barrier != barrier { + return raftcore.LeadershipEvidence{}, raftcore.ErrTransfer + } + k := raftstore.Key{Namespace: raftstore.Enrollment, Key: cs.MemberKey(node).Key} + rows, _, err := store.Read(ctx, app.Applied, []raftstore.Key{k}) + if err != nil { + return raftcore.LeadershipEvidence{}, err + } + var m cs.MemberRecord + if cs.Decode(rows[k], &m) != nil || m.Node != node || m.Retired || m.Protocol != raftcore.ProtocolVersion { + return raftcore.LeadershipEvidence{}, raftcore.ErrTransfer + } + if m.Device != "" { + k := raftstore.Key{Namespace: raftstore.State, Key: cs.DeviceKey(m.Device).Key} + rows, _, err := store.Read(ctx, app.Applied, []raftstore.Key{k}) + if err != nil { + return raftcore.LeadershipEvidence{}, err + } + if rows[k] != nil { + var d cs.Device + if cs.Decode(rows[k], &d) != nil || d.Revoked { + return raftcore.LeadershipEvidence{}, raftcore.ErrTransfer + } + } + } + return raftcore.LeadershipEvidence{Node: node, Revision: revision, Applied: app.Applied, Protocol: m.Protocol, Durable: true}, nil +} + +// PlanLeadership rechecks the leader's committed authorization, independently +// of a previously authenticated target reply. Revocation may commit after that +// reply was produced; a lagging target cannot vouch its old authority back in. +func PlanLeadership(ctx context.Context, store *raftstore.SQLite, core *raftcore.Core, evidence raftcore.LeadershipEvidence) (cs.Membership, raftcore.LeadershipPlan, error) { + h, err := LeadershipState(ctx, store) + if err != nil { + return h, raftcore.LeadershipPlan{}, err + } + if evidence.Node != 0 { + app, _, err := store.ApplicationView(ctx) + if err != nil { + return h, raftcore.LeadershipPlan{}, err + } + k := raftstore.Key{Namespace: raftstore.Enrollment, Key: cs.MemberKey(evidence.Node).Key} + rows, _, err := store.Read(ctx, app.Applied, []raftstore.Key{k}) + if err != nil { + return h, raftcore.LeadershipPlan{}, err + } + var member cs.MemberRecord + if cs.Decode(rows[k], &member) != nil || member.Node != evidence.Node { + return h, raftcore.LeadershipPlan{}, raftcore.ErrTransfer + } + var device []byte + if member.Device != "" { + k = raftstore.Key{Namespace: raftstore.State, Key: cs.DeviceKey(member.Device).Key} + rows, _, err = store.Read(ctx, app.Applied, []raftstore.Key{k}) + if err != nil { + return h, raftcore.LeadershipPlan{}, err + } + device = rows[k] + } + peer, err := cs.PeerFromRecord(member, device, raftcore.Configuration(app.ConfState)) + if err != nil || peer.Revoked { + return h, raftcore.LeadershipPlan{}, raftcore.ErrTransfer + } + } + plan, err := core.PlanLeadership(h, evidence) + return h, plan, err +} diff --git a/internal/raftdirectory/leadership_test.go b/internal/raftdirectory/leadership_test.go new file mode 100644 index 0000000000..851669eab4 --- /dev/null +++ b/internal/raftdirectory/leadership_test.go @@ -0,0 +1,374 @@ +package raftdirectory_test + +import ( + "context" + "crypto/ed25519" + "errors" + "fmt" + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/reqsign" + "net/http" + "net/http/httptest" + "path/filepath" + "slices" + "testing" + "time" +) + +func personalGroup(t *testing.T) *group { + extra := []raftstore.Write{} + for _, name := range []string{"paired-a", "paired-b"} { + pub := key(name).Public().(ed25519.PublicKey) + id := cs.DeviceIDOfKey(pub) + extra = append(extra, raftstore.Write{Key: raftstore.Key{Namespace: raftstore.State, Key: cs.DeviceKey(id).Key}, Value: encoded(t, cs.Device{V: 1})}, raftstore.Write{Key: raftstore.Key{Namespace: raftstore.Enrollment, Key: cs.AuthorityKey(id).Key}, Value: encoded(t, cs.Authority{Cluster: cluster, Key: pub, Personal: true})}) + } + g := newGroup(t, func() time.Time { return time.UnixMilli(1000) }, extra...) + anchors := []cs.MemberRecord{} + g.keys = map[string]ed25519.PrivateKey{} + for _, name := range []string{"paired-a", "paired-b"} { + g.keys[cs.DeviceIDOfKey(key(name).Public().(ed25519.PublicKey))] = key(name) + } + for id := uint64(1); id <= 3; id++ { + k := key(fmt.Sprint("anchor-", id)) + g.keys[cs.AnchorActor(id)] = k + anchors = append(anchors, cs.MemberRecord{Node: id, Key: k.Public().(ed25519.PublicKey), Incarnation: 1, Protocol: 1, Anchor: true}) + } + membershipChange(t, g, cs.Mutation{InitMembership: &cs.InitMembership{Anchors: anchors, Limit: cs.MaxMemberRecords}}) + path := filepath.Join(t.TempDir(), "personal.db") + s, e := raftstore.Open(path, raftstore.Identity{ClusterUUID: cluster, NodeID: 4, Protocol: 1}) + must(t, e) + n, e := raftcore.New(raftcore.Config{ClusterUUID: cluster, NodeID: 4, Seed: 4, Protocol: 1}, raftcore.RecoveredState{}) + must(t, e) + g.paths = append(g.paths, path) + g.stores = append(g.stores, s) + g.nodes = append(g.nodes, n) + g.status = append(g.status, raftcore.Status{}) + g.results = append(g.results, nil) + g.absent[4] = true + h := registry(t, g, g.leader()) + membershipChange(t, g, cs.Mutation{EnrollMember: &cs.EnrollMember{Node: 4, Device: cs.DeviceIDOfKey(key("paired-b").Public().(ed25519.PublicKey)), Protocol: 1, Revision: h.Revision}}) + reconcileMembership(t, g) + leader := g.leader() + snapshot, e := g.stores[leader].CreateSnapshot(ctx) + must(t, e) + installNativeSnapshot(t, g, leader, 3, snapshot) + g.absent[4] = false + // The second certified home also joins without a vote. Both can sleep + // together without creating a two-voter default or changing quorum. + path = filepath.Join(t.TempDir(), "second-personal.db") + s, e = raftstore.Open(path, raftstore.Identity{ClusterUUID: cluster, NodeID: 5, Protocol: 1}) + must(t, e) + n, e = raftcore.New(raftcore.Config{ClusterUUID: cluster, NodeID: 5, Seed: 5, Protocol: 1}, raftcore.RecoveredState{}) + must(t, e) + g.paths = append(g.paths, path) + g.stores = append(g.stores, s) + g.nodes = append(g.nodes, n) + g.status = append(g.status, raftcore.Status{}) + g.results = append(g.results, nil) + g.absent[5] = true + h = registry(t, g, g.leader()) + membershipChange(t, g, cs.Mutation{EnrollMember: &cs.EnrollMember{Node: 5, Device: cs.DeviceIDOfKey(key("paired-a").Public().(ed25519.PublicKey)), Protocol: 1, Revision: h.Revision}}) + reconcileMembership(t, g) + leader = g.leader() + snapshot, e = g.stores[leader].CreateSnapshot(ctx) + must(t, e) + installNativeSnapshot(t, g, leader, 4, snapshot) + g.absent[5] = false + g.drive(g.leader(), raftcore.Tick{}) + g.drain() + return g +} + +type leaderBackend struct{ g *group } + +func (b leaderBackend) Status(ctx context.Context, actor string) (clusterctl.Status, error) { + i, app, e := b.g.barrier(actor) + if e != nil { + return clusterctl.Status{}, e + } + h, e := raftdirectory.LeadershipState(ctx, b.g.stores[i]) + if e != nil { + return clusterctl.Status{}, e + } + k := raftstore.Key{Namespace: raftstore.Membership, Key: cs.MemberDeviceKey(actor).Key} + rows, _, e := b.g.stores[i].Read(ctx, app.Applied, []raftstore.Key{k}) + if e != nil { + return clusterctl.Status{}, e + } + var mapping cs.DeviceMember + if rows[k] != nil { + if e = cs.Decode(rows[k], &mapping); e != nil { + return clusterctl.Status{}, e + } + } + self := mapping.Node + return clusterctl.Status{Cluster: cluster, Self: self, Leader: uint64(i + 1), Membership: h, Configuration: raftcore.Configuration(app.ConfState)}, nil +} +func (b leaderBackend) Lead(ctx context.Context, c cs.Command) (cs.LeadershipReceipt, error) { + r, e := b.g.submit(c) + if e != nil { + return cs.LeadershipReceipt{}, e + } + if r.Code != cs.OK { + return cs.LeadershipReceipt{}, clusterctl.CommittedRefusal{Code: r.Code, Sequence: c.Sequence} + } + for n := 0; n < 30; n++ { + g := b.g + i := g.leader() + if i < 0 { + for j := range g.nodes { + g.drive(j, raftcore.Tick{}) + } + g.drain() + continue + } + h, e := raftdirectory.LeadershipState(ctx, g.stores[i]) + if e != nil { + return cs.LeadershipReceipt{}, e + } + l := h.Leadership + if l == nil { + return cs.LeadershipReceipt{}, clusterctl.ErrUnavailable + } + if i == int(l.Node)-1 && l.Settled { + g.drive(i, raftcore.Read{RequestID: fmt.Sprint("lead-proof-", n)}) + g.drain() + confirmed, e := g.nodes[i].LeadershipReceipt(h) + if e != nil { + return cs.LeadershipReceipt{}, e + } + if l.Completed { + return confirmed, nil + } + if !l.Completed { + r, e = g.Change(ctx, cs.AnchorActor(1), cs.Mutation{CompleteLeadership: &cs.CompleteLeadership{Revision: h.Revision}}) + if e != nil || r.Code != cs.OK { + return cs.LeadershipReceipt{}, errors.New("completion did not commit") + } + h, e = raftdirectory.LeadershipState(ctx, g.stores[i]) + if e != nil { + return cs.LeadershipReceipt{}, e + } + g.drive(i, raftcore.Read{RequestID: "completed-leadership"}) + g.drain() + } + return g.nodes[i].LeadershipReceipt(h) + } + target := l.Node + if l.Restore { + target = l.Standby + } + evidence, e := raftdirectory.DurableLeadershipEvidence(ctx, g.stores[target-1], target, h.Revision, l.Barrier) + if e != nil { + g.drive(i, raftcore.Tick{}) + g.drain() + continue + } + _, p, e := raftdirectory.PlanLeadership(ctx, g.stores[i], g.nodes[i], evidence) + if errors.Is(e, raftcore.ErrTransfer) || errors.Is(e, raftcore.ErrBusy) { + g.drive(i, raftcore.Tick{}) + g.drain() + continue + } + if e != nil { + return cs.LeadershipReceipt{}, e + } + if p.Configure { + g.drive(i, raftcore.Configure{Change: p.Change}) + g.drain() + } + if p.Settle { + r, e = g.Change(ctx, cs.AnchorActor(1), cs.Mutation{SettleLeadership: &cs.SettleLeadership{Revision: h.Revision}}) + if e != nil || r.Code != cs.OK { + return cs.LeadershipReceipt{}, errors.New("settlement did not commit") + } + } + if p.Transfer != 0 { + g.drive(i, raftcore.Transfer{Target: p.Transfer}) + g.drain() + } + } + return cs.LeadershipReceipt{}, clusterctl.ErrUnavailable +} + +// R6/R9 exercise the authenticated status/lead door from two disposable saved +// homes, across actual FULL SQLite stores and the production planner. The thin +// leader receives only coordination metadata; no blob route is even mounted. +func TestR6R7R9NativeLeadershipControlDoorAndRestart(t *testing.T) { + g := personalGroup(t) + homes := []migrationHome{savedMigrationHome(t, "paired-a"), savedMigrationHome(t, "paired-b")} + ids := []string{cs.DeviceIDOfKey(key("paired-a").Public().(ed25519.PublicKey)), cs.DeviceIDOfKey(key("paired-b").Public().(ed25519.PublicKey))} + backend := leaderBackend{g} + calls := 0 + server := httptest.NewServer(clusterctl.Handler(func(r *http.Request, raw []byte) (string, string, error) { + calls++ + if r.URL.Path != clusterctl.Path { + t.Fatal("thin leader fetched content", r.URL.Path) + } + id, dev, e := reqsign.Verify(r, raw, g.clock()) + if e != nil { + return id, dev, e + } + if dev != ids[0] && dev != ids[1] { + return "", "", directory.ErrUnauthorized + } + return id, dev, nil + }, func(id string) (clusterctl.Backend, error) { + if id != homes[0].id.ID() { + return nil, directory.ErrUnauthorized + } + return backend, nil + })) + defer server.Close() + clients := []clusterctl.Client{{Base: server.URL, Sign: reqsign.SignFor(homes[0], g.clock), HTTP: server.Client()}, {Base: server.URL, Sign: reqsign.SignFor(homes[1], g.clock), HTTP: server.Client()}} + s, e := clients[1].Status(ctx) + must(t, e) + if s.Self != 4 || len(s.Configuration.Voters) != 3 { + t.Fatal(s) + } + c, e := cs.Sign(cs.Command{Schema: 1, Cluster: cluster, Actor: ids[1], Session: "lead", Sequence: 1, Now: 1000, Change: cs.Mutation{RequestLeadership: &cs.RequestLeadership{Revision: s.Membership.Revision, Node: 4, Device: ids[1]}}}, key("paired-b")) + must(t, e) + receipt, e := clients[1].Lead(ctx, c) + must(t, e) + if receipt.Node != 4 || g.leader() != 3 || receipt.Barrier == 0 { + t.Fatal(receipt) + } + h := registry(t, g, 3) + if !h.Leadership.Completed { + t.Fatal("successful transfer not durably completed") + } + if _, e := raftdirectory.DurableLeadershipEvidence(ctx, g.stores[0], 4, h.Revision, h.Leadership.Barrier); e == nil { + t.Fatal("a different member attested P's store") + } + // An exact retry of the completed request returns another fresh read receipt + // and cannot replay the old voter swap or increment configuration revision. + again, e := clients[1].Lead(ctx, c) + must(t, e) + if again.Node != 4 || again.Revision != receipt.Revision { + t.Fatal(again) + } + + for i, store := range g.stores { + recovered, err := store.Recover(ctx) + must(t, err) + if !slices.Equal(recovered.Core.Anchors, []uint64{1, 2, 3}) { + t.Fatal("R7/R14 anchor ranks lost during C swap", i, recovered.Core.Anchors) + } + } + recoverMembership(t, g) + status, e := clients[0].Status(ctx) + must(t, e) + if status.Membership.Revision != h.Revision || len(status.Configuration.Voters) != 3 { + t.Fatal(status) + } + if calls < 4 { + t.Fatal("control did not cross the real HTTP door") + } + // The new anchor may restore C; it must not automatically transfer to a + // returning P merely because P remains a caught-up voter before restoration. + if g.leader() != 3 { + p, e := g.nodes[g.leader()].PlanLeadership(registry(t, g, g.leader()), raftcore.LeadershipEvidence{}) + if e != nil || !p.Fallback || p.Transfer != 0 { + t.Fatal(p, e) + } + } + g.absent[4], g.absent[5] = true, true + for n := 0; n < 30 && g.leader() < 0; n++ { + for id := 0; id < 3; id++ { + g.drive(id, raftcore.Tick{}) + } + g.drain() + } + if g.leader() < 0 { + t.Fatal("R7 A+B did not elect with both homes asleep") + } + h = registry(t, g, g.leader()) + r, e := g.Change(ctx, cs.AnchorActor(1), cs.Mutation{RestoreAnchors: &cs.RestoreAnchors{Revision: h.Revision}}) + must(t, e) + if r.Code != cs.OK { + t.Fatal(r) + } + for n := 0; n < 30; n++ { + i := g.leader() + h = registry(t, g, i) + if h.Leadership == nil { + break + } + evidence, err := raftdirectory.DurableLeadershipEvidence(ctx, g.stores[2], 3, h.Revision, h.Leadership.Barrier) + if err != nil { + g.drive(i, raftcore.Tick{}) + g.drain() + continue + } + _, plan, err := raftdirectory.PlanLeadership(ctx, g.stores[i], g.nodes[i], evidence) + if errors.Is(err, raftcore.ErrTransfer) { + g.drive(i, raftcore.Tick{}) + g.drain() + continue + } + must(t, err) + if plan.Configure { + g.drive(i, raftcore.Configure{Change: plan.Change}) + g.drain() + } + if plan.Settle { + r, e = g.Change(ctx, cs.AnchorActor(1), cs.Mutation{SettleLeadership: &cs.SettleLeadership{Revision: h.Revision}}) + must(t, e) + if r.Code != cs.OK { + t.Fatal(r) + } + } + } + h = registry(t, g, g.leader()) + if h.Leadership != nil { + t.Fatal("R7 anchor restoration did not finish") + } + app, _, e := g.stores[g.leader()].ApplicationView(ctx) + must(t, e) + if len(app.ConfState.GetVoters()) != 3 || len(app.ConfState.GetLearners()) != 2 { + t.Fatal("R9 two sleeping homes changed the default", app.ConfState) + } + for i, store := range g.stores[:3] { + _, err := store.CreateSnapshot(ctx) + must(t, err) + recovered, err := store.Recover(ctx) + must(t, err) + if !slices.Equal(recovered.Core.Anchors, []uint64{1, 2, 3}) { + t.Fatal("R7/R14 snapshot/restoration lost anchor ranks", i) + } + } + g.absent[4], g.absent[5] = false, false + g.drive(g.leader(), raftcore.Tick{}) + g.drain() + if g.leader() == 3 || g.leader() == 4 { + t.Fatal("R7 returning home automatically regained leadership") + } + +} + +// R6/R16 refuse a target whose reply predates committed revocation. A target's +// local ready barrier cannot override the leader's newer authorization state. +func TestR6R16PromotionRechecksCommittedRevocation(t *testing.T) { + g := personalGroup(t) + h := registry(t, g, g.leader()) + device := cs.DeviceIDOfKey(key("paired-b").Public().(ed25519.PublicKey)) + membershipChange(t, g, cs.Mutation{RequestLeadership: &cs.RequestLeadership{Revision: h.Revision, Node: 4, Device: device}}) + h = registry(t, g, g.leader()) + evidence, e := raftdirectory.DurableLeadershipEvidence(ctx, g.stores[3], 4, h.Revision, h.Leadership.Barrier) + must(t, e) + g.absent[4] = true + r, e := g.Change(ctx, "a", cs.Mutation{Revoke: &cs.DeviceID{Device: device}}) + must(t, e) + if r.Code != cs.OK { + t.Fatal(r) + } + _, plan, e := raftdirectory.PlanLeadership(ctx, g.stores[g.leader()], g.nodes[g.leader()], evidence) + if !errors.Is(e, raftcore.ErrTransfer) || plan.Configure { + t.Fatal("stale readiness admitted revoked target", plan, e) + } +} diff --git a/internal/raftstore/recover.go b/internal/raftstore/recover.go index 33fbf58e06..b318180908 100644 --- a/internal/raftstore/recover.go +++ b/internal/raftstore/recover.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" + "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/raftcore" "github.com/Agent-Field/codeaf/internal/raftsnapshot" "go.etcd.io/raft/v3" @@ -221,7 +222,34 @@ func (s *SQLite) Recover(ctx context.Context) (Recovery, error) { if a.RecordCount != appRows || a.RecordsHash != recordsHash || (a.Applied == 0 && appRows != 0) { return ErrCorrupt } - out.Core = raftcore.RecoveredState{HardState: h, ConfState: a.ConfState, Snapshot: snapshot, Entries: entries, Applied: a.Applied} + var anchors []uint64 + var registry []byte + err = tx.QueryRowContext(ctx, `SELECT data FROM records WHERE namespace='membership' AND key='registry'`).Scan(®istry) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + if registry != nil { + var membership clusterstate.Membership + if clusterstate.Decode(registry, &membership) != nil || membership.Validate() != nil || membership.Index > a.Applied { + return ErrCorrupt + } + anchors = membership.Anchors + } + if registry == nil { + var imported []byte + err = tx.QueryRowContext(ctx, `SELECT data FROM records WHERE namespace='membership' AND key='legacy-import'`).Scan(&imported) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + if imported != nil { + var manifest clusterstate.LegacyManifest + if clusterstate.Decode(imported, &manifest) != nil || manifest.Validate() != nil || manifest.Config.Cluster != s.identity.ClusterUUID { + return ErrCorrupt + } + anchors = manifest.Config.Anchors + } + } + out.Core = raftcore.RecoveredState{Anchors: anchors, HardState: h, ConfState: a.ConfState, Snapshot: snapshot, Entries: entries, Applied: a.Applied} if snapIndex == 0 { out.Core.Snapshot = nil } diff --git a/internal/raftstore/sqlite.go b/internal/raftstore/sqlite.go index 7ef5a145e4..5a831a4753 100644 --- a/internal/raftstore/sqlite.go +++ b/internal/raftstore/sqlite.go @@ -630,3 +630,7 @@ func (s *SQLite) Read(ctx context.Context, expectedApplied uint64, keys []Key) ( } return out, a, nil } + +// MemberIdentity exposes the immutable store binding so a target cannot attest +// durable application on behalf of a different member. +func (s *SQLite) MemberIdentity() Identity { return s.identity } diff --git a/internal/syncsetup/cluster.go b/internal/syncsetup/cluster.go new file mode 100644 index 0000000000..6ba5f62220 --- /dev/null +++ b/internal/syncsetup/cluster.go @@ -0,0 +1,147 @@ +package syncsetup + +import ( + "context" + "encoding/json" + "errors" + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/filelock" + "github.com/Agent-Field/codeaf/internal/fsreplace" + "github.com/Agent-Field/codeaf/internal/reqsign" + "net/http" + "os" + "path/filepath" + "time" +) + +func (s *Sync) clusterClient() clusterctl.Client { + return clusterctl.Client{Base: s.Relay, Sign: reqsign.SignFor(deviceSigner{s.Identity, s.Device}, time.Now), HTTP: gated(s.Home, &http.Client{Timeout: 30 * time.Second})} +} +func (s *Sync) ClusterStatus(ctx context.Context) (clusterctl.Status, error) { + return s.clusterClient().Status(ctx) +} + +type leadershipSession struct { + Sequence uint64 + Pending *cs.Command +} + +// LeadCluster saves the exact signed request before transmission. A lost reply +// retries that same operation after restart, including its observed revision; +// it cannot silently promote a newer target or allocate a second request. +func (s *Sync) LeadCluster(ctx context.Context, node uint64) (cs.LeadershipReceipt, error) { + var receipt cs.LeadershipReceipt + if Quiet(s.Home) { + return receipt, ErrQuiet + } + path := filepath.Join(s.Home, "cluster-leadership-"+s.Device.ID()+".json") + lock, err := os.OpenFile(path+".lock", os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return receipt, err + } + defer lock.Close() + if err = filelock.Lock(lock, true, true); err != nil { + return receipt, errors.New("another window is requesting leadership on this computer") + } + defer filelock.Unlock(lock) + var saved leadershipSession + b, err := os.ReadFile(path) + if err == nil { + err = json.Unmarshal(b, &saved) + } + if err != nil && !errors.Is(err, os.ErrNotExist) { + return receipt, err + } + if saved.Pending != nil && (saved.Pending.Change.RequestLeadership == nil || saved.Pending.Actor != s.Device.ID() || saved.Pending.Session != "leadership" || saved.Pending.Sequence != saved.Sequence+1) { + return receipt, errors.New("this computer has an unreadable pending leadership request; its record has been kept") + } + client := s.clusterClient() + if saved.Pending == nil { + status, e := client.Status(ctx) + if e != nil { + return receipt, e + } + if node == 0 { + node = status.Self + } + if node == 0 { + return receipt, errors.New("this computer must join the cluster with persistent storage before it can lead") + } + if saved.Sequence == ^uint64(0) { + return receipt, errors.New("this computer's leadership request sequence is full") + } + c := cs.Command{Schema: cs.SchemaVersion, Cluster: status.Cluster, Actor: s.Device.ID(), Session: "leadership", Sequence: saved.Sequence + 1, Ack: saved.Sequence, Now: time.Now().UnixMilli(), Change: cs.Mutation{RequestLeadership: &cs.RequestLeadership{Revision: status.Membership.Revision, Node: node, Device: s.Device.ID()}}} + // A personal device may request another enrolled target, but must bind its + // committed device ID rather than guessing it from a node number. + if node != status.Self { + return receipt, errors.New("run codeaf cluster lead on the computer that should lead") + } + c.RequestDigest, e = cs.Digest(c) + if e != nil { + return receipt, e + } + signed, e := cs.SigningBytes(c) + if e != nil { + return receipt, e + } + c.Signature = s.Device.Sign(signed) + saved.Pending = &c + if err = saveLeadership(path, saved); err != nil { + return receipt, err + } + } else if node != 0 && saved.Pending.Change.RequestLeadership.Node != node { + return receipt, errors.New("a leadership request is still pending; retry it before choosing another computer") + } + receipt, err = client.Lead(ctx, *saved.Pending) + if err != nil { + var refused clusterctl.CommittedRefusal + if errors.As(err, &refused) && refused.Sequence == saved.Pending.Sequence { + saved.Sequence = saved.Pending.Sequence + saved.Pending = nil + if e := saveLeadership(path, saved); e != nil { + return receipt, e + } + } + return receipt, err + } + saved.Sequence = saved.Pending.Sequence + saved.Pending = nil + if err = saveLeadership(path, saved); err != nil { + return cs.LeadershipReceipt{}, err + } + return receipt, nil +} +func saveLeadership(path string, v leadershipSession) error { + b, err := json.Marshal(v) + if err != nil { + return err + } + f, err := os.CreateTemp(filepath.Dir(path), ".cluster-lead-") + if err != nil { + return err + } + defer os.Remove(f.Name()) + if err = f.Chmod(0600); err == nil { + _, err = f.Write(b) + } + if err == nil { + err = f.Sync() + } + closeErr := f.Close() + if err == nil { + err = closeErr + } + if err != nil { + return err + } + if err = fsreplace.Rename(f.Name(), path); err != nil { + return err + } + d, err := os.Open(filepath.Dir(path)) + if err != nil { + return err + } + defer d.Close() + return d.Sync() +} diff --git a/internal/syncsetup/cluster_test.go b/internal/syncsetup/cluster_test.go new file mode 100644 index 0000000000..677b64433d --- /dev/null +++ b/internal/syncsetup/cluster_test.go @@ -0,0 +1,107 @@ +package syncsetup + +import ( + "context" + "crypto/ed25519" + "encoding/hex" + "errors" + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/reqsign" + "net/http" + "net/http/httptest" + "reflect" + "testing" + "time" +) + +type leadershipBackend struct { + t *testing.T + commands []cs.Command + fail bool +} + +func (b *leadershipBackend) Status(context.Context, string) (clusterctl.Status, error) { + return clusterctl.Status{Cluster: "retry-cluster", Self: 4, Membership: cs.Membership{Revision: 1}}, nil +} +func (b *leadershipBackend) Lead(_ context.Context, c cs.Command) (cs.LeadershipReceipt, error) { + b.commands = append(b.commands, c) + if b.fail { + b.fail = false + return cs.LeadershipReceipt{}, errors.New("reply lost after commit") + } + return cs.LeadershipReceipt{Node: 4, Term: 3, Revision: 2, Barrier: 12, Voters: []uint64{1, 2, 4}}, nil +} + +// R6/R17/R23 persist the original signed request across a lost reply and a new +// Sync handle. A later sync-off decision stops both status and lead traffic. +func TestR6R17R23LeadershipRequestRetryAndQuietWire(t *testing.T) { + backend := &leadershipBackend{t: t, fail: true} + calls := 0 + server := httptest.NewServer(clusterctl.Handler(func(r *http.Request, b []byte) (string, string, error) { + calls++ + return reqsign.Verify(r, b, time.Now()) + }, func(string) (clusterctl.Backend, error) { return backend, nil })) + defer server.Close() + home := machine(t, server.URL) + s, ok, e := Open(home) + if e != nil || !ok { + t.Fatal(e) + } + if _, e = s.LeadCluster(context.Background(), 0); e == nil { + t.Fatal("lost reply reported success") + } + again, ok, e := Open(home) + if e != nil || !ok { + t.Fatal(e) + } + receipt, e := again.LeadCluster(context.Background(), 0) + if e != nil || receipt.Node != 4 { + t.Fatal(receipt, e) + } + if len(backend.commands) != 2 || !reflect.DeepEqual(backend.commands[0], backend.commands[1]) { + t.Fatal("retry changed committed intent") + } + c := backend.commands[0] + b, e := cs.SigningBytes(c) + if e != nil { + t.Fatal(e) + } + pub, e := hex.DecodeString(s.Device.Cert.Device) + if e != nil { + t.Fatal(e) + } + if !ed25519.Verify(pub, b, c.Signature) { + t.Fatal("request lacks original personal signature") + } + n := calls + t.Setenv(URLVar, "off") + if _, e = again.ClusterStatus(context.Background()); e == nil { + t.Fatal("off status succeeded") + } + if _, e = again.LeadCluster(context.Background(), 0); e == nil { + t.Fatal("off lead succeeded") + } + if calls != n { + t.Fatal("R23 sync-off sent cluster traffic") + } + t.Setenv(URLVar, server.URL) + solo := t.TempDir() + if _, e = identity.EnsureSolo(solo); e != nil { + t.Fatal(e) + } + s, ok, e = Open(solo) + if e != nil || !ok { + t.Fatal(e) + } + if _, e = s.ClusterStatus(context.Background()); e == nil { + t.Fatal("solo status succeeded") + } + if _, e = s.LeadCluster(context.Background(), 0); e == nil { + t.Fatal("solo lead succeeded") + } + if calls != n { + t.Fatal("R23 never-paired home sent cluster traffic") + } +} diff --git a/relay/hosted/src/raft/abi.js b/relay/hosted/src/raft/abi.js index 169d3c904e..5cf472ba2d 100644 --- a/relay/hosted/src/raft/abi.js +++ b/relay/hosted/src/raft/abi.js @@ -32,12 +32,14 @@ export function response(bytes) { const error = r.str(); if (error) { r.end(); throw new Error(error); } return r; } -export function createFrame({ cluster, id, seed, bootstrap = [], hard = new Uint8Array(), conf = new Uint8Array(), snapshot = new Uint8Array(), entries = [], applied = 0n }) { +export function createFrame({ cluster, id, seed, bootstrap = [], anchors = [], hard = new Uint8Array(), conf = new Uint8Array(), snapshot = new Uint8Array(), entries = [], applied = 0n }) { const w = frame(1).str(cluster).u64(id).u64(seed).u32(1).u32(bootstrap.length); for (const id of bootstrap) w.u64(id); w.blob(hard).blob(conf).blob(snapshot).u32(entries.length); for (const e of entries) w.blob(e); - return w.u64(applied).finish(); + w.u64(applied); + if (anchors.length) { w.u32(anchors.length); for (const id of anchors) w.u64(id); } + return w.finish(); } export function stepFrame(handle, tag, value) { const w = frame(3).u32(handle).u32(tag); diff --git a/relay/hosted/src/raft/actor.js b/relay/hosted/src/raft/actor.js index b5a7dbe651..6386da4714 100644 --- a/relay/hosted/src/raft/actor.js +++ b/relay/hosted/src/raft/actor.js @@ -1,5 +1,6 @@ import { CORE_MEMORY_BUDGET, Reactor, HostCodec, createFrame, response, effects, frame, stepFrame } from './abi.js'; import { MemberStorage, DEFAULT_ROWS_BYTES, unb64 } from './storage.js'; +import { decodeCanonical, encodeCanonical } from './codec.js'; import { committedRegistry } from './membership.js'; const RESERVATION = 6 * 1024 * 1024; @@ -39,7 +40,12 @@ export class MemberActor { const recovered = await this.store.open({ cluster: config.cluster, id: String(config.id), protocol: 1 }, config.ceiling ?? DEFAULT_ROWS_BYTES); const cache = recovered.entries.reduce((n, b) => n + b.length, 0); if (cache > CACHE_BYTES) throw new Error('raft member: log cache capacity; recover with checkpoint'); - const r = response(this.reactor.call(createFrame({ ...config, bootstrap: recovered.entries.length || recovered.snapshot.length || recovered.hard.length ? [] : config.bootstrap, ...recovered }))); + const registry = this.store.read('membership', 'registry'); + const imported = this.store.read('membership', 'legacy-import'); + const provisioned = (config.anchors ?? config.bootstrap ?? []).map(a => String(typeof a === 'object' ? a.node : a)); + const anchors = registry ? decodeCanonical(new TextDecoder().decode(registry)).anchors : imported ? decodeCanonical(new TextDecoder().decode(imported)).config.anchors : provisioned; + if (provisioned.length && anchors.join(',') !== provisioned.join(',')) throw new Error('raft member: conflicting anchor manifest'); + const r = response(this.reactor.call(createFrame({ ...config, anchors, bootstrap: recovered.entries.length || recovered.snapshot.length || recovered.hard.length ? [] : config.bootstrap, ...recovered }))); this.handle = r.u32(); r.end(); this.config = config; this.generation++; this.cache = cache; return this.drive(0); } catch (e) { @@ -101,6 +107,33 @@ export class MemberActor { } return { send: [], reads: [], status: this.status, settle }; }); } + // Evidence is an internal authenticated target reply, derived under the + // durable member queue. No public request may set its own applied index. + leadershipEvidence(revision, barrier) { return this.serial(() => { + if(this.stopped||this.store.stopped||!this.handle)throw new Error('raft member: stopped; recover from durable storage'); + const header = this.store.read('membership', 'registry'); + if (!header) throw new Error('raft member: enrollment is absent'); + const h = JSON.parse(new TextDecoder().decode(header)), app = this.store.application(); + if (h.revision !== String(revision) || !h.leadership || h.leadership.barrier !== String(barrier) || BigInt(app.applied) < BigInt(barrier) || committedRegistry(this.reactor,this.store)[this.config.id]?.revoked !== false) throw new Error('raft member: leadership barrier not applied'); + return {node:this.config.id,revision:String(revision),applied:String(app.applied),protocol:this.store.identity.protocol,durable:true}; + }); } + leadership(evidence) { evidence = structuredClone(evidence); return this.serial(async () => { + const header = this.store.read('membership', 'registry'); + if (!header) throw new Error('raft member: enrollment is absent'); + if(BigInt(evidence.node)!==0n){const target=committedRegistry(this.reactor,this.store)[String(evidence.node)];if(!target||target.revoked)throw new Error('raft member: leadership target is unavailable');} + const raw = new TextEncoder().encode(encodeCanonical(evidence)); + const r = response(this.reactor.call(frame(21).u32(this.handle).blob(header).blob(raw).finish())); + const plan = JSON.parse(new TextDecoder().decode(r.blob())); r.end(); + if (plan.configure) return {...(await this.drive(8,unb64(plan.change ?? ''))),plan}; + if (BigInt(plan.transfer)) return {...(await this.drive(7,BigInt(plan.transfer))),plan}; + return {send:[],reads:[],status:this.status,plan}; + }); } + leadershipReceipt() { return this.serial(() => { + const header = this.store.read('membership', 'registry'); + if (!header) throw new Error('raft member: enrollment is absent'); + const r = response(this.reactor.call(frame(22).u32(this.handle).blob(header).finish())); + const receipt = JSON.parse(new TextDecoder().decode(r.blob()));r.end();return receipt; + }); } joinStatus(advertised) { return this.serial(() => { const raw = this.store.read('enrollment', 'node/' + this.config.id); if (!raw || committedRegistry(this.reactor, this.store)[this.config.id]?.revoked) return { ready: false }; diff --git a/relay/hosted/src/raft/directory.js b/relay/hosted/src/raft/directory.js index 5f697abf37..0b67c558eb 100644 --- a/relay/hosted/src/raft/directory.js +++ b/relay/hosted/src/raft/directory.js @@ -9,7 +9,7 @@ export function directoryEvaluator(runtime, cluster, ceiling) { const keys = Array.from({ length: r.u32() }, () => ({ namespace: r.str(), key: r.str() })); r.end(); const w = (app.index === undefined ? frame(10) : frame(15)).blob(command).str(cluster).u64(app.now).u64(app.version).u64(app.size).u64(ceiling).u32(keys.length); for (const k of keys) { const data = read(k.namespace, k.key); w.str(k.namespace).str(k.key).u32(data == null ? 0 : 1).blob(data ?? new Uint8Array()); } - if (app.index !== undefined) w.u64(app.index).blob(unb64(app.conf)); + if (app.index !== undefined) w.u64(app.index).blob(unb64(app.conf)).u64(app.term ?? 0); r = response(runtime.call(w.finish())); const now = r.u64(), version = r.u64(), result = r.blob(); const writes = Array.from({ length: r.u32() }, () => ({ namespace: r.str(), key: r.str(), data: r.blob() })); r.end(); @@ -24,3 +24,11 @@ export class RaftDirectoryFacade { async command(originalSignedBytes) { return this.host.commit(originalSignedBytes); } async read(authenticatedRequest) { return this.host.barrierRead(authenticatedRequest); } } + +// The status facade cannot change voters. Its host authorizes a fresh quorum +// read and drives the committed intent, target election and completion barrier. +export class RaftLeadershipFacade { + constructor(host) { this.host = host; } + async status(authenticatedRequest) { return this.host.clusterStatus(authenticatedRequest); } + async lead(originalSignedBytes) { return this.host.leadership(originalSignedBytes); } +} diff --git a/relay/hosted/src/raft/member.js b/relay/hosted/src/raft/member.js index fe454d2aab..3cb5aaecaa 100644 --- a/relay/hosted/src/raft/member.js +++ b/relay/hosted/src/raft/member.js @@ -74,6 +74,9 @@ export class RaftMemberDO extends DurableObject { const actor = new MemberActor(this.ctx.storage, sharedRuntime(), this.evaluator(config)); this.recoveredEffects = await actor.init(config); this.config = config; this.registry = actor.store.read('membership', 'registry') ? committedRegistry(sharedRuntime(), actor.store) : await this.ctx.storage.get('raft-registry'); this.actor = actor; } + async leadershipEvidence(revision,barrier) { await this.load(); return this.actor.leadershipEvidence(revision,barrier); } + async leadership(evidence) { await this.load(); return this.actor.leadership(evidence); } + async leadershipReceipt() { await this.load(); return this.actor.leadershipReceipt(); } async membership() { await this.load(); return this.actor.membership(); } async joinStatus(advertised) { await this.load(); return this.actor.joinStatus(advertised); } async drainRecovery() { diff --git a/relay/hosted/src/raft/storage.js b/relay/hosted/src/raft/storage.js index bbf2470a2d..b6b68e0529 100644 --- a/relay/hosted/src/raft/storage.js +++ b/relay/hosted/src/raft/storage.js @@ -133,7 +133,7 @@ export class MemberStorage { if (e.type === 0 && e.data.length) { check(evaluate, 'raft storage: application evaluator is absent'); let readBytes = 0, writeBytes = 0; - const change = evaluate(e.data, (ns, k) => { const data = this.read(ns, k); readBytes += data?.length ?? 0; check(readBytes <= ROW_BYTES, 'raft storage: evaluation read-set capacity'); return data; }, { ...app, index: e.index }); + const change = evaluate(e.data, (ns, k) => { const data = this.read(ns, k); readBytes += data?.length ?? 0; check(readBytes <= ROW_BYTES, 'raft storage: evaluation read-set capacity'); return data; }, { ...app, index: e.index, term: e.term }); check(change && !change.then && Array.isArray(change.writes)); for (const r of change.writes) { check(namespaces.includes(r.namespace) && typeof r.key === 'string' && enc.encode(r.key).length > 0 && enc.encode(r.key).length <= ROW_BYTES && r.data instanceof Uint8Array && r.data.length <= ROW_BYTES); From a83a0954b884f0647d8b77c38323d42a348b8124 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 03:22:56 -0400 Subject: [PATCH 030/103] Prove ranked fallback through conditional writes and both runtimes. Retain every original split-vote seed and all joint crash cases, measure every seeded conditional acquire and preserve old exact replay. Exercise native process death, WASI promotion, snapshot restoration and minority refusal under the existing laptop resource ceilings. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/raftsim/README.md | 21 +- internal/raftsim/leadership_test.go | 273 ++++++++++++++++++ internal/raftsim/replay.go | 5 + internal/raftsim/simulator.go | 68 ++++- .../leadership/r7-split-vote-seeds.json | 192 ++++++++++++ relay/hosted/test/raft/actor-worker.js | 7 + relay/hosted/test/raft/actors-run.sh | 3 +- relay/hosted/test/raft/actors.test.js | 50 +++- relay/hosted/test/raft/build.mjs | 2 +- relay/hosted/test/raft/membership.test.js | 40 ++- 10 files changed, 633 insertions(+), 28 deletions(-) create mode 100644 internal/raftsim/leadership_test.go create mode 100644 internal/raftsim/testdata/leadership/r7-split-vote-seeds.json diff --git a/internal/raftsim/README.md b/internal/raftsim/README.md index 4b4030bd5a..a7e4073c9f 100644 --- a/internal/raftsim/README.md +++ b/internal/raftsim/README.md @@ -38,11 +38,13 @@ sh scripts/raft-proof-footprint.sh nice -n 5 env GOMEMLIMIT=128MiB RAFTSIM_SEEDS -run '^TestR' -count=1 -timeout=180s -v ``` -The 6–12 tick election timeout bounds one attempt. Split votes and omitted +Each ranked election window bounds one attempt (6–7/8–9/10–11 ticks for +anchors and 12–13 for personal voters). Split votes and omitted messages can require several attempts; the campaign allows eight maximum timeout windows while retaining the 2,000-event ceiling. A separate seeded check asserts one delivered tick per live node per round and each election -attempt within 6–12 ticks. This budget is not a phone-fallback latency claim. +anchor attempt within 6–12 ticks. The separate R7 campaign measures the +phone-fallback deadline through a committed conditional operation. Lost-reply retries first wait for the original request's durable session result without submitting a second proposal. An absent result after a timeout remains unknown; it cannot be compared to a later successful retry as an empty result. @@ -133,3 +135,18 @@ a fresh snapshot and removes it with delayed responses. The unchanged 2,000-even ceiling still applies. Separate signed-policy tests prove sequential admission, its 33rd-learner refusal, revision races and retained ID tombstones; the raw simulator configuration action is not a public admission policy. + +## Ranked fallback timing + +Current recordings explicitly carry TimingVersion=2. Reading older unversioned +recordings selects their original version 1, including exact RNG consumption +and event digests. This compatibility is internal replay input, never a product +feature switch. New groups always default to version 2 in both runtimes. + +R7 keeps all 190 original split-vote seeds in testdata/leadership. The seeded +leadership campaign measures through a real committed conditional acquire and +reports p50/p95/max; it never accepts an elected role alone as successful +fallback. The B+C campaign separately requires a conditional operation within +50 seconds after A and learner P disappear in restored anchor topology. If +C is still a learner and the voter set is A+B+P, simultaneous P+A loss remains +unavailable under R8: no election window can replace a missing majority. diff --git a/internal/raftsim/leadership_test.go b/internal/raftsim/leadership_test.go new file mode 100644 index 0000000000..a0e8a5c208 --- /dev/null +++ b/internal/raftsim/leadership_test.go @@ -0,0 +1,273 @@ +package raftsim + +import ( + "encoding/json" + "fmt" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsim/model" + "os" + "path/filepath" + "slices" + "testing" +) + +func leadershipHeader(s *Simulator) cs.Membership { + l := s.Leader() + idx := s.Nodes[l].Stable.Applied + return cs.Membership{Revision: 3, Index: idx, NextNode: 6, Anchors: []uint64{1, 2, 3}, Nodes: []uint64{1, 2, 3, 4, 5}, Live: 2, Limit: 4096, Leadership: &cs.LeadershipIntent{Node: 4, Standby: 3, Revision: 3, Barrier: idx, Term: s.Nodes[l].Status.Term}} +} +func simEvidence(s *Simulator, h cs.Membership) raftcore.LeadershipEvidence { + node := h.Leadership.Node + if h.Leadership.Restore { + node = h.Leadership.Standby + } + return raftcore.LeadershipEvidence{Node: node, Revision: h.Revision, Applied: s.Nodes[node-1].Stable.Applied, Protocol: 1, Durable: true} +} +func simLeadership(t *testing.T, s *Simulator, h cs.Membership) { + t.Helper() + leader := elect(t, s) + e := simEvidence(s, h) + s.Do(Action{Kind: "leadership", Node: leader, Leadership: &h, Evidence: &e}) + s.Drain() + if s.Err != nil { + finish(t, s) + } +} +func swapSim(t *testing.T, s *Simulator, h cs.Membership) { + t.Helper() + for n := 0; n < 20; n++ { + leader := elect(t, s) + p, e := s.Nodes[leader].Core.PlanLeadership(h, simEvidence(s, h)) + if e == raftcore.ErrTransfer || e == raftcore.ErrBusy { + s.Ticks(1) + continue + } + if e != nil { + t.Fatal(e) + } + if p.Settle { + return + } + if p.Fallback { + t.Fatal("unexpected fallback while applying recorded restore") + } + simLeadership(t, s, h) + } + t.Fatal("swap failed") +} + +// R6/R7/R8/R9 exercise the real core and durable completion protocol for each +// seed. P leads without content; P and every worker disappear, leaving only +// A+B voting. Standby C catches up and re-enters through joint consensus. A +// simultaneous P+anchor loss remains unavailable. Histories are independently +// checked, including the operation after restoration. +func leadershipSeed(t *testing.T, seed int) int { + s := group(t, Config{Seed: uint64(seed), Nodes: 5}) + initial := create(t, s, "a") + leader := s.Leader() + s.Do(Action{Kind: "configure", Node: leader, Count: 5}) + s.Drain() + s.Ticks(1) + h := leadershipHeader(s) + swapSim(t, s, h) + h.Leadership.Settled = true + s.Ticks(1) + simLeadership(t, s, h) + if s.Leader() != 3 { + t.Fatal("R6 requested P is not leader") + } + // Read receipt must come from P's current term, not its old learner state. + submit(t, s, 3, model.Call{Kind: "read", Actor: "owner", Cell: "a"}) + if _, e := s.Nodes[3].Core.LeadershipReceipt(h); e != nil { + t.Fatal("R6 missing actual quorum barrier", e) + } + h.Leadership.Completed = true + s.Do(Action{Kind: "crash", Node: 3, Value: 1}) + s.Do(Action{Kind: "crash", Node: 4, Value: 1}) + // Minority never manufactures a successful coordination result. + if seed%2 == 0 { + s.Do(Action{Kind: "crash", Node: 1, Value: 1}) + s.Ticks(13) + if s.Leader() >= 0 { + t.Fatal("R8 minority became writable") + } + s.Do(Action{Kind: "restart", Node: 1}) + s.Drain() + } + startAction := len(s.Actions) + healthy := 0 + for ; healthy < 13 && s.Leader() < 0; healthy++ { + s.Ticks(1) + } + if s.Leader() < 0 { + t.Fatalf("R7 fallback exceeded 65 seconds: %d ticks", healthy) + } + // A+B commit immediately, before C is promoted and without P's approval. + taken := submit(t, s, s.Leader(), model.Call{Kind: "acquire", Actor: "fallback", Cell: "a", Expected: initial, Force: true, At: 1000}) + if taken.Code != "ok" || taken.Cell.Fence != initial.Fence+1 { + t.Fatal("R7 surviving A+B could not commit a new conditional take", taken) + } + fallbackTicks := 0 + for _, action := range s.Actions[startAction:] { + if action.Kind == "ticks" { + fallbackTicks += action.Count + } + } + if fallbackTicks > 13 { + t.Fatalf("R7 conditional acquire exceeded 65 seconds: %d ticks", fallbackTicks) + } + p, e := s.Nodes[s.Leader()].Core.PlanLeadership(h, raftcore.LeadershipEvidence{}) + if e != nil || !p.Fallback || p.Transfer != 0 { + t.Fatal("R7 did not request anchor restoration", p, e) + } + h.Revision++ + h.Index = s.Nodes[s.Leader()].Stable.Applied + h.Leadership = &cs.LeadershipIntent{Node: 4, Standby: 3, Revision: h.Revision, Barrier: h.Index, Restore: true} + swapSim(t, s, h) + conf := s.Nodes[s.Leader()].Stable.Conf + if !slices.Equal(conf.GetVoters(), []uint64{1, 2, 3}) || !slices.Contains(conf.GetLearners(), 4) { + t.Fatal("R7 topology not restored", conf) + } + submit(t, s, s.Leader(), model.Call{Kind: "advance_time", Actor: "owner", At: 2000}) + // Rejoining P remains a learner and cannot replay its old transfer intent. + s.Do(Action{Kind: "restart", Node: 3}) + s.Drain() + s.Ticks(2) + if s.Leader() == 3 { + t.Fatal("R7 automatic leadership regain") + } + finish(t, s) + return fallbackTicks * 5 +} + +func TestR6R7R8R9SeededLeadershipAndFallback(t *testing.T) { + seconds := []int{} + for seed := 0; seed < seedCount(t); seed++ { + t.Run(fmt.Sprintf("seed-%03d", seed), func(t *testing.T) { seconds = append(seconds, leadershipSeed(t, seed)) }) + } + slices.Sort(seconds) + if len(seconds) > 0 { + t.Logf("R7 conditional acquire: n=%d p50=%ds p95=%ds max=%ds", len(seconds), seconds[(len(seconds)-1)/2], seconds[(95*len(seconds)+99)/100-1], seconds[len(seconds)-1]) + } +} + +// R7/T3 retain EVERY original healthy split-vote failure as fixed regression +// seeds. They run the current timing contract, while versioned old evidence +// separately continues to replay with its original event digests. +func TestR7RankedWindowsRetainedSplitVoteSeeds(t *testing.T) { + data, err := os.ReadFile(filepath.Join("testdata", "leadership", "r7-split-vote-seeds.json")) + if err != nil { + t.Fatal(err) + } + var seeds []int + if json.Unmarshal(data, &seeds) != nil || len(seeds) != 190 { + t.Fatal("R7 missing retained counterexamples") + } + for _, seed := range seeds { + t.Run(fmt.Sprintf("seed-%03d", seed), func(t *testing.T) { leadershipSeed(t, seed) }) + } +} + +// R12 cuts every production swap boundary, then resumes the committed intent. +// The fixture does not remove a voter to escape an inconvenient joint quorum. +func TestR6R7R12LeadershipJointSwapCrashBoundaries(t *testing.T) { + for _, stage := range []string{"enter", "leave", "reverse-enter", "reverse-leave"} { + for node := 0; node < 4; node++ { + for _, phase := range []string{"persist-write", "persist-flush", "persist-ack", "apply-write", "apply-flush", "apply-ack"} { + t.Run(fmt.Sprintf("%s/%d/%s", stage, node, phase), func(t *testing.T) { + s := group(t, Config{Seed: 1312, Nodes: 5}) + s.Do(Action{Kind: "configure", Node: s.Leader(), Count: 5}) + s.Drain() + s.Ticks(1) + h := leadershipHeader(s) + if stage == "reverse-enter" || stage == "reverse-leave" { + swapSim(t, s, h) + h.Revision++ + h.Index = s.Nodes[s.Leader()].Stable.Applied + h.Leadership = &cs.LeadershipIntent{Node: 4, Standby: 3, Revision: h.Revision, Barrier: h.Index, Restore: true} + s.Ticks(1) + } + if stage == "leave" || stage == "reverse-leave" { + simLeadership(t, s, h) + } + s.Do(Action{Kind: "arm-cut", Node: node, Other: 1, Request: model.Call{Kind: phase}}) + simLeadership(t, s, h) + if !s.CutHit { + t.Fatal("R12 requested cut not reached") + } + s.Do(Action{Kind: "restart", Node: node}) + s.Drain() + s.Ticks(1) + swapSim(t, s, h) + finish(t, s) + }) + } + } + } +} + +// R7/R12 failing event schedules must remain exactly replayable when the host +// advances its in-memory workflow after an earlier intent was recorded. +func TestR7R12LeadershipWorkflowReplayRetainsEarlierIntent(t *testing.T) { + s := group(t, Config{Seed: 1313, Nodes: 5}) + s.Do(Action{Kind: "configure", Node: s.Leader(), Count: 5}) + s.Drain() + s.Ticks(1) + h := leadershipHeader(s) + simLeadership(t, s, h) + h.Leadership.Settled = true + h.Leadership.Completed = true + replay, err := Run(Replay{Config: s.initial, Actions: s.Actions}) + if err != nil || replay.Err != nil || replay.Digest() != s.Digest() || replay.Events != s.Events { + t.Fatalf("R7 earlier intent changed during replay: %v %v", err, replay.Err) + } + finish(t, s) +} + +// R7/R8/T3 distinguish losing two active voters from losing A after C is +// restored. In the latter topology B+C have quorum and B must ask first, +// committing a conditional operation within 50 seconds. P remains a learner. +func TestR7R8SeededRestoredAnchorsWithAAndPAbsent(t *testing.T) { + seconds := []int{} + for seed := 0; seed < seedCount(t); seed++ { + t.Run(fmt.Sprintf("seed-%03d", seed), func(t *testing.T) { + s := group(t, Config{Seed: uint64(seed), Nodes: 4}) + initial := create(t, s, "a") + s.Do(Action{Kind: "configure", Node: s.Leader(), Count: 5}) + s.Drain() + s.Ticks(1) + if s.Leader() != 0 { + t.Fatal("R7 rank A did not lead healthy anchors") + } + s.Do(Action{Kind: "crash", Node: 0, Value: 1}) + s.Do(Action{Kind: "crash", Node: 3, Value: 1}) + start := len(s.Actions) + for tick := 0; tick < 10 && s.Leader() < 0; tick++ { + s.Ticks(1) + } + if s.Leader() != 1 { + t.Fatal("R7 B did not lead B+C within 50 seconds", s.Leader()) + } + taken := submit(t, s, 1, model.Call{Kind: "acquire", Actor: "fallback", Cell: "a", Expected: initial, Force: true, At: 1000}) + if taken.Code != "ok" || taken.Cell.Fence != initial.Fence+1 { + t.Fatal("R7 B+C conditional acquire failed", taken) + } + ticks := 0 + for _, a := range s.Actions[start:] { + if a.Kind == "ticks" { + ticks += a.Count + } + } + if ticks > 10 { + t.Fatal("R7 B+C conditional acquire exceeded 50 seconds", ticks) + } + seconds = append(seconds, 5*ticks) + finish(t, s) + }) + } + slices.Sort(seconds) + if len(seconds) > 0 { + t.Logf("R7 B+C conditional acquire: n=%d p50=%ds p95=%ds max=%ds", len(seconds), seconds[(len(seconds)-1)/2], seconds[(95*len(seconds)+99)/100-1], seconds[len(seconds)-1]) + } +} diff --git a/internal/raftsim/replay.go b/internal/raftsim/replay.go index b06684f048..9ad042aaee 100644 --- a/internal/raftsim/replay.go +++ b/internal/raftsim/replay.go @@ -28,6 +28,11 @@ type Evidence struct { } func Run(replay Replay) (*Simulator, error) { + // Unversioned evidence predates ranked election windows. Preserve those + // bytes and event digests; new recordings always carry their timing version. + if replay.Config.TimingVersion == 0 { + replay.Config.TimingVersion = 1 + } s, err := New(replay.Config) if err != nil { return s, err diff --git a/internal/raftsim/simulator.go b/internal/raftsim/simulator.go index d5a95f3e17..f2aebc881d 100644 --- a/internal/raftsim/simulator.go +++ b/internal/raftsim/simulator.go @@ -21,12 +21,13 @@ const clusterUUID = "raftsim-contract" const MaxEvents = 2000 type Config struct { - PaddingBytes int - CutSnapshot bool - RecordDisk bool - Seed uint64 - Nodes int - MaxEvents int + TimingVersion uint16 + PaddingBytes int + CutSnapshot bool + RecordDisk bool + Seed uint64 + Nodes int + MaxEvents int // Fault rates are per thousand and apply only to network messages. Drop int Duplicate int @@ -37,12 +38,14 @@ type Config struct { Broken string } type Action struct { - Kind string - Node int - Other int - Count int - Value int64 - Request model.Call + Leadership *clusterstate.Membership `json:",omitempty"` + Evidence *raftcore.LeadershipEvidence `json:",omitempty"` + Kind string + Node int + Other int + Count int + Value int64 + Request model.Call } type Trace struct { Order int64 @@ -152,6 +155,9 @@ type Simulator struct { } func New(config Config) (*Simulator, error) { + if config.TimingVersion == 0 { + config.TimingVersion = raftcore.ElectionTimingVersion + } if config.Nodes == 0 { config.Nodes = 3 } @@ -203,7 +209,7 @@ func (s *Simulator) start(node int) { s.fail("recovery: %v", err) return } - cfg := raftcore.Config{ClusterUUID: clusterUUID, NodeID: uint64(node + 1), Protocol: raftcore.ProtocolVersion, Seed: s.random(^uint64(0))} + cfg := raftcore.Config{TimingVersion: s.initial.TimingVersion, Anchors: []uint64{1, 2, 3}, ClusterUUID: clusterUUID, NodeID: uint64(node + 1), Protocol: raftcore.ProtocolVersion, Seed: s.random(^uint64(0))} if n.Stable.Hard == nil && len(n.Stable.Entries) == 0 && n.Stable.Snapshot == nil && node < 3 { cfg.Bootstrap = []uint64{1, 2, 3} } @@ -605,6 +611,26 @@ func (s *Simulator) run(count int, until int64) { // Do records every external input for exact replay. Network deliveries and disk // completions are derived from the recorded seed and these ordered actions. func (s *Simulator) Do(a Action) int { + // Replay owns immutable input bytes. A host updating the next leadership + // stage must not rewrite the intent that drove an earlier configuration. + if a.Leadership != nil { + h := *a.Leadership + h.Anchors = slices.Clone(h.Anchors) + h.Nodes = slices.Clone(h.Nodes) + if h.Leadership != nil { + l := *h.Leadership + h.Leadership = &l + } + if h.Pending != nil { + p := *h.Pending + h.Pending = &p + } + a.Leadership = &h + } + if a.Evidence != nil { + e := *a.Evidence + a.Evidence = &e + } s.Actions = append(s.Actions, a) s.stamp() if s.Err != nil { @@ -658,6 +684,22 @@ func (s *Simulator) Do(a Action) int { s.Config.Delay = a.Value case "snapshot": s.checkpoint(a.Node) + case "leadership": + if a.Leadership == nil || a.Evidence == nil { + s.fail("missing leadership action evidence") + return -1 + } + plan, err := s.Nodes[a.Node].Core.PlanLeadership(*a.Leadership, *a.Evidence) + if err != nil { + s.fail("leadership plan: %v", err) + return -1 + } + if plan.Configure { + s.step(a.Node, raftcore.Configure{Change: plan.Change}) + } + if plan.Transfer != 0 { + s.step(a.Node, raftcore.Transfer{Target: plan.Transfer}) + } case "configure": var change *pb.ConfChangeV2 switch a.Count { diff --git a/internal/raftsim/testdata/leadership/r7-split-vote-seeds.json b/internal/raftsim/testdata/leadership/r7-split-vote-seeds.json new file mode 100644 index 0000000000..77d58c56f6 --- /dev/null +++ b/internal/raftsim/testdata/leadership/r7-split-vote-seeds.json @@ -0,0 +1,192 @@ +[ + 3, + 7, + 22, + 81, + 87, + 89, + 93, + 98, + 113, + 120, + 123, + 127, + 132, + 139, + 143, + 152, + 165, + 173, + 179, + 180, + 189, + 191, + 211, + 235, + 251, + 260, + 262, + 267, + 269, + 273, + 281, + 288, + 296, + 309, + 327, + 350, + 363, + 367, + 381, + 391, + 405, + 412, + 415, + 417, + 445, + 449, + 450, + 460, + 463, + 467, + 490, + 505, + 507, + 509, + 519, + 521, + 525, + 548, + 549, + 587, + 589, + 603, + 605, + 610, + 626, + 643, + 656, + 657, + 665, + 684, + 687, + 690, + 711, + 713, + 721, + 745, + 759, + 767, + 768, + 770, + 785, + 835, + 845, + 878, + 879, + 899, + 900, + 911, + 916, + 919, + 925, + 927, + 943, + 961, + 977, + 991, + 994, + 995, + 1013, + 1033, + 1059, + 1078, + 1079, + 1081, + 1083, + 1092, + 1093, + 1102, + 1108, + 1118, + 1125, + 1155, + 1169, + 1181, + 1197, + 1203, + 1204, + 1206, + 1211, + 1221, + 1247, + 1291, + 1298, + 1319, + 1344, + 1352, + 1360, + 1369, + 1379, + 1395, + 1403, + 1411, + 1416, + 1417, + 1465, + 1466, + 1469, + 1475, + 1476, + 1479, + 1481, + 1483, + 1487, + 1502, + 1515, + 1527, + 1539, + 1541, + 1578, + 1603, + 1611, + 1615, + 1629, + 1644, + 1647, + 1653, + 1659, + 1670, + 1685, + 1687, + 1691, + 1699, + 1719, + 1723, + 1729, + 1731, + 1733, + 1739, + 1755, + 1759, + 1765, + 1766, + 1788, + 1799, + 1801, + 1805, + 1813, + 1815, + 1817, + 1820, + 1839, + 1853, + 1881, + 1893, + 1959, + 1965, + 1969, + 1983, + 1989, + 1997 +] diff --git a/relay/hosted/test/raft/actor-worker.js b/relay/hosted/test/raft/actor-worker.js index a5de74de17..f70d55b1a7 100644 --- a/relay/hosted/test/raft/actor-worker.js +++ b/relay/hosted/test/raft/actor-worker.js @@ -3,6 +3,7 @@ import core from './core.wasm'; import { installRaftModule, runtimeMetrics, MemberActor, sharedRuntime } from '../../src/raft/actor.js'; import { RaftMemberDO } from '../../src/raft/member.js'; +import { encodeCanonical } from '../../src/raft/codec.js'; import { frame, response } from '../../src/raft/abi.js'; import { MemberInbox } from '../../src/raft/inbox.js'; import { b64, unb64 } from '../../src/raft/storage.js'; @@ -58,6 +59,12 @@ export class ActorFixture extends RaftMemberDO { if (url.pathname === '/init') return reply(await this.initialize(b)); await this.load(); if (url.pathname === '/input') return reply(await this.actor.input(b.tag, [2,3,8,11].includes(b.tag) ? unb64(b.value) : [5,6,7,9].includes(b.tag) ? BigInt(b.value) : b.value)); + if (url.pathname === '/leadership-plan' || url.pathname === '/leadership-receipt') { + const w=frame(url.pathname.endsWith('-plan')?21:22).u32(this.actor.handle).blob(unb64(b.header)); + if(url.pathname.endsWith('-plan'))w.blob(new TextEncoder().encode(encodeCanonical(b.value))); + const r=response(sharedRuntime().call(w.finish())),v=JSON.parse(new TextDecoder().decode(r.blob()));r.end();return reply(v); + } + if (url.pathname === '/close') return reply(await this.actor.close()); if (url.pathname === '/row') return reply(this.actor.store.read('state', 'cell')); if (url.pathname === '/application') return reply(this.actor.store.application()); if (url.pathname === '/checkpoint') return reply(await this.actor.checkpoint()); diff --git a/relay/hosted/test/raft/actors-run.sh b/relay/hosted/test/raft/actors-run.sh index d87971a572..6438df67a8 100644 --- a/relay/hosted/test/raft/actors-run.sh +++ b/relay/hosted/test/raft/actors-run.sh @@ -8,4 +8,5 @@ sh scripts/raft-proof-footprint.sh go test -p=1 ./internal/raftabi ./internal/ra cd relay/hosted sh ../../scripts/raft-proof-footprint.sh sh test/local.sh env CODEAF_RAFT_ACTORS=1 node test/raft/build.mjs "$out" sh ../../scripts/raft-proof-footprint.sh sh test/local.sh env CODEAF_RAFT_PROOF_DIR="$out" node --max-old-space-size=32 test/raft/actor-unit.test.js -sh ../../scripts/raft-proof-footprint.sh sh test/local.sh env CODEAF_RAFT_PROOF_DIR="$out" timeout 120s node --max-old-space-size=24 test/raft/actors.test.js +# Bound coordinator garbage so native peers and workerd keep their shared RSS headroom. +sh ../../scripts/raft-proof-footprint.sh sh test/local.sh env CODEAF_RAFT_PROOF_DIR="$out" timeout 120s node --max-old-space-size=16 test/raft/actors.test.js diff --git a/relay/hosted/test/raft/actors.test.js b/relay/hosted/test/raft/actors.test.js index bc8500332e..05d541b747 100644 --- a/relay/hosted/test/raft/actors.test.js +++ b/relay/hosted/test/raft/actors.test.js @@ -13,7 +13,7 @@ const sha = b => createHash('sha256').update(b).digest('hex'); const b64 = b => Buffer.from(b).toString('base64'); const unb64 = s => Buffer.from(s, 'base64'); const clockTicks = Number(execFileSync('getconf', ['CLK_TCK'], { encoding: 'utf8' }).trim()); -const report = { calls: [], requirements: ['R3','R4','R10','R12','R13','R14','R16','R27'], samples: [], phases: [] }; +const report = { calls: [], requirements: ['R3','R4','R6','R7','R8','R10','R12','R13','R14','R16','R27'], samples: [], phases: [] }; async function sample() { const ids = [process.pid, ...(await readFile(`/proc/${process.pid}/task/${process.pid}/children`, 'utf8')).trim().split(/\s+/).filter(Boolean).map(Number)]; const processes = []; @@ -47,6 +47,7 @@ class Group { for (let i = 1; i <= 4; i++) { const k = generateKeyPairSync('ed25519'); this.keys[i] = k.privateKey; this.view[i] = { key: k.publicKey.export({ format: 'der', type: 'spki' }).subarray(-32).toString('base64'), incarnation: '1', revoked: false, voter: true }; } } async call(id, op, value = {}) { + if(op==='leadership-plan')value={...value,value:JSON.parse(encodeCanonical(value.value))}; let r; const start = performance.now(); if (this.wasm.includes(id)) { const res = await this.worker.dispatchFetch(`http://proof/actor/${this.cluster}-${id}/${op}`, { method: 'POST', body: JSON.stringify(value), signal: AbortSignal.timeout(5000) }); assert.equal(res.status, 200, await res.clone().text()); r = await res.json(); @@ -58,7 +59,7 @@ class Group { accept(id, r) { if (!r) return; if (r.status) this.nodes.set(id, r.status); this.queue.push(...(r.send ?? []).map(m => ({ ...m, from: id }))); this.reads.push(...(r.reads ?? []).map(r => ({ ...r, node: id }))); } async init() { for (let id = 1; id <= 3; id++) this.accept(id, await this.call(id, 'init', { cluster: this.cluster, id: String(id), seed: String(100 + id), bootstrap: ['1','2','3'], scope: this.scope })); - for (const id of this.wasm) await this.call(id, 'enrollment', { view: this.view, applied: this.nodes.get(id).applied }); + for (const id of this.wasm.filter(id=>this.nodes.has(id))) await this.call(id, 'enrollment', { view: this.view, applied: this.nodes.get(id).applied }); await this.pump(); await this.tick(16); assert(this.leader(), JSON.stringify([...this.nodes])); } envelope(from, to, payload, chunk = null) { @@ -94,8 +95,8 @@ class Group { async sameRows() { const rows = []; for (const id of this.nodes.keys()) rows.push(await this.call(id, 'row')); assert(rows[0]); assert.equal(rows[0], rows[1]); assert.equal(rows[0], rows[2]); for (const row of rows) assert.equal(row, rows[0]); return rows[0]; } } -test('R3 R4 R10 R13 R14 R27 hosted actors commit, ReadIndex, crash and snapshot in both directions', { timeout: 120000 }, async () => { - const dir = await mkdtemp(join(tmpdir(), 'p08-')), native = new Native(join(dir, 'native')); +test('R3 R4 R6 R7 R8 R10 R13 R14 R27 hosted actors commit, ReadIndex, crash and snapshot in both directions', { timeout: 120000 }, async () => { + const dir = await mkdtemp(join(tmpdir(), 'p08-')); let native = new Native(join(dir, 'native')); let worker, inspector, samplingTask, samplingError, samplingPaused = false; const heaps = []; // A sample owns its inspector call until completion. Drain it before each // intentional process death so a late timeout cannot escape the test. @@ -148,6 +149,47 @@ test('R3 R4 R10 R13 R14 R27 hosted actors commit, ReadIndex, crash and snapshot await j.step(j.leader(), 8, ''); await j.pump(); await j.tick(2); assert(await j.read(j.leader(), 'left-joint')); await j.sameRows(); const left = unb64(await j.call(1, 'checkpoint')); const leftConf = fields(unb64(JSON.parse(fields(left).get(1).toString()).conf)); assert(!leftConf.has(3)); assert(!leftConf.has(4)); assert.notEqual(j.leader(), 3, 'removed voter remains a learner after leave/restart'); + // R6/R7/R8/R12 use the production planner, not literal voter edits, for + // transfer completion and reverse restoration with P absent. The synthetic + // policy header belongs only to this host-mechanism fixture; signed policy + // application is covered separately by the actual directory evaluator. + let header={revision:'3',index:j.nodes.get(j.leader()).applied,next_node:'5',anchors:['1','2','3'],nodes:['1','2','3','4'],live:1,limit:4096,leadership:{node:'4',standby:'3',revision:'3',barrier:j.nodes.get(j.leader()).applied,term:j.nodes.get(j.leader()).term,restore:false,settled:true,completed:false}}; + const plan=async(node,target)=>j.call(node,'leadership-plan',{header:b64(Buffer.from(encodeCanonical(header))),value:{node:String(target),revision:header.revision,applied:j.nodes.get(target).applied,protocol:1,durable:true}}); + const receipt=async node=>j.call(node,'leadership-receipt',{header:b64(Buffer.from(encodeCanonical(header)))}); + await j.tick(1); + let p=await plan(j.leader(),4);assert.equal(p.transfer,'4');await j.step(j.leader(),7,p.transfer);await j.pump();assert.equal(j.leader(),4); + await assert.rejects(receipt(4),/caught-up voter/);assert(await j.read(4,'native-personal-leader'));assert.equal((await receipt(4)).node,'4'); + header.leadership.completed=true; + j.block.add(4);j.block.add(3);native.child.kill('SIGKILL');await new Promise(r=>native.child.once('exit',r)); + j.block.add(2);await j.tick(13);assert.equal(j.leader(),undefined,'R8 personal plus anchor loss must be unavailable');j.block.delete(2); + for(let n=0;n<24&&!j.leader();n++)await j.tick(1);assert([1,2].includes(j.leader())); + report.phases.push('R8 personal + anchor loss unavailable; no second writable cluster'); + const beforeFallbackWrite=j.nodes.get(j.leader()).applied;await j.mutate(3);assert(BigInt(j.nodes.get(j.leader()).applied)>BigInt(beforeFallbackWrite),'R7 A+B conditional operation did not commit'); + p=await plan(j.leader(),3);assert.equal(p.fallback,true);assert.equal(p.transfer,'0'); + native=new Native(join(dir,'native'));j.native=native;g.native=native; + j.accept(3,await j.call(3,'init',{cluster:'joint',seed:'103',bootstrap:['1','2','3']}));j.block.delete(3);await j.pump(); + // C catches up through the committed restore barrier while P remains + // absent. A+B alone decide both sides of the reverse joint transition. + header={...header,revision:'4',index:j.nodes.get(j.leader()).applied,leadership:{...header.leadership,revision:'4',barrier:j.nodes.get(j.leader()).applied,restore:true,settled:false,completed:false}}; + await j.tick(1); + p=await plan(j.leader(),3);assert(p.configure);await j.step(j.leader(),8,p.change);await j.pump(); + p=await plan(j.leader(),3);assert(p.configure);await j.step(j.leader(),8,p.change??'');await j.pump(); + p=await plan(j.leader(),3);assert(p.settle);assert(await j.read(j.leader(),'restored-without-personal')); + const reverse=JSON.parse(fields(unb64(await j.call(1,'checkpoint'))).get(1).toString()); + assert(!fields(unb64(reverse.conf)).has(3)); + j.accept(4,await j.call(4,'init',{cluster:'joint',seed:'104',bootstrap:['1','2','3']}));j.block.delete(4);await j.tick(2);assert.notEqual(j.leader(),4); + report.phases.push('R6 native P elected + current-term read; R7 A+B restore C with P absent'); + await j.transfer(3);await j.call(1,'close');await j.call(2,'close'); + // R6 reverses the runtime roles: P is WASI and two anchors are native. + const k=new Group(worker,native,'wasm-personal',[1,4]);await k.init(); + k.accept(4,await k.call(4,'init',{cluster:k.cluster,id:'4',seed:'104',bootstrap:[],scope:k.scope}));await k.call(4,'enrollment',{view:k.view,applied:k.nodes.get(4).applied}); + await k.step(k.leader(),8,b64(Uint8Array.of(18,4,8,3,16,4)));await k.pump();await k.tick(2); + let kh={revision:'3',index:k.nodes.get(k.leader()).applied,next_node:'5',anchors:['1','2','3'],nodes:['1','2','3','4'],live:1,limit:4096,leadership:{node:'4',standby:'3',revision:'3',barrier:k.nodes.get(k.leader()).applied,term:k.nodes.get(k.leader()).term,restore:false,settled:false,completed:false}}; + const kp=async()=>k.call(k.leader(),'leadership-plan',{header:b64(Buffer.from(encodeCanonical(kh))),value:{node:'4',revision:'3',applied:k.nodes.get(4).applied,protocol:1,durable:true}}); + p=await kp();assert(p.configure);await k.step(k.leader(),8,p.change);await k.pump();p=await kp();assert(p.configure);await k.step(k.leader(),8,p.change??'');await k.pump();await k.tick(2);for(let n=0;n<24&&!k.leader();n++)await k.tick(1);assert(k.leader()); + p=await kp();assert(p.settle);kh.leadership.settled=true;p=await kp();assert.equal(p.transfer,'4');await k.step(k.leader(),7,'4');await k.pump();assert.equal(k.leader(),4);assert(await k.read(4,'wasm-personal-read')); + assert.equal((await k.call(4,'leadership-receipt',{header:b64(Buffer.from(encodeCanonical(kh)))})).node,'4'); + report.phases.push('R6 WASI P promoted + elected + current-term read'); report.phases.push('two wasm + native peers; full joint checkpoint/restart/leave'); // All three anchors plus two other identities share exactly one runtime. const h = new Group(worker, native, 'co-resident', [1,2,3]); await h.init(); await h.create(); assert(await h.read(h.leader(), 'all-hosted')); await h.sameRows(); diff --git a/relay/hosted/test/raft/build.mjs b/relay/hosted/test/raft/build.mjs index 85c8abaa9d..6cdcbc4254 100644 --- a/relay/hosted/test/raft/build.mjs +++ b/relay/hosted/test/raft/build.mjs @@ -38,7 +38,7 @@ async function walk(path) { for (const p of ['../../internal/raftsnapshot', '../../internal/raftcore', '../../internal/raftabi', '../../internal/clusterstate', '../../cmd/codeaf-raft-wasm', '../../third_party/raft', '../../third_party/raft-patch', 'src/raft']) await walk(resolve(p)); for (const p of ['../../go.mod','../../go.sum']) source.push([relative(resolve('../..'),resolve(p)),sha(await readFile(p))]); source.sort((a,b)=>a[0].localeCompare(b[0])); -const manifest = { abi: 1, coreProtocol: 1, rowSchema: 1, snapshotSchema: 1, snapshotFormatSHA256: sha(await readFile('../../internal/raftsnapshot/format.go')), go: 'go1.26.5', sourceHash: sha(JSON.stringify(source)), wasmSHA256: sha(wasm), sources: source }; +const manifest = { abi: 1, coreProtocol: 1, electionTiming: 2, rowSchema: 1, snapshotSchema: 1, snapshotFormatSHA256: sha(await readFile('../../internal/raftsnapshot/format.go')), go: 'go1.26.5', sourceHash: sha(JSON.stringify(source)), wasmSHA256: sha(wasm), sources: source }; await writeFile(resolve(out, 'manifest.json'), JSON.stringify(manifest, null, 2)+'\n'); const compressed = { wasm: gzipSync(wasm,{level:9}).length, javascript: Math.max(gzipSync(js,{level:9}).length, gzipSync(productionJS,{level:9}).length), storageFixture:gzipSync(storage,{level:9}).length, manifest: gzipSync(JSON.stringify(manifest),{level:9}).length }; const report = { imports, rawWasm: wasm.length, rawJavascript: js.length, rawProductionJavascript: productionJS.length, productionJavascriptCompressed: gzipSync(productionJS,{level:9}).length, compressed, totalCompressed: Object.values(compressed).reduce((a,b)=>a+b,0), limit: 3*1024*1024, wasmSHA256: manifest.wasmSHA256, sourceHash: manifest.sourceHash }; diff --git a/relay/hosted/test/raft/membership.test.js b/relay/hosted/test/raft/membership.test.js index 19a06a535c..ea1e3c7295 100644 --- a/relay/hosted/test/raft/membership.test.js +++ b/relay/hosted/test/raft/membership.test.js @@ -29,7 +29,7 @@ class Native { async close() { this.child.stdin.end(); if (this.child.exitCode === null) await new Promise(r => this.child.once('exit',r)); } } -test('R9 R11 R12 R13 R14 R16 R17 hosted learner intents, exact retries, tombstones and native policy parity', { timeout:30000 }, async () => { +test('R6 R7 R9 R11 R12 R13 R14 R16 R17 hosted learner intents, exact retries, tombstones and native policy parity', { timeout:30000 }, async () => { const reactor = new Reactor(new WebAssembly.Module(readFileSync(out + '/core.wasm'))), native = new Native(), calls = []; const policy = { call(b) { const result = reactor.call(b); calls.push([b.slice(),result.slice()]); return result; } }; const keys = Object.fromEntries(['a','b','x1','x2','x3'].map(id => [id,generateKeyPairSync('ed25519')])); @@ -45,7 +45,7 @@ test('R9 R11 R12 R13 R14 R16 R17 hosted learner intents, exact retries, tombston const configs = [], absent = new Set(); const accept = e => queue.push(...e.send); const pump = async () => { for (let n=0;queue.length;n++) { assert(n<2000); const m=queue.shift(), id=Number(m.to); if (!actors[id-1] || absent.has(id)) continue; accept(await actors[id-1].input(2,m.bytes)); } }; - const leader = () => actors.findIndex(a => a.status.role===2); + const leader = () => actors.findIndex((a,i) => !absent.has(i+1) && a.status.role===2); const add = async id => { const host = new Host(); hosts.push(host); const evaluator = (raw,read,app) => { @@ -54,13 +54,14 @@ test('R9 R11 R12 R13 R14 R16 R17 hosted learner intents, exact retries, tombston if (!results[id-1]) results[id-1]=[]; results[id-1].push(JSON.parse(dec.decode(r.result))); return {...r,writes:[...base,...r.writes]}; }; const config = {cluster,id:String(id),seed:String(id+44),bootstrap:id<=3?['1','2','3']:[]}; configs[id-1]=config; - const actor = new MemberActor(host,reactor,evaluator); actors[id-1]=actor; accept(await actor.init(config)); + const actor = new MemberActor(host,policy,evaluator); actors[id-1]=actor; accept(await actor.init(config)); }; - const command = change => { - const sequence=++seq.a, c={schema:1,cluster,actor:'a',session:'api',sequence:String(sequence),request_digest:'',now:1000,change,signature:null}; + const command = (change,signer='a') => { + const actor=signer.startsWith('x')?'anchor/'+signer.slice(1):signer; + const sequence=seq[actor]=(seq[actor]??0)+1, c={schema:1,cluster,actor,session:'api',sequence:String(sequence),request_digest:'',now:1000,change,signature:null}; if (sequence>1)c.ack=String(sequence-1); c.request_digest=sha(bytes({cluster,actor:c.actor,session:c.session,sequence:c.sequence,change})); - c.signature=sign(null,Buffer.concat([Buffer.from('codeaf.directory.command.v1\0'),bytes(c)]),keys.a.privateKey).toString('base64');return bytes(c); + c.signature=sign(null,Buffer.concat([Buffer.from('codeaf.directory.command.v1\0'),bytes(c)]),keys[signer].privateKey).toString('base64');return bytes(c); }; const submit = async raw => {const i=leader(),n=results[i]?.length??0;accept(await actors[i].input(3,raw));await pump();assert.equal(results[i].length,n+1);return results[i].at(-1);}; const registry = () => committedRegistry(policy,actors[leader()].store); @@ -87,11 +88,36 @@ test('R9 R11 R12 R13 R14 R16 R17 hosted learner intents, exact retries, tombston for(let n=0;n<12&&actors[3].status.applied{const r=directoryEvaluator(policy,cluster,8*1024*1024)(...args);results[3]??=[];results[3].push(JSON.parse(dec.decode(r.result)));return r;};const restored=new MemberActor(hosts[3],policy,restoredEvaluate);actors[3]=restored;accept(await restored.init(configs[3]));await pump();assert.deepEqual(await restored.joinStatus(r.membership.barrier),{ready:true}); + // R6/R7 use actual Go policy, persisted wasm actors and the production + // planner. The native peer replays every signed application evaluation. + h=header();r=await submit(command({request_leadership:{revision:h.revision,node:'4',device:'b'}}));assert.equal(r.code,'ok'); + const tickAll=async()=>{for(let n=0;n{ + for(let n=0;n<40;n++){ + if(leader()<0){await tickAll();continue;} + const h=header(),l=h.leadership;if(!l)return; + if(!l.restore&&l.settled&&leader()===3){ + accept(await actors[3].input(4,'leadership-proof'));await pump();assert.equal((await actors[3].leadershipReceipt()).node,'4'); + const r=await submit(command({complete_leadership:{revision:h.revision}},'x1'));assert.equal(r.code,'ok');return; + } + const target=Number(l.restore?l.standby:l.node)-1;let evidence; + try{evidence=await actors[target].leadershipEvidence(h.revision,l.barrier);}catch{await tickAll();continue;} + const e=await actors[leader()].leadership(evidence);accept(e);await pump(); + if(e.plan.settle){const r=await submit(command({settle_leadership:{revision:h.revision}},'x1'));assert.equal(r.code,'ok');} + } + assert.fail('leadership did not finish'); + }; + await reconcileLead();assert.equal(leader(),3);assert.equal(registry()['3'].voter,false);assert(header().leadership.completed); + absent.add(4);for(let n=0;n<30&&leader()<0;n++)await tickAll();assert([0,1].includes(leader())); + h=header();let plan=await actors[leader()].leadership({node:'0',revision:h.revision,applied:'0',protocol:1,durable:false});assert(plan.plan.fallback);assert.equal(plan.plan.transfer,'0'); + r=await submit(command({restore_anchors:{revision:h.revision}},'x1'));assert.equal(r.code,'ok');await tickAll();await reconcileLead();assert.equal(registry()['4'].voter,false);assert.equal(header().leadership,undefined); + absent.delete(4);await tickAll();assert.notEqual(leader(),3); h=header();absent.add(4);r=await submit(command({remove_member:{revision:h.revision,node:'4',device:'b'}}));assert.equal(r.code,'ok');await settle();assert.equal(registry()['4'].revoked,true); await actors[leader()].checkpoint();assert(actors[leader()].store.read('tombstones','node/4')); h=header();r=await submit(command({enroll_member:{node:h.next_node,revision:h.revision,device:'b',protocol:1}}));assert.equal(r.code,'ok');assert.equal(r.membership.node,'5'); r=await submit(command({revoke:{device:'b'}}));assert.equal(r.code,'ok');assert.equal(registry()['5'].revoked,true); + await assert.rejects(actors[leader()].leadership({node:'5',revision:header().revision,applied:'0',protocol:1,durable:true}),/leadership target is unavailable/); for(const [b,expected] of calls)assert.deepEqual(await native.call(b),expected); assert(calls.length>30); } finally {await native.close();for(const h of hosts)h.db.close();} From 60408c8b596e3f1b90acf316ee17a97e92a2d63c Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 04:00:16 -0400 Subject: [PATCH 031/103] Commit fresh hold evidence and durable idle barriers. Keep renewal authority in the shared state machine because a socket cannot extend ownership outside quorum. Require every voter to apply a generation before timer suppression, and invalidate it across wake and recovery. Prove the policy with native persistence and seeded partial-delivery histories. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterstate/apply.go | 37 +++++- internal/clusterstate/liveness.go | 135 ++++++++++++++++++++ internal/clusterstate/liveness_test.go | 144 +++++++++++++++++++++ internal/clusterstate/types.go | 3 + internal/raftabi/abi.go | 10 ++ internal/raftabi/scheduler.go | 84 +++++++++++++ internal/raftcore/README.md | 37 +++++- internal/raftcore/core.go | 26 +++- internal/raftcore/scheduler.go | 141 +++++++++++++++++++++ internal/raftcore/scheduler_test.go | 158 ++++++++++++++++++++++++ internal/raftcore/types.go | 6 + internal/raftdirectory/liveness.go | 98 +++++++++++++++ internal/raftdirectory/liveness_test.go | 125 +++++++++++++++++++ internal/raftsim/application.go | 19 ++- internal/raftsim/quiescence.go | 13 ++ internal/raftsim/quiescence_test.go | 102 +++++++++++++++ internal/raftsim/simulator.go | 70 +++++++++++ 17 files changed, 1193 insertions(+), 15 deletions(-) create mode 100644 internal/clusterstate/liveness.go create mode 100644 internal/clusterstate/liveness_test.go create mode 100644 internal/raftabi/scheduler.go create mode 100644 internal/raftcore/scheduler.go create mode 100644 internal/raftcore/scheduler_test.go create mode 100644 internal/raftdirectory/liveness.go create mode 100644 internal/raftdirectory/liveness_test.go create mode 100644 internal/raftsim/quiescence.go create mode 100644 internal/raftsim/quiescence_test.go diff --git a/internal/clusterstate/apply.go b/internal/clusterstate/apply.go index 2f2e9ca458..b0b65500af 100644 --- a/internal/clusterstate/apply.go +++ b/internal/clusterstate/apply.go @@ -137,8 +137,14 @@ func Sign(c Command, key ed25519.PrivateKey) (Command, error) { // Dependencies declares every possible row before evaluation. Hosts may page // these immutable records, but must distinguish absent rows from unread rows. func Dependencies(c Command) []RecordKey { - keys := []RecordKey{IdentityKey(), DeviceKey(c.Actor), AuthorityKey(c.Actor), SessionKey(c.Actor, c.Session), SessionsKey()} + keys := []RecordKey{QuiescenceKey(), IdentityKey(), DeviceKey(c.Actor), AuthorityKey(c.Actor), SessionKey(c.Actor, c.Session), SessionsKey()} m := c.Change + if m.RenewHolds != nil { + keys = append(keys, DeviceKey(m.RenewHolds.Device), AuthorityKey(m.RenewHolds.Device)) + } + if m.PrepareQuiescence != nil { + keys = append(keys, MembershipKey()) + } keys = append(keys, membershipDependencies(m)...) if m.PutDevice != nil { keys = append(keys, DeviceKey(m.PutDevice.ID)) @@ -186,6 +192,12 @@ func Dependencies(c Command) []RecordKey { } func cellIDs(m Mutation) []string { switch { + case m.RenewHolds != nil: + out := []string{} + for _, h := range m.RenewHolds.Items { + out = append(out, h.Cell) + } + return out case m.Create != nil: return []string{m.Create.Cell} case m.Acquire != nil: @@ -251,6 +263,8 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er target = new(Device) case k.Namespace == "state" && len(k.Key) >= 5 && k.Key[:5] == "cell/": target = new(Cell) + case k == QuiescenceKey(): + target = new(Quiescence) case k == MembershipKey(): var h Membership if Decode(b, &h) != nil || h.Validate() != nil { @@ -308,11 +322,11 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er out.Result.Code = Revoked return out, nil } - service := c.Change.RecordUpload != nil || c.Change.AdmitJoin != nil || c.Change.CompleteJoin != nil - if service != auth.Effects || auth.Personal && auth.Effects || auth.Anchor && (auth.Personal || auth.Effects || c.Change.SettleMembership == nil && c.Change.SettleLeadership == nil && c.Change.CompleteLeadership == nil && c.Change.RestoreAnchors == nil && c.Change.CloseSession == nil) { + service := c.Change.RenewHolds != nil || c.Change.RecordUpload != nil || c.Change.AdmitJoin != nil || c.Change.CompleteJoin != nil + if service != auth.Effects || auth.Personal && auth.Effects || auth.Anchor && (auth.Personal || auth.Effects || c.Change.PrepareQuiescence == nil && c.Change.SettleMembership == nil && c.Change.SettleLeadership == nil && c.Change.CompleteLeadership == nil && c.Change.RestoreAnchors == nil && c.Change.CloseSession == nil) { return out, nil } - if (c.Change.RestoreAnchors != nil || c.Change.CompleteLeadership != nil) && !auth.Anchor { + if (c.Change.PrepareQuiescence != nil || c.Change.RestoreAnchors != nil || c.Change.CompleteLeadership != nil) && !auth.Anchor { return out, nil } var last session @@ -353,7 +367,17 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er if identity.Rotation != nil && c.Change.Rotate == nil && c.Change.AdvanceTime == nil && c.Change.DeviceSeen == nil && c.Change.CloseSession == nil && c.Change.CompleteJoin == nil { code = Rotated } else { - if service { + if c.Change.PrepareQuiescence != nil { + v := c.Change.PrepareQuiescence + if meta.Index == 0 || v.Barrier != meta.Index-1 || v.Term != meta.Term || len(v.Voters) == 0 || !reflect.DeepEqual(v.Voters, meta.Configuration.Voters) || meta.Configuration.Joint() { + code = Conflict + } else { + put(QuiescenceKey(), Quiescence{Index: meta.Index, Term: meta.Term, Barrier: v.Barrier, Voters: v.Voters}) + } + } else if c.Change.RenewHolds != nil { + code = renewHolds(c, out.Now, rows, put, &out.Result) + out.Result.Index = meta.Index + } else if service { code = applyEffect(c, out.Now, rows, put) } else { if membershipMutation(c.Change) { @@ -388,6 +412,9 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er out.Version = meta.Version out.Result.Version = meta.Version } + if rows[QuiescenceKey()] != nil && c.Change.PrepareQuiescence == nil { + put(QuiescenceKey(), Quiescence{}) + } if !known { counts.Live++ counts.Retained++ diff --git a/internal/clusterstate/liveness.go b/internal/clusterstate/liveness.go new file mode 100644 index 0000000000..3b2d2b04e5 --- /dev/null +++ b/internal/clusterstate/liveness.go @@ -0,0 +1,135 @@ +package clusterstate + +// Coordination periods are shared by committed policy and both host schedulers. +const ( + RenewEveryMillis int64 = 30_000 + PresenceEveryMillis int64 = 10_000 + EvidenceWindowMillis int64 = PresenceEveryMillis * 5 / 2 + TickEveryMillis int64 = 5_000 + IdleAfterMillis int64 = 60_000 + MaxRenewHolds = 64 +) + +// RenewHolds is signed only by the restricted ingress effects authority after +// authenticating fresh socket evidence. Evidence time fixes the expiry: replay +// can never turn an old ping into another ninety seconds of ownership. +type RenewHolds struct { + Device string `json:"device"` + SeenAt int64 `json:"seen_at"` + Items []HoldFence `json:"items"` +} +type HoldFence struct { + Cell string `json:"cell"` + Fence uint64 `json:"fence"` +} + +// Quiescence is a durable application marker, not a lease or permission to read. +// Index is its log position; any subsequent command invalidates the marker. +type Quiescence struct { + Index uint64 `json:"index"` + Term uint64 `json:"term"` + Barrier uint64 `json:"barrier"` + Voters []uint64 `json:"voters"` +} +type PrepareQuiescence struct { + Barrier uint64 `json:"barrier"` + Term uint64 `json:"term"` + Voters []uint64 `json:"voters"` +} + +func QuiescenceKey() RecordKey { return RecordKey{"membership", "quiescence"} } + +func renewHolds(c Command, now int64, rows map[RecordKey][]byte, put func(RecordKey, any), result *Result) Code { + v := c.Change.RenewHolds + if v.Device == "" || len(v.Items) == 0 || len(v.Items) > MaxRenewHolds || v.SeenAt <= 0 || v.SeenAt > now || now-v.SeenAt > EvidenceWindowMillis { + return BadRequest + } + var actor Device + var auth Authority + if Decode(rows[AuthorityKey(v.Device)], &auth) != nil || !auth.Personal || auth.Cluster != c.Cluster { + return Unauthorized + } + if raw := rows[DeviceKey(v.Device)]; raw != nil && Decode(raw, &actor) != nil { + return BadRequest + } + if actor.Revoked { + return Revoked + } + seen := map[string]bool{} + for _, h := range v.Items { + if h.Cell == "" || h.Fence == 0 || seen[h.Cell] { + return BadRequest + } + seen[h.Cell] = true + } + for _, h := range v.Items { + var cell Cell + code := OK + raw := rows[CellKey(h.Cell)] + switch { + case raw == nil: + code = NotFound + case Decode(raw, &cell) != nil: + return BadRequest + case cell.Lease.Device != v.Device || cell.Lease.Fence != h.Fence: + code = FenceStale + case cell.Lease.Expires == 0 || v.SeenAt <= cell.Lease.ReleasedAt: + code = FenceStale + default: + // A recent publish already covers this evidence; retain its later + // expiry without creating a redundant application write. + until := v.SeenAt + LeaseTTLMillis + if until > cell.Lease.Expires { + cell.Lease.Expires = until + put(CellKey(h.Cell), cell) + } + } + item := ItemResult{Cell: h.Cell, Code: code} + if code == OK { + item.View = &CellView{Cell: cell, Now: now} + } + result.Items = append(result.Items, item) + } + return OK +} + +// RecordActive is evaluated one immutable row at a time so large imported +// identities never have to be materialized to decide whether they may sleep. +func RecordActive(key RecordKey, raw []byte, now int64) (bool, error) { + if key.Namespace == "state" && len(key.Key) >= 5 && key.Key[:5] == "cell/" { + var c Cell + if Decode(raw, &c) != nil { + return false, ErrDependency + } + return c.Lease.Expires > now, nil + } + if key == MembershipKey() { + var h Membership + if Decode(raw, &h) != nil || h.Validate() != nil { + return false, ErrDependency + } + return h.Pending != nil || h.Leadership != nil && !h.Leadership.Completed, nil + } + if key.Namespace == "outbox" && len(key.Key) >= 5 && key.Key[:5] == "join/" { + var request JoinRequest + if Decode(raw, &request) != nil { + return false, ErrDependency + } + return request.Decision != nil && request.Outcome == "", nil + } + return false, nil +} + +// RenewalCovered suppresses a redundant batch item after an authoritative +// read. A recent publish may already cover this sample; this never invents a +// receipt or extends a released, displaced or expired lease. +func RenewalCovered(raw []byte, device string, hold HoldFence, seen int64) (bool, error) { + if raw == nil { + return false, nil + } + var c Cell + if err := Decode(raw, &c); err != nil { + return false, err + } + return seen > 0 && seen <= int64(MaxSafeInteger)-LeaseTTLMillis && c.Lease.Device == device && c.Lease.Fence == hold.Fence && c.Lease.Expires >= seen+LeaseTTLMillis, nil +} diff --git a/internal/clusterstate/liveness_test.go b/internal/clusterstate/liveness_test.go new file mode 100644 index 0000000000..eee7860769 --- /dev/null +++ b/internal/clusterstate/liveness_test.go @@ -0,0 +1,144 @@ +package clusterstate_test + +import ( + "crypto/ed25519" + "fmt" + "math/rand" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +// R22 pins freshness, release-before-delayed-renew, exact fences and committed +// time without importing a socket into the authorization rule. +func TestR22CommittedRenewalEvidence(t *testing.T) { + for seed := 0; seed < seedCount(t); seed++ { + t.Run(fmt.Sprintf("seed-%04d", seed), func(t *testing.T) { + a := newApplication(t) + a.put(cs.AuthorityKey("ingress"), cs.Authority{Cluster: "test", Key: actorKey("ingress").Public().(ed25519.PublicKey), Effects: true}) + now := int64(1_000_000 + seed) + a.run("a", now, cs.Mutation{Create: &cs.CreateCell{Cell: "chat", Init: cs.CellInit{Head: "sealed"}}}) + renew := func(device string, seen int64, fence uint64) cs.Result { + a.meta.Index++ + return a.run("ingress", now, cs.Mutation{RenewHolds: &cs.RenewHolds{Device: device, SeenAt: seen, Items: []cs.HoldFence{{Cell: "chat", Fence: fence}}}}) + } + before := a.meta.Version + now += 30_000 + r := renew("a", now, 1) + if r.Code != cs.OK || r.Index == 0 || len(r.Items) != 1 || r.Items[0].Code != cs.OK || a.cell().Lease.Expires != now+90_000 || a.meta.Version != before { + t.Fatal("R22 committed expiry/version", r, a.cell()) + } + until := a.cell().Lease.Expires + now += 10_000 + r = renew("a", now-10_000, 1) + if r.Code != cs.OK || a.cell().Lease.Expires != until { + t.Fatal("R22 replay extended expiry", r) + } + r = renew("a", now-25_001, 1) + if r.Code != cs.BadRequest { + t.Fatal("R22 stale evidence", r) + } + r = renew("a", now+1, 1) + if r.Code != cs.BadRequest { + t.Fatal("R22 future evidence", r) + } + r = renew("a", now, 2) + if r.Items[0].Code != cs.FenceStale || a.cell().Lease.Expires != until { + t.Fatal("R22 wrong fence", r) + } + for _, quiet := range []bool{false, true} { + a.run("a", now, cs.Mutation{Release: &cs.ReleaseCell{Cell: "chat", Fence: 1, Quiet: quiet}}) + r = renew("a", now-1, 1) + if r.Items[0].Code != cs.FenceStale || a.cell().Lease.Expires != 0 { + t.Fatal("R22 delayed renew revived release", r) + } + } + r = a.run("b", now, cs.Mutation{Acquire: &cs.AcquireCell{Cell: "chat", Expect: cs.LeaseExpectation{Device: "a", Fence: 1}}}) + if r.Code != cs.OK { + t.Fatal(r) + } + r = renew("a", now, 1) + if r.Items[0].Code != cs.FenceStale || a.cell().Lease.Fence != 2 { + t.Fatal("R22 displaced holder renewed", r) + } + a.run("b", now, cs.Mutation{Revoke: &cs.DeviceID{Device: "a"}}) + r = renew("a", now, 1) + if r.Code != cs.Revoked { + t.Fatal("R22 revoked evidence", r) + } + rng := rand.New(rand.NewSource(int64(seed))) + // The independent expiry oracle uses only the accepted evidence + // sample, never the implementation's resulting cell to predict it. + expected := a.cell().Lease.Expires + for step := 0; step < 12; step++ { + now += int64(rng.Intn(9000)) + seen := now - int64(rng.Intn(40_000)) + r = renew("b", seen, 2) + if now-seen > 25_000 { + if r.Code != cs.BadRequest { + t.Fatal("R22 randomized stale sample", r) + } + } else { + expected = max(expected, seen+90_000) + if r.Code != cs.OK || r.Items[0].Code != cs.OK { + t.Fatal("R22 randomized fresh sample", r) + } + } + if a.cell().Lease.Expires != expected || a.cell().Lease.Fence != 2 { + t.Fatal("R22 randomized evidence changed authority", seen, expected, a.cell()) + } + } + + }) + } +} + +func TestR26QuiescenceMarkerCancelsOnCommand(t *testing.T) { + a := newApplication(t) + a.put(cs.AuthorityKey("anchor"), cs.Authority{Cluster: "test", Key: actorKey("anchor").Public().(ed25519.PublicKey), Anchor: true}) + a.meta.Index = 11 + a.meta.Term = 2 + a.meta.Configuration = cs.Configuration{Voters: []uint64{1, 2, 3}} + r := a.run("anchor", 1000, cs.Mutation{PrepareQuiescence: &cs.PrepareQuiescence{Barrier: 10, Term: 2, Voters: []uint64{1, 2, 3}}}) + if r.Code != cs.OK { + t.Fatal(r) + } + var marker cs.Quiescence + if cs.Decode(a.rows[cs.QuiescenceKey()], &marker) != nil || marker.Index != 11 { + t.Fatal(marker) + } + a.meta.Index++ + a.run("a", 1001, cs.Mutation{AdvanceTime: &cs.AdvanceTime{}}) + if cs.Decode(a.rows[cs.QuiescenceKey()], &marker) != nil || marker.Index != 0 { + t.Fatal("R26 mutation retained old generation", marker) + } + a.meta.Index++ + r = a.run("anchor", 1002, cs.Mutation{PrepareQuiescence: &cs.PrepareQuiescence{Barrier: 10, Term: 2, Voters: []uint64{1, 2, 3}}}) + if r.Code != cs.Conflict { + t.Fatal("R26 stale barrier accepted", r) + } +} + +// R26/R27 retain scheduled committed delivery work without holding an idle +// pairing request or a completed historical outbox row active forever. +func TestR26R27CommittedOutboxPreventsSleep(t *testing.T) { + key := cs.JoinKey("fixture") + for _, v := range []struct { + request cs.JoinRequest + active bool + }{ + {cs.JoinRequest{Device: "a", Expires: 5000}, false}, + {cs.JoinRequest{Device: "a", Decision: &cs.JoinDecision{Deny: true}}, true}, + {cs.JoinRequest{Device: "a", Decision: &cs.JoinDecision{Deny: true}, Outcome: "delivered"}, false}, + {cs.JoinRequest{Device: "a", Decision: &cs.JoinDecision{Deny: true}, Outcome: "gone"}, false}, + } { + raw, err := cs.Encode(v.request) + if err != nil { + t.Fatal(err) + } + active, err := cs.RecordActive(key, raw, 1000) + if err != nil || active != v.active { + t.Fatal(v, active, err) + } + } +} diff --git a/internal/clusterstate/types.go b/internal/clusterstate/types.go index 0388b49e25..42d16dab05 100644 --- a/internal/clusterstate/types.go +++ b/internal/clusterstate/types.go @@ -44,6 +44,8 @@ type Command struct { // their typed operations here; unknown fields fail decoding on older members. // There is no arbitrary JSON payload in which task text or secrets can hide. type Mutation struct { + RenewHolds *RenewHolds `json:"renew_holds,omitempty"` + PrepareQuiescence *PrepareQuiescence `json:"prepare_quiescence,omitempty"` CompleteLeadership *CompleteLeadership `json:"complete_leadership,omitempty"` RequestLeadership *RequestLeadership `json:"request_leadership,omitempty"` RestoreAnchors *RestoreAnchors `json:"restore_anchors,omitempty"` @@ -144,6 +146,7 @@ type DeviceSeen struct { // no free-form diagnostic text, so logging a refusal cannot disclose payloads. // Now and Version are application time/version, never a local clock/Raft index. type Result struct { + Index uint64 `json:"index,omitempty"` Membership *MembershipResult `json:"membership,omitempty"` Schema uint16 `json:"schema"` Code Code `json:"code"` diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index 679123f988..a09a5a37d0 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -164,6 +164,14 @@ func (r *Runtime) Call(b []byte) (out []byte) { func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { e := encoder{} switch op { + case 23: + return r.schedule(d) + case 24: + return activeRecord(d) + case 25: + return coordinationPolicy(d) + case 26: + return renewalCovered(d) case 21, 22: return r.leadership(d, op == 22) case 20: @@ -272,6 +280,8 @@ func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { return legacyAuthorities(d) case 13: return legacyGenesis(d) + case 15: + in = raftcore.IdleRead{RequestID: d.str()} case 12: in = raftcore.TransportResult{Peer: d.u64(), Snapshot: d.u32() != 0, Success: d.u32() != 0} case 11: diff --git a/internal/raftabi/scheduler.go b/internal/raftabi/scheduler.go new file mode 100644 index 0000000000..1264c19e7a --- /dev/null +++ b/internal/raftabi/scheduler.go @@ -0,0 +1,84 @@ +package raftabi + +import ( + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" +) + +func (r *Runtime) schedule(d *decoder) ([]byte, error) { + id, raw := d.u32(), d.blob() + var in raftcore.ScheduleInput + if err := clusterstate.Decode(raw, &in); err != nil { + return nil, err + } + if err := d.end(); err != nil { + return nil, err + } + node := r.nodes[id] + if node == nil { + return nil, errFrame + } + p, err := node.Schedule(in) + if err != nil { + return nil, err + } + b, err := clusterstate.Encode(p) + if err != nil { + return nil, err + } + e := encoder{} + e.blob(b) + return e.b, nil +} +func activeRecord(d *decoder) ([]byte, error) { + key := clusterstate.RecordKey{Namespace: d.str(), Key: d.str()} + raw, now := d.blob(), d.u64() + if err := d.end(); err != nil || now > clusterstate.MaxSafeInteger { + return nil, errFrame + } + active, err := clusterstate.RecordActive(key, raw, int64(now)) + if err != nil { + return nil, err + } + e := encoder{} + e.flag(active) + return e.b, nil +} + +func coordinationPolicy(d *decoder) ([]byte, error) { + if err := d.end(); err != nil { + return nil, err + } + b, err := clusterstate.Encode(struct { + TickEvery int64 `json:"tick_every"` + RenewEvery int64 `json:"renew_every"` + PresenceEvery int64 `json:"presence_every"` + EvidenceWindow int64 `json:"evidence_window"` + IdleAfter int64 `json:"idle_after"` + MaxRenewHolds int `json:"max_renew_holds"` + MaxMessageBytes int `json:"max_message_bytes"` + }{clusterstate.TickEveryMillis, clusterstate.RenewEveryMillis, clusterstate.PresenceEveryMillis, clusterstate.EvidenceWindowMillis, clusterstate.IdleAfterMillis, clusterstate.MaxRenewHolds, clusterstate.MaxCommandBytes}) + if err != nil { + return nil, err + } + e := encoder{} + e.blob(b) + return e.b, nil +} + +func renewalCovered(d *decoder) ([]byte, error) { + raw, device, cell, fence, seen := d.blob(), d.str(), d.str(), d.u64(), d.u64() + if err := d.end(); err != nil || seen > clusterstate.MaxSafeInteger { + return nil, errFrame + } + if len(raw) == 0 { + raw = nil + } + covered, err := clusterstate.RenewalCovered(raw, device, clusterstate.HoldFence{Cell: cell, Fence: fence}, int64(seen)) + if err != nil { + return nil, err + } + e := encoder{} + e.flag(covered) + return e.b, nil +} diff --git a/internal/raftcore/README.md b/internal/raftcore/README.md index 76ff1e136a..7d6db33b2b 100644 --- a/internal/raftcore/README.md +++ b/internal/raftcore/README.md @@ -90,7 +90,7 @@ Do not use this package to load a full application identity into wasm: P01 rows and P09 paged apply remain outside the consensus cache. Quiescence scheduling, clock samples, retry deadlines and application logical time are explicit host inputs in later packets. `NeedNextTick` currently means every non-stopped handle -needs ticks; the safe quiescence protocol belongs to P14. +needs ticks unless the explicit committed quiescence protocol below stops them. ## Replay contract @@ -205,3 +205,38 @@ only another personal request may promote it. Public handlers accept the typed signed request and never expose Configure or host-supplied readiness evidence. P14 schedules ticks/wake/housekeeping, and P17 assembles these host/control seams with the personal member lifecycle and the one production directory route. + +## Committed liveness and idle scheduling (P14) + +`Schedule` takes monotonic elapsed milliseconds, conservative host activity, +its durable `clusterstate.Quiescence` row and authenticated applied voter +acknowledgments. It returns at most one tick, the next delay, wake destinations, +or a barrier/prepare plan. Run it under the member queue. The host includes +queued commands, outboxes, assignments, holds and membership/transfer work in +activity; `raftdirectory.Schedule` and `MemberActor.schedule` page the current +application's hold/intent dependencies using the same Go activity rule. + +After sixty idle seconds the elected leader needs an internal `IdleRead` before +preparing a marker bound to its applied index, term and exact current voters. +`IdleRead` preserves elapsed idle time but still performs the ordinary safe +quorum read. Only a committed, durably applied anchor command creates the marker. +Every current voter must acknowledge that exact durable index/term before any +reached member can stop its timer. Unreached members remain active. Operations +cancel the generation; delayed final delivery cannot put it back to sleep. +Recovery always starts awake as a follower. An elapsed-hours sample yields one +tick, never hours of tick debt. A remembered leader never bypasses ReadIndex. + +Application `RenewHolds` is restricted ingress authority, with exact device and +fence and a fresh ingress sample. Its expiry is sample + ordinary lease TTL, +never receipt-time + TTL. Released leases remain released. A recent publish can +cover a sample; the authoritative read plus shared `RenewalCovered` helper skips +that item before logging. Successful receipt indexes escape only after durable +application. Presence pongs alone do not suppress client heartbeats. + +The router's timer driver must resume the existing P12/P13 committed workflow +and P10 outboxes before idle preparation. It must authenticate target evidence, +finish joint stages, settle, transfer, obtain the elected target's fresh read, +commit completion and obtain a new receipt. Returning completed intents never +regain leadership automatically. `QuiescenceCoordinator` sequences the idle +part without adding another coordination writer. P17 supplies the sole product +host and the outbound router assembly; no deployment or rollout switch exists. diff --git a/internal/raftcore/core.go b/internal/raftcore/core.go index 3930a5ec36..3198dade4d 100644 --- a/internal/raftcore/core.go +++ b/internal/raftcore/core.go @@ -46,6 +46,7 @@ type readRequest struct { // queued implicitly. Storage failure or an unknown committed command stops the // handle. Recovery always starts as a follower from supplied durable bytes. type Core struct { + schedule scheduling cfg Config node *raft.RawNode store *storage @@ -150,7 +151,7 @@ func New(cfg Config, recovered RecoveredState) (core *Core, err error) { if conf == nil { conf = &pb.ConfState{} } - c := &Core{cfg: cfg, electionRank: rank, rng: splitMix64{state: cfg.Seed}, applied: recovered.Applied, reads: make(map[string]readRequest), barriers: make(map[string]uint64)} + c := &Core{schedule: scheduling{cancelled: recovered.Applied}, cfg: cfg, electionRank: rank, rng: splitMix64{state: cfg.Seed}, applied: recovered.Applied, reads: make(map[string]readRequest), barriers: make(map[string]uint64)} c.store = &storage{MemoryStorage: raft.NewMemoryStorage(), conf: conf} if !raft.IsEmptySnap(snapshot) { if err := c.store.ApplySnapshot(snapshot); err != nil { @@ -216,11 +217,20 @@ func (c *Core) Step(input Input) (effects Effects, err error) { err = ErrStopped } effects.Status = c.status() - effects.NeedNextTick = !c.stopped + effects.NeedNextTick = !c.stopped && !c.schedule.sleeping if !c.stopped && c.pending == nil { effects.SnapshotNeeded = c.SnapshotNeeded() } }() + if idle, ok := input.(IdleRead); ok { + active, sleeping := c.schedule.active, c.schedule.sleeping + input = Read{RequestID: idle.RequestID} + defer func() { + if err == nil && !sleeping { + c.schedule.active = active + } + }() + } if c.stopped { return effects, ErrStopped } @@ -268,6 +278,14 @@ func (c *Core) Step(input Input) (effects Effects, err error) { if c.pending != nil { return effects, ErrBusy } + switch input.(type) { + case Read, Propose, Configure, Transfer: + c.wakeScheduling() + case PeerMessage: + if c.schedule.sleeping { + c.wakeScheduling() + } + } switch in := input.(type) { case CompactSnapshot: p := in.Snapshot @@ -302,7 +320,9 @@ func (c *Core) Step(input Input) (effects Effects, err error) { } case Poll: case Tick: - c.node.Tick() + if !c.schedule.sleeping { + c.node.Tick() + } case PeerMessage: m := &pb.Message{} if len(in.Bytes) > MaxUncommittedBytes || proto.Unmarshal(in.Bytes, m) != nil || !c.validPeer(m) { diff --git a/internal/raftcore/scheduler.go b/internal/raftcore/scheduler.go new file mode 100644 index 0000000000..40f74df6da --- /dev/null +++ b/internal/raftcore/scheduler.go @@ -0,0 +1,141 @@ +package raftcore + +import ( + "slices" + + "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +// ScheduleInput is an internal host call under the same member queue as Step. +// Now is elapsed monotonic milliseconds, never accumulated missed tick debt. +// Busy includes work, assignments, holds, ingress requests and outbox work. +// Marker and acknowledgments come from durable application/authenticated peers; +// they are never accepted from a public coordination request. +type ScheduleInput struct { + Now int64 `json:"now"` + Busy bool `json:"busy"` + Marker clusterstate.Quiescence `json:"marker"` + Acks []SleepAck `json:"acks"` +} +type SleepAck struct { + Node uint64 `json:"node"` + Index uint64 `json:"index"` + Term uint64 `json:"term"` +} +type SchedulePlan struct { + Barrier bool `json:"barrier"` + Tick bool `json:"tick"` + Prepare bool `json:"prepare"` + Sleeping bool `json:"sleeping"` + Wake []uint64 `json:"wake"` + After int64 `json:"after"` +} +type scheduling struct { + started bool + now int64 + active int64 + next int64 + marker uint64 + cancelled uint64 + sleeping bool + wake bool +} + +// wakeScheduling invalidates the observed sleep generation before processing +// an operation. A delayed final acknowledgment cannot put that generation back +// to sleep. Core reads still use ReadOnlySafe, even with a remembered leader. +func (c *Core) wakeScheduling() { + s := &c.schedule + s.cancelled = max(s.cancelled, s.marker, c.applied) + s.sleeping = false + s.active = s.now + s.next = s.now + clusterstate.TickEveryMillis + s.wake = true + c.quorumIndex, c.quorumTerm = 0, 0 +} + +// Schedule suppresses scheduling only after ALL current voters have durably +// applied one committed marker. It does not use TickQuiesced, a clock lease or +// a reduced quorum. An unreachable member retains its ordinary active timer. +func (c *Core) Schedule(in ScheduleInput) (SchedulePlan, error) { + var p SchedulePlan + if c.stopped { + return p, ErrStopped + } + if c.pending != nil { + return p, ErrBusy + } + if in.Now < 0 || in.Now > int64(clusterstate.MaxSafeInteger)-clusterstate.IdleAfterMillis { + return p, ErrConfig + } + s := &c.schedule + if !s.started { + s.started = true + s.now = in.Now + s.active = in.Now + s.next = in.Now + clusterstate.TickEveryMillis + } + // A backward host sample cannot move a deadline into the past. + s.now = max(s.now, in.Now) + conf := Configuration(c.store.conf) + voters := slices.Clone(conf.Voters) + for _, id := range conf.Outgoing { + if !slices.Contains(voters, id) { + voters = append(voters, id) + } + } + slices.Sort(voters) + if s.wake { + p.Wake = voters + s.wake = false + } + last, err := c.store.LastIndex() + if err != nil { + return p, err + } + busy := in.Busy || conf.Joint() || len(c.reads) != 0 || last != c.applied || c.status().Commit != c.applied + m := in.Marker + valid := m.Index != 0 && m.Index == c.applied && m.Term == c.status().Term && m.Barrier+1 == m.Index && slices.Equal(m.Voters, voters) + if valid { + s.marker = max(s.marker, m.Index) + } + if busy { + s.cancelled = max(s.cancelled, s.marker, c.applied) + s.active = s.now + } + if (busy || !valid) && s.sleeping { + c.wakeScheduling() + p.Wake = voters + s.wake = false + } + if !busy && valid && m.Index > s.cancelled { + complete := len(in.Acks) == len(voters) + seen := map[uint64]bool{} + for _, ack := range in.Acks { + if !slices.Contains(voters, ack.Node) || seen[ack.Node] || ack.Index != m.Index || ack.Term != m.Term { + complete = false + } + seen[ack.Node] = true + } + if complete { + s.sleeping = true + } + } + p.Sleeping = s.sleeping + if s.sleeping { + return p, nil + } + p.After = max(int64(1), s.next-s.now) + if s.now >= s.next { + p.Tick = true + s.next = s.now + clusterstate.TickEveryMillis + p.After = clusterstate.TickEveryMillis + } + // Fresh ReadIndex is required before preparing the durable marker. Host + // activity checks must also cover application intents and unexpired holds. + eligible := !busy && c.leader() && s.now-s.active >= clusterstate.IdleAfterMillis && (!valid || m.Index <= s.cancelled) + fresh := c.quorumTerm == c.status().Term && c.quorumIndex >= c.applied + p.Barrier = eligible && !fresh + p.Prepare = eligible && fresh + return p, nil +} diff --git a/internal/raftcore/scheduler_test.go b/internal/raftcore/scheduler_test.go new file mode 100644 index 0000000000..ddb8cb0f2a --- /dev/null +++ b/internal/raftcore/scheduler_test.go @@ -0,0 +1,158 @@ +package raftcore + +import ( + "bytes" + "encoding/json" + "go.etcd.io/raft/v3" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +// R26 uses actual durable Ready completions and safe reads; sleep certificates +// cannot be inferred from majority transport delivery or a remembered leader. +func TestR26PartialQuiescenceWakeAndRestart(t *testing.T) { + g := newGroup(t) + g.elect() + l := g.leader() + c := g.nodes[l-1] + s := c.status() + marker := cs.Quiescence{Index: s.Applied, Term: s.Term, Barrier: s.Applied - 1, Voters: []uint64{1, 2, 3}} + acks := []SleepAck{} + for _, node := range g.nodes { + acks = append(acks, SleepAck{Node: node.cfg.NodeID, Index: s.Applied, Term: s.Term}) + } + for _, n := range g.nodes { + if _, e := n.Schedule(ScheduleInput{Now: 0}); e != nil { + t.Fatal(e) + } + } + p, e := c.Schedule(ScheduleInput{Now: 60_000, Marker: marker, Acks: acks[:2]}) + if e != nil || p.Sleeping { + t.Fatal("R26 partial barrier slept", p, e) + } + p, e = c.Schedule(ScheduleInput{Now: 60_000, Marker: marker, Acks: acks}) + if e != nil || !p.Sleeping || p.Tick || p.After != 0 { + t.Fatal("R26 complete barrier did not sleep", p, e) + } + // A one-hour scheduler delay injects no tick debt; the operation wakes it. + g.call(l, Read{RequestID: "wake"}) + g.pump() + p, e = c.Schedule(ScheduleInput{Now: 3_600_000, Marker: marker, Acks: acks}) + if e != nil || p.Sleeping || len(p.Wake) != 3 { + t.Fatal("R26 wake reused certificate", p, e) + } + d := g.durable[l-1] + recovered, e := New(Config{ClusterUUID: "test-cluster", NodeID: l, Seed: 88, Protocol: 1, Anchors: []uint64{1, 2, 3}}, d) + if e != nil { + t.Fatal(e) + } + p, e = recovered.Schedule(ScheduleInput{Now: 0, Marker: marker, Acks: acks}) + if e != nil || p.Sleeping { + t.Fatal("R26 recovery reused sleep certificate", p, e) + } +} + +// R27 counts virtual whole days with no wall sleeps. A delayed scheduler turn +// runs at most one tick; stable heartbeats create no application/log writes. +func TestR27VirtualDayCounters(t *testing.T) { + g := newGroup(t) + g.elect() + l := g.leader() + c := g.nodes[l-1] + st := c.status() + marker := cs.Quiescence{Index: st.Applied, Term: st.Term, Barrier: st.Applied - 1, Voters: []uint64{1, 2, 3}} + acks := []SleepAck{{1, st.Applied, st.Term}, {2, st.Applied, st.Term}, {3, st.Applied, st.Term}} + c.Schedule(ScheduleInput{}) + p, e := c.Schedule(ScheduleInput{Now: 60_000, Marker: marker, Acks: acks}) + if e != nil || !p.Sleeping { + t.Fatal(p, e) + } + for now := int64(65_000); now <= 86_400_000; now += 5_000 { + p, e = c.Schedule(ScheduleInput{Now: now, Marker: marker, Acks: acks}) + if e != nil || p.Tick || p.After != 0 { + t.Fatal("R27 recurring idle turn", p, e) + } + } + // A busy identity's scheduler is bounded independently of learner fanout. + g.call(l, Read{RequestID: "active"}) + g.pump() + ticks := 0 + for now := int64(86_400_000); now <= 172_800_000; now += 5_000 { + p, e = c.Schedule(ScheduleInput{Now: now, Busy: true}) + if e != nil { + t.Fatal(e) + } + if p.Tick { + ticks++ + } + } + if ticks != 17_280 { + t.Fatal("R27 active tick budget", ticks) + } + p, e = c.Schedule(ScheduleInput{Now: 500_000_000, Busy: false}) + if e != nil || !p.Tick { + t.Fatal("R27 resumed scheduler", p, e) + } + p, e = c.Schedule(ScheduleInput{Now: 500_000_000, Busy: false}) + if e != nil || p.Tick { + t.Fatal("R27 replayed tick debt", p, e) + } +} + +// R26's internal idle barrier preserves the sixty-second idle interval while +// still using the real quorum read; a public read begins a new idle interval. +func TestR26IdlePreparationNeedsFreshQuorum(t *testing.T) { + g := newGroup(t) + l := g.elect() + c := g.nodes[l-1] + c.Schedule(ScheduleInput{Now: 0}) + p, e := c.Schedule(ScheduleInput{Now: 60_000}) + if e != nil || !p.Barrier || p.Prepare { + t.Fatal(p, e) + } + g.call(l, IdleRead{RequestID: "sleep-preparation"}) + g.pump() + p, e = c.Schedule(ScheduleInput{Now: 60_000}) + if e != nil || p.Barrier || !p.Prepare { + t.Fatal("R26 idle barrier reset elapsed idle time", p, e) + } + g.call(l, Read{RequestID: "public-operation"}) + g.pump() + p, e = c.Schedule(ScheduleInput{Now: 60_001}) + if e != nil || p.Prepare { + t.Fatal("R26 public operation retained idle timer", p, e) + } +} + +func TestR27StableHeartbeatHasNoLogWrites(t *testing.T) { + g := newGroup(t) + l := g.elect() + g.tick(2) + before := g.nodes[l-1].status().Applied + retained := len(g.durable[l-1].Entries) + g.trace.Reset() + g.tick(12) + if g.nodes[l-1].status().Applied != before || len(g.durable[l-1].Entries) != retained { + t.Fatal("R27 empty heartbeat wrote log") + } + decoder := json.NewDecoder(bytes.NewReader(g.trace.Bytes())) + messages, writes := 0, 0 + for decoder.More() { + var e struct { + Send []Outbound + Persist *PersistBatch + } + if err := decoder.Decode(&e); err != nil { + t.Fatal(err) + } + messages += len(e.Send) + if p := e.Persist; p != nil && (len(p.Entries) != 0 || !raft.IsEmptyHardState(p.HardState)) { + writes++ + } + } + if messages != 12*4 || writes != 0 { + t.Fatal("R27 heartbeat envelope", messages, writes) + } + t.Logf("R27 24-hour active voter envelope: ticks=%d messages=%d log_writes=%d", 3*17280, messages/12*17280, writes) +} diff --git a/internal/raftcore/types.go b/internal/raftcore/types.go index 3440590de4..3498802797 100644 --- a/internal/raftcore/types.go +++ b/internal/raftcore/types.go @@ -96,6 +96,12 @@ type Propose struct{ Command []byte } type Configure struct{ Change []byte } type Read struct{ RequestID string } +// IdleRead is an internal quiescence barrier; it cannot authorize a client +// operation and preserves only the already elapsed idle interval. +type IdleRead struct{ RequestID string } + +func (IdleRead) coreInput() {} + // CancelRead marks a host timeout without producing a successful read. A // submitted context retains its admission slot until Raft finishes that round; // otherwise repeated cancellation could create unbounded hidden work. diff --git a/internal/raftdirectory/liveness.go b/internal/raftdirectory/liveness.go new file mode 100644 index 0000000000..f583374b81 --- /dev/null +++ b/internal/raftdirectory/liveness.go @@ -0,0 +1,98 @@ +package raftdirectory + +import ( + "context" + "errors" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/dirwatch" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftstore" +) + +// Renew is called only with authenticated ingress evidence. The host cannot +// return from CommitEffect until quorum and application persistence complete. +// Socket pongs remain advisory while this call waits or fails. +func (e Effects) Renew(ctx context.Context, now int64, evidence clusterstate.RenewHolds) ([]dirwatch.HoldReceipt, error) { + if len(evidence.Items) > clusterstate.MaxRenewHolds { + return nil, errors.New("raftdirectory: hold batch capacity") + } + items := make([]clusterstate.HoldFence, 0, len(evidence.Items)) + for _, h := range evidence.Items { + raw, err := e.Host.ReadEffect(ctx, clusterstate.CellKey(h.Cell)) + if err != nil { + return nil, err + } + covered, err := clusterstate.RenewalCovered(raw, evidence.Device, h, evidence.SeenAt) + if err != nil { + return nil, err + } + if !covered { + items = append(items, h) + } + } + if len(items) == 0 { + return nil, nil + } + evidence.Items = items + c, err := clusterstate.Sign(clusterstate.Command{Schema: clusterstate.SchemaVersion, Cluster: e.Cluster, Actor: e.Actor, Session: "holds", Sequence: 1, Now: now, Change: clusterstate.Mutation{RenewHolds: &evidence}}, e.Key) + if err != nil { + return nil, err + } + r, err := e.Host.CommitEffect(ctx, c) + if err != nil { + return nil, err + } + if r.Code != clusterstate.OK || r.Index == 0 { + return nil, errors.New("raftdirectory: hold renewal did not commit") + } + receipts := []dirwatch.HoldReceipt{} + for _, item := range r.Items { + if item.Code == clusterstate.OK && item.View != nil { + receipts = append(receipts, dirwatch.HoldReceipt{Cell: item.Cell, Fence: item.View.Cell.Lease.Fence, Index: r.Index, At: r.Now, Until: item.View.Cell.Lease.Expires}) + } + } + return receipts, nil +} + +// Schedule reads local durable activity only to decide timer suppression. It +// authorizes no operation. Callers hold the actor queue for this paged scan and +// include queued ingress, assignments and external work in busy. +func Schedule(ctx context.Context, store *raftstore.SQLite, core *raftcore.Core, elapsed, ingressNow int64, busy bool, acks []raftcore.SleepAck) (raftcore.SchedulePlan, error) { + app, _, err := store.ApplicationView(ctx) + if err != nil { + return raftcore.SchedulePlan{}, err + } + for _, ns := range []raftstore.Namespace{raftstore.State, raftstore.Membership, raftstore.Outbox} { + after := "" + for { + rows, _, err := store.ScanPage(ctx, app.Applied, ns, after) + if err != nil { + return raftcore.SchedulePlan{}, err + } + for _, row := range rows { + active, err := clusterstate.RecordActive(clusterstate.RecordKey{Namespace: string(ns), Key: row.Key.Key}, row.Value, ingressNow) + if err != nil { + return raftcore.SchedulePlan{}, err + } + busy = busy || active + after = row.Key.Key + } + if len(rows) == 0 { + break + } + } + } + key := raftstore.Key{Namespace: raftstore.Membership, Key: clusterstate.QuiescenceKey().Key} + rows, _, err := store.Read(ctx, app.Applied, []raftstore.Key{key}) + if err != nil { + return raftcore.SchedulePlan{}, err + } + var marker clusterstate.Quiescence + if raw := rows[key]; raw != nil { + if err = clusterstate.Decode(raw, &marker); err != nil { + return raftcore.SchedulePlan{}, err + } + } + return core.Schedule(raftcore.ScheduleInput{Now: elapsed, Busy: busy, Marker: marker, Acks: acks}) +} diff --git a/internal/raftdirectory/liveness_test.go b/internal/raftdirectory/liveness_test.go new file mode 100644 index 0000000000..82746d0dce --- /dev/null +++ b/internal/raftdirectory/liveness_test.go @@ -0,0 +1,125 @@ +package raftdirectory_test + +import ( + "crypto/ed25519" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/raftstore" +) + +// R22/R26 use the real three-native-store application adapter. Renew receipts +// escape only after durable commit; markers and exact expiry survive snapshots. +func TestR22R26NativeCommittedLivenessAndSleep(t *testing.T) { + now := int64(1_000_000) + extra := raftstore.Write{Key: raftstore.Key{Namespace: raftstore.Enrollment, Key: cs.AuthorityKey("anchor").Key}, Value: encoded(t, cs.Authority{Cluster: cluster, Key: key("anchor").Public().(ed25519.PublicKey), Anchor: true})} + g := newGroup(t, func() time.Time { return time.UnixMilli(now) }, extra) + r, e := g.Change(ctx, "a", cs.Mutation{Create: &cs.CreateCell{Cell: "chat", V1: true, Init: cs.CellInit{Head: "sealed"}}}) + must(t, e) + if r.Code != cs.OK { + t.Fatal(r) + } + effects := raftdirectory.Effects{Host: &effectHost{g: g}, Cluster: cluster, Actor: "effects", Key: key("effects")} + now += 30_000 + receipts, e := effects.Renew(ctx, now, cs.RenewHolds{Device: "a", SeenAt: now, Items: []cs.HoldFence{{Cell: "chat", Fence: 1}}}) + must(t, e) + if len(receipts) != 1 || receipts[0].Index == 0 || receipts[0].Until != now+90_000 { + t.Fatal(receipts) + } + g.blocked = true + if _, err := effects.Renew(ctx, now+30_000, cs.RenewHolds{Device: "a", SeenAt: now + 30_000, Items: []cs.HoldFence{{Cell: "chat", Fence: 1}}}); err == nil { + t.Fatal("R22 disconnected leader returned renewal") + } + g.blocked = false + g.drive(g.leader(), raftcore.Tick{}) + g.drain() + before := g.status[g.leader()].Applied + receipts, e = effects.Renew(ctx, now, cs.RenewHolds{Device: "a", SeenAt: now, Items: []cs.HoldFence{{Cell: "chat", Fence: 1}}}) + must(t, e) + if len(receipts) != 0 || g.status[g.leader()].Applied != before { + t.Fatal("R22 covered sample wrote another entry") + } + r, e = g.Change(ctx, "a", cs.Mutation{Release: &cs.ReleaseCell{Cell: "chat", Fence: 1}}) + must(t, e) + if r.Code != cs.OK { + t.Fatal(r) + } + receipts, e = effects.Renew(ctx, now+1, cs.RenewHolds{Device: "a", SeenAt: now, Items: []cs.HoldFence{{Cell: "chat", Fence: 1}}}) + must(t, e) + if len(receipts) != 0 { + t.Fatal("R22 delayed release renewal", receipts) + } + i, app, e := g.barrier("a") + must(t, e) + for node := range g.nodes { + _, e = raftdirectory.Schedule(ctx, g.stores[node], g.nodes[node], 0, now, false, nil) + must(t, e) + } + r, e = g.Change(ctx, "anchor", cs.Mutation{PrepareQuiescence: &cs.PrepareQuiescence{Barrier: app.Applied, Term: g.status[i].Term, Voters: []uint64{1, 2, 3}}}) + must(t, e) + if r.Code != cs.OK { + t.Fatal(r) + } + g.drive(g.leader(), raftcore.Tick{}) + g.drain() + i = g.leader() + index := g.status[i].Applied + term := g.status[i].Term + acks := []raftcore.SleepAck{{Node: 1, Index: index, Term: term}, {Node: 2, Index: index, Term: term}, {Node: 3, Index: index, Term: term}} + for node := range g.nodes { + p, e := raftdirectory.Schedule(ctx, g.stores[node], g.nodes[node], 60_000, now, false, acks) + must(t, e) + if !p.Sleeping { + t.Fatal("R26 durable native marker did not sleep", node, p) + } + } + // Actual SQLite snapshot/reopen must never restore serving leader/sleep state. + restartEffects(t, g) + for node := range g.nodes { + p, e := raftdirectory.Schedule(ctx, g.stores[node], g.nodes[node], 0, now, false, acks) + must(t, e) + if p.Sleeping { + t.Fatal("R26 recovered sleep generation") + } + } + v, e := g.ReadCell(ctx, "a", "chat") + must(t, e) + if v.Cell.Lease.Expires != 0 || v.Cell.Lease.Fence != 1 { + t.Fatal("R22 released state changed", v) + } +} + +// R26/R27 a committed undelivered pairing result is scheduled work. Even a +// stale host's final sleep certificate cannot suspend its delivery timer. +func TestR26R27NativePendingOutboxKeepsTimerActive(t *testing.T) { + decision := cs.JoinDecision{Code: "fixture", Deny: true} + extra := []raftstore.Write{ + {Key: raftstore.Key{Namespace: raftstore.Enrollment, Key: cs.AuthorityKey("anchor").Key}, Value: encoded(t, cs.Authority{Cluster: cluster, Key: key("anchor").Public().(ed25519.PublicKey), Anchor: true})}, + {Key: raftstore.Key{Namespace: raftstore.Outbox, Key: cs.JoinKey("fixture").Key}, Value: encoded(t, cs.JoinRequest{Device: "a", Decision: &decision})}, + } + g := newGroup(t, func() time.Time { return time.UnixMilli(1000) }, extra...) + _, e := g.Change(ctx, "a", cs.Mutation{AdvanceTime: &cs.AdvanceTime{}}) + must(t, e) + i := g.leader() + barrier := g.status[i].Applied + term := g.status[i].Term + r, e := g.Change(ctx, "anchor", cs.Mutation{PrepareQuiescence: &cs.PrepareQuiescence{Barrier: barrier, Term: term, Voters: []uint64{1, 2, 3}}}) + must(t, e) + if r.Code != cs.OK { + t.Fatal(r) + } + g.drive(i, raftcore.Tick{}) + g.drain() + index := g.status[i].Applied + acks := []raftcore.SleepAck{{Node: 1, Index: index, Term: term}, {Node: 2, Index: index, Term: term}, {Node: 3, Index: index, Term: term}} + for node := range g.nodes { + p, e := raftdirectory.Schedule(ctx, g.stores[node], g.nodes[node], 60_000, 1000, false, acks) + must(t, e) + if p.Sleeping || p.Prepare { + t.Fatal("R26 pending committed delivery slept", node, p) + } + } +} diff --git a/internal/raftsim/application.go b/internal/raftsim/application.go index 351db93e49..1238a15e1f 100644 --- a/internal/raftsim/application.go +++ b/internal/raftsim/application.go @@ -16,12 +16,13 @@ import ( // authorized writer. Session policy here is a contract fixture, not a second // production implementation. The checker uses its own rules and state types. type ProbeApplication struct { - Now int64 - Vault string - Cells map[string]clusterstate.Cell - Revoked map[string]bool - Sessions map[string]probeSession - Padding []byte + Coordination map[string][]byte `json:",omitempty"` + Now int64 + Vault string + Cells map[string]clusterstate.Cell + Revoked map[string]bool + Sessions map[string]probeSession + Padding []byte } type probeSession struct { Sequence uint64 @@ -35,6 +36,12 @@ func freshApp() ProbeApplication { } func (a ProbeApplication) copy() ProbeApplication { out := ProbeApplication{Now: a.Now, Vault: a.Vault, Cells: map[string]clusterstate.Cell{}, Revoked: map[string]bool{}, Sessions: map[string]probeSession{}, Padding: bytes.Clone(a.Padding)} + if a.Coordination != nil { + out.Coordination = map[string][]byte{} + for k, v := range a.Coordination { + out.Coordination[k] = bytes.Clone(v) + } + } for key, cell := range a.Cells { cell.Frames = slices.Clone(cell.Frames) if cell.Keys != nil { diff --git a/internal/raftsim/quiescence.go b/internal/raftsim/quiescence.go new file mode 100644 index 0000000000..fdcfd1e88c --- /dev/null +++ b/internal/raftsim/quiescence.go @@ -0,0 +1,13 @@ +package raftsim + +import ( + "crypto/ed25519" + "crypto/sha256" +) + +// This public deterministic fixture key belongs only to the simulator's narrow +// anchor authority. It never authenticates a real identity or peer. +func quiescenceKey() ed25519.PrivateKey { + seed := sha256.Sum256([]byte("raftsim quiescence anchor fixture")) + return ed25519.NewKeyFromSeed(seed[:]) +} diff --git a/internal/raftsim/quiescence_test.go b/internal/raftsim/quiescence_test.go new file mode 100644 index 0000000000..4a2b87f60c --- /dev/null +++ b/internal/raftsim/quiescence_test.go @@ -0,0 +1,102 @@ +package raftsim + +import ( + "fmt" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsim/model" +) + +func sleepMarker(t *testing.T, s *Simulator, node int) cs.Quiescence { + t.Helper() + var m cs.Quiescence + if cs.Decode(s.Nodes[node].Stable.App.Coordination["membership/quiescence"], &m) != nil { + t.Fatal("R26 missing durable marker") + } + return m +} + +// R26 covers all partial final-delivery sets, cold followers, old certificates, +// first minority reads and first conditional writes after wake. Every marker +// uses the real production evaluator and every ack follows its durable apply. +func TestR26SeededPartialSleepAndWake(t *testing.T) { + for seed := 0; seed < seedCount(t); seed++ { + t.Run(fmt.Sprintf("seed-%04d", seed), func(t *testing.T) { + s := group(t, Config{Seed: uint64(seed)}) + initial := create(t, s, "chat") + l := s.Leader() + submit(t, s, l, model.Call{Kind: "release", Actor: "owner", Cell: "chat", Fence: initial.Fence, At: 1000}) + submit(t, s, l, model.Call{Kind: "read", Actor: "owner", Cell: "chat"}) + for node := range s.Nodes { + s.Do(Action{Kind: "schedule", Node: node, Schedule: &raftcore.ScheduleInput{Now: 0}}) + } + s.Do(Action{Kind: "quiesce", Node: l, Value: 1000}) + s.Drain() + s.Ticks(1) + if s.Err != nil { + finish(t, s) + } + marker := sleepMarker(t, s, l) + acks := []raftcore.SleepAck{} + for node, n := range s.Nodes { + local := sleepMarker(t, s, node) + if local.Index != marker.Index || n.Stable.Applied != marker.Index { + t.Fatal("R26 premature applied ack", node, local, n.Stable.Applied) + } + acks = append(acks, raftcore.SleepAck{Node: uint64(node + 1), Index: local.Index, Term: local.Term}) + } + // A majority applied acknowledgment cannot suppress even the leader's timer. + s.Do(Action{Kind: "schedule", Node: l, Schedule: &raftcore.ScheduleInput{Now: 60_000, Marker: marker, Acks: acks[:2]}}) + s.Drain() + if s.Nodes[l].SchedulePlan.Sleeping { + t.Fatal("R26 partial acknowledgments slept") + } + // The final certificate can reach any subset. Unreached members remain active. + for node := range s.Nodes { + if seed%8&(1< Date: Sat, 24 Oct 2026 04:00:16 -0400 Subject: [PATCH 032/103] Require committed lease receipts before suppressing heartbeats. Keep presence advisory because pongs cannot prove durable renewal. Recheck quiet mode for retained clients and watch streams so a later sync-off decision closes every sending path. Document the ownership and idle limits beside the continuation journey. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/cellsync/liveness.go | 10 +- internal/cellsync/liveness_test.go | 38 ++++++-- internal/directory/client.go | 2 +- internal/directory/hold.go | 3 + internal/directory/holdreceipt_test.go | 28 ++++++ internal/directory/watchclient.go | 10 +- internal/directory/watchclient_test.go | 6 +- internal/dirwatch/doc.go | 5 +- internal/dirwatch/events.go | 2 +- internal/dirwatch/feed.go | 23 +++-- internal/dirwatch/holder.go | 38 ++++++-- internal/dirwatch/holder_test.go | 11 ++- internal/dirwatch/receipts_test.go | 46 +++++++++ internal/dirwatch/stream.go | 16 +++- .../continuing-a-chat-on-another-computer.md | 9 ++ internal/syncsetup/cluster.go | 2 +- internal/syncsetup/cluster_quiet_test.go | 96 +++++++++++++++++++ internal/syncsetup/quiet.go | 35 +++++++ internal/syncsetup/syncsetup.go | 24 ++++- 19 files changed, 357 insertions(+), 47 deletions(-) create mode 100644 internal/directory/holdreceipt_test.go create mode 100644 internal/dirwatch/receipts_test.go create mode 100644 internal/syncsetup/cluster_quiet_test.go diff --git a/internal/cellsync/liveness.go b/internal/cellsync/liveness.go index 4ddcf7d6b5..69d8980fd0 100644 --- a/internal/cellsync/liveness.go +++ b/internal/cellsync/liveness.go @@ -1,8 +1,7 @@ package cellsync -// Liveness is a watch socket that can vouch for the leases this process holds -// . While it does, the relay counts the socket's -// pings as the holder's proof of life, so the Batcher need not heartbeat. +// Liveness follows committed lease-renewal receipts over the watch socket. +// Presence alone never suppresses the Batcher's ordinary heartbeat. type Liveness interface { // Keep names the lease of cell at fence on the socket and answers how to // read whether it is vouched for. The socket dials again to name it. @@ -11,9 +10,8 @@ type Liveness interface { // Keeping is one lease as the socket names it. type Keeping interface { - // Healthy is true only while the relay is certain to be vouching: the - // socket is open, the relay said it vouches, the socket names this lease, - // and it showed a sign of life lately. Anything less, and the holder beats. + // Healthy requires a recent committed receipt for this cell and fence, + // with enough hold-until time left for an ordinary heartbeat on expiry. Healthy() bool // Changes signals, without blocking, that Healthy may have changed. Changes() <-chan struct{} diff --git a/internal/cellsync/liveness_test.go b/internal/cellsync/liveness_test.go index 8130355bec..ccdaa90c5a 100644 --- a/internal/cellsync/liveness_test.go +++ b/internal/cellsync/liveness_test.go @@ -151,14 +151,15 @@ func TestBeatsOnRelayWithoutVouching(t *testing.T) { // vouchingStream is a socket that vouches, speaks once and then stays quiet. type vouchingStream struct { - spoke bool - done chan struct{} + receipts []dirwatch.HoldReceipt + spoke bool + done chan struct{} } func (s *vouchingStream) Next(ctx context.Context) (dirwatch.Frame, error) { if !s.spoke { s.spoke = true - return dirwatch.Frame{Version: 1}, nil + return dirwatch.Frame{Version: 1, Holds: s.receipts}, nil } select { case <-ctx.Done(): @@ -187,21 +188,30 @@ func TestBeatsForCellsBeyondMaxHolds(t *testing.T) { } } -// The same socket with room left vouches, and the heartbeats stop. -func TestBeatsStopForCellsWithinMaxHolds(t *testing.T) { - holder := dirwatch.Hold("within-max-holds", 2, vouchingDial) +// R22 suppresses a normal heartbeat only after the socket carries a committed +// exact-fence receipt. Socket openness alone has its separate negative case. +func TestR22BeatsStopForCommittedHoldsWithinCap(t *testing.T) { + dial := func(_ context.Context, holds []directory.Hold) (dirwatch.Stream, error) { + now := time.Now().UnixMilli() + receipts := []dirwatch.HoldReceipt{} + for _, h := range holds { + receipts = append(receipts, dirwatch.HoldReceipt{Cell: h.Cell, Fence: h.Fence, Index: 1, At: now, Until: now + 90_000}) + } + return &vouchingStream{done: make(chan struct{}), receipts: receipts}, nil + } + holder := dirwatch.Hold("within-max-holds-committed", 2, dial) h := runHeld(t, holdSocket{holder}) probe := holder.Take(directory.Hold{Cell: cellID, Fence: h.drv.Fence}) defer probe.Release() for deadline := time.Now().Add(time.Second); !probe.Healthy(); time.Sleep(time.Millisecond) { if time.Now().After(deadline) { - t.Fatal("the socket never vouched") + t.Fatal("R22 the socket never delivered its committed receipt") } } before := h.dirA.beats.Load() - h.ticks(10) + h.ticks(1) if got := h.dirA.beats.Load() - before; got != 0 { - t.Fatalf("%d heartbeats with the socket vouching, want none", got) + t.Fatalf("%d heartbeats covered by a committed receipt, want none", got) } } @@ -213,3 +223,13 @@ type holdSocket struct { func (s holdSocket) Keep(cell string, fence uint64) Keeping { return s.h.Take(directory.Hold{Cell: cell, Fence: fence}) } + +// R22 keeps ordinary beats on a healthy legacy socket with no committed receipt. +func TestR22BeatsContinueWithoutCommittedReceipt(t *testing.T) { + holder := dirwatch.Hold("within-max-holds-no-receipt", 2, vouchingDial) + h := runHeld(t, holdSocket{holder}) + h.ticks(4) + if got := h.dirA.beats.Load(); got != 4 { + t.Fatalf("R22 %d heartbeats without a committed receipt, want 4", got) + } +} diff --git a/internal/directory/client.go b/internal/directory/client.go index 387c4f90c5..661ebdf406 100644 --- a/internal/directory/client.go +++ b/internal/directory/client.go @@ -77,7 +77,7 @@ type Directory interface { // two missed beats. const ( LeaseTTL = time.Duration(clusterstate.LeaseTTLMillis) * time.Millisecond - HeartbeatEvery = 30 * time.Second + HeartbeatEvery = time.Duration(clusterstate.RenewEveryMillis) * time.Millisecond ) // The errors a Client returns. Callers compare with errors.Is. diff --git a/internal/directory/hold.go b/internal/directory/hold.go index 40280a9d4d..7e3a86a68b 100644 --- a/internal/directory/hold.go +++ b/internal/directory/hold.go @@ -148,3 +148,6 @@ func watchQuery(holds []Hold, events bool) string { } return q + "&" + EventsQuery } + +// HoldIdentity binds committed watch receipts to the exact lease. +func (h Hold) HoldIdentity() (string, uint64) { return h.Cell, h.Fence } diff --git a/internal/directory/holdreceipt_test.go b/internal/directory/holdreceipt_test.go new file mode 100644 index 0000000000..c3642b01e1 --- /dev/null +++ b/internal/directory/holdreceipt_test.go @@ -0,0 +1,28 @@ +package directory + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/dirwatch" +) + +// R22's explicit receipt is distinct from pong/version frames and preserves +// full-width indexes/fences; a maximum hold set fits the client's read budget. +func TestR22CommittedHoldReceiptWire(t *testing.T) { + receipts := make([]dirwatch.HoldReceipt, MaxHolds) + for i := range receipts { + receipts[i] = dirwatch.HoldReceipt{Cell: strings.Repeat("x", MaxHoldCellBytes), Fence: ^uint64(0), Index: ^uint64(0), At: 1000, Until: 91000} + } + raw, e := json.Marshal(struct { + Holds []dirwatch.HoldReceipt `json:"holds"` + }{receipts}) + if e != nil || len(raw) > watchLimit { + t.Fatal("R22 receipt read budget", len(raw), e) + } + frame, e := decodeFrame(raw) + if e != nil || len(frame.Holds) != MaxHolds || frame.Holds[0].Fence != ^uint64(0) || frame.Holds[0].Index != ^uint64(0) { + t.Fatal(frame, e) + } +} diff --git a/internal/directory/watchclient.go b/internal/directory/watchclient.go index eaf1f280ac..54476f5a62 100644 --- a/internal/directory/watchclient.go +++ b/internal/directory/watchclient.go @@ -50,7 +50,7 @@ const closeSilent websocket.StatusCode = dirwatch.CloseSilent // watchLimit is the most a frame may weigh. The frames are a few bytes, so a // larger one is a fault and is refused before it is buffered. -const watchLimit = 1 << 10 +const watchLimit = MaxHolds * (MaxHoldCellBytes*6 + 256) // Watch opens the signed watch socket of a screen: it names no holds, so it // vouches for nothing, and it asks for events (EventsQuery), so its Stream @@ -74,6 +74,9 @@ func (h *HTTP) dial(ctx context.Context, holds []Hold, events bool) (dirwatch.St return nil, err } req.URL.RawQuery = withBeat(watchQuery(holds, events)) + if len(holds) != 0 { + req.URL.RawQuery += "&receipts=1" + } h.sign(req, nil) // The library applies the client's Timeout to the dial alone and clears it // for the socket, so the shared client is safe to hand over. @@ -173,13 +176,16 @@ func decodeFrame(data []byte) (dirwatch.Frame, error) { return dirwatch.Frame{Pong: true}, nil } var f struct { - V *uint64 `json:"v"` + V *uint64 `json:"v"` + Holds []dirwatch.HoldReceipt `json:"holds"` dirwatch.Event } if err := json.Unmarshal(data, &f); err != nil { return dirwatch.Frame{}, fmt.Errorf("directory: undecodable watch frame %q", data) } switch { + case f.Holds != nil: + return dirwatch.Frame{Holds: f.Holds}, nil case f.V != nil: return dirwatch.Frame{Version: *f.V}, nil case f.T != "": diff --git a/internal/directory/watchclient_test.go b/internal/directory/watchclient_test.go index 2a5052808e..d9c3f17923 100644 --- a/internal/directory/watchclient_test.go +++ b/internal/directory/watchclient_test.go @@ -202,7 +202,11 @@ func TestWatchHoldingNamesHoldsInTheSignedURLAndReadsTheVouchWord(t *testing.T) t.Fatal(err) } defer s.Close() - if got := (<-seen).URL.Query()["hold"]; len(got) != 2 || got[0] != "c:1:7" || got[1] != "d:2" { + request := <-seen + if request.URL.Query().Get("receipts") != "1" { + t.Fatal("R22 holding watch omitted committed receipt negotiation") + } + if got := request.URL.Query()["hold"]; len(got) != 2 || got[0] != "c:1:7" || got[1] != "d:2" { t.Fatalf("holds in the query = %q", got) } if !s.Vouching() { diff --git a/internal/dirwatch/doc.go b/internal/dirwatch/doc.go index ea03a77550..7037fef4e7 100644 --- a/internal/dirwatch/doc.go +++ b/internal/dirwatch/doc.go @@ -2,8 +2,9 @@ // shows the directory learns of another device's change when it happens rather // than by asking on a timer. // -// The socket carries one fact: the directory's version. It never carries -// records, so a frame is only ever a reason to read the list once. This package +// The socket carries directory versions, presence hints and committed hold +// receipts. Versions prompt a fresh list read; receipts cover only an exact +// lease and never replace an authority read. This package // owns the connection (dialling, keeping it alive, noticing it died, coming // back with jittered waits) and nothing else; what to do with a new version, // and what to do while the socket is down, stay with the surface that draws diff --git a/internal/dirwatch/events.go b/internal/dirwatch/events.go index 1c38235fe5..3f06234c8d 100644 --- a/internal/dirwatch/events.go +++ b/internal/dirwatch/events.go @@ -117,7 +117,7 @@ func (v view) forgotten() view { // through the view, and any other frame is a version. func (fr Frame) apply(st *State, v *view) { switch { - case fr.Pong: + case fr.Pong, fr.Holds != nil: case fr.Event != nil: *v = v.withEvent(*fr.Event) st.Online, st.Joined = v.online, v.joined diff --git a/internal/dirwatch/feed.go b/internal/dirwatch/feed.go index bfe32ca4c7..402cb07cd3 100644 --- a/internal/dirwatch/feed.go +++ b/internal/dirwatch/feed.go @@ -63,13 +63,15 @@ type Feed struct { // redial asks the serving loop to end the socket it has and dial again. redial chan struct{} - mu sync.Mutex - state State - events view // what the events told; State.Online and State.Joined are its two parts - heard time.Time // when the open socket last showed a sign of life - subs map[*sub]struct{} - cancel context.CancelFunc - done chan struct{} + mu sync.Mutex + state State + events view // what the events told; State.Online and State.Joined are its two parts + receipts []HoldReceipt + receiptAt time.Time + heard time.Time // when the open socket last showed a sign of life + subs map[*sub]struct{} + cancel context.CancelFunc + done chan struct{} } // newFeed builds a feed that is not yet connected. @@ -126,7 +128,7 @@ func (f *Feed) set(edit func(*State)) { defer f.mu.Unlock() before := f.state edit(&f.state) - if f.state.equal(before) { + if f.state.equal(before) && len(f.receipts) == 0 { return } for s := range f.subs { @@ -236,6 +238,7 @@ func (f *Feed) serve(ctx context.Context, s Stream) error { defer cancel() defer f.set(func(st *State) { st.Up, st.Vouching = false, false + f.receipts = nil f.events = f.events.forgotten() st.Online = f.events.online }) @@ -295,6 +298,10 @@ func (f *Feed) hear(fr Frame, vouching bool) { f.retry.reset() f.mu.Lock() f.heard = f.clock.Now() + if fr.Holds != nil { + f.receipts = slices.Clone(fr.Holds) + f.receiptAt = f.heard + } f.mu.Unlock() f.set(func(st *State) { st.Up, st.Vouching = true, vouching diff --git a/internal/dirwatch/holder.go b/internal/dirwatch/holder.go index 380beca078..dce1404409 100644 --- a/internal/dirwatch/holder.go +++ b/internal/dirwatch/holder.go @@ -5,6 +5,8 @@ import ( "math/rand/v2" "slices" "sync" + + "github.com/Agent-Field/codeaf/internal/clusterstate" ) // ── THE HOLDING SOCKET ────────────────────────────────────────────────────── @@ -128,22 +130,42 @@ func (h *Holder[H]) Take(hold H) *Holding[H] { return w } -// Healthy says whether the relay is vouching for the hold: the socket is open, -// its server said it vouches, the socket was dialled naming this hold, and it -// showed a sign of life within DeadAfter. Every part matters. A socket that is -// down proves nothing; a server that did not say so is an old one; a socket -// dialled before the hold was taken does not name it; and a link that went -// quiet may be half open, which the feed only notices at its own deadline. +// Healthy requires a committed exact-fence receipt. Ping/pong and the old +// upgrade header alone are advisory presence and cannot suppress heartbeats. func (w *Holding[H]) Healthy() bool { st := w.feed.snapshot() - if !st.Up || !st.Vouching { + if !st.Up { + return false + } + identity, ok := any(w.hold).(HoldIdentity) + if !ok { return false } + cell, fence := identity.HoldIdentity() h := w.holder h.mu.Lock() named := slices.Contains(h.dialled, w.hold) h.mu.Unlock() - return named && h.clock.Now().Sub(w.feed.lastHeard()) <= DeadAfter + if !named { + return false + } + w.feed.mu.Lock() + receipts, received := slices.Clone(w.feed.receipts), w.feed.receiptAt + w.feed.mu.Unlock() + now := h.clock.Now() + if now.Before(received) || now.Sub(w.feed.lastHeard()) > DeadAfter { + return false + } + for _, r := range receipts { + if r.Cell != cell || r.Fence != fence || r.Index == 0 || r.At <= 0 || r.Until <= r.At || r.Until-r.At > clusterstate.LeaseTTLMillis { + continue + } + // The receipt's absolute expiry and a bounded local lifetime both apply. + // Falling back one renewal period early leaves time for an ordinary beat. + remaining := min(r.Until-now.UnixMilli(), r.Until-r.At-now.Sub(received).Milliseconds()) + return remaining > clusterstate.RenewEveryMillis + } + return false } // Changes signals that the socket's state changed, coalesced as a Follower's diff --git a/internal/dirwatch/holder_test.go b/internal/dirwatch/holder_test.go index 0c7319c467..158ef743e6 100644 --- a/internal/dirwatch/holder_test.go +++ b/internal/dirwatch/holder_test.go @@ -19,6 +19,8 @@ type hold struct { // holderRig is a holder on a fake clock dialling scripted sockets, and the // holds each dial named. +func (h hold) HoldIdentity() (string, uint64) { return h.cell, h.fence } + type holderRig struct { t *testing.T clock *fakeClock @@ -65,7 +67,8 @@ func (r *holderRig) dials() [][]hold { // speak makes the socket say something and waits until the holder believes it. func (r *holderRig) speak(s *fakeStream, w *Holding[hold]) { r.t.Helper() - s.send(Frame{Version: 1}, nil) + cell, fence := w.hold.HoldIdentity() + s.send(Frame{Version: 1, Holds: []HoldReceipt{{Cell: cell, Fence: fence, Index: 1, At: r.clock.Now().UnixMilli(), Until: r.clock.Now().UnixMilli() + 90_000}}}, nil) for deadline := time.Now().Add(5 * time.Second); !w.feed.snapshot().Up; time.Sleep(time.Millisecond) { if time.Now().After(deadline) { r.t.Fatal("the holder was never told the socket is up") @@ -106,14 +109,14 @@ func TestHolderBeatsWhileSocketDown(t *testing.T) { } } -func TestHolderIsNotHealthyWithoutTheVouchingWord(t *testing.T) { +func TestR22HolderReceiptDoesNotRequireLegacyVouchHeader(t *testing.T) { r := newHolderRig(t, 4) r.vouch = false w := r.holder.Take(hold{"c1", 7}) defer w.Release() r.speak(r.socket(), w) - if w.Healthy() { - t.Fatal("healthy on a socket whose server did not say it vouches") + if !w.Healthy() { + t.Fatal("committed receipt refused because legacy header is absent") } } diff --git a/internal/dirwatch/receipts_test.go b/internal/dirwatch/receipts_test.go new file mode 100644 index 0000000000..2df12db214 --- /dev/null +++ b/internal/dirwatch/receipts_test.go @@ -0,0 +1,46 @@ +package dirwatch + +import ( + "testing" + "time" +) + +// R22 starts with a live legacy-vouching socket and no receipt. Neither its +// version frame nor pongs authorize suppressing an ordinary durable heartbeat. +func TestR22PongsCannotReplaceCommittedReceipt(t *testing.T) { + r := newHolderRig(t, 4) + w := r.holder.Take(hold{"chat", 7}) + defer w.Release() + s := r.socket() + s.send(Frame{Version: 1}, nil) + for deadline := time.Now().Add(5 * time.Second); !w.feed.snapshot().Up; { + if time.Now().After(deadline) { + t.Fatal("socket did not open") + } + time.Sleep(time.Millisecond) + } + if w.Healthy() { + t.Fatal("R22 socket-only renewal") + } + now := r.clock.Now().UnixMilli() + w.feed.hear(Frame{Holds: []HoldReceipt{{Cell: "chat", Fence: 6, Index: 1, At: now, Until: now + 90000}}}, true) + if w.Healthy() { + t.Fatal("R22 wrong fence covered") + } + w.feed.hear(Frame{Holds: []HoldReceipt{{Cell: "chat", Fence: 7, Index: 1, At: now, Until: now + 90000}}}, true) + if !w.Healthy() { + t.Fatal("R22 committed receipt missing") + } + r.clock.mu.Lock() + r.clock.now = r.clock.now.Add(61 * time.Second) + r.clock.mu.Unlock() + w.feed.hear(Frame{Pong: true}, true) + if w.Healthy() { + t.Fatal("R22 pongs renewed expiring receipt") + } + // A delayed old receipt cannot reset its absolute expiry. + w.feed.hear(Frame{Holds: []HoldReceipt{{Cell: "chat", Fence: 7, Index: 1, At: now, Until: now + 90000}}}, true) + if w.Healthy() { + t.Fatal("R22 delayed receipt renewed expiry") + } +} diff --git a/internal/dirwatch/stream.go b/internal/dirwatch/stream.go index 5a358bdc29..0bf2a6708f 100644 --- a/internal/dirwatch/stream.go +++ b/internal/dirwatch/stream.go @@ -4,6 +4,8 @@ import ( "context" "errors" "time" + + "github.com/Agent-Field/codeaf/internal/clusterstate" ) // Stream is one open socket, as the feed needs it. @@ -23,7 +25,19 @@ type Stream interface { } // Frame is what arrived: the directory's version, an event, or only a pong. +type HoldReceipt struct { + Cell string `json:"cell"` + Fence uint64 `json:"fence,string"` + Index uint64 `json:"index,string"` + At int64 `json:"at"` + Until int64 `json:"until"` +} + +// HoldIdentity identifies the exact lease named by a committed receipt. +type HoldIdentity interface{ HoldIdentity() (string, uint64) } + type Frame struct { + Holds []HoldReceipt Version uint64 Pong bool // Event is set for an event frame (it has no version); nil otherwise. @@ -65,7 +79,7 @@ const ( // two halves cannot drift. The relay answers a ping without waking // anything, so it costs nothing, and it keeps a middlebox that drops idle // connections from dropping this one. - KeepEvery = 10 * time.Second + KeepEvery = time.Duration(clusterstate.PresenceEveryMillis) * time.Millisecond // DeadAfter is how long a socket may stay silent before it is dead: 2.5 // beats, 25 s. That tolerates one lost pong and a late second one, the // same factor the relay uses for its presence window. diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index 5326202240..2113ecb2cf 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -287,3 +287,12 @@ Network partitions or scheduling stalls can take longer. If the personal leader and either remaining relay voter are both unavailable before the third anchor is restored, writes wait for a majority to return. Returning computers stay learners; leadership returns only when you request it again. + + +## Does an open connection keep my chat running on this computer? + +The sync connection carries presence hints and committed lease acknowledgments. An open socket or an answered ping alone cannot keep ownership of a chat. codeaf skips an ordinary heartbeat only while it has a recent acknowledgment for that exact chat and ownership number, with enough time left before expiry. Without that acknowledgment it keeps sending ordinary heartbeats. A delayed acknowledgment for an earlier owner cannot take the chat back, and a delayed renewal cannot undo a clean quit. + +## Does syncing keep sending requests when nothing is happening? + +A home that has never paired sends no sync traffic. `CODEAF_SYNC_URL=off` also blocks new sync requests from clients that were opened before the setting changed. Presence pings on an open view are hints; they do not write chat ownership. On the consensus directory, a group with no work or held chats can stop its periodic coordination timer after an idle barrier has reached every voter. The first operation wakes coordination and must obtain a fresh majority response. A saved leader address cannot answer that operation by itself. Losing the majority leaves syncing unavailable and keeps local work intact. diff --git a/internal/syncsetup/cluster.go b/internal/syncsetup/cluster.go index 6ba5f62220..eca4322888 100644 --- a/internal/syncsetup/cluster.go +++ b/internal/syncsetup/cluster.go @@ -16,7 +16,7 @@ import ( ) func (s *Sync) clusterClient() clusterctl.Client { - return clusterctl.Client{Base: s.Relay, Sign: reqsign.SignFor(deviceSigner{s.Identity, s.Device}, time.Now), HTTP: gated(s.Home, &http.Client{Timeout: 30 * time.Second})} + return clusterctl.Client{Base: s.Relay, Sign: reqsign.SignFor(deviceSigner{s.Identity, s.Device}, time.Now), HTTP: s.ClusterHTTP(&http.Client{Timeout: 30 * time.Second})} } func (s *Sync) ClusterStatus(ctx context.Context) (clusterctl.Status, error) { return s.clusterClient().Status(ctx) diff --git a/internal/syncsetup/cluster_quiet_test.go b/internal/syncsetup/cluster_quiet_test.go new file mode 100644 index 0000000000..cca792a2ff --- /dev/null +++ b/internal/syncsetup/cluster_quiet_test.go @@ -0,0 +1,96 @@ +package syncsetup + +import ( + "context" + "crypto/ed25519" + "errors" + "io" + "net/http" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/dirwatch" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/rafttransport" +) + +type clusterWireProbe struct{ calls int } + +func (p *clusterWireProbe) RoundTrip(r *http.Request) (*http.Response, error) { + p.calls++ + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"generation":"1","messages":[]}`)), Request: r}, nil +} + +// R23 intercepts the actual peer client at the wire, before DNS, including a +// handle opened while solo and one retained after the person turns sync off. +func TestR23ClusterWireChecksQuietAtEveryUse(t *testing.T) { + home := t.TempDir() + if _, err := identity.EnsureSolo(home); err != nil { + t.Fatal(err) + } + probe := &clusterWireProbe{} + s := &Sync{Home: home} + client := s.ClusterHTTP(&http.Client{Transport: probe}) + peer := rafttransport.Client{URL: "http://sync.invalid", HTTP: client} + req := rafttransport.Request{Scope: rafttransport.Scope{Identity: "id", Cluster: "cluster", Epoch: 1, Protocol: 1}, Node: 1, Incarnation: 1, Generation: 1} + if err := req.Sign(ed25519.NewKeyFromSeed(make([]byte, 32))); err != nil { + t.Fatal(err) + } + if _, err := peer.Exchange(context.Background(), req); !errors.Is(err, ErrQuiet) { + t.Fatal("R23 solo exchange", err) + } + for _, path := range []string{"/v2/cluster/enroll", "/v2/cluster/status", "/v2/cluster/socket"} { + r, _ := http.NewRequest("GET", "http://sync.invalid"+path, nil) + if _, err := client.Do(r); !errors.Is(err, ErrQuiet) { + t.Fatal(err) + } + } + if probe.calls != 0 { + t.Fatal("R23 solo reached transport") + } + // The existing paired fixture supplies a real saved personal identity. + paired := machine(t, "http://sync.invalid") + s.Home = paired + existing := s.ClusterHTTP(&http.Client{Transport: probe}) + peer.HTTP = existing + if _, err := peer.Exchange(context.Background(), req); err != nil { + t.Fatal("R23 active fixture", err) + } + n := probe.calls + t.Setenv(URLVar, "off") + if _, err := peer.Exchange(context.Background(), req); !errors.Is(err, ErrQuiet) { + t.Fatal("R23 retained handle after off", err) + } + if probe.calls != n { + t.Fatal("R23 off reached transport") + } +} + +type clusterStreamProbe struct{ pings, reads, closed int } + +func (p *clusterStreamProbe) Next(context.Context) (dirwatch.Frame, error) { + p.reads++ + return dirwatch.Frame{Pong: true}, nil +} +func (p *clusterStreamProbe) Ping(context.Context) error { p.pings++; return nil } +func (p *clusterStreamProbe) Vouching() bool { return true } +func (p *clusterStreamProbe) Close() { p.closed++ } + +func TestR23ExistingWatchStopsSendingAfterOff(t *testing.T) { + home := machine(t, "http://sync.invalid") + probe := &clusterStreamProbe{} + stream := quietStream{Stream: probe, home: home} + if err := stream.Ping(context.Background()); err != nil { + t.Fatal(err) + } + t.Setenv(URLVar, "off") + if err := stream.Ping(context.Background()); !errors.Is(err, ErrQuiet) { + t.Fatal(err) + } + if _, err := stream.Next(context.Background()); !errors.Is(err, ErrQuiet) { + t.Fatal(err) + } + if probe.pings != 1 || probe.reads != 0 || probe.closed == 0 || stream.Vouching() { + t.Fatal("R23 retained watch bypassed off", probe) + } +} diff --git a/internal/syncsetup/quiet.go b/internal/syncsetup/quiet.go index 8ce2e4f266..0ce14e509b 100644 --- a/internal/syncsetup/quiet.go +++ b/internal/syncsetup/quiet.go @@ -17,6 +17,7 @@ import ( "net/http" "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/dirwatch" "github.com/Agent-Field/codeaf/internal/identity" ) @@ -65,3 +66,37 @@ func (s *Sync) list(ctx context.Context) (directory.Listing, error) { } return s.Dir.List(ctx) } + +// ClusterHTTP supplies the same use-time quiet gate for enrollment, discovery, +// exchange and socket upgrades. Constructing it performs no network operation; +// handles created before pairing or sync-off recheck the decision on every use. +func (s *Sync) ClusterHTTP(client *http.Client) *http.Client { + if client == nil { + client = &http.Client{} + } + copyClient := *client + return gated(s.Home, ©Client) +} + +// quietStream rechecks an existing socket at every read/ping, so a later off +// decision cannot retain a second transport path around quietWire. +type quietStream struct { + dirwatch.Stream + home string +} + +func (s quietStream) Next(ctx context.Context) (dirwatch.Frame, error) { + if Quiet(s.home) { + s.Stream.Close() + return dirwatch.Frame{}, ErrQuiet + } + return s.Stream.Next(ctx) +} +func (s quietStream) Ping(ctx context.Context) error { + if Quiet(s.home) { + s.Stream.Close() + return ErrQuiet + } + return s.Stream.Ping(ctx) +} +func (s quietStream) Vouching() bool { return !Quiet(s.home) && s.Stream.Vouching() } diff --git a/internal/syncsetup/syncsetup.go b/internal/syncsetup/syncsetup.go index 00fcac5f65..3009eb21aa 100644 --- a/internal/syncsetup/syncsetup.go +++ b/internal/syncsetup/syncsetup.go @@ -214,7 +214,16 @@ func (s *Sync) Follow() dirwatch.Follower { if !ok { return nil } - return dirwatch.Follow(s.Ledger, w.Watch) + return dirwatch.Follow(s.Ledger, func(ctx context.Context) (dirwatch.Stream, error) { + if Quiet(s.Home) { + return nil, ErrQuiet + } + stream, err := w.Watch(ctx) + if err != nil { + return nil, err + } + return quietStream{Stream: stream, home: s.Home}, nil + }) } // holdKey keeps the holding socket's feed apart from the home screen's, which @@ -222,7 +231,7 @@ func (s *Sync) Follow() dirwatch.Follower { func (s *Sync) holdKey() string { return s.Ledger + "#holding" } // Liveness is the process's holding socket for this identity, which lets the -// chats it holds stop heartbeating while the relay vouches for them. It answers +// chats it holds stop heartbeating while committed hold receipts cover them. It answers // nil when the directory client cannot name holds (a self-hosted directory), so // those chats heartbeat as they always did. func (s *Sync) Liveness() cellsync.Liveness { @@ -230,7 +239,16 @@ func (s *Sync) Liveness() cellsync.Liveness { if !ok { return nil } - return holdSocket{dirwatch.Hold(s.holdKey(), directory.MaxHolds, w.WatchHolding)} + return holdSocket{dirwatch.Hold(s.holdKey(), directory.MaxHolds, func(ctx context.Context, holds []directory.Hold) (dirwatch.Stream, error) { + if Quiet(s.Home) { + return nil, ErrQuiet + } + stream, err := w.WatchHolding(ctx, holds) + if err != nil { + return nil, err + } + return quietStream{Stream: stream, home: s.Home}, nil + })} } // holdSocket adapts the holding socket to the Batcher's view of it. From 0d649f4d9a16eb83e49979c8536628032027c9d8 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 04:00:17 -0400 Subject: [PATCH 033/103] Drive hosted renewal and idle scheduling through the shared Go policy. Serialize timer decisions with durable dispatch because a concurrent wake must retain its alarm. Bound evidence pages and transport retention, and require exact batch acknowledgments across restart. Exercise the production actor, alarms and native replay locally. Co-Authored-By: Claude Opus 5.5 (1M context) --- relay/hosted/src/raft/README.md | 29 ++++- relay/hosted/src/raft/abi.js | 2 +- relay/hosted/src/raft/actor.js | 25 +++++ relay/hosted/src/raft/liveness.js | 101 ++++++++++++++++++ relay/hosted/src/raft/member.js | 67 +++++++++++- relay/hosted/src/routes.js | 2 +- relay/hosted/src/tenant.js | 4 +- relay/hosted/src/watch.js | 32 +++++- relay/hosted/test/raft/liveness.test.js | 43 ++++++++ relay/hosted/test/raft/membership-worker.js | 12 +++ .../test/raft/membership-workerd.test.js | 25 ++++- relay/hosted/test/raft/scheduler.test.js | 97 +++++++++++++++++ relay/hosted/test/watch.test.js | 26 ++++- 13 files changed, 451 insertions(+), 14 deletions(-) create mode 100644 relay/hosted/src/raft/liveness.js create mode 100644 relay/hosted/test/raft/liveness.test.js create mode 100644 relay/hosted/test/raft/scheduler.test.js diff --git a/relay/hosted/src/raft/README.md b/relay/hosted/src/raft/README.md index 3f972e600b..0860e48616 100644 --- a/relay/hosted/src/raft/README.md +++ b/relay/hosted/src/raft/README.md @@ -55,7 +55,7 @@ observed reactor memory must fit 64 MiB. Retained core log cache is capped at 2 MiB, with lossless checkpoints at 1 MiB or 1,024 applied entries. This earlier local checkpoint keeps the cache below the shared 4 MiB retained-log ceiling. A leader is never evicted to meet admission. Only an explicitly quiescent, -nonleader, drained actor may close; P14 must coordinate actual quiescence/wake +nonleader, drained actor may close; the P14 scheduler coordinates quiescence/wake and alarm scheduling. `drainRecovery` exposes recovered dependent effects for an internal router to deliver rather than silently dropping them. @@ -115,3 +115,30 @@ settle an existing intent or close their request session. They cannot enroll, remove, read directory content, mutate chats, pair or rotate. Thus a crashed requester need not return for an already authorized admission to finish. The settlement still requires the exact applied configuration and pending revision. + + +## Committed holds, timers and wake + +ABI operations 23–26 supply scheduling, one-row activity checks, shared periods +and covered-renewal checks; step tag 15 is the internal idle ReadIndex. Copy all +ABI results before awaits. `CommittedHolds` samples authenticated watch evidence, +pages at most the Go batch limit per turn, skips publish-covered items after a +fresh authoritative read and emits exact-fence/index/hold-until receipts only +after application commit. `CommittedWatchers` supplies advisory presence without +the old socket-vouch header or lifted expiry. A failed renewal emits no receipt. + +`MemberAlarms` holds the member queue across the timer decision, durable dispatch +and alarm update. Dispatch enqueues messages; it must never await their Raft +responses while holding that queue. `QuiescenceCoordinator` performs host +maintenance, internal safe read, bound marker commit, durable applied acks and +final sleep delivery. Lost final delivery leaves that member active. A wake or +new command invalidates the old generation, and recovered handles start awake. + +The DO alarm handler retains at most one bounded durable transport batch, with a +monotonic batch ID. `scheduled` rereads that same batch after a lost reply; +`acknowledgeScheduled` removes only the matching ID. A stalled router backpressures +further turns with bounded retry delay. Draining this batch is transport delivery, +never append or application success. The sole P17 router must consume those +original peer bytes through the authenticated exchange bridge, propagate wake to +current voters, and supply queued assignment/outbox activity conservatively. +No host listener, provider service or alternate directory writer is added here. diff --git a/relay/hosted/src/raft/abi.js b/relay/hosted/src/raft/abi.js index 5cf472ba2d..dceeee84e4 100644 --- a/relay/hosted/src/raft/abi.js +++ b/relay/hosted/src/raft/abi.js @@ -44,7 +44,7 @@ export function createFrame({ cluster, id, seed, bootstrap = [], anchors = [], h export function stepFrame(handle, tag, value) { const w = frame(3).u32(handle).u32(tag); if ([2, 3, 8, 11].includes(tag)) w.blob(value); - else if ([4, 10].includes(tag)) w.str(value); + else if ([4, 10, 15].includes(tag)) w.str(value); else if ([5, 6, 7, 9].includes(tag)) w.u64(value); if (tag === 12) w.u64(value.peer).u32(value.snapshot ? 1 : 0).u32(value.success ? 1 : 0); return w.finish(); diff --git a/relay/hosted/src/raft/actor.js b/relay/hosted/src/raft/actor.js index 6386da4714..e7b3993b34 100644 --- a/relay/hosted/src/raft/actor.js +++ b/relay/hosted/src/raft/actor.js @@ -141,6 +141,31 @@ export class MemberActor { const r = response(this.reactor.call(frame(18).blob(raw).blob(unb64(app.conf)).u64(app.applied).u64(advertised).finish())); const ready = r.u32() !== 0; r.end(); return { ready }; }); } + // Timer decisions use the shared Go scheduler and paged Go activity rules. + // An internal caller supplies queued assignment/outbox work conservatively. + schedule(now, ingressNow, busy, acks = []) { acks = structuredClone(acks); return this.serial(() => this.scheduleInQueue(now,ingressNow,busy,acks)); } + async scheduleInQueue(now, ingressNow, busy, acks = []) { + busy = busy || this.pending > 1; + for (const row of this.store.rows("SELECT namespace,key FROM records WHERE namespace='state' OR (namespace='membership' AND key='registry') OR (namespace='outbox' AND key LIKE 'join/%') ORDER BY namespace,key")) { + const raw = this.store.read(row.namespace,row.key); + const r = response(this.reactor.call(frame(24).str(row.namespace).str(row.key).blob(raw).u64(ingressNow).finish())); + busy = (r.u32() !== 0) || busy; r.end(); + } + const raw = this.store.read('membership','quiescence'); + const marker = raw ? decodeCanonical(new TextDecoder().decode(raw)) : {index:'0',term:'0',barrier:'0',voters:null}; + const request = new TextEncoder().encode(encodeCanonical({now,busy,marker,acks})); + const r = response(this.reactor.call(frame(23).u32(this.handle).blob(request).finish())); + const plan = decodeCanonical(new TextDecoder().decode(r.blob())); r.end(); + const effects = plan.tick ? await this.drive(1) : {send:[],reads:[],status:this.status}; + return { ...effects, plan }; + } + // A final acknowledgment is local durable application, not transport receipt. + sleepAck(index, term) { return this.serial(() => { + const raw = this.store.read('membership','quiescence'); + const m = raw && decodeCanonical(new TextDecoder().decode(raw)); + if (!m || m.index !== String(index) || m.term !== String(term) || this.status.applied !== BigInt(index) || this.status.term !== BigInt(term)) throw new Error('raft member: sleep barrier not applied'); + return {node:this.config.id,index:String(index),term:String(term)}; + }); } begin(data) { data = data.slice(); return this.serial(() => this.store.beginSnapshot(data)); } chunk(id, offset, data, hash) { data = data.slice(); return this.serial(() => this.store.putChunk(id, offset, data, hash)); } complete(id) { return this.serial(() => this.store.completeSnapshot(id)); } diff --git a/relay/hosted/src/raft/liveness.js b/relay/hosted/src/raft/liveness.js new file mode 100644 index 0000000000..71ff710ecc --- /dev/null +++ b/relay/hosted/src/raft/liveness.js @@ -0,0 +1,101 @@ +import { frame, response } from './abi.js'; +import { decodeCanonical } from './codec.js'; + +// Policy numbers come from the same Go constants as native scheduling/apply. +export function coordinationPolicy(runtime) { + const r = response(runtime.call(frame(25).finish())); + const policy = decodeCanonical(new TextDecoder().decode(r.blob())); r.end(); + policy.covered=(raw,device,hold,seen)=>{ + const q=response(runtime.call(frame(26).blob(raw??new Uint8Array()).str(device).str(hold.cell).u64(hold.fence).u64(seen).finish())); + const covered=q.u32()!==0;q.end();return covered; + }; + return policy; +} + +// The bridge is installed only beside the consensus facade. Watch evidence is +// ingress-authenticated; commitEffect returns durable application results. A +// socket auto-response alone never changes the directory or its visible version. +export class CommittedHolds { + constructor(host, policy) { this.host = host; this.policy = policy; this.next = null; this.running = null; this.page = null; } + opened(now, hasHolds) { if (hasHolds && this.next === null) this.next = now; return this.next; } + renew(now, sockets) { + if (this.running) return this.running; + this.running = this.run(now, sockets).finally(() => { this.running = null; }); return this.running; + } + async run(now, sockets) { + if (this.next !== null && now < this.next) return this.next; + if (!this.page) { + // A cycle retains at most the admitted sockets' bounded routing tuples. + // Copy before any await so a replaced socket cannot rewrite old evidence. + this.page = { sockets:sockets.filter(s=>s.holds.length).map(s=>({...s,holds:structuredClone(s.holds)})), socket:0, offset:0 }; + } + const page=this.page; let remaining=this.policy.max_renew_holds; + while(page.socket0) { + const socket=page.sockets[page.socket]; + if(socket.seenAt<=0 || socket.seenAt>now || now-socket.seenAt>this.policy.evidence_window) {page.socket++;page.offset=0;continue;} + const chunk=socket.holds.slice(page.offset,page.offset+remaining); + page.offset+=chunk.length;remaining-=chunk.length; + if(page.offset===socket.holds.length){page.socket++;page.offset=0;} + const items=[]; + try { for(const h of chunk) { + const key=Array.from(new TextEncoder().encode(h.cell),b=>b.toString(16).padStart(2,'0')).join(''); + const raw=await this.host.barrierRecord({namespace:'state',key:'cell/'+key}); + if(!this.policy.covered(raw,socket.device,h,socket.seenAt))items.push({cell:h.cell,fence:String(h.fence)}); + } } catch {socket.failed?.();continue;} + if(!items.length)continue; + let result; + try { result=await this.host.commitEffect({renew_holds:{device:socket.device,seen_at:socket.seenAt,items}},now); } + catch {socket.failed?.();continue;} + if(result.code!=='ok'||!result.index||result.index==='0'){socket.failed?.();continue;} + const holds=(result.items??[]).filter(item=>item.code==='ok'&&item.view).map(item=>({cell:item.cell,fence:item.view.cell.lease.fence,index:result.index,at:result.now,until:item.view.cell.lease.expires})); + if(holds.length)socket.send(JSON.stringify({holds})); + } + if(page.sockettrue) { this.storage=storage;this.actor=actor;this.dispatch=dispatch;this.activity=activity;this.ready=ready;this.turn=Promise.resolve(); } + run(elapsed, ingressNow, acks=[]) { + const task=this.turn.then(()=>this.actor.serial(async()=>{ + if(!await this.ready())return null; + const busy=await this.activity(); + const effects=await this.actor.scheduleInQueue(elapsed,ingressNow,busy,acks); + await this.dispatch(effects); + if(effects.plan.sleeping) await this.storage.deleteAlarm(); + else await this.storage.setAlarm(ingressNow+effects.plan.after); + return effects; + })); + this.turn=task.catch(()=>{});return task; + } +} + +// The router resumes committed membership/leadership/outbox work before idle +// preparation. Each callback uses the existing serialized member doors; this +// driver has no voter-list editor, lease writer or successful read shortcut. +export class QuiescenceCoordinator { + constructor(host) { this.host=host;this.tail=Promise.resolve(); } + run(elapsed,now) { + const task=this.tail.then(async()=>{ + const busy=await this.host.maintain(); + let effects=await this.host.schedule(elapsed,now,busy); + await this.host.dispatch(effects); + if(effects.plan.wake?.length)await this.host.wake(effects.plan.wake); + if(effects.plan.barrier){ + await this.host.idleBarrier(); + effects=await this.host.schedule(elapsed,now,busy);await this.host.dispatch(effects); + } + if(!effects.plan.prepare)return effects.plan; + const marker=await this.host.prepare(); + const acks=await this.host.appliedAcks(marker); + await this.host.finish(marker,acks,elapsed,now); + return effects.plan; + }); + this.tail=task.catch(()=>{});return task; + } +} diff --git a/relay/hosted/src/raft/member.js b/relay/hosted/src/raft/member.js index 3cb5aaecaa..4c79632a56 100644 --- a/relay/hosted/src/raft/member.js +++ b/relay/hosted/src/raft/member.js @@ -1,10 +1,11 @@ import { DurableObject } from 'cloudflare:workers'; import { MemberActor, sharedRuntime } from './actor.js'; -import { MemberInbox } from './inbox.js'; +import { MemberInbox, INBOX_BYTES } from './inbox.js'; import { directoryEvaluator } from './directory.js'; -import { DEFAULT_ROWS_BYTES, MemberStorage } from './storage.js'; +import { DEFAULT_ROWS_BYTES, MemberStorage, b64 } from './storage.js'; import { HostCodec } from './abi.js'; import { committedRegistry, provisionedRegistry } from './membership.js'; +import { MemberAlarms, coordinationPolicy } from './liveness.js'; import { stageLegacyImport, finishLegacyImport } from './legacy.js'; export { installRaftModule, runtimeMetrics } from './actor.js'; @@ -74,6 +75,68 @@ export class RaftMemberDO extends DurableObject { const actor = new MemberActor(this.ctx.storage, sharedRuntime(), this.evaluator(config)); this.recoveredEffects = await actor.init(config); this.config = config; this.registry = actor.store.read('membership', 'registry') ? committedRegistry(sharedRuntime(), actor.store) : await this.ctx.storage.get('raft-registry'); this.actor = actor; } + // Alarm effects wait in one bounded durable transport batch. The router + // drains original bytes and acknowledges the batch ID; arrival is never an + // append acknowledgment. A stalled router backpressures further alarm turns. + async scheduled() { await this.load(); return this.actor.serial(async()=>{ + const manifest=await this.ctx.storage.get('raft-scheduled');if(!manifest)return null; + const policy=coordinationPolicy(sharedRuntime()); + if(manifest.bytes>INBOX_BYTES||manifest.parts!==Math.ceil(manifest.bytes/policy.max_message_bytes))throw new Error('raft member: damaged scheduled batch'); + const raw=new Uint8Array(manifest.bytes); + for(let n=0;n{ + await this.ctx.storage.transaction(async tx=>{ + const batch=await tx.get('raft-scheduled');if(batch?.id!==String(id))return; + for(let n=0;nNumber(!view[String(a.to)]?.voter)-Number(!view[String(b.to)]?.voter)); + const id=String(BigInt(await this.ctx.storage.get('raft-scheduled-sequence')??'0')+1n); + const batch={id,send:[],wake:effects.plan.wake},dropped=[]; + let size=new TextEncoder().encode(JSON.stringify(batch)).length; + for(const m of sends){ + const entry={to:String(m.to),bytes:b64(m.bytes)},cost=new TextEncoder().encode(JSON.stringify(entry)).length+1; + if(size+cost>INBOX_BYTES){dropped.push(m);continue;} + batch.send.push(entry);size+=cost; + } + const raw=new TextEncoder().encode(JSON.stringify(batch)); + const manifest={id,bytes:raw.length,parts:Math.ceil(raw.length/policy.max_message_bytes),hash:this.actor.store.hash(raw)}; + await this.ctx.storage.transaction(async tx=>{ + for(let n=0;n{ + const dropped=await this.retainScheduled(effects,policy); + for(const m of dropped){const inspected=this.actor.codec.inspect(4,m.bytes);await this.actor.drive(12,{peer:BigInt(m.to),snapshot:inspected.snapshot.length>0,success:false});} + }; + const ready=async()=>{ + if(await this.ctx.storage.get('raft-scheduled')){await this.ctx.storage.setAlarm(now+policy.idle_after);return false;} + return true; + }; + const timer=new MemberAlarms(this.ctx.storage,this.actor,dispatch,async()=>false,ready); + await timer.run(Math.floor(performance.now()),now); + } + async schedule(elapsed, ingressNow, busy, acks=[]) { await this.load(); return this.actor.schedule(elapsed, ingressNow, busy, acks); } + async sleepAck(index,term) { await this.load(); return this.actor.sleepAck(index,term); } async leadershipEvidence(revision,barrier) { await this.load(); return this.actor.leadershipEvidence(revision,barrier); } async leadership(evidence) { await this.load(); return this.actor.leadership(evidence); } async leadershipReceipt() { await this.load(); return this.actor.leadershipReceipt(); } diff --git a/relay/hosted/src/routes.js b/relay/hosted/src/routes.js index 023b4a4ce4..d79cdbbd0e 100644 --- a/relay/hosted/src/routes.js +++ b/relay/hosted/src/routes.js @@ -46,7 +46,7 @@ function watching(c) { const { searchParams } = new URL(c.request.url); const holds = parseHolds(searchParams.getAll('hold'), c.tenant.limits.maxHolds); const beat = parseBeat(searchParams.getAll('beat')); - return c.tenant.watch(c.device, { holds, beat, events: searchParams.get('events') === '1' }); + return c.tenant.watch(c.device, { holds, beat, events: searchParams.get('events') === '1', receipts: searchParams.get('receipts') === '1' }); } // The directory's version rides on a presence answer as on a list. diff --git a/relay/hosted/src/tenant.js b/relay/hosted/src/tenant.js index d464bb2d95..fb95fbeb7d 100644 --- a/relay/hosted/src/tenant.js +++ b/relay/hosted/src/tenant.js @@ -110,11 +110,11 @@ export class Tenant { } /** watch opens a socket for `device` that is told the directory's version now and on every visible change, and that vouches for the leases it names in `holds`. `beat` is the period it declared its pings at (contract 21.12.3). */ - watch(device, { holds, beat, events = false }) { + watch(device, { holds, beat, events = false, receipts = false }) { this.dir.assertLegacy(); const now = this.clock(); this.dir.seen(device, now); - return this.watchers.accept(device, this.dir.version, now, { holds, beat, events }); + return this.watchers.accept(device, this.dir.version, now, { holds, beat, events, receipts }); } /** join stores the device a paired device approved, and tells the identity's other sockets it joined. */ diff --git a/relay/hosted/src/watch.js b/relay/hosted/src/watch.js index 8c6d42fda2..6803655c77 100644 --- a/relay/hosted/src/watch.js +++ b/relay/hosted/src/watch.js @@ -18,7 +18,7 @@ export const CLOSE_SILENT = 4408; const ANSWER_HEADERS = { 'Codeaf-Vouch': '1', 'Codeaf-Presence': '1' }; /** What a socket carries through hibernation; an attachment from before a field existed reads as its default. */ -const FRESH = { at: 0, holds: [], events: false, beat: PRESENCE.defaultBeat, lapsed: false }; +const FRESH = { receipts: false, at: 0, holds: [], events: false, beat: PRESENCE.defaultBeat, lapsed: false }; const attachmentOf = (ws) => ({ ...FRESH, ...ws.deserializeAttachment() }); /** @@ -82,11 +82,11 @@ export class Watchers { * it is also the viewer that makes the presence sweep worth arming (contract 21.12.4). `beat` is the ping period the * client declared, from which the socket's window follows. */ - accept(device, version, now, { holds, beat, events = false }) { + accept(device, version, now, { holds, beat, events = false, receipts = false }) { if (this.ctx.getWebSockets().length >= this.cap) throw new Wire('too_many_watchers', 429); const { 0: client, 1: server } = new WebSocketPair(); this.ctx.acceptWebSocket(server, [device]); - server.serializeAttachment({ ...FRESH, at: now, holds, events, beat }); + server.serializeAttachment({ ...FRESH, at: now, holds, events, beat, receipts }); server.send(frame(version)); if (events) for (const text of this.liveness.snapshot(device)) server.send(text); this.liveness.opened(device); @@ -104,6 +104,21 @@ export class Watchers { return vouchedUntil(sockets, cell, fence, this.ttlMs); } + // Only the committed-renewal bridge consumes this snapshot. The platform's + // hibernating ping timestamp remains advisory until its exact holds commit. + holdEvidence(now) { + return this.ctx.getWebSockets().map(ws => { + const att = attachmentOf(ws); + // Presence may forgive a closing socket, but renewal evidence must + // fail closed when the platform cannot supply its timestamp. + let seenAt=0; + try {seenAt=Math.max(att.at,this.ctx.getWebSocketAutoResponseTimestamp(ws)?.getTime()??0);} catch {} + return {device:this.ctx.getTags(ws)[0],holds:att.lapsed ? [] : att.holds.map(([cell,fence])=>({cell,fence:String(fence)})), + seenAt, + send:text=>{if(att.receipts)attempt(()=>ws.send(text));},failed:()=>attempt(()=>ws.close(1013,'retry'))}; + }); + } + /** broadcast tells every socket of the identity the new version. */ broadcast(version) { const text = frame(version); @@ -182,3 +197,14 @@ function attempt(fn) { /* already closing */ } } + +// Consensus watches never derive a lease from a socket. V1 callers keep normal +// beats; updated callers may suppress them only after a committed hold receipt. +export class CommittedWatchers extends Watchers { + vouchedUntil() { return 0; } + accept(...args) { + const reply=super.accept(...args); + const headers=new Headers(reply.headers);headers.delete('Codeaf-Vouch'); + return new Response(null,{status:101,webSocket:reply.webSocket,headers}); + } +} diff --git a/relay/hosted/test/raft/liveness.test.js b/relay/hosted/test/raft/liveness.test.js new file mode 100644 index 0000000000..227de887c5 --- /dev/null +++ b/relay/hosted/test/raft/liveness.test.js @@ -0,0 +1,43 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { CommittedHolds, MemberAlarms } from '../../src/raft/liveness.js'; + +// R22's protocol bridge must wait for commit, preserve sample times, close on +// unavailable and keep presence pongs separate from committed hold receipts. +test('R22 committed hold receipts and failure', async()=>{ + const policy={renew_every:30000,evidence_window:25000,max_renew_holds:2,covered:()=>false}; + let complete, begin, calls=[], sent=[],failed=0; const started=new Promise(r=>begin=r); + const host={barrierRecord:async()=>null,commitEffect:(change,now)=>{calls.push([change,now]);begin();return new Promise(r=>complete=r)}}; + const bridge=new CommittedHolds(host,policy); + const socket={device:'a',seenAt:1000,holds:[{cell:'c',fence:'7'}],send:b=>sent.push(JSON.parse(b)),failed:()=>failed++}; + bridge.opened(1000,true); + const pending=bridge.renew(1000,[socket]);await started;assert.equal(sent.length,0); + complete({code:'ok',index:'42',now:1000,items:[{cell:'c',code:'ok',view:{cell:{lease:{fence:'7',expires:91000}}}}]}); + assert.equal(await pending,31000);assert.equal(sent[0].holds[0].index,'42');assert.equal(calls[0][0].renew_holds.seen_at,1000); + await bridge.renew(2000,[socket]);assert.equal(calls.length,1); + await bridge.renew(31000,[socket]);assert.equal(calls.length,1,'stale evidence renewed'); + socket.seenAt=61000;host.commitEffect=async()=>{throw Error('minority')}; + await bridge.renew(61000,[socket]);assert.equal(failed,1);assert.equal(sent.length,1); + assert.equal(await bridge.renew(91000,[]),null); +}); + +test('R26 R27 alarms stop only after dispatch and durable sleep',async()=>{ + const ops=[],storage={setAlarm:async at=>ops.push(['arm',at]),deleteAlarm:async()=>ops.push(['stop'])}; + let sleeping=false; + const actor={serial:async fn=>fn(),scheduleInQueue:async(now,wall,busy)=>({send:[],plan:{sleeping,after:5000},now,wall,busy})}; + const timer=new MemberAlarms(storage,actor,async e=>ops.push(['dispatch',e.busy]),async()=>false); + await timer.run(0,1000);assert.deepEqual(ops,[['dispatch',false],['arm',6000]]); + sleeping=true;await timer.run(60000,61000);assert.deepEqual(ops.slice(-2),[['dispatch',false],['stop']]); + const failed=new MemberAlarms(storage,actor,async()=>{throw Error('backpressure')},async()=>true); + const n=ops.length;await assert.rejects(failed.run(1,1));assert.equal(ops.length,n,'failed dispatch claimed timer completion'); +}); + +test('R27 hold evidence overflow is paged without starving later sockets',async()=>{ + let calls=0;const policy={renew_every:30000,evidence_window:25000,max_renew_holds:2,covered:()=>false}; + const bridge=new CommittedHolds({barrierRecord:async()=>null,commitEffect:async change=>{calls++;return {code:'ok',index:'1',items:[]}}},policy); + const sockets=Array.from({length:5},(_,i)=>({device:'d'+i,seenAt:1000,holds:[{cell:'c'+i,fence:'1'}],send(){}})); + bridge.opened(1000,true); + assert.equal(await bridge.renew(1000,sockets),1000);assert.equal(calls,2); + assert.equal(await bridge.renew(1000,sockets),1000);assert.equal(calls,4); + assert.equal(await bridge.renew(1000,sockets),31000);assert.equal(calls,5); +}); diff --git a/relay/hosted/test/raft/membership-worker.js b/relay/hosted/test/raft/membership-worker.js index 03f1c54577..d55b32c690 100644 --- a/relay/hosted/test/raft/membership-worker.js +++ b/relay/hosted/test/raft/membership-worker.js @@ -5,6 +5,7 @@ import { RaftMemberDO } from '../../src/raft/member.js'; import { installRaftModule, sharedRuntime } from '../../src/raft/actor.js'; import { directoryEvaluator } from '../../src/raft/directory.js'; import { committedRegistry } from '../../src/raft/membership.js'; +import { MemberAlarms, coordinationPolicy } from '../../src/raft/liveness.js'; import { b64, unb64 } from '../../src/raft/storage.js'; export class ProofStore {} installRaftModule(core); @@ -26,6 +27,17 @@ export class ActorFixture extends RaftMemberDO { await this.load(); if (b.op==='input') return reply(await this.actor.input(b.tag,[2,3,8].includes(b.tag)?unb64(b.value):b.value)); if (b.op==='peer') return reply(await this.peer(unb64(b.value))); + if (b.op==='schedule') return reply(await this.schedule(b.elapsed,b.now,b.busy,b.acks)); + if (b.op==='sleep-ack') return reply(await this.sleepAck(b.index,b.term)); + if (b.op==='retain-fixture') return reply(await this.actor.serial(()=>this.retainScheduled({send:b.send.map(m=>({to:BigInt(m.to),bytes:unb64(m.bytes)})),plan:{wake:[]}},coordinationPolicy(sharedRuntime())))); + if (b.op==='scheduled') return reply(await this.scheduled()); + if (b.op==='ack-scheduled') return reply(await this.acknowledgeScheduled(b.id)); + if (b.op==='fire-alarm') return reply(await this.alarm()); + if (b.op==='alarm-state') return reply(await this.ctx.storage.getAlarm()); + if (b.op==='turn') { + const timer=new MemberAlarms(this.ctx.storage,this.actor,async()=>{},async()=>b.busy); + return reply(await timer.run(b.elapsed,b.now,b.acks)); + } if (b.op==='membership') return reply(await this.membership()); if (b.op==='checkpoint') return reply(await this.actor.checkpoint()); if (b.op==='registry') return reply(await this.refreshEnrollment()); diff --git a/relay/hosted/test/raft/membership-workerd.test.js b/relay/hosted/test/raft/membership-workerd.test.js index f14cd8ad85..932105b1e3 100644 --- a/relay/hosted/test/raft/membership-workerd.test.js +++ b/relay/hosted/test/raft/membership-workerd.test.js @@ -10,7 +10,7 @@ import { openWorkerd } from './workerd.js'; // R11/R12/R16 exercise the ACTUAL hosted peer and committed-registry doors. // A learner remains absent during writes/compaction and both process restarts. -test('R9 R11 R12 R13 R16 actual workerd admits only committed learners and retains retired authority across restarts',{timeout:45000},async()=>{ +test('R9 R11 R12 R13 R16 R26 R27 actual workerd admits only committed learners and retains retired authority across restarts',{timeout:45000},async()=>{ const proof=process.env.CODEAF_RAFT_PROOF_DIR??'/tmp/codeaf-p12',out=await mkdtemp(join(tmpdir(),'p12-wasm-')),disk=await mkdtemp(join(tmpdir(),'p12-disk-')); const bundle=await build({entryPoints:['test/raft/membership-worker.js'],bundle:true,format:'esm',target:'es2022',external:['core.wasm','cloudflare:*'],write:false}); await writeFile(join(out,'worker.js'),bundle.outputFiles[0].contents);await copyFile(join(proof,'core.wasm'),join(out,'core.wasm'));await writeFile(join(out,'storage-worker.js'),'export class ProofStore{}'); @@ -39,6 +39,29 @@ test('R9 R11 R12 R13 R16 actual workerd admits only committed learners and retai await pump();for(let n=0;n<30&&!leader();n++)await tick();assert(leader()); await assert.rejects(call(leader(),'enrollment',{view:{},applied:'0'}),/absent/); await submit(command({init_membership:{anchors,limit:4096}})); + // R26/R27 use actual DO storage, committed Go markers and the production + // scheduler. Partial final delivery and process eviction cannot confer reads. + for(const id of [1,2,3])accept(id,await call(id,'schedule',{elapsed:0,now:1000,busy:false,acks:[]}));await pump(); + const asleepLeader=leader(); + accept(asleepLeader,await call(asleepLeader,'input',{tag:4,value:'idle-barrier'}));await pump(); + await submit(command({prepare_quiescence:{barrier:status[leader()].applied,term:status[leader()].term,voters:['1','2','3']}},String(leader()))); + await tick(); + const marker=JSON.parse(Buffer.from(await call(leader(),'row',{namespace:'membership',key:'quiescence'}),'base64').toString()); + const acks=[];for(const id of [1,2,3])acks.push(await call(id,'sleep-ack',{index:marker.index,term:marker.term})); + let timed=await call(leader(),'turn',{elapsed:60000,now:Date.now(),busy:false,acks:acks.slice(0,2)});accept(leader(),timed);await pump();assert.equal(timed.plan.sleeping,false);assert(await call(leader(),'alarm-state')); + for(const id of [1,2,3]){timed=await call(id,'turn',{elapsed:60000,now:Date.now(),busy:false,acks});accept(id,timed);assert.equal(timed.plan.sleeping,true);assert.equal(await call(id,'alarm-state'),null)}await pump(); + for(const id of [1,2,3]){await call(id,'fire-alarm');assert.equal(await call(id,'alarm-state'),null)} + accept(leader(),await call(leader(),'input',{tag:4,value:'wake-before-alarm'}));await pump(); + const queuedAt=leader();await call(queuedAt,'fire-alarm');const queued=await call(queuedAt,'scheduled');assert(queued?.id);assert.equal(queued.wake.length,3); + await call(queuedAt,'fire-alarm');assert.deepEqual(await call(queuedAt,'scheduled'),queued); + await call(queuedAt,'ack-scheduled',{id:'wrong'});assert.deepEqual(await call(queuedAt,'scheduled'),queued); + await restart();assert.deepEqual(await call(queuedAt,'scheduled'),queued);await call(queuedAt,'ack-scheduled',{id:queued.id});assert.equal(await call(queuedAt,'scheduled'),null); + const maxPacket=Buffer.alloc(64*1024,7).toString('base64');await call(queuedAt,'retain-fixture',{send:[{to:'1',bytes:maxPacket},{to:'2',bytes:maxPacket}]}); + const large=await call(queuedAt,'scheduled');assert.equal(large.send.length,2);assert.equal(large.send[0].bytes,maxPacket);assert(JSON.stringify(large).length>64*1024); + await restart();assert.deepEqual(await call(queuedAt,'scheduled'),large);await call(queuedAt,'ack-scheduled',{id:large.id});assert.equal(await call(queuedAt,'scheduled'),null); +for(const id of [1,2,3])accept(id,await call(id,'status'));await pump(); + for(const id of [1,2,3]){timed=await call(id,'schedule',{elapsed:0,now:1000,busy:false,acks});accept(id,timed);assert.equal(timed.plan.sleeping,false)}await pump(); + for(let n=0;n<30&&!leader();n++)await tick();assert(leader()); let h=await header();const enrolled=command({enroll_member:{node:h.next_node,revision:h.revision,device:'b',protocol:1}});await submit(enrolled);await settle(); let view=await call(leader(),'registry');assert.equal(view['4'].voter,false);assert.equal(view['4'].revoked,false); await submit(command({set_vault:{old:'',new:'sealed-ref'}}));await call(leader(),'checkpoint'); diff --git a/relay/hosted/test/raft/scheduler.test.js b/relay/hosted/test/raft/scheduler.test.js new file mode 100644 index 0000000000..59f937e3f9 --- /dev/null +++ b/relay/hosted/test/raft/scheduler.test.js @@ -0,0 +1,97 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { spawn } from 'node:child_process'; +import { DatabaseSync } from 'node:sqlite'; +import { generateKeyPairSync,createHash,sign } from 'node:crypto'; +import { Reactor } from '../../src/raft/abi.js'; +import { MemberActor } from '../../src/raft/actor.js'; +import { directoryEvaluator } from '../../src/raft/directory.js'; +import { encodeCanonical,decodeCanonical } from '../../src/raft/codec.js'; +import { coordinationPolicy,CommittedHolds,MemberAlarms,QuiescenceCoordinator } from '../../src/raft/liveness.js'; + +const out=process.env.CODEAF_RAFT_PROOF_DIR??'/tmp/codeaf-p14'; +const enc=new TextEncoder(),dec=new TextDecoder(),bytes=v=>enc.encode(encodeCanonical(v)); +class Host { + constructor(){this.db=new DatabaseSync(':memory:');this.sql={exec:(q,...p)=>{const rows=this.db.prepare(q).all(...p);return {toArray:()=>rows,[Symbol.iterator]:()=>rows[Symbol.iterator]()}}};this.alarms=[];} + transactionSync(fn){this.db.exec('BEGIN');try{const r=fn();this.db.exec('COMMIT');return r}catch(e){this.db.exec('ROLLBACK');throw e}} + async sync(){} + async setAlarm(at){this.alarms.push(at)} + async deleteAlarm(){this.alarms.push(null)} +} +class Native { + constructor(){this.child=spawn(out+'/native-peer',[],{stdio:['pipe','pipe','inherit'],env:{...process.env,GOMEMLIMIT:'32MiB'}});this.buffer=Buffer.alloc(0); + this.child.stdout.on('data',b=>{this.buffer=Buffer.concat([this.buffer,b]);if(!this.waiter||this.buffer.length<4)return;const n=this.buffer.readUInt32LE();if(this.buffer.length{if(this.waiter){clearTimeout(this.waiter.timer);this.waiter.reject(Error('native exited '+code))}});} + call(b){return new Promise((resolve,reject)=>{assert(!this.waiter);this.waiter={resolve,reject,timer:setTimeout(()=>reject(Error('native timeout')),5000)};const h=Buffer.alloc(4);h.writeUInt32LE(b.length);this.child.stdin.write(h);this.child.stdin.write(b)})} + async close(){this.child.stdin.end();if(this.child.exitCode===null)await new Promise(r=>this.child.once('exit',r))} +} + +// R22/R26 exercise production Go policy and actual persisted WASI actors. All +// ABI inputs, including timer decisions, replay byte-for-byte natively. +test('R22 R26 R27 committed renewal, durable sleep, cold wake and native parity',{timeout:30000},async()=>{ + const reactor=new Reactor(new WebAssembly.Module(readFileSync(out+'/core.wasm'))),native=new Native(),calls=[]; + const runtime={call(b){const r=reactor.call(b);calls.push([b.slice(),r.slice()]);return r}}; + const policy=coordinationPolicy(runtime); + const cluster='p14-hosted',actors=[],hosts=[],configs=[],queue=[],results=[]; + const keys=Object.fromEntries(['a','b','ingress','anchor'].map(id=>[id,generateKeyPairSync('ed25519')])); + const pub=id=>keys[id].publicKey.export({format:'der',type:'spki'}).subarray(-32).toString('base64'); + const genesis=Object.keys(keys).map(id=>({namespace:'enrollment',key:'actor/'+Buffer.from(id).toString('hex'),data:bytes({cluster,key:pub(id),personal:id==='a'||id==='b',...(id==='ingress'?{effects:true}:{}),...(id==='anchor'?{anchor:true}:{})})})); + genesis.push({namespace:'state',key:'identity',data:bytes({V:1,identity:'fixture'})}); + const size=rows=>rows.reduce((n,r)=>n+12+r.namespace.length+r.key.length+r.data.length,0); + const accept=e=>queue.push(...e.send),leader=()=>actors.findIndex(a=>a.status.role===2); + const pump=async()=>{for(let n=0;queue.length;n++){assert(n<2000);const m=queue.shift();accept(await actors[Number(m.to)-1].input(2,m.bytes))}}; + const add=async id=>{const host=new Host();hosts[id-1]=host;configs[id-1]={cluster,id:String(id),seed:String(100+id),bootstrap:['1','2','3']}; + const evaluate=(raw,read,app)=>{const base=read('state','identity')?[]:genesis;const r=directoryEvaluator(runtime,cluster,8*1024*1024)(raw,(ns,k)=>base.find(v=>v.namespace===ns&&v.key===k)?.data??read(ns,k),{...app,size:app.size+size(base)});results[id-1]??=[];results[id-1].push(decodeCanonical(dec.decode(r.result)));return {...r,writes:[...base,...r.writes]}}; + const a=new MemberActor(host,runtime,evaluate);actors[id-1]=a;accept(await a.init(configs[id-1]));}; + const sequences={}; + const command=(change,now=1000000,actor='a')=>{const n=sequences[actor]=(sequences[actor]??0)+1;const c={schema:1,cluster,actor,session:'test',sequence:String(n),...(n>1?{ack:String(n-1)}:{}),request_digest:'',now,change,signature:null};c.request_digest=createHash('sha256').update(bytes({cluster,actor,session:c.session,sequence:c.sequence,change})).digest('hex');c.signature=sign(null,Buffer.concat([Buffer.from('codeaf.directory.command.v1\0'),bytes(c)]),keys[actor].privateKey).toString('base64');return bytes(c)}; + const submit=async(raw)=>{const l=leader(),n=results[l]?.length??0;accept(await actors[l].input(3,raw));await pump();assert.equal(results[l].length,n+1);return results[l].at(-1)}; + try{ + for(let id=1;id<=3;id++)await add(id);await pump();for(let n=0;n<20&&leader()<0;n++){for(const a of actors)accept(await a.input(1));await pump()}assert(leader()>=0); + let r=await submit(command({create:{cell:'chat',v1:true,init:{head:'sealed',class:'host-bound',size:'0',keys:null},receipt:''}}));assert.equal(r.code,'ok'); + const bridge=new CommittedHolds({barrierRecord:async key=>{ + const l=leader();accept(await actors[l].input(4,'renew-read-'+calls.length));await pump();return actors[l].store.read(key.namespace,key.key); + },commitEffect:async(change,now)=>submit(command(change,now,'ingress'))},policy); + const receipts=[];bridge.opened(1030000,true);await bridge.renew(1030000,[{device:'a',seenAt:1030000,holds:[{cell:'chat',fence:'1'}],send:s=>receipts.push(JSON.parse(s))}]); + assert.equal(receipts[0].holds[0].until,1120000);assert(BigInt(receipts[0].holds[0].index)>0n); + r=await submit(command({release:{cell:'chat',fence:'1',quiet:false}},1030001));assert.equal(r.code,'ok'); + r=await submit(command({renew_holds:{device:'a',seen_at:1030000,items:[{cell:'chat',fence:'1'}]}},1030002,'ingress'));assert.equal(r.items[0].code,'fence_stale'); + for(const a of actors){accept(await a.schedule(0,1030002,false));await pump()} + const l=leader(),status=actors[l].status; + r=await submit(command({prepare_quiescence:{barrier:String(status.applied),term:String(status.term),voters:['1','2','3']}},1030003,'anchor'));assert.equal(r.code,'ok'); + for(const a of actors){accept(await a.input(1));await pump()} + const marker=decodeCanonical(dec.decode(actors[l].store.read('membership','quiescence'))),acks=[]; + for(const a of actors)acks.push(await a.sleepAck(marker.index,marker.term)); + let e=await actors[l].schedule(60000,1030003,false,acks.slice(0,2));accept(e);await pump();assert.equal(e.plan.sleeping,false); + for(let n=0;n{accept(e)},async()=>false); + e=await alarms.run(60000,1030003,acks);await pump();assert.equal(e.plan.sleeping,true);assert.equal(hosts[n].alarms.at(-1),null); + } + // One cold member cannot reinstate the old certificate after eviction. + const cold=(l+1)%3,old=actors[cold],host=hosts[cold],evaluate=old.evaluate; + await old.close();actors[cold]=new MemberActor(host,runtime,evaluate);accept(await actors[cold].init(configs[cold]));await pump(); + e=await actors[cold].schedule(0,1030004,false,acks);accept(e);await pump();assert.equal(e.plan.sleeping,false); + accept(await actors[l].input(4,'first-wake-read'));await pump(); + e=await actors[l].schedule(60001,1030004,false,acks);accept(e);await pump();assert.equal(e.plan.sleeping,false);assert.equal(e.plan.wake.length,3); + r=await submit(command({acquire:{cell:'chat',force:false,expect:{device:'a',fence:'1'}}},1030005,'b'));assert.equal(r.code,'ok');assert.equal(r.view.cell.lease.fence,'2'); + r=await submit(command({release:{cell:'chat',fence:'2',quiet:false}},1030006,'b'));assert.equal(r.code,'ok'); + let sleeps=0,barriers=0; + const driver=new QuiescenceCoordinator({ + maintain:async()=>false, + schedule:(elapsed,now,busy)=>actors[leader()].schedule(elapsed,now,busy), + dispatch:async e=>{accept(e);await pump()}, + wake:async voters=>assert.equal(voters.length,3), + idleBarrier:async()=>{barriers++;accept(await actors[leader()].input(15,'idle-auto'));await pump()}, + prepare:async()=>{ + const s=actors[leader()].status; + const r=await submit(command({prepare_quiescence:{barrier:String(s.applied),term:String(s.term),voters:['1','2','3']}},1030007,'anchor'));assert.equal(r.code,'ok'); + accept(await actors[leader()].input(1));await pump();return decodeCanonical(dec.decode(actors[leader()].store.read('membership','quiescence'))); + }, + appliedAcks:async m=>{const a=[];for(const actor of actors)a.push(await actor.sleepAck(m.index,m.term));return a}, + finish:async(m,acks,elapsed,now)=>{for(const actor of actors){const e=await actor.schedule(elapsed,now,false,acks);accept(e);await pump();assert(e.plan.sleeping);sleeps++}}, + }); + const automatic=await driver.run(120002,1030007);assert(automatic.prepare);assert.equal(barriers,1);assert.equal(sleeps,3); + const start=calls.length;assert(start>100);for(const [request,expected]of calls)assert.deepEqual(await native.call(request),expected,'R14 native/WASI scheduling divergence'); + }finally{await native.close();for(const h of hosts)h.db.close()} +}); diff --git a/relay/hosted/test/watch.test.js b/relay/hosted/test/watch.test.js index 6f9429665d..a2bf8c19b1 100644 --- a/relay/hosted/test/watch.test.js +++ b/relay/hosted/test/watch.test.js @@ -4,7 +4,8 @@ import { Directory } from '../src/directory.js'; import { STAGE1 } from '../src/rules.js'; import { windowMs } from '../src/limits.js'; import { parseBeat } from '../src/presence.js'; -import { CLOSE_REVOKED, CLOSE_ROTATED, CLOSE_SILENT, Watchers } from '../src/watch.js'; +import { parseHolds } from '../src/vouch.js'; +import { CLOSE_REVOKED, CLOSE_ROTATED, CLOSE_SILENT, Watchers, CommittedWatchers } from '../src/watch.js'; import { memorySql } from './sql.js'; import { init } from './helpers.js'; @@ -170,7 +171,7 @@ function platform() { } /** A relay of watchers over the stand-in, answering the response the 101 would carry. */ -function relay() { +function relay(WatchType = Watchers) { const real = { Date: Date.now, Pair: globalThis.WebSocketPair, Response: globalThis.Response }; const clock = { now: 5_000_000 }; const p = platform(); @@ -187,7 +188,7 @@ function relay() { Date.now = () => clock.now; const armed = []; const stamped = []; - const watchers = new Watchers(p.ctx, 1000, 90_000, (at) => armed.push(at), (d, at) => stamped.push([d, at])); + const watchers = new WatchType(p.ctx, 1000, 90_000, (at) => armed.push(at), (d, at) => stamped.push([d, at])); const restore = () => ((Date.now = real.Date), (globalThis.WebSocketPair = real.Pair), (globalThis.Response = real.Response)); const dial = (device, beat, events = false) => { const answer = watchers.accept(device, 1, clock.now, { holds: [], beat, events }); @@ -299,3 +300,22 @@ test('an attachment from before presence reads as a live-by-default beat of 30 a r.clock.now += 1_001; assert.deepEqual([...r.watchers.online()], []); })); + + +// R22 preserves advisory presence without giving old clients a socket-only +// lease promise. Fresh platform evidence still needs the committed bridge. +test('R22 consensus watch supplies evidence without lifted leases or a vouch header',()=>{ + const r=relay(CommittedWatchers); + try{ + const answer=r.watchers.accept('dev_a',1,r.clock.now,{holds:parseHolds(['chat:7'],16),beat:10,receipts:true}); + assert.equal(answer.headers.get('Codeaf-Presence'),'1');assert.equal(answer.headers.get('Codeaf-Vouch'),null); + r.clock.now+=10000;r.p.open[0].pinged=r.clock.now; + const evidence=r.watchers.holdEvidence(r.clock.now);assert.equal(evidence[0].device,'dev_a');assert.equal(evidence[0].seenAt,r.clock.now);assert.deepEqual(evidence[0].holds,[{cell:'chat',fence:'7'}]); + const before=r.p.open[0].sent.length;evidence[0].send('{"holds":[]}');assert.equal(r.p.open[0].sent.length,before+1); + r.watchers.accept('dev_old',1,r.clock.now,{holds:parseHolds(['chat:7'],16),beat:10}); + const old=r.watchers.holdEvidence(r.clock.now)[1];old.send('{"holds":[]}');assert.equal(r.p.open[1].sent.length,1,'old client received a frame it cannot decode'); + assert.equal(r.watchers.vouchedUntil('dev_a','chat',7),0); + r.p.ctx.getWebSocketAutoResponseTimestamp=()=>{throw Error('closing')}; + assert.equal(r.watchers.holdEvidence(r.clock.now)[0].seenAt,0,'unreadable timestamp invented fresh ownership evidence'); + }finally{r.restore()} +}); From 07feddef11c80e37ed377cdccf30d3863397cab3 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 04:27:20 -0400 Subject: [PATCH 034/103] Delegate selected chats without exporting personal identity authority. Bind expiring worker certificates to exact chat scopes and retain sealed grants. Commit registration, availability, drain and revocation rules with incarnation fencing so replacement processes cannot reuse old authority. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterstate/README.md | 39 ++ internal/clusterstate/apply.go | 55 ++- internal/clusterstate/codec.go | 5 + internal/clusterstate/effects.go | 32 +- internal/clusterstate/leadership.go | 5 +- internal/clusterstate/liveness.go | 35 +- internal/clusterstate/membership.go | 114 ++++- internal/clusterstate/types.go | 26 +- internal/clusterstate/worker.go | 408 ++++++++++++++++ internal/clusterstate/worker_certificate.go | 58 +++ internal/clusterstate/worker_read.go | 68 +++ internal/clusterstate/worker_test.go | 509 ++++++++++++++++++++ internal/identity/worker.go | 90 ++++ internal/identity/worker_test.go | 115 +++++ 14 files changed, 1507 insertions(+), 52 deletions(-) create mode 100644 internal/clusterstate/worker.go create mode 100644 internal/clusterstate/worker_certificate.go create mode 100644 internal/clusterstate/worker_read.go create mode 100644 internal/clusterstate/worker_test.go create mode 100644 internal/identity/worker.go create mode 100644 internal/identity/worker_test.go diff --git a/internal/clusterstate/README.md b/internal/clusterstate/README.md index 1950d9660c..72182d84c3 100644 --- a/internal/clusterstate/README.md +++ b/internal/clusterstate/README.md @@ -53,3 +53,42 @@ export: its visible version deliberately lives outside the v1 body. no production types or apply functions. P06 adds the Porcupine adapter, unknown timeout outcomes and replay/history machinery; later assignment packets extend the independent domain. P01 does not claim simulator or consensus proof. + +Worker delegation (P15) is a fourth exclusive Authority role. Personal pairing +Cert remains unchanged; WorkerCertificate uses its own signed domain and binds +the root public identity, cluster UUID, worker key, sorted selected cell IDs, +exclusive expiry and explicit learner consent. EnrollWorker commits only sealed +per-cell grants; it cannot replace personal authority or clear revocation. +WorkerRegistry retains at most 4,096 delegations independently of learner quota; +existing rows, grants and node tombstones are never deleted on pressure/expiry. + +RegisterWorker requires a monotonic incarnation CAS. WorkerIncarnation is signed +and included in the request digest on every worker command. Authorization checks +it before dedup, with the narrow predecessor exception for an exact registration +retry. DrainWorker is sticky across registration and blocks WorkerCanClaim; +held work can save/release. WorkerBeatBatch retains original per-worker signed, +sequenced, fresh evidence, returns per-item codes and updates availability only. +Idle workers need no recurring beat. Availability and work hints do not acquire +or renew a chat. RenewHolds and upload receipts bind worker scopes/incarnations. + +A delegated worker may beat/publish/release an already held selected chat, close +its request session, register, heartbeat or drain itself. It cannot use ordinary +Acquire, create/archive, pairing, vault, rotation or membership/leadership doors. +P16 supplies the conditional assignment claim and must extend the allowlist and +assignment dependencies there; it must bind an assignment's claimant incarnation +before granting execution, even when a chat fence survives a worker restart. + +ReadWorkerCell returns one selected chat and its sealed grant after the host +commits current ingress time and passes a fresh quorum ReadIndex barrier. The +native AuthorizedWorkerRead and hosted workerCell adapters have the SAME host +prerequisite, not a follower/local read path. Identity-wide AuthorizedRead still +refuses workers. The public setup/client/provisioning lifecycle remains P17. + +Scoped API workers never receive the identity log. Explicit learner consent +permits its routing metadata, never content keys outside selected sealed grants. +MemberRecord binds worker expiry/incarnation; CommittedViewAt/MembershipViewAt +and hosted ABI-16's optional tail derive current revocation. Hosts MUST use the +time-aware view for worker learners; the older view fails closed for them. A new +worker incarnation retires the old peer view; removal/re-enrollment uses a fresh +node ID. Worker voters and promotion are refused. Rotation/removal cannot erase +bytes already downloaded by a worker. diff --git a/internal/clusterstate/apply.go b/internal/clusterstate/apply.go index b0b65500af..cae16322c2 100644 --- a/internal/clusterstate/apply.go +++ b/internal/clusterstate/apply.go @@ -50,11 +50,12 @@ func JoinKey(code string) RecordKey { return RecordKey{"outbox", "join/" + segme // Personal directory authority and restricted effect authority are exclusive. // Delegated worker scopes are admitted separately. type Authority struct { - Cluster string `json:"cluster"` - Key []byte `json:"key"` - Personal bool `json:"personal"` - Effects bool `json:"effects,omitempty"` - Anchor bool `json:"anchor,omitempty"` + Worker *WorkerCertificate `json:"worker,omitempty"` + Cluster string `json:"cluster"` + Key []byte `json:"key"` + Personal bool `json:"personal"` + Effects bool `json:"effects,omitempty"` + Anchor bool `json:"anchor,omitempty"` } type session struct { Sequence uint64 `json:"sequence"` @@ -101,12 +102,13 @@ type Evaluation struct { // newer ingress time; its operation and durable sequence must remain identical. func Digest(c Command) (string, error) { b, err := Encode(struct { - Cluster string `json:"cluster"` - Actor string `json:"actor"` - Session string `json:"session"` - Sequence uint64 `json:"sequence"` - Change Mutation `json:"change"` - }{c.Cluster, c.Actor, c.Session, c.Sequence, c.Change}) + WorkerIncarnation uint64 `json:"worker_incarnation,omitempty"` + Cluster string `json:"cluster"` + Actor string `json:"actor"` + Session string `json:"session"` + Sequence uint64 `json:"sequence"` + Change Mutation `json:"change"` + }{c.WorkerIncarnation, c.Cluster, c.Actor, c.Session, c.Sequence, c.Change}) if err != nil { return "", err } @@ -146,6 +148,7 @@ func Dependencies(c Command) []RecordKey { keys = append(keys, MembershipKey()) } keys = append(keys, membershipDependencies(m)...) + keys = append(keys, workerDependencies(c)...) if m.PutDevice != nil { keys = append(keys, DeviceKey(m.PutDevice.ID)) } @@ -263,6 +266,10 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er target = new(Device) case k.Namespace == "state" && len(k.Key) >= 5 && k.Key[:5] == "cell/": target = new(Cell) + case k == WorkerRegistryKey(): + target = new(WorkerRegistry) + case k.Namespace == "state" && len(k.Key) > 7 && k.Key[:7] == "worker/": + target = new(WorkerRecord) case k == QuiescenceKey(): target = new(Quiescence) case k == MembershipKey(): @@ -313,7 +320,7 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er } digest, _ := Digest(c) signed, e := SigningBytes(c) - if e != nil || c.Cluster != meta.Cluster || auth.Cluster != c.Cluster || (!auth.Personal && !auth.Effects && !auth.Anchor) || len(auth.Key) != ed25519.PublicKeySize || !ed25519.Verify(auth.Key, signed, c.Signature) || digest != c.RequestDigest { + if e != nil || c.Cluster != meta.Cluster || auth.Cluster != c.Cluster || !auth.ValidRole() || len(auth.Key) != ed25519.PublicKeySize || !ed25519.Verify(auth.Key, signed, c.Signature) || digest != c.RequestDigest { return out, nil } // Revocation precedes dedup: a saved old response cannot authorize a revoked @@ -322,7 +329,16 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er out.Result.Code = Revoked return out, nil } - service := c.Change.RenewHolds != nil || c.Change.RecordUpload != nil || c.Change.AdmitJoin != nil || c.Change.CompleteJoin != nil + // Worker incarnation and scope precede dedup so an old successful result + // cannot restart activity after a process replacement, expiry or revocation. + if auth.Worker != nil { + code := authorizeWorkerCommand(c, max(meta.Now, c.Now), identity, auth, rows) + if code != OK { + out.Result.Code = code + return out, nil + } + } + service := c.Change.RenewHolds != nil || c.Change.RecordUpload != nil || c.Change.AdmitJoin != nil || c.Change.CompleteJoin != nil || auth.Effects && c.Change.WorkerBeatBatch != nil if service != auth.Effects || auth.Personal && auth.Effects || auth.Anchor && (auth.Personal || auth.Effects || c.Change.PrepareQuiescence == nil && c.Change.SettleMembership == nil && c.Change.SettleLeadership == nil && c.Change.CompleteLeadership == nil && c.Change.RestoreAnchors == nil && c.Change.CloseSession == nil) { return out, nil } @@ -377,10 +393,14 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er } else if c.Change.RenewHolds != nil { code = renewHolds(c, out.Now, rows, put, &out.Result) out.Result.Index = meta.Index + } else if c.Change.WorkerBeatBatch != nil { + code = applyWorker(c, out.Now, identity, rows, put, &out.Result) } else if service { code = applyEffect(c, out.Now, rows, put) } else { - if membershipMutation(c.Change) { + if workerMutation(c.Change) { + code = applyWorker(c, out.Now, identity, rows, put, &out.Result) + } else if membershipMutation(c.Change) { code = applyMembership(c, meta, rows, put, &out.Result) } else { code = applyMutation(c, out.Now, identity, rows, put, &out.Result) @@ -639,7 +659,7 @@ func applyMutation(c Command, now int64, identity IdentityRec, rows map[RecordKe if b != nil { return Exists } - if !m.Create.V1 && !validReceipt(rows, m.Create.Receipt, c.Actor, m.Create.Init.Head, m.Create.Init.Frames) { + if !m.Create.V1 && !validReceipt(rows, m.Create.Receipt, c.Actor, m.Create.Init.Head, m.Create.Init.Frames, id, c.WorkerIncarnation) { return BadRequest } cell = Created(m.Create.Init, c.Actor, now) @@ -666,7 +686,7 @@ func applyMutation(c Command, now int64, identity IdentityRec, rows map[RecordKe cell, err = Heartbeat(cell, c.Actor, m.Beat.Beat, now) case m.Publish != nil: cell, err = PublishTo(cell, c.Actor, m.Publish.Publish, now) - if err == nil && !m.Publish.V1 && !validReceipt(rows, m.Publish.Receipt, c.Actor, m.Publish.Publish.Head, m.Publish.Publish.Frames) { + if err == nil && !m.Publish.V1 && !validReceipt(rows, m.Publish.Receipt, c.Actor, m.Publish.Publish.Head, m.Publish.Publish.Frames, id, c.WorkerIncarnation) { return BadRequest } case m.Release != nil: @@ -704,6 +724,9 @@ func visibleChange(k RecordKey, old, next []byte, now int64) bool { return true } switch { + case k == WorkerRegistryKey() || len(k.Key) > 7 && k.Key[:7] == "worker/": + // Availability is advisory and does not repaint the chat listing. + return false case len(k.Key) > 5 && k.Key[:5] == "cell/": var a, b Cell if Decode(old, &a) != nil || Decode(next, &b) != nil { diff --git a/internal/clusterstate/codec.go b/internal/clusterstate/codec.go index ac6043e6f6..41ac404016 100644 --- a/internal/clusterstate/codec.go +++ b/internal/clusterstate/codec.go @@ -325,6 +325,11 @@ func (r Result) Validate() error { if !r.Code.valid() { return ErrCodec } + for _, worker := range r.Workers { + if !worker.Code.valid() { + return ErrCodec + } + } for _, item := range r.Items { if !item.Code.valid() { return ErrCodec diff --git a/internal/clusterstate/effects.go b/internal/clusterstate/effects.go index 42baa8a07b..7c038af468 100644 --- a/internal/clusterstate/effects.go +++ b/internal/clusterstate/effects.go @@ -19,10 +19,12 @@ const ( // store before proposing it; neither a personal device nor a worker can mint it. // Frames remain the existing incremental hint, not a plaintext closure manifest. type UploadReceipt struct { - Device string `json:"device"` - Head string `json:"head"` - HeadRID string `json:"head_rid"` - Frames []string `json:"frames,omitempty"` + Cell string `json:"cell,omitempty"` + WorkerIncarnation uint64 `json:"worker_incarnation,omitempty"` + Device string `json:"device"` + Head string `json:"head"` + HeadRID string `json:"head_rid"` + Frames []string `json:"frames,omitempty"` } type AdmitJoin struct { Code string `json:"code"` @@ -56,9 +58,9 @@ func IdentityID(public []byte) string { return "id_" + hex.EncodeToString(h[:16]) } -func validReceipt(rows map[RecordKey][]byte, id, actor, head string, frames []string) bool { +func validReceipt(rows map[RecordKey][]byte, id, actor, head string, frames []string, cell string, incarnation uint64) bool { var r UploadReceipt - return id != "" && Decode(rows[UploadKey(id)], &r) == nil && UploadID(r) == id && r.Device == actor && r.Head == head && reflect.DeepEqual(normalFrames(r.Frames), normalFrames(frames)) + return id != "" && Decode(rows[UploadKey(id)], &r) == nil && UploadID(r) == id && r.Device == actor && r.Head == head && (incarnation == 0 || r.Cell == cell && r.WorkerIncarnation == incarnation) && reflect.DeepEqual(normalFrames(r.Frames), normalFrames(frames)) } func normalFrames(f []string) []string { if len(f) == 0 { @@ -119,9 +121,25 @@ func applyEffect(c Command, now int64, rows map[RecordKey][]byte, put func(Recor if d.Revoked { return Revoked } - if Decode(rows[AuthorityKey(r.Device)], &auth) != nil || !auth.Personal || auth.Cluster != c.Cluster { + if Decode(rows[AuthorityKey(r.Device)], &auth) != nil || !auth.ValidRole() || auth.Cluster != c.Cluster || !auth.Personal && auth.Worker == nil { return Unauthorized } + if auth.Worker != nil { + var identity IdentityRec + if Decode(rows[IdentityKey()], &identity) != nil { + return BadRequest + } + a, w, code := workerFromRows(c.Cluster, r.Device, now, identity, rows) + if code != OK { + return code + } + if r.WorkerIncarnation == 0 || r.WorkerIncarnation != w.Incarnation { + return Conflict + } + if !a.Worker.Allows(r.Cell) { + return Unauthorized + } + } k := UploadKey(UploadID(r)) if rows[k] == nil { put(k, r) diff --git a/internal/clusterstate/leadership.go b/internal/clusterstate/leadership.go index db8dbf9661..8921f980b0 100644 --- a/internal/clusterstate/leadership.go +++ b/internal/clusterstate/leadership.go @@ -114,6 +114,9 @@ func applyLeadership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, if Decode(rows[MemberKey(v.Node)], &member) != nil { return NotFound } + if member.Worker { + return Unauthorized + } if member.Anchor || member.Retired || member.Protocol != ClusterProtocol || member.ReadyIndex == 0 || member.Node != v.Node || member.Device != v.Device { return Conflict } @@ -121,7 +124,7 @@ func applyLeadership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, // checked independently of the owner who requests the transfer. var auth Authority var device Device - if Decode(rows[AuthorityKey(member.Device)], &auth) != nil || !auth.Personal || auth.Anchor || auth.Effects || auth.Cluster != c.Cluster { + if Decode(rows[AuthorityKey(member.Device)], &auth) != nil || !auth.ValidRole() || !auth.Personal || auth.Anchor || auth.Effects || auth.Cluster != c.Cluster { return Unauthorized } if b := rows[DeviceKey(member.Device)]; b != nil { diff --git a/internal/clusterstate/liveness.go b/internal/clusterstate/liveness.go index 3b2d2b04e5..0692abf381 100644 --- a/internal/clusterstate/liveness.go +++ b/internal/clusterstate/liveness.go @@ -14,9 +14,10 @@ const ( // authenticating fresh socket evidence. Evidence time fixes the expiry: replay // can never turn an old ping into another ninety seconds of ownership. type RenewHolds struct { - Device string `json:"device"` - SeenAt int64 `json:"seen_at"` - Items []HoldFence `json:"items"` + WorkerIncarnation uint64 `json:"worker_incarnation,omitempty"` + Device string `json:"device"` + SeenAt int64 `json:"seen_at"` + Items []HoldFence `json:"items"` } type HoldFence struct { Cell string `json:"cell"` @@ -46,7 +47,7 @@ func renewHolds(c Command, now int64, rows map[RecordKey][]byte, put func(Record } var actor Device var auth Authority - if Decode(rows[AuthorityKey(v.Device)], &auth) != nil || !auth.Personal || auth.Cluster != c.Cluster { + if Decode(rows[AuthorityKey(v.Device)], &auth) != nil || !auth.ValidRole() || auth.Cluster != c.Cluster || !auth.Personal && auth.Worker == nil { return Unauthorized } if raw := rows[DeviceKey(v.Device)]; raw != nil && Decode(raw, &actor) != nil { @@ -55,6 +56,24 @@ func renewHolds(c Command, now int64, rows map[RecordKey][]byte, put func(Record if actor.Revoked { return Revoked } + if auth.Worker != nil { + var identity IdentityRec + if Decode(rows[IdentityKey()], &identity) != nil { + return BadRequest + } + a, w, code := workerFromRows(c.Cluster, v.Device, now, identity, rows) + if code != OK { + return code + } + if v.WorkerIncarnation == 0 || v.WorkerIncarnation != w.Incarnation { + return Conflict + } + for _, h := range v.Items { + if !a.Worker.Allows(h.Cell) { + return Unauthorized + } + } + } seen := map[string]bool{} for _, h := range v.Items { if h.Cell == "" || h.Fence == 0 || seen[h.Cell] { @@ -96,6 +115,14 @@ func renewHolds(c Command, now int64, rows map[RecordKey][]byte, put func(Record // RecordActive is evaluated one immutable row at a time so large imported // identities never have to be materialized to decide whether they may sleep. func RecordActive(key RecordKey, raw []byte, now int64) (bool, error) { + if key.Namespace == "state" && len(key.Key) > 7 && key.Key[:7] == "worker/" { + var worker WorkerRecord + if Decode(raw, &worker) != nil { + return false, ErrDependency + } + return len(worker.Work) > 0 && worker.AvailableUntil > now, nil + } + if key.Namespace == "state" && len(key.Key) >= 5 && key.Key[:5] == "cell/" { var c Cell if Decode(raw, &c) != nil { diff --git a/internal/clusterstate/membership.go b/internal/clusterstate/membership.go index ae717a0b34..1e1a89779c 100644 --- a/internal/clusterstate/membership.go +++ b/internal/clusterstate/membership.go @@ -37,14 +37,16 @@ func hasID(ids []uint64, id uint64) bool { // MemberRecord contains protocol authority only. Retired IDs are never deleted; // a device that rejoins receives a fresh ID and a fresh transport incarnation. type MemberRecord struct { - Node uint64 `json:"node"` - Device string `json:"device"` - Key []byte `json:"key"` - Incarnation uint64 `json:"incarnation"` - Protocol uint16 `json:"protocol"` - Anchor bool `json:"anchor"` - Retired bool `json:"retired"` - ReadyIndex uint64 `json:"ready_index"` + Worker bool `json:"worker,omitempty"` + WorkerExpires int64 `json:"worker_expires,omitempty"` + Node uint64 `json:"node"` + Device string `json:"device"` + Key []byte `json:"key"` + Incarnation uint64 `json:"incarnation"` + Protocol uint16 `json:"protocol"` + Anchor bool `json:"anchor"` + Retired bool `json:"retired"` + ReadyIndex uint64 `json:"ready_index"` } type MembershipIntent struct { Node uint64 `json:"node"` @@ -147,12 +149,12 @@ func applyMembership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, seen := map[string]bool{} h = Membership{Revision: 1, Index: meta.Index, Limit: v.Limit} for i, a := range v.Anchors { - if !validNode(a.Node) || !validNode(a.Node+1) || i > 0 && a.Node <= v.Anchors[i-1].Node || !hasID(meta.Configuration.Voters, a.Node) || len(a.Key) != ed25519.PublicKeySize || seen[string(a.Key)] || a.Device != "" || !a.Anchor || a.Retired || a.Incarnation != 1 || a.Protocol != ClusterProtocol || a.ReadyIndex != 0 || rows[MemberKey(a.Node)] != nil { + if !validNode(a.Node) || !validNode(a.Node+1) || i > 0 && a.Node <= v.Anchors[i-1].Node || !hasID(meta.Configuration.Voters, a.Node) || len(a.Key) != ed25519.PublicKeySize || seen[string(a.Key)] || a.Device != "" || !a.Anchor || a.Worker || a.WorkerExpires != 0 || a.Retired || a.Incarnation != 1 || a.Protocol != ClusterProtocol || a.ReadyIndex != 0 || rows[MemberKey(a.Node)] != nil { return BadRequest } var old Authority if b := rows[AuthorityKey(AnchorActor(a.Node))]; b != nil { - if Decode(b, &old) != nil || !old.Anchor || old.Personal || old.Effects || old.Cluster != c.Cluster || !bytes.Equal(old.Key, a.Key) { + if Decode(b, &old) != nil || !old.ValidRole() || !old.Anchor || old.Personal || old.Effects || old.Cluster != c.Cluster || !bytes.Equal(old.Key, a.Key) { return Conflict } } @@ -190,7 +192,7 @@ func applyMembership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, var authority Authority var device Device var prior DeviceMember - if Decode(rows[AuthorityKey(v.Device)], &authority) != nil || authority.Cluster != c.Cluster || !authority.Personal || authority.Effects || len(authority.Key) != ed25519.PublicKeySize { + if Decode(rows[AuthorityKey(v.Device)], &authority) != nil || authority.Cluster != c.Cluster || !authority.ValidRole() || !authority.Personal && authority.Worker == nil || len(authority.Key) != ed25519.PublicKeySize { return Unauthorized } if b := rows[DeviceKey(v.Device)]; b != nil { @@ -201,6 +203,23 @@ func applyMembership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, return Revoked } } + incarnation := uint64(1) + workerExpires := int64(0) + if authority.Worker != nil { + var identity IdentityRec + if Decode(rows[IdentityKey()], &identity) != nil { + return BadRequest + } + _, w, code := workerFromRows(c.Cluster, v.Device, result.Now, identity, rows) + if code != OK { + return code + } + if !authority.Worker.Learner || w.Incarnation == 0 || w.Draining { + return Unauthorized + } + incarnation = w.Incarnation + workerExpires = authority.Worker.Expires + } if b := rows[MemberDeviceKey(v.Device)]; b != nil { if Decode(b, &prior) != nil { return BadRequest @@ -227,7 +246,7 @@ func applyMembership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, h.Live++ h.Nodes = append(h.Nodes, node) h.Pending = &MembershipIntent{Node: node, Revision: h.Revision} - put(MemberKey(node), MemberRecord{Node: node, Device: v.Device, Key: bytes.Clone(authority.Key), Incarnation: 1, Protocol: ClusterProtocol}) + put(MemberKey(node), MemberRecord{Node: node, Device: v.Device, Key: bytes.Clone(authority.Key), Incarnation: incarnation, Protocol: ClusterProtocol, Worker: authority.Worker != nil, WorkerExpires: workerExpires}) put(MemberDeviceKey(v.Device), DeviceMember{Node: node}) put(MembershipKey(), h) result.Membership = &MembershipResult{Node: node, Revision: h.Revision} @@ -352,7 +371,7 @@ func AnchorEnrollments(anchors []MemberRecord, voters []uint64) ([]PeerEnrollmen seen := map[string]bool{} peers := []PeerEnrollment{} for i, m := range anchors { - if !validNode(m.Node) || i > 0 && anchors[i-1].Node >= m.Node || !hasID(voters, m.Node) || !m.Anchor || m.Device != "" || m.Retired || m.ReadyIndex != 0 || m.Protocol != ClusterProtocol || m.Incarnation != 1 || len(m.Key) != ed25519.PublicKeySize || seen[string(m.Key)] { + if !validNode(m.Node) || i > 0 && anchors[i-1].Node >= m.Node || !hasID(voters, m.Node) || !m.Anchor || m.Worker || m.WorkerExpires != 0 || m.Device != "" || m.Retired || m.ReadyIndex != 0 || m.Protocol != ClusterProtocol || m.Incarnation != 1 || len(m.Key) != ed25519.PublicKeySize || seen[string(m.Key)] { return nil, ErrDependency } seen[string(m.Key)] = true @@ -404,7 +423,7 @@ func PeerFromRecord(m MemberRecord, device []byte, conf Configuration) (PeerEnro if !validNode(m.Node) || m.Protocol != ClusterProtocol || len(m.Key) != ed25519.PublicKeySize || m.Incarnation == 0 { return PeerEnrollment{}, ErrDependency } - revoked := m.Retired + revoked := m.Retired || m.Worker if m.Device != "" && device != nil { var d Device if Decode(device, &d) != nil { @@ -414,3 +433,70 @@ func PeerFromRecord(m MemberRecord, device []byte, conf Configuration) (PeerEnro } return PeerEnrollment{Node: m.Node, Incarnation: m.Incarnation, Key: bytes.Clone(m.Key), Voter: hasID(conf.Voters, m.Node) || hasID(conf.Outgoing, m.Node), Revoked: revoked}, nil } + +// PeerFromRecordAt narrows worker routing using the same committed scope and +// incarnation as application commands. A worker can never be a voting peer. +func PeerFromRecordAt(m MemberRecord, device []byte, conf Configuration, now int64, identity IdentityRec, authority Authority, worker WorkerRecord) (PeerEnrollment, error) { + p, err := PeerFromRecord(m, device, conf) + if err != nil || !m.Worker { + return p, err + } + var d Device + if device != nil && Decode(device, &d) != nil { + return p, ErrDependency + } + p.Revoked = m.Retired || p.Voter || worker.Incarnation != m.Incarnation || m.WorkerExpires != authorityExpiry(authority) || workerAuthority(authority.Cluster, m.Device, now, identity, authority, d) != OK || authority.Worker == nil || !authority.Worker.Learner || !bytes.Equal(m.Key, authority.Key) + return p, nil +} +func authorityExpiry(a Authority) int64 { + if a.Worker != nil { + return a.Worker.Expires + } + return 0 +} + +// MembershipViewAt includes expiring delegated peers; the old metadata-only +// view deliberately refuses them until its host supplies committed time. +func MembershipViewAt(read ReadRecord, conf Configuration, applied uint64, now int64) (Membership, []PeerEnrollment, error) { + h, peers, err := MembershipView(read, conf, applied) + if err != nil { + return h, peers, err + } + for i, p := range peers { + var m MemberRecord + b, e := read(MemberKey(p.Node)) + if e != nil { + return h, nil, e + } + if Decode(b, &m) != nil { + return h, nil, ErrDependency + } + if !m.Worker { + continue + } + var identity IdentityRec + var a Authority + var w WorkerRecord + for _, v := range []struct { + k RecordKey + into any + }{{IdentityKey(), &identity}, {AuthorityKey(m.Device), &a}, {WorkerKey(m.Device), &w}} { + b, e = read(v.k) + if e != nil { + return h, nil, e + } + if Decode(b, v.into) != nil { + return h, nil, ErrDependency + } + } + d, e := read(DeviceKey(m.Device)) + if e != nil { + return h, nil, e + } + peers[i], e = PeerFromRecordAt(m, d, conf, now, identity, a, w) + if e != nil { + return h, nil, e + } + } + return h, peers, nil +} diff --git a/internal/clusterstate/types.go b/internal/clusterstate/types.go index 42d16dab05..ec5ad8c619 100644 --- a/internal/clusterstate/types.go +++ b/internal/clusterstate/types.go @@ -28,22 +28,27 @@ var ( // client sequence. Signature authentication and committed authorization belong // to ingress and apply respectively; decoding alone grants no authority. type Command struct { - Schema uint16 `json:"schema"` - Cluster string `json:"cluster"` - Actor string `json:"actor"` - Session string `json:"session"` - Sequence uint64 `json:"sequence"` - RequestDigest string `json:"request_digest"` - Ack uint64 `json:"ack,omitempty"` - Now int64 `json:"now"` - Change Mutation `json:"change"` - Signature []byte `json:"signature"` + WorkerIncarnation uint64 `json:"worker_incarnation,omitempty"` + Schema uint16 `json:"schema"` + Cluster string `json:"cluster"` + Actor string `json:"actor"` + Session string `json:"session"` + Sequence uint64 `json:"sequence"` + RequestDigest string `json:"request_digest"` + Ack uint64 `json:"ack,omitempty"` + Now int64 `json:"now"` + Change Mutation `json:"change"` + Signature []byte `json:"signature"` } // Mutation is a closed union: exactly one field is present. Future packets add // their typed operations here; unknown fields fail decoding on older members. // There is no arbitrary JSON payload in which task text or secrets can hide. type Mutation struct { + EnrollWorker *EnrollWorker `json:"enroll_worker,omitempty"` + RegisterWorker *RegisterWorker `json:"register_worker,omitempty"` + WorkerBeatBatch *WorkerBeatBatch `json:"worker_beat_batch,omitempty"` + DrainWorker *DrainWorker `json:"drain_worker,omitempty"` RenewHolds *RenewHolds `json:"renew_holds,omitempty"` PrepareQuiescence *PrepareQuiescence `json:"prepare_quiescence,omitempty"` CompleteLeadership *CompleteLeadership `json:"complete_leadership,omitempty"` @@ -146,6 +151,7 @@ type DeviceSeen struct { // no free-form diagnostic text, so logging a refusal cannot disclose payloads. // Now and Version are application time/version, never a local clock/Raft index. type Result struct { + Workers []WorkerResult `json:"workers,omitempty"` Index uint64 `json:"index,omitempty"` Membership *MembershipResult `json:"membership,omitempty"` Schema uint16 `json:"schema"` diff --git a/internal/clusterstate/worker.go b/internal/clusterstate/worker.go new file mode 100644 index 0000000000..6112e6f969 --- /dev/null +++ b/internal/clusterstate/worker.go @@ -0,0 +1,408 @@ +package clusterstate + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "encoding/hex" +) + +const MaxWorkerBeatItems = MaxRenewHolds + +// WorkerRegistry charges every retained delegation, including expired/revoked +// workers. Capacity refusal never deletes records or reuses their authority. +type WorkerRegistry struct { + Retained uint32 `json:"retained"` +} + +type EnrollWorker struct { + Certificate WorkerCertificate `json:"certificate"` + Grants map[string][]byte `json:"grants"` +} +type RegisterWorker struct { + Previous uint64 `json:"previous"` + Incarnation uint64 `json:"incarnation"` + Caps Caps `json:"caps"` + SealedLabels []byte `json:"sealed_labels"` +} +type DrainWorker struct { + Worker string `json:"worker"` + Incarnation uint64 `json:"incarnation"` +} + +// WorkerRecord carries only the existing public capability shape and opaque +// encrypted metadata. Availability does not confer a lease or authorize tools. +type WorkerRecord struct { + Incarnation uint64 `json:"incarnation"` + HeartbeatDigest string `json:"heartbeat_digest,omitempty"` + Heartbeat uint64 `json:"heartbeat"` + SeenAt int64 `json:"seen_at"` + AvailableUntil int64 `json:"available_until"` + Draining bool `json:"draining"` + Caps Caps `json:"caps"` + SealedLabels []byte `json:"sealed_labels"` + Grants map[string][]byte `json:"grants"` + Work []HoldFence `json:"work"` +} + +// A batch retains each worker's original signed evidence. A scheduling client +// may collect proofs, but cannot manufacture availability on a worker's behalf. +type WorkerHeartbeat struct { + Worker string `json:"worker"` + Incarnation uint64 `json:"incarnation"` + Sequence uint64 `json:"sequence"` + SeenAt int64 `json:"seen_at"` + Work []HoldFence `json:"work"` + Signature []byte `json:"signature"` +} +type WorkerBeatBatch struct { + Items []WorkerHeartbeat `json:"items"` +} +type WorkerResult struct { + Worker string `json:"worker"` + Code Code `json:"code"` + Incarnation uint64 `json:"incarnation"` + AvailableUntil int64 `json:"available_until"` + Draining bool `json:"draining"` +} + +func WorkerHeartbeatBytes(cluster string, h WorkerHeartbeat) ([]byte, error) { + h.Signature = nil + b, err := Encode(struct { + Cluster string `json:"cluster"` + Beat WorkerHeartbeat `json:"beat"` + }{cluster, h}) + return append([]byte("codeaf/worker-beat/v1\x00"), b...), err +} +func WorkerKey(worker string) RecordKey { return RecordKey{"state", "worker/" + segment(worker)} } +func WorkerRegistryKey() RecordKey { return RecordKey{"state", "worker-registry"} } + +// ValidRole keeps service, personal, anchor and delegated credentials mutually +// exclusive, including when rows are supplied by genesis or an old snapshot. +func (a Authority) ValidRole() bool { + n := 0 + for _, yes := range []bool{a.Personal, a.Effects, a.Anchor, a.Worker != nil} { + if yes { + n++ + } + } + return n == 1 +} +func workerMutation(m Mutation) bool { + return m.EnrollWorker != nil || m.RegisterWorker != nil || m.WorkerBeatBatch != nil || m.DrainWorker != nil +} +func workerDependencies(c Command) []RecordKey { + keys := []RecordKey{WorkerKey(c.Actor)} + add := func(id string) { keys = append(keys, WorkerKey(id), AuthorityKey(id), DeviceKey(id)) } + if v := c.Change.EnrollWorker; v != nil { + add(v.Certificate.DeviceID()) + keys = append(keys, WorkerRegistryKey()) + } + if v := c.Change.DrainWorker; v != nil { + add(v.Worker) + } + if v := c.Change.WorkerBeatBatch; v != nil { + for _, h := range v.Items { + add(h.Worker) + for _, work := range h.Work { + keys = append(keys, CellKey(work.Cell)) + } + } + } + if v := c.Change.RecordUpload; v != nil { + add(v.Device) + } + if v := c.Change.RenewHolds; v != nil { + add(v.Device) + } + if v := c.Change.EnrollMember; v != nil { + add(v.Device) + } + return keys +} + +func workerAuthority(cluster, worker string, now int64, identity IdentityRec, a Authority, device Device) Code { + if device.Revoked { + return Revoked + } + if identity.Rotation != nil { + return Rotated + } + if !a.ValidRole() || a.Worker == nil || a.Cluster != cluster || !bytes.Equal(a.Key, a.Worker.Key) || a.Worker.DeviceID() != worker || VerifyWorkerCertificate(*a.Worker, identity.SigningKey, cluster, now) != nil { + return Unauthorized + } + return OK +} +func workerFromRows(cluster, worker string, now int64, identity IdentityRec, rows map[RecordKey][]byte) (Authority, WorkerRecord, Code) { + var a Authority + var d Device + var w WorkerRecord + if Decode(rows[AuthorityKey(worker)], &a) != nil || Decode(rows[WorkerKey(worker)], &w) != nil { + return a, w, Unauthorized + } + if b := rows[DeviceKey(worker)]; b != nil && Decode(b, &d) != nil { + return a, w, BadRequest + } + return a, w, workerAuthority(cluster, worker, now, identity, a, d) +} +func authorizeWorkerCommand(c Command, now int64, identity IdentityRec, a Authority, rows map[RecordKey][]byte) Code { + _, w, code := workerFromRows(c.Cluster, c.Actor, now, identity, rows) + if code != OK { + return code + } + if v := c.Change.RegisterWorker; v != nil { + // A successful registration changes the incarnation itself. Admit its + // exact predecessor retry to dedup, but never a superseded generation. + if c.WorkerIncarnation == w.Incarnation || v.Previous == c.WorkerIncarnation && v.Previous < ^uint64(0) && v.Incarnation == v.Previous+1 && v.Incarnation == w.Incarnation { + return OK + } + return Conflict + } + if c.WorkerIncarnation != w.Incarnation { + return Conflict + } + if w.Incarnation == 0 { + return Conflict + } + if c.Change.CloseSession != nil { + return OK + } + if c.Change.DrainWorker != nil { + if c.Change.DrainWorker.Worker == c.Actor { + return OK + } + return Unauthorized + } + if v := c.Change.WorkerBeatBatch; v != nil { + for _, h := range v.Items { + if h.Worker != c.Actor { + return Unauthorized + } + } + return OK + } + // Ownership acquisition is reserved for P16's committed conditional claim. + // Drain does not block saving/releasing a fence already held by this worker. + if c.Change.Publish == nil && c.Change.Beat == nil && c.Change.BeatBatch == nil && c.Change.Release == nil { + return Unauthorized + } + for _, cell := range cellIDs(c.Change) { + if !a.Worker.Allows(cell) { + return Unauthorized + } + } + if c.Change.Publish != nil && c.Change.Publish.V1 { + return Unauthorized + } + return OK +} +func applyWorker(c Command, now int64, identity IdentityRec, rows map[RecordKey][]byte, put func(RecordKey, any), result *Result) Code { + if v := c.Change.EnrollWorker; v != nil { + cert := v.Certificate + id := cert.DeviceID() + if VerifyWorkerCertificate(cert, identity.SigningKey, c.Cluster, now) != nil || len(v.Grants) != len(cert.Cells) { + return BadRequest + } + for _, cell := range cert.Cells { + if len(v.Grants[cell]) == 0 || len(v.Grants[cell]) > MaxWorkerSealedBytes { + return BadRequest + } + } + var d Device + if b := rows[DeviceKey(id)]; b != nil { + if Decode(b, &d) != nil { + return BadRequest + } + if d.Revoked { + return Revoked + } + } + if b := rows[AuthorityKey(id)]; b != nil { + var a Authority + var w WorkerRecord + if Decode(b, &a) != nil || Decode(rows[WorkerKey(id)], &w) != nil { + return Conflict + } + old, _ := Encode(a.Worker) + incoming, _ := Encode(&cert) + grants, _ := Encode(w.Grants) + want, _ := Encode(v.Grants) + if !a.ValidRole() || a.Worker == nil || !bytes.Equal(old, incoming) || !bytes.Equal(grants, want) { + return Conflict + } + return OK + } + var registry WorkerRegistry + if b := rows[WorkerRegistryKey()]; b != nil && Decode(b, ®istry) != nil { + return BadRequest + } + if registry.Retained >= MaxWorkerRecords { + return Capacity + } + registry.Retained++ + put(WorkerRegistryKey(), registry) + put(AuthorityKey(id), Authority{Cluster: c.Cluster, Key: cert.Key, Worker: &cert}) + put(WorkerKey(id), WorkerRecord{Grants: v.Grants}) + put(DeviceKey(id), Stamped(d, rows[DeviceKey(id)] != nil, Device{V: 1}, now)) + return OK + } + if v := c.Change.RegisterWorker; v != nil { + _, w, code := workerFromRows(c.Cluster, c.Actor, now, identity, rows) + if code != OK { + return code + } + if v.Previous != w.Incarnation || w.Incarnation == ^uint64(0) || v.Incarnation != w.Incarnation+1 { + return Conflict + } + if !publicCaps(v.Caps) || len(v.SealedLabels) > MaxWorkerSealedBytes { + return BadRequest + } + w.Incarnation = v.Incarnation + w.Heartbeat = 0 + w.HeartbeatDigest = "" + w.SeenAt = 0 + w.AvailableUntil = 0 + w.Work = nil + w.Caps = v.Caps + w.SealedLabels = v.SealedLabels + put(WorkerKey(c.Actor), w) + result.Workers = []WorkerResult{{Worker: c.Actor, Code: OK, Incarnation: w.Incarnation, Draining: w.Draining}} + return OK + } + if v := c.Change.DrainWorker; v != nil { + _, w, code := workerFromRows(c.Cluster, v.Worker, now, identity, rows) + if code != OK { + return code + } + if w.Incarnation == 0 || v.Incarnation != w.Incarnation { + return Conflict + } + w.Draining = true + put(WorkerKey(v.Worker), w) + result.Workers = []WorkerResult{{Worker: v.Worker, Code: OK, Incarnation: w.Incarnation, AvailableUntil: w.AvailableUntil, Draining: true}} + return OK + } + if v := c.Change.WorkerBeatBatch; v != nil { + if len(v.Items) == 0 || len(v.Items) > MaxWorkerBeatItems { + return BadRequest + } + seen := map[string]bool{} + for _, h := range v.Items { + if h.Worker == "" || seen[h.Worker] || len(h.Work) > MaxWorkerScopes { + return BadRequest + } + seen[h.Worker] = true + } + for _, h := range v.Items { + a, w, code := workerFromRows(c.Cluster, h.Worker, now, identity, rows) + if code == OK { + code = checkWorkerHeartbeat(c.Cluster, h, now, a, w, rows) + } + item := WorkerResult{Worker: h.Worker, Code: code} + if code == OK { + w.Heartbeat = h.Sequence + w.HeartbeatDigest = workerHeartbeatDigest(c.Cluster, h) + w.SeenAt = h.SeenAt + w.AvailableUntil = min(h.SeenAt+LeaseTTLMillis, a.Worker.Expires) + w.Work = h.Work + put(WorkerKey(h.Worker), w) + item.Incarnation = w.Incarnation + item.AvailableUntil = w.AvailableUntil + item.Draining = w.Draining + } + result.Workers = append(result.Workers, item) + } + return OK + } + return BadRequest +} +func checkWorkerHeartbeat(cluster string, h WorkerHeartbeat, now int64, a Authority, w WorkerRecord, rows map[RecordKey][]byte) Code { + if w.Incarnation == 0 || h.Incarnation != w.Incarnation || h.Sequence == 0 || h.Sequence < w.Heartbeat { + return Conflict + } + b, err := WorkerHeartbeatBytes(cluster, h) + if err != nil || !ed25519.Verify(a.Key, b, h.Signature) { + return Unauthorized + } + // Exact evidence may be replayed after its freshness window, but keeps + // the original deadline. A different proof cannot reuse that sequence. + if h.Sequence == w.Heartbeat { + if w.HeartbeatDigest != "" && w.HeartbeatDigest == workerHeartbeatDigest(cluster, h) { + return OK + } + return Conflict + } + if h.SeenAt <= 0 || h.SeenAt > now || now-h.SeenAt > EvidenceWindowMillis || h.SeenAt < w.SeenAt { + return BadRequest + } + seen := map[string]bool{} + for _, work := range h.Work { + if !a.Worker.Allows(work.Cell) { + return Unauthorized + } + if seen[work.Cell] || work.Fence == 0 { + return BadRequest + } + seen[work.Cell] = true + var cell Cell + if Decode(rows[CellKey(work.Cell)], &cell) != nil { + return NotFound + } + if cell.Lease.Device != h.Worker || cell.Lease.Fence != work.Fence { + return FenceStale + } + } + return OK +} + +// Public capability values are closed vocabularies. Free text such as a host +// label, repository path, GPU model or scheduling requirement stays sealed. +func publicCaps(c Caps) bool { + allowed := func(s string, values ...string) bool { + for _, v := range values { + if s == v { + return true + } + } + return false + } + pointer := func(s *string, values ...string) bool { return s == nil || allowed(*s, values...) } + return allowed(c.OS, "", "linux", "darwin", "windows", "ios", "android", "other") && allowed(c.Arch, "", "amd64", "arm64", "386", "arm", "wasm", "other") && pointer(c.Sandbox, "seatbelt", "landlock") && pointer(c.Container, "none", "container") && pointer(c.GPU, "none", "present") && allowed(c.Cow, "", "apfs", "btrfs", "xfs-reflink", "overlayfs", "none") +} + +// WorkerCanClaim is the common P16 prerequisite at the claim's log position. +// A registry row, socket or downloaded grant alone never starts work. +func WorkerCanClaim(cluster, worker, cell string, incarnation uint64, now int64, identity IdentityRec, a Authority, d Device, w WorkerRecord) Code { + if code := workerAuthority(cluster, worker, now, identity, a, d); code != OK { + return code + } + if !a.Worker.Allows(cell) { + return Unauthorized + } + if incarnation == 0 || incarnation != w.Incarnation || w.Draining || w.AvailableUntil <= now { + return Conflict + } + return OK +} + +// WorkerCanRead admits only the selected chat at the current committed +// incarnation. Staging is allowed before claiming; availability and draining +// govern new claims, not retrieval or saving work already held. +func WorkerCanRead(cluster, worker, cell string, incarnation uint64, now int64, identity IdentityRec, a Authority, d Device, w WorkerRecord) Code { + if code := workerAuthority(cluster, worker, now, identity, a, d); code != OK { + return code + } + if !a.Worker.Allows(cell) { + return Unauthorized + } + if incarnation == 0 || incarnation != w.Incarnation { + return Conflict + } + return OK +} + +func workerHeartbeatDigest(cluster string, h WorkerHeartbeat) string { + b, _ := WorkerHeartbeatBytes(cluster, h) + digest := sha256.Sum256(b) + return hex.EncodeToString(digest[:]) +} diff --git a/internal/clusterstate/worker_certificate.go b/internal/clusterstate/worker_certificate.go new file mode 100644 index 0000000000..686a6c416b --- /dev/null +++ b/internal/clusterstate/worker_certificate.go @@ -0,0 +1,58 @@ +package clusterstate + +import ( + "crypto/ed25519" + "sort" +) + +const ( + MaxWorkerScopes = 32 + MaxWorkerRecords = 4096 + MaxWorkerSealedBytes = 4096 +) + +// WorkerCertificate delegates selected chats without exporting identity secrets. +// Learner is explicit consent to see the identity's routing log; it never +// grants a vote. Human labels and execution credentials belong in sealed grants. +type WorkerCertificate struct { + V uint16 `json:"v"` + Identity string `json:"identity"` + Cluster string `json:"cluster"` + Key []byte `json:"key"` + Made int64 `json:"made"` + Expires int64 `json:"expires"` + Cells []string `json:"cells"` + Learner bool `json:"learner"` + Signature []byte `json:"signature"` +} + +// WorkerCertificateBytes has a separate domain from personal certificates, so +// delegation can never be presented as approval for personal-device pairing. +func WorkerCertificateBytes(c WorkerCertificate) ([]byte, error) { + c.Signature = nil + b, err := Encode(c) + return append([]byte("codeaf/worker-cert/v1\x00"), b...), err +} +func (c WorkerCertificate) DeviceID() string { return DeviceIDOfKey(c.Key) } +func (c WorkerCertificate) Allows(cell string) bool { + i := sort.SearchStrings(c.Cells, cell) + return i < len(c.Cells) && c.Cells[i] == cell +} + +// VerifyWorkerCertificate uses committed cluster time, including the exclusive +// expiry boundary. There is no local clock or shared-root worker credential. +func VerifyWorkerCertificate(c WorkerCertificate, public []byte, cluster string, now int64) error { + if c.V != 1 || c.Cluster == "" || c.Cluster != cluster || len(public) != ed25519.PublicKeySize || c.Identity != IdentityID(public) || len(c.Key) != ed25519.PublicKeySize || c.Made < 0 || c.Expires <= c.Made || c.Expires > int64(MaxSafeInteger)-LeaseTTLMillis || now < c.Made || now >= c.Expires || len(c.Cells) == 0 || len(c.Cells) > MaxWorkerScopes { + return ErrCodec + } + for i, cell := range c.Cells { + if cell == "" || len(cell) > 256 || i > 0 && c.Cells[i-1] >= cell { + return ErrCodec + } + } + b, err := WorkerCertificateBytes(c) + if err != nil || !ed25519.Verify(public, b, c.Signature) { + return ErrCodec + } + return nil +} diff --git a/internal/clusterstate/worker_read.go b/internal/clusterstate/worker_read.go new file mode 100644 index 0000000000..0219961999 --- /dev/null +++ b/internal/clusterstate/worker_read.go @@ -0,0 +1,68 @@ +package clusterstate + +import "bytes" + +// WorkerRead is a selected-chat response. The sealed grant is returned only +// after current committed authorization, never as part of an identity listing. +type WorkerRead struct { + Code Code `json:"code"` + View *CellView `json:"view,omitempty"` + Grant []byte `json:"grant,omitempty"` +} + +func WorkerReadKeys(worker, cell string) []RecordKey { + return []RecordKey{IdentityKey(), AuthorityKey(worker), DeviceKey(worker), WorkerKey(worker), CellKey(cell)} +} + +// ReadWorkerCell is shared by native and hosted adapters. The host owns the +// fresh quorum barrier and supplies one immutable applied read-set, with time +// already committed. Neither a local replica nor this evaluator is a barrier. +func ReadWorkerCell(cluster, worker, cell string, incarnation uint64, now int64, read ReadRecord) (WorkerRead, error) { + rows := make([][]byte, 5) + size := 0 + for i, k := range WorkerReadKeys(worker, cell) { + b, e := read(k) + if e != nil { + return WorkerRead{}, e + } + size += len(k.Namespace) + len(k.Key) + len(b) + if size > MaxEvaluationBytes { + return WorkerRead{}, ErrDependency + } + rows[i] = b + } + var identity IdentityRec + var a Authority + var d Device + var w WorkerRecord + var c Cell + for _, v := range []struct { + i int + into any + }{{0, &identity}, {1, &a}, {3, &w}} { + if rows[v.i] == nil { + return WorkerRead{Code: Unauthorized}, nil + } + if Decode(rows[v.i], v.into) != nil { + return WorkerRead{}, ErrDependency + } + } + if rows[2] != nil && Decode(rows[2], &d) != nil { + return WorkerRead{}, ErrDependency + } + code := WorkerCanRead(cluster, worker, cell, incarnation, now, identity, a, d, w) + if code != OK { + return WorkerRead{Code: code}, nil + } + if rows[4] == nil { + return WorkerRead{Code: NotFound}, nil + } + if Decode(rows[4], &c) != nil { + return WorkerRead{}, ErrDependency + } + grant := w.Grants[cell] + if len(grant) == 0 { + return WorkerRead{}, ErrDependency + } + return WorkerRead{Code: OK, View: &CellView{Now: now, Cell: c}, Grant: bytes.Clone(grant)}, nil +} diff --git a/internal/clusterstate/worker_test.go b/internal/clusterstate/worker_test.go new file mode 100644 index 0000000000..2599291a03 --- /dev/null +++ b/internal/clusterstate/worker_test.go @@ -0,0 +1,509 @@ +package clusterstate_test + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/ed25519" + "fmt" + "math/rand" + "reflect" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +type workerFixture struct { + a *application + id string + key ed25519.PrivateKey + cert cs.WorkerCertificate + serial int +} + +func sealedWorker(t *testing.T, text string) []byte { + t.Helper() + b, e := aes.NewCipher(bytes.Repeat([]byte{3}, 32)) + if e != nil { + t.Fatal(e) + } + g, e := cipher.NewGCM(b) + if e != nil { + t.Fatal(e) + } + nonce := bytes.Repeat([]byte{7}, g.NonceSize()) + return append(nonce, g.Seal(nil, nonce, []byte(text), nil)...) +} +func workerFixtureFor(t *testing.T, learner bool) *workerFixture { + a := membershipApp(t, cs.MaxMemberRecords) + k := actorKey("worker-key") + cert := cs.WorkerCertificate{V: 1, Identity: cs.IdentityID(actorKey("root").Public().(ed25519.PublicKey)), Cluster: "test", Key: k.Public().(ed25519.PublicKey), Made: 1000, Expires: 1000000, Cells: []string{"chat"}, Learner: learner} + raw, e := cs.WorkerCertificateBytes(cert) + if e != nil { + t.Fatal(e) + } + cert.Signature = ed25519.Sign(actorKey("root"), raw) + f := &workerFixture{a: a, id: cert.DeviceID(), key: k, cert: cert} + r := f.owner(1000, cs.Mutation{EnrollWorker: &cs.EnrollWorker{Certificate: cert, Grants: map[string][]byte{"chat": sealedWorker(t, "selected-chat-key-and-credential")}}}) + if r.Code != cs.OK { + t.Fatal("R16 enrollment", r) + } + return f +} +func (f *workerFixture) command(actor string, k ed25519.PrivateKey, inc uint64, now int64, m cs.Mutation) cs.Command { + f.serial++ + c, e := cs.Sign(cs.Command{Schema: 1, Cluster: "test", Actor: actor, Session: fmt.Sprint("request-", f.serial), Sequence: 1, Now: now, WorkerIncarnation: inc, Change: m}, k) + if e != nil { + f.a.t.Fatal(e) + } + return c +} +func (f *workerFixture) eval(c cs.Command) cs.Result { f.a.meta.Index++; return f.a.evaluate(c).Result } +func (f *workerFixture) owner(now int64, m cs.Mutation) cs.Result { + return f.eval(f.command("a", actorKey("a"), 0, now, m)) +} +func (f *workerFixture) run(inc uint64, now int64, m cs.Mutation) cs.Result { + return f.eval(f.command(f.id, f.key, inc, now, m)) +} +func (f *workerFixture) record() cs.WorkerRecord { + var w cs.WorkerRecord + if e := cs.Decode(f.a.rows[cs.WorkerKey(f.id)], &w); e != nil { + f.a.t.Fatal(e) + } + return w +} +func (f *workerFixture) authority() cs.Authority { + var a cs.Authority + _ = cs.Decode(f.a.rows[cs.AuthorityKey(f.id)], &a) + return a +} +func (f *workerFixture) identity() cs.IdentityRec { + var i cs.IdentityRec + _ = cs.Decode(f.a.rows[cs.IdentityKey()], &i) + return i +} +func (f *workerFixture) register(previous uint64, now int64) cs.Result { + return f.run(previous, now, cs.Mutation{RegisterWorker: &cs.RegisterWorker{Previous: previous, Incarnation: previous + 1, Caps: cs.Caps{OS: "linux", Arch: "arm64", Cow: "none"}, SealedLabels: sealedWorker(f.a.t, "secret-worker-label requirement /private/repo")}}) +} +func (f *workerFixture) proof(inc, seq uint64, at int64, work ...cs.HoldFence) cs.WorkerHeartbeat { + h := cs.WorkerHeartbeat{Worker: f.id, Incarnation: inc, Sequence: seq, SeenAt: at, Work: work} + b, e := cs.WorkerHeartbeatBytes("test", h) + if e != nil { + f.a.t.Fatal(e) + } + h.Signature = ed25519.Sign(f.key, b) + return h +} +func (f *workerFixture) beat(h cs.WorkerHeartbeat, now int64) cs.Result { + return f.owner(now, cs.Mutation{WorkerBeatBatch: &cs.WorkerBeatBatch{Items: []cs.WorkerHeartbeat{h}}}) +} +func expectWorker(t *testing.T, r cs.Result, code cs.Code) { + t.Helper() + if r.Code != code { + t.Fatalf("want %s: %+v", code, r) + } +} + +// R15/R16/R18 allow only scoped held-work mutations. A worker cannot acquire +// through the personal door, edit membership, pair, rotate or obtain the vault. +func TestR15R16R18WorkerScopeAndAuthorityDoors(t *testing.T) { + f := workerFixtureFor(t, false) + expectWorker(t, f.register(0, 1001), cs.OK) + denied := []cs.Mutation{ + {EnrollWorker: &cs.EnrollWorker{Certificate: f.cert}}, {DecideJoin: &cs.JoinDecision{}}, {AdmitJoin: &cs.AdmitJoin{}}, {Rotate: &cs.RotateRequest{Op: "freeze"}}, {SetVault: &cs.VaultCAS{}}, {PutDevice: &cs.PutDevice{ID: f.id}}, {Revoke: &cs.DeviceID{Device: "b"}}, {EnrollMember: &cs.EnrollMember{Device: f.id}}, {RequestLeadership: &cs.RequestLeadership{Device: f.id}}, {InitMembership: &cs.InitMembership{}}, {AdvanceTime: &cs.AdvanceTime{}}, {Create: &cs.CreateCell{Cell: "chat"}}, {Acquire: &cs.AcquireCell{Cell: "chat"}}, {Archive: &cs.CellID{Cell: "chat"}}, {DeviceSeen: &cs.DeviceSeen{Device: f.id}}, + {Publish: &cs.PublishCell{Cell: "outside"}}, {Beat: &cs.BeatCell{Cell: "outside"}}, {Release: &cs.ReleaseCell{Cell: "outside"}}, {BeatBatch: &cs.BeatBatch{Items: []cs.BeatCell{{Cell: "chat"}, {Cell: "outside"}}}}, {DrainWorker: &cs.DrainWorker{Worker: "b", Incarnation: 1}}, + } + for i, m := range denied { + t.Run(fmt.Sprint(i), func(t *testing.T) { + before, _ := cs.Encode(f.record()) + expectWorker(t, f.run(1, 1002, m), cs.Unauthorized) + after, _ := cs.Encode(f.record()) + if !bytes.Equal(before, after) { + t.Fatal("refusal changed worker") + } + }) + } + // The worker may save/release a fence obtained through P16's future claim. + f.a.put(cs.CellKey("chat"), cs.Created(cs.CellInit{Head: "old"}, f.id, 1000)) + receipt := cs.UploadReceipt{Device: f.id, Cell: "chat", WorkerIncarnation: 1, Head: "new", HeadRID: "opaque-rid"} + f.a.put(cs.UploadKey(cs.UploadID(receipt)), receipt) + expectWorker(t, f.run(1, 1003, cs.Mutation{Publish: &cs.PublishCell{Cell: "chat", Receipt: cs.UploadID(receipt), Publish: cs.Publish{Fence: 1, OldHead: "old", Head: "new"}}}), cs.OK) + expectWorker(t, f.run(1, 1004, cs.Mutation{Release: &cs.ReleaseCell{Cell: "chat", Fence: 1}}), cs.OK) + // Reclassifying an enrollment cannot smuggle in a second role. + a := f.authority() + a.Personal = true + f.a.put(cs.AuthorityKey(f.id), a) + expectWorker(t, f.run(1, 1005, cs.Mutation{Rotate: &cs.RotateRequest{Op: "freeze"}}), cs.Unauthorized) +} + +// R16/R18 require replacement, drain and revoke to beat delayed already-signed +// work. Drain is permanent for this delegation, while held work can still save. +func TestR16R17R18WorkerIncarnationDrainAndRevoke(t *testing.T) { + f := workerFixtureFor(t, true) + expectWorker(t, f.register(0, 1001), cs.OK) + h := f.proof(1, 1, 1002) + r := f.beat(h, 1002) + expectWorker(t, r, cs.OK) + if r.Workers[0].AvailableUntil != 91002 { + t.Fatal(r) + } + if code := cs.WorkerCanClaim("test", f.id, "chat", 1, 1003, f.identity(), f.authority(), cs.Device{}, f.record()); code != cs.OK { + t.Fatal("fresh worker unavailable", code) + } + old := f.command(f.id, f.key, 1, 1003, cs.Mutation{Beat: &cs.BeatCell{Cell: "chat", Beat: cs.Beat{Fence: 1}}}) + f.a.put(cs.CellKey("chat"), cs.Created(cs.CellInit{Head: "old"}, f.id, 1000)) + saved := f.eval(old) + expectWorker(t, saved, cs.OK) + expectWorker(t, f.register(1, 1004), cs.OK) + expectWorker(t, f.eval(old), cs.Conflict) + if r := f.beat(f.proof(1, 2, 1004), 1004); r.Workers[0].Code != cs.Conflict { + t.Fatal("old incarnation refreshed availability", r) + } + if f.record().AvailableUntil != 0 { + t.Fatal("restart inherited availability") + } + expectWorker(t, f.owner(1005, cs.Mutation{DrainWorker: &cs.DrainWorker{Worker: f.id, Incarnation: 1}}), cs.Conflict) + expectWorker(t, f.owner(1005, cs.Mutation{DrainWorker: &cs.DrainWorker{Worker: f.id, Incarnation: 2}}), cs.OK) + expectWorker(t, f.register(2, 1006), cs.OK) + if !f.record().Draining { + t.Fatal("restart cleared drain") + } + expectWorker(t, f.beat(f.proof(3, 1, 1007), 1007), cs.OK) + if code := cs.WorkerCanClaim("test", f.id, "chat", 3, 1007, f.identity(), f.authority(), cs.Device{}, f.record()); code != cs.Conflict { + t.Fatal("draining worker could claim", code) + } + expectWorker(t, f.run(3, 1008, cs.Mutation{Release: &cs.ReleaseCell{Cell: "chat", Fence: 1}}), cs.OK) + pending := f.command(f.id, f.key, 3, 1008, cs.Mutation{Beat: &cs.BeatCell{Cell: "chat", Beat: cs.Beat{Fence: 1}}}) + expectWorker(t, f.owner(1009, cs.Mutation{Revoke: &cs.DeviceID{Device: f.id}}), cs.OK) + expectWorker(t, f.eval(pending), cs.Revoked) + if r := f.beat(f.proof(3, 2, 1009), 1009); r.Workers[0].Code != cs.Revoked { + t.Fatal(r) + } + expectWorker(t, f.owner(1010, cs.Mutation{EnrollWorker: &cs.EnrollWorker{Certificate: f.cert, Grants: f.record().Grants}}), cs.Revoked) +} + +// R15/R16/R27 keep secret labels sealed, reject free-text capability fields and +// retain all delegations when capacity fills or certificates expire. +func TestR15R16R27WorkerSealedMetadataAndLimits(t *testing.T) { + f := workerFixtureFor(t, false) + expectWorker(t, f.register(0, 1001), cs.OK) + for _, caps := range []cs.Caps{{OS: "private-host-name"}, {Arch: "/private/repo"}, {GPU: func() *string { s := "secret GPU model"; return &s }()}} { + expectWorker(t, f.run(1, 1002, cs.Mutation{RegisterWorker: &cs.RegisterWorker{Previous: 1, Incarnation: 2, Caps: caps}}), cs.BadRequest) + } + for _, b := range f.a.rows { + for _, sentinel := range []string{"secret-worker-label", "/private/repo", "selected-chat-key-and-credential"} { + if bytes.Contains(b, []byte(sentinel)) { + t.Fatal("R15 plaintext leaked", sentinel) + } + } + } + before := append([]byte(nil), f.a.rows[cs.WorkerKey(f.id)]...) + expectWorker(t, f.run(1, f.cert.Expires, cs.Mutation{Release: &cs.ReleaseCell{Cell: "chat", Fence: 1}}), cs.Unauthorized) + if !bytes.Equal(before, f.a.rows[cs.WorkerKey(f.id)]) { + t.Fatal("expiry deleted record") + } + f.a.put(cs.WorkerRegistryKey(), cs.WorkerRegistry{Retained: cs.MaxWorkerRecords}) + other := f.cert + other.Key = actorKey("another-worker").Public().(ed25519.PublicKey) + b, _ := cs.WorkerCertificateBytes(other) + other.Signature = ed25519.Sign(actorKey("root"), b) + expectWorker(t, f.owner(1003, cs.Mutation{EnrollWorker: &cs.EnrollWorker{Certificate: other, Grants: map[string][]byte{"chat": {1}}}}), cs.Capacity) + if !bytes.Equal(before, f.a.rows[cs.WorkerKey(f.id)]) { + t.Fatal("admission evicted retained record") + } +} + +// R9/R12/R16 forbid learner-by-default delegation and worker promotion. Exact +// incarnation and committed expiry also fence already-open protocol traffic. +func TestR9R12R16WorkerLearnerConsentAndPeerRevocation(t *testing.T) { + for _, learner := range []bool{false, true} { + t.Run(fmt.Sprint(learner), func(t *testing.T) { + f := workerFixtureFor(t, learner) + expectWorker(t, f.register(0, 1001), cs.OK) + r := enroll(f.a, f.id) + if !learner { + expectWorker(t, r, cs.Unauthorized) + return + } + expectWorker(t, r, cs.OK) + expectWorker(t, settle(f.a), cs.OK) + read := func(k cs.RecordKey) ([]byte, error) { return f.a.rows[k], nil } + peer := func(now int64) cs.PeerEnrollment { + _, p, e := cs.MembershipViewAt(read, f.a.meta.Configuration, f.a.meta.Index, now) + if e != nil { + t.Fatal(e) + } + return p[3] + } + if p := peer(1002); p.Revoked || p.Voter { + t.Fatal("worker is not an authorized learner", p) + } + if !peer(f.cert.Expires).Revoked { + t.Fatal("expired peer authorized") + } + f.a.meta.Term = 2 + expectWorker(t, f.owner(1003, cs.Mutation{RequestLeadership: &cs.RequestLeadership{Node: 4, Device: f.id, Revision: memberHeader(f.a).Revision}}), cs.Unauthorized) + expectWorker(t, f.register(1, 1004), cs.OK) + if !peer(1004).Revoked { + t.Fatal("old worker socket survived incarnation change") + } + }) + } +} + +// R16/R18 use an independent small registry oracle for reordered register, +// drain, signed-heartbeat and revoke events. Every seed honors RAFTSIM_SEEDS. +func TestR16R18WorkerRegistryOrderingCampaign(t *testing.T) { + for seed := 0; seed < seedCount(t); seed++ { + t.Run(fmt.Sprintf("seed-%04d", seed), func(t *testing.T) { + f := workerFixtureFor(t, false) + rng := rand.New(rand.NewSource(int64(seed))) + var inc, beat uint64 + var draining, revoked bool + now := int64(1000) + for step := 0; step < 18; step++ { + now += int64(1 + rng.Intn(100)) + switch rng.Intn(4) { + case 0: + prev := inc + if rng.Intn(3) == 0 && prev > 0 { + prev-- + } + want := cs.OK + if revoked { + want = cs.Revoked + } else if prev != inc { + want = cs.Conflict + } + expectWorker(t, f.register(prev, now), want) + if want == cs.OK { + inc++ + beat = 0 + } + case 1: + target := inc + if rng.Intn(3) == 0 && target > 0 { + target-- + } + want := cs.OK + if revoked { + want = cs.Revoked + } else if target == 0 || target != inc { + want = cs.Conflict + } + expectWorker(t, f.owner(now, cs.Mutation{DrainWorker: &cs.DrainWorker{Worker: f.id, Incarnation: target}}), want) + if want == cs.OK { + draining = true + } + case 2: + target := inc + if rng.Intn(3) == 0 && target > 0 { + target-- + } + n := beat + 1 + if rng.Intn(4) == 0 { + n = beat + } + want := cs.OK + if revoked { + want = cs.Revoked + } else if inc == 0 || target != inc || n <= beat { + want = cs.Conflict + } + r := f.beat(f.proof(target, n, now), now) + expectWorker(t, r, cs.OK) + if r.Workers[0].Code != want { + t.Fatal("independent heartbeat oracle", want, r) + } + if want == cs.OK { + beat = n + } + case 3: + if step > 10 { + expectWorker(t, f.owner(now, cs.Mutation{Revoke: &cs.DeviceID{Device: f.id}}), cs.OK) + revoked = true + } + } + w := f.record() + if w.Incarnation != inc || w.Heartbeat != beat || w.Draining != draining { + t.Fatal("independent registry oracle differs", w, inc, beat, draining) + } + } + }) + } +} + +// R17 retries keep the exact committed response and the worker-incarnation +// binding is part of the durable request digest, even across process restart. +func TestR17WorkerExactResultAndDigestBinding(t *testing.T) { + f := workerFixtureFor(t, false) + c := f.command(f.id, f.key, 0, 1001, cs.Mutation{RegisterWorker: &cs.RegisterWorker{Incarnation: 1}}) + r := f.eval(c) + expectWorker(t, r, cs.OK) + // Register is an operation by the predecessor. Exact retry must survive the + // incarnation change caused by that SAME registration. + got := f.eval(c) + if !reflect.DeepEqual(got, r) { + t.Fatal("R17 lost registration result", r, got) + } + c.WorkerIncarnation = 1 + digest, e := cs.Digest(c) + if e != nil || digest == c.RequestDigest { + t.Fatal("incarnation not bound to digest") + } +} + +// R15/R16/R22 prove signed availability is not ownership, and freshness, +// per-item refusals and exact incarnation bind both uploads and hold renewals. +func TestR15R16R22WorkerEvidenceAndEffectsScopes(t *testing.T) { + f := workerFixtureFor(t, false) + expectWorker(t, f.register(0, 1001), cs.OK) + f.a.put(cs.AuthorityKey("effects"), cs.Authority{Cluster: "test", Key: actorKey("effects").Public().(ed25519.PublicKey), Effects: true}) + f.a.put(cs.CellKey("chat"), cs.Created(cs.CellInit{Head: "old"}, f.id, 1000)) + effect := func(now int64, m cs.Mutation) cs.Result { + return f.eval(f.command("effects", actorKey("effects"), 0, now, m)) + } + before := append([]byte(nil), f.a.rows[cs.CellKey("chat")]...) + h := f.proof(1, 1, 1002, cs.HoldFence{Cell: "chat", Fence: 1}) + r := f.beat(h, 1002) + if r.Workers[0].Code != cs.OK { + t.Fatal(r) + } + if !bytes.Equal(before, f.a.rows[cs.CellKey("chat")]) { + t.Fatal("availability changed ownership") + } + for _, change := range []func(*cs.WorkerHeartbeat){ + func(h *cs.WorkerHeartbeat) { h.Signature[0] ^= 1 }, + func(h *cs.WorkerHeartbeat) { h.Work = []cs.HoldFence{{Cell: "outside", Fence: 1}} }, + func(h *cs.WorkerHeartbeat) { h.SeenAt = 1004 }, + func(h *cs.WorkerHeartbeat) { h.Sequence = 1 }, + } { + bad := f.proof(1, 2, 1003) + change(&bad) + r := f.beat(bad, 1003) + if r.Workers[0].Code == cs.OK { + t.Fatal("bad evidence accepted", r) + } + if f.record().AvailableUntil != 91002 { + t.Fatal("replayed evidence moved availability") + } + } + stale := f.proof(1, 2, 1003) + r = f.beat(stale, 1003+cs.EvidenceWindowMillis+1) + if r.Workers[0].Code != cs.BadRequest { + t.Fatal("old signed evidence accepted", r) + } + r = f.owner(1004, cs.Mutation{WorkerBeatBatch: &cs.WorkerBeatBatch{Items: []cs.WorkerHeartbeat{f.proof(1, 2, 30000), {Worker: "unknown", Incarnation: 1}}}}) + if r.Workers[0].Code == cs.OK || r.Workers[1].Code != cs.Unauthorized { + t.Fatal("batch per-worker results", r) + } + // Use committed time after the deliberate stale-evidence advance. + at := int64(30010) + receipt := cs.UploadReceipt{Device: f.id, Cell: "outside", WorkerIncarnation: 1, Head: "new", HeadRID: "rid"} + expectWorker(t, effect(at, cs.Mutation{RecordUpload: &receipt}), cs.Unauthorized) + receipt.Cell = "chat" + receipt.WorkerIncarnation = 2 + expectWorker(t, effect(at, cs.Mutation{RecordUpload: &receipt}), cs.Conflict) + receipt.WorkerIncarnation = 1 + expectWorker(t, effect(at, cs.Mutation{RecordUpload: &receipt}), cs.OK) + expectWorker(t, effect(at, cs.Mutation{RenewHolds: &cs.RenewHolds{Device: f.id, WorkerIncarnation: 1, SeenAt: at, Items: []cs.HoldFence{{Cell: "outside", Fence: 1}}}}), cs.Unauthorized) + renew := cs.RenewHolds{Device: f.id, WorkerIncarnation: 1, SeenAt: at, Items: []cs.HoldFence{{Cell: "chat", Fence: 1}}} + expectWorker(t, effect(at, cs.Mutation{RenewHolds: &renew}), cs.OK) + expectWorker(t, f.run(1, at+1, cs.Mutation{Release: &cs.ReleaseCell{Cell: "chat", Fence: 1}}), cs.OK) + r = effect(at+2, cs.Mutation{RenewHolds: &renew}) + if r.Items[0].Code != cs.FenceStale { + t.Fatal("delayed renew undid release", r) + } +} + +// R27 idle registry availability does not pin timers. Active work does, while +// its proof lasts; the registry itself never deletes or expires saved records. +func TestR27WorkerActivityAndScopedRead(t *testing.T) { + f := workerFixtureFor(t, false) + expectWorker(t, f.register(0, 1001), cs.OK) + expectWorker(t, f.beat(f.proof(1, 1, 1002), 1002), cs.OK) + active, e := cs.RecordActive(cs.WorkerKey(f.id), f.a.rows[cs.WorkerKey(f.id)], 1003) + if e != nil || active { + t.Fatal("idle worker kept timer active", e) + } + f.a.put(cs.CellKey("chat"), cs.Created(cs.CellInit{Head: "head"}, f.id, 1000)) + expectWorker(t, f.beat(f.proof(1, 2, 1003, cs.HoldFence{Cell: "chat", Fence: 1}), 1003), cs.OK) + active, e = cs.RecordActive(cs.WorkerKey(f.id), f.a.rows[cs.WorkerKey(f.id)], 1004) + if e != nil || !active { + t.Fatal("active worker did not keep timer", e) + } + active, e = cs.RecordActive(cs.WorkerKey(f.id), f.a.rows[cs.WorkerKey(f.id)], 91003) + if e != nil || active { + t.Fatal("availability never went stale", e) + } + read := func(k cs.RecordKey) ([]byte, error) { return f.a.rows[k], nil } + r, e := cs.ReadWorkerCell("test", f.id, "chat", 1, 1004, read) + if e != nil || r.Code != cs.OK || r.View.Cell.Head != "head" || len(r.Grant) == 0 { + t.Fatal(r, e) + } + for _, cell := range []string{"outside", ""} { + r, e = cs.ReadWorkerCell("test", f.id, cell, 1, 1004, read) + if e != nil || r.Code != cs.Unauthorized || r.Grant != nil || r.View != nil { + t.Fatal("scope response disclosed data", r, e) + } + } + r, e = cs.ReadWorkerCell("test", f.id, "chat", 1, f.cert.Expires, read) + if e != nil || r.Code != cs.Unauthorized { + t.Fatal("expired certificate read", r, e) + } +} + +// R16 checks a CURRENT worker peer, independently of incarnation retirement. +// An already signed request loses to the committed device revoke everywhere. +func TestR16WorkerActivePeerRevoke(t *testing.T) { + f := workerFixtureFor(t, true) + expectWorker(t, f.register(0, 1001), cs.OK) + expectWorker(t, enroll(f.a, f.id), cs.OK) + expectWorker(t, settle(f.a), cs.OK) + read := func(k cs.RecordKey) ([]byte, error) { return f.a.rows[k], nil } + _, before, e := cs.MembershipViewAt(read, f.a.meta.Configuration, f.a.meta.Index, 1002) + if e != nil || before[3].Revoked { + t.Fatal(before, e) + } + pending := f.command(f.id, f.key, 1, 1002, cs.Mutation{RegisterWorker: &cs.RegisterWorker{Previous: 1, Incarnation: 2}}) + expectWorker(t, f.owner(1003, cs.Mutation{Revoke: &cs.DeviceID{Device: f.id}}), cs.OK) + expectWorker(t, f.eval(pending), cs.Revoked) + _, after, e := cs.MembershipViewAt(read, f.a.meta.Configuration, f.a.meta.Index, 1003) + if e != nil || !after[3].Revoked { + t.Fatal("revoked active peer admitted", after, e) + } +} + +// R16/R17 allow the restricted ingress service to batch original worker proofs +// without root authority. Replaying one proof cannot extend availability. +func TestR16R17WorkerHeartbeatCollectorAndReplay(t *testing.T) { + f := workerFixtureFor(t, false) + expectWorker(t, f.register(0, 1001), cs.OK) + f.a.put(cs.AuthorityKey("effects"), cs.Authority{Cluster: "test", Key: actorKey("effects").Public().(ed25519.PublicKey), Effects: true}) + h := f.proof(1, 1, 1002) + command := f.command("effects", actorKey("effects"), 0, 1002, cs.Mutation{WorkerBeatBatch: &cs.WorkerBeatBatch{Items: []cs.WorkerHeartbeat{h}}}) + r := f.eval(command) + expectWorker(t, r, cs.OK) + if r.Workers[0].Code != cs.OK { + t.Fatal(r) + } + command.Now = 1002 + cs.EvidenceWindowMillis + 1 + var e error + command, e = cs.Sign(command, actorKey("effects")) + if e != nil { + t.Fatal(e) + } + r = f.eval(command) + expectWorker(t, r, cs.OK) + if r.Workers[0].Code != cs.OK || r.Workers[0].AvailableUntil != 91002 || f.record().AvailableUntil != 91002 { + t.Fatal("exact proof replay moved availability", r) + } + // The collector cannot alter the proof while retaining its sequence. + h.SeenAt = command.Now + b, _ := cs.WorkerHeartbeatBytes("test", h) + h.Signature = ed25519.Sign(f.key, b) + r = f.eval(f.command("effects", actorKey("effects"), 0, command.Now, cs.Mutation{WorkerBeatBatch: &cs.WorkerBeatBatch{Items: []cs.WorkerHeartbeat{h}}})) + if r.Workers[0].Code != cs.Conflict { + t.Fatal("same sequence accepted different proof", r) + } +} diff --git a/internal/identity/worker.go b/internal/identity/worker.go new file mode 100644 index 0000000000..e302791e03 --- /dev/null +++ b/internal/identity/worker.go @@ -0,0 +1,90 @@ +package identity + +import ( + "bytes" + "crypto/ed25519" + "encoding/hex" + "errors" + "sort" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +// WorkerCert is deliberately a different wire type and signing domain from +// Cert. A delegated worker cannot be passed to Seed.Join or personal pairing. +type WorkerCert = clusterstate.WorkerCertificate + +// IssueWorkerCert signs an expiring, selected-chat delegation. The caller seals +// selected chat keys and execution credentials separately for the worker's key. +// Learner consent discloses routing metadata for the whole identity. +func IssueWorkerCert(id Identity, public ed25519.PublicKey, cluster string, cells []string, made, expires time.Time, learner bool) (WorkerCert, error) { + cells = append([]string(nil), cells...) + sort.Strings(cells) + c := WorkerCert{V: 1, Identity: id.ID(), Cluster: cluster, Key: bytes.Clone(public), Made: made.UnixMilli(), Expires: expires.UnixMilli(), Cells: cells, Learner: learner} + b, err := clusterstate.WorkerCertificateBytes(c) + if err != nil { + return WorkerCert{}, err + } + c.Signature = id.Sign(b) + if err := clusterstate.VerifyWorkerCertificate(c, id.PublicKey(), cluster, made.UnixMilli()); err != nil { + return WorkerCert{}, err + } + return c, nil +} + +// Worker holds only a worker's own private seed and its public delegation. It +// contains none of the identity seed, dedup secret or identity-wide content key. +type Worker struct { + Certificate WorkerCert + seed key +} + +func (s Seed) JoinWorker(c WorkerCert, public ed25519.PublicKey, cluster string, now time.Time) (Worker, error) { + if !bytes.Equal(c.Key, s.Public()) { + return Worker{}, errors.New("identity: the certificate is for another worker") + } + if err := clusterstate.VerifyWorkerCertificate(c, public, cluster, now.UnixMilli()); err != nil { + return Worker{}, err + } + // Detach the signed scope from the enrollment caller's mutable buffers. + raw, _ := clusterstate.Encode(c) + var owned WorkerCert + if err := clusterstate.Decode(raw, &owned); err != nil { + return Worker{}, err + } + return Worker{Certificate: owned, seed: s.seed}, nil +} +func (w Worker) ID() string { return w.Certificate.DeviceID() } +func (w Worker) Sign(b []byte) []byte { return ed25519.Sign(ed25519.NewKeyFromSeed(w.seed[:]), b) } + +// Marshal persists only this worker's own seed and public delegation. Like +// Dev.Marshal, these private bytes are local credentials, never an enrollment +// payload or part of a log/snapshot. No identity-wide secret is present. +func (w Worker) Marshal() ([]byte, error) { + return clusterstate.Encode(workerDocument{V: 1, Seed: hex.EncodeToString(w.seed[:]), Certificate: w.Certificate}) +} + +type workerDocument struct { + V uint16 `json:"v"` + Seed string `json:"worker_seed"` + Certificate WorkerCert `json:"certificate"` +} + +// UnmarshalWorker resumes a locally retained worker without loading or making +// a personal identity. Expired/tampered delegations retain their saved bytes but +// cannot become active credentials again. +func UnmarshalWorker(raw []byte, public ed25519.PublicKey, cluster string, now time.Time) (Worker, error) { + var doc workerDocument + if err := clusterstate.Decode(raw, &doc); err != nil { + return Worker{}, err + } + if doc.V != 1 { + return Worker{}, errors.New("identity: unsupported worker credential") + } + var seed key + if err := decodeKey(doc.Seed, &seed); err != nil { + return Worker{}, err + } + return (Seed{seed: seed}).JoinWorker(doc.Certificate, public, cluster, now) +} diff --git a/internal/identity/worker_test.go b/internal/identity/worker_test.go new file mode 100644 index 0000000000..f63fafdadc --- /dev/null +++ b/internal/identity/worker_test.go @@ -0,0 +1,115 @@ +package identity + +import ( + "bytes" + "crypto/ed25519" + "encoding/hex" + "encoding/json" + "reflect" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +// R15/R16 require a delegated worker to join using its own key and a public +// certificate, without the root/content/dedup secrets used by personal pairing. +func TestR15R16DelegatedWorkerCertificateWithoutRootExport(t *testing.T) { + id := sample() + seed, err := NewSeed() + if err != nil { + t.Fatal(err) + } + cells := []string{"chat-b", "chat-a"} + cert, err := IssueWorkerCert(id, seed.Public(), "cluster", cells, time.UnixMilli(1000), time.UnixMilli(2000), false) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(cells, []string{"chat-b", "chat-a"}) { + t.Fatal("issuance mutated caller scopes") + } + w, err := seed.JoinWorker(cert, id.PublicKey(), "cluster", time.UnixMilli(1000)) + if err != nil { + t.Fatal(err) + } + + rawPrivate, err := w.Marshal() + if err != nil { + t.Fatal(err) + } + restored, err := UnmarshalWorker(rawPrivate, id.PublicKey(), "cluster", time.UnixMilli(1001)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(restored.Sign([]byte("resume")), w.Sign([]byte("resume"))) { + t.Fatal("worker key did not survive restart") + } + for _, secret := range [][]byte{id.seed[:], id.cell[:], id.dedup[:]} { + if bytes.Contains(rawPrivate, []byte(hex.EncodeToString(secret))) { + t.Fatal("worker credential contains an identity-wide secret") + } + } + if _, err := UnmarshalWorker(rawPrivate, id.PublicKey(), "cluster", time.UnixMilli(2000)); err == nil { + t.Fatal("expired credential resumed") + } + msg := []byte("worker proof") + if !ed25519.Verify(seed.Public(), msg, w.Sign(msg)) || w.ID() != cert.DeviceID() { + t.Fatal("worker lost its own key") + } + cert.Cells[0] = "changed" + if !w.Certificate.Allows("chat-a") { + t.Fatal("caller changed retained delegation") + } + raw, _ := json.Marshal(w) + for _, secret := range [][]byte{id.seed[:], id.cell[:], id.dedup[:]} { + if bytes.Contains(raw, secret) { + t.Fatal("R15 worker exported identity secret") + } + } + var personal Cert + if json.Unmarshal(raw, &personal) != nil || personal.Verify(id.PublicKey()) == nil { + t.Fatal("delegation became personal certificate") + } + if reflect.TypeOf(w).NumField() != 2 { + t.Fatal("review worker secret inventory") + } +} + +// R16 binds every permission and the exclusive expiry boundary to the root +// signature; changing any scope, key, role or lifetime must refuse enrollment. +func TestR16DelegatedWorkerCertificateTamperingAndExpiry(t *testing.T) { + id := sample() + s, _ := NewSeed() + other, _ := NewSeed() + c, e := IssueWorkerCert(id, s.Public(), "cluster", []string{"a", "b"}, time.UnixMilli(1000), time.UnixMilli(2000), true) + if e != nil { + t.Fatal(e) + } + tests := map[string]func(*WorkerCert){ + "version": func(c *WorkerCert) { c.V++ }, "cluster": func(c *WorkerCert) { c.Cluster = "other" }, "identity": func(c *WorkerCert) { c.Identity = "other" }, "key": func(c *WorkerCert) { c.Key = other.Public() }, "scope": func(c *WorkerCert) { c.Cells[0] = "outside" }, "learner": func(c *WorkerCert) { c.Learner = false }, "made": func(c *WorkerCert) { c.Made-- }, "expires": func(c *WorkerCert) { c.Expires++ }, "signature": func(c *WorkerCert) { c.Signature[0] ^= 1 }, + } + for name, change := range tests { + t.Run(name, func(t *testing.T) { + raw, _ := cs.Encode(c) + var modified WorkerCert + _ = cs.Decode(raw, &modified) + change(&modified) + if cs.VerifyWorkerCertificate(modified, id.PublicKey(), "cluster", 1000) == nil { + t.Fatal("R16 accepted altered permission") + } + }) + } + for _, now := range []int64{999, 2000, 3000} { + if _, e := s.JoinWorker(c, id.PublicKey(), "cluster", time.UnixMilli(now)); e == nil { + t.Fatal("R16 lifetime boundary accepted", now) + } + } + if _, e := other.JoinWorker(c, id.PublicKey(), "cluster", time.UnixMilli(1000)); e == nil { + t.Fatal("other seed joined") + } + for _, cells := range [][]string{nil, {"a", "a"}, {""}} { + if _, e := IssueWorkerCert(id, s.Public(), "cluster", cells, time.UnixMilli(1000), time.UnixMilli(2000), false); e == nil { + t.Fatal("invalid scopes issued") + } + } +} From e70467393ae89ba2960913fa7d3b6ea178a4ded1 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 04:27:20 -0400 Subject: [PATCH 035/103] Keep native and hosted worker access behind committed policy. Return only selected chat grants after the host quorum barrier and derive learner routing from current worker scope, expiry and incarnation. Exercise real native recovery and byte-identical WASI policy to preserve the same authorization boundary in both runtimes. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/raftabi/README.md | 8 ++ internal/raftabi/abi.go | 2 + internal/raftabi/application.go | 18 ++++ internal/raftabi/worker.go | 29 ++++++ internal/raftdirectory/apply.go | 2 +- internal/raftdirectory/worker.go | 41 ++++++++ internal/raftdirectory/worker_test.go | 138 ++++++++++++++++++++++++++ internal/rafttransport/README.md | 8 ++ internal/rafttransport/registry.go | 17 ++++ relay/hosted/src/raft/membership.js | 6 +- relay/hosted/src/raft/worker.js | 14 +++ relay/hosted/test/raft/worker.test.js | 110 ++++++++++++++++++++ 12 files changed, 391 insertions(+), 2 deletions(-) create mode 100644 internal/raftabi/worker.go create mode 100644 internal/raftdirectory/worker.go create mode 100644 internal/raftdirectory/worker_test.go create mode 100644 relay/hosted/src/raft/worker.js create mode 100644 relay/hosted/test/raft/worker.test.js diff --git a/internal/raftabi/README.md b/internal/raftabi/README.md index 44015d1ad7..af029da776 100644 --- a/internal/raftabi/README.md +++ b/internal/raftabi/README.md @@ -183,3 +183,11 @@ bootstrap manifest before initialization. They must never use the current voter list, which changes when C becomes a standby. The core uses election timing version 2 in native and WASI. Rank survives snapshots and restarts as registry state; absent provisioning identifies a personal learner, never an invented rank. + +P15 adds operation 27: cluster/worker/cell strings, incarnation/committed-time +uint64s, then five blobs in WorkerReadKeys order (empty means absent). It returns +one canonical WorkerRead blob. The host must pass its fresh current-term quorum +barrier before returning these bytes; the ABI supplies policy, not consensus. +Operation 16 accepts an optional tail: committed time uint64, then identity, +authority and worker-record blobs. It is required for delegated peers; the old +metadata-only form returns them revoked. Personal/anchor encodings are unchanged. diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index a09a5a37d0..8450e05fad 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -164,6 +164,8 @@ func (r *Runtime) Call(b []byte) (out []byte) { func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { e := encoder{} switch op { + case 27: + return workerRead(d) case 23: return r.schedule(d) case 24: diff --git a/internal/raftabi/application.go b/internal/raftabi/application.go index 3df4d4caf0..87dec81c29 100644 --- a/internal/raftabi/application.go +++ b/internal/raftabi/application.go @@ -93,6 +93,15 @@ func registryCodec(d *decoder) ([]byte, error) { conf := new(pb.ConfState) d.message(conf) header, member, device := d.blob(), d.blob(), d.blob() + supplied := len(d.b) > 0 + now := int64(0) + var identity, authority, worker []byte + if supplied { + now = int64(d.u64()) + identity = d.blob() + authority = d.blob() + worker = d.blob() + } if err := d.end(); err != nil { return nil, err } @@ -112,6 +121,15 @@ func registryCodec(d *decoder) ([]byte, error) { return nil, clusterstate.ErrDependency } peer, err := clusterstate.PeerFromRecord(m, device, raftcore.Configuration(conf)) + if supplied && m.Worker { + var id clusterstate.IdentityRec + var a clusterstate.Authority + var w clusterstate.WorkerRecord + if clusterstate.Decode(identity, &id) != nil || clusterstate.Decode(authority, &a) != nil || clusterstate.Decode(worker, &w) != nil { + return nil, clusterstate.ErrDependency + } + peer, err = clusterstate.PeerFromRecordAt(m, device, raftcore.Configuration(conf), now, id, a, w) + } if err != nil { return nil, err } diff --git a/internal/raftabi/worker.go b/internal/raftabi/worker.go new file mode 100644 index 0000000000..01f5bf7b63 --- /dev/null +++ b/internal/raftabi/worker.go @@ -0,0 +1,29 @@ +package raftabi + +import "github.com/Agent-Field/codeaf/internal/clusterstate" + +func workerRead(d *decoder) ([]byte, error) { + cluster, worker, cell, incarnation, now := d.str(), d.str(), d.str(), d.u64(), d.u64() + rows := map[clusterstate.RecordKey][]byte{} + for _, k := range clusterstate.WorkerReadKeys(worker, cell) { + b := d.blob() + if len(b) == 0 { + b = nil + } + rows[k] = b + } + if err := d.end(); err != nil || now > clusterstate.MaxSafeInteger { + return nil, errFrame + } + result, err := clusterstate.ReadWorkerCell(cluster, worker, cell, incarnation, int64(now), func(k clusterstate.RecordKey) ([]byte, error) { return rows[k], nil }) + if err != nil { + return nil, err + } + raw, err := clusterstate.Encode(result) + if err != nil { + return nil, err + } + e := encoder{} + e.blob(raw) + return e.b, nil +} diff --git a/internal/raftdirectory/apply.go b/internal/raftdirectory/apply.go index 5945dc969e..ef9bac71f4 100644 --- a/internal/raftdirectory/apply.go +++ b/internal/raftdirectory/apply.go @@ -114,7 +114,7 @@ func authorizedRead(ctx context.Context, store *raftstore.SQLite, cluster, actor } var auth clusterstate.Authority var device clusterstate.Device - if clusterstate.Decode(rows[keys[0]], &auth) != nil || auth.Cluster != cluster || (auth.Personal == effects) || auth.Effects != effects || auth.Anchor || len(auth.Key) != ed25519.PublicKeySize { + if clusterstate.Decode(rows[keys[0]], &auth) != nil || auth.Cluster != cluster || !auth.ValidRole() || (auth.Personal == effects) || auth.Effects != effects || auth.Anchor || len(auth.Key) != ed25519.PublicKeySize { return app, errors.New("raftdirectory: unauthorized") } if raw := rows[keys[1]]; raw != nil { diff --git a/internal/raftdirectory/worker.go b/internal/raftdirectory/worker.go new file mode 100644 index 0000000000..88298e9dbf --- /dev/null +++ b/internal/raftdirectory/worker.go @@ -0,0 +1,41 @@ +package raftdirectory + +import ( + "context" + "errors" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftstore" +) + +// AuthorizedWorkerRead runs under the actor lock after CURRENT committed time +// and a fresh quorum barrier, just like AuthorizedRead. It exposes one selected +// chat and its sealed grant, never a directory listing or the identity vault. +func AuthorizedWorkerRead(ctx context.Context, store *raftstore.SQLite, cluster, worker, cell string, incarnation uint64) (cs.CellView, []byte, error) { + app, _, err := store.ApplicationView(ctx) + if err != nil { + return cs.CellView{}, nil, err + } + source := cs.WorkerReadKeys(worker, cell) + keys := make([]raftstore.Key, len(source)) + for i, k := range source { + keys[i] = raftstore.Key{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key} + } + rows, _, err := store.Read(ctx, app.Applied, keys) + if err != nil { + return cs.CellView{}, nil, err + } + result, err := cs.ReadWorkerCell(cluster, worker, cell, incarnation, app.LogicalTime, func(k cs.RecordKey) ([]byte, error) { + return rows[raftstore.Key{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key}], nil + }) + if err != nil { + return cs.CellView{}, nil, err + } + if result.Code == cs.NotFound { + return cs.CellView{}, nil, cs.ErrNotFound + } + if result.Code != cs.OK { + return cs.CellView{}, nil, errors.New("raftdirectory: unauthorized worker") + } + return *result.View, result.Grant, nil +} diff --git a/internal/raftdirectory/worker_test.go b/internal/raftdirectory/worker_test.go new file mode 100644 index 0000000000..7c48b15a06 --- /dev/null +++ b/internal/raftdirectory/worker_test.go @@ -0,0 +1,138 @@ +package raftdirectory_test + +import ( + "bytes" + "crypto/ed25519" + "fmt" + "reflect" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/rafttransport" +) + +// R3/R13/R15/R16/R17/R18 exercise real quorum/application persistence, selected +// grant retrieval, learner protocol authority and delayed worker commands across +// drain/incarnation/revoke, with exact retry after snapshot/compaction/restart. +func TestR3R13R15R16R17R18WorkerRegistryNative(t *testing.T) { + now := int64(1000) + g := newGroup(t, func() time.Time { return time.UnixMilli(now) }) + k := key("delegated-native-worker") + id := cs.DeviceIDOfKey(k.Public().(ed25519.PublicKey)) + serial := 0 + command := func(actor string, key ed25519.PrivateKey, inc uint64, m cs.Mutation) cs.Command { + serial++ + c, e := cs.Sign(cs.Command{Schema: 1, Cluster: cluster, Actor: actor, Session: fmt.Sprint("worker-", serial), Sequence: 1, Now: now, WorkerIncarnation: inc, Change: m}, key) + must(t, e) + return c + } + submit := func(c cs.Command, want cs.Code) cs.Result { + t.Helper() + r, e := g.submit(c) + must(t, e) + if r.Code != want { + t.Fatalf("want %s: %+v", want, r) + } + return r + } + owner := func(m cs.Mutation, want cs.Code) cs.Result { return submit(command("a", key("a"), 0, m), want) } + cert := cs.WorkerCertificate{V: 1, Identity: cs.IdentityID(key("root").Public().(ed25519.PublicKey)), Cluster: cluster, Key: k.Public().(ed25519.PublicKey), Made: 1000, Expires: 1000000, Cells: []string{"selected"}, Learner: true} + body, e := cs.WorkerCertificateBytes(cert) + must(t, e) + cert.Signature = ed25519.Sign(key("root"), body) + grant := []byte{0xde, 0xad, 0xbe, 0xef} + owner(cs.Mutation{EnrollWorker: &cs.EnrollWorker{Certificate: cert, Grants: map[string][]byte{"selected": grant}}}, cs.OK) + registration := command(id, k, 0, cs.Mutation{RegisterWorker: &cs.RegisterWorker{Incarnation: 1, Caps: cs.Caps{OS: "linux", Arch: "amd64", Cow: "none"}, SealedLabels: []byte{1, 2, 3}}}) + first := submit(registration, cs.OK) + owner(cs.Mutation{Create: &cs.CreateCell{V1: true, Cell: "selected", Init: cs.CellInit{Head: "head"}}}, cs.OK) + owner(cs.Mutation{Create: &cs.CreateCell{V1: true, Cell: "outside", Init: cs.CellInit{Head: "other"}}}, cs.OK) + i := g.leader() + read := func(i int, k cs.RecordKey) ([]byte, error) { + app, _, e := g.stores[i].ApplicationView(ctx) + if e != nil { + return nil, e + } + sk := raftstore.Key{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key} + rows, _, e := g.stores[i].Read(ctx, app.Applied, []raftstore.Key{sk}) + return rows[sk], e + } + // The fresh owner barrier is the host's explicit prerequisite to scoped reads. + _, _, e = g.barrier("a") + must(t, e) + view, got, e := raftdirectory.AuthorizedWorkerRead(ctx, g.stores[i], cluster, id, "selected", 1) + must(t, e) + if view.Cell.Head != "head" || !bytes.Equal(got, grant) { + t.Fatal("selected grant/head lost") + } + if _, _, e = raftdirectory.AuthorizedWorkerRead(ctx, g.stores[i], cluster, id, "outside", 1); e == nil { + t.Fatal("R16 cross-scope read succeeded") + } + if _, e = raftdirectory.AuthorizedRead(ctx, g.stores[i], cluster, id); e == nil { + t.Fatal("worker read identity-wide directory") + } + anchors := []cs.MemberRecord{} + for n := uint64(1); n <= 3; n++ { + anchors = append(anchors, cs.MemberRecord{Node: n, Key: key(fmt.Sprint("anchor-", n)).Public().(ed25519.PublicKey), Incarnation: 1, Protocol: 1, Anchor: true}) + } + owner(cs.Mutation{InitMembership: &cs.InitMembership{Anchors: anchors, Limit: cs.MaxMemberRecords}}, cs.OK) + h := registry(t, g, i) + owner(cs.Mutation{EnrollMember: &cs.EnrollMember{Node: h.NextNode, Revision: h.Revision, Device: id, Protocol: 1}}, cs.OK) + // A real learner may be absent; it cannot pin owner commits or compaction. + plan, e := g.nodes[i].PlanMembership(registry(t, g, i)) + must(t, e) + g.drive(i, raftcore.Configure{Change: plan.Change}) + g.drain() + h = registry(t, g, g.leader()) + owner(cs.Mutation{SettleMembership: &cs.SettleMembership{Revision: h.Revision, Node: 4}}, cs.OK) + peers := func() rafttransport.View { + i := g.leader() + app, _, e := g.stores[i].ApplicationView(ctx) + must(t, e) + v, e := rafttransport.CommittedViewAt(rafttransport.Scope{Identity: cert.Identity, Cluster: cluster, Epoch: 1, Protocol: 1}, 1, app.Applied, raftcore.Configuration(app.ConfState), app.LogicalTime, func(k cs.RecordKey) ([]byte, error) { return read(i, k) }) + must(t, e) + return v + } + if v := peers(); v.Members[3].Revoked || v.Members[3].Voter { + t.Fatal("worker is not a learner", v) + } + for j, s := range g.stores { + m, e := s.CreateSnapshot(ctx) + must(t, e) + must(t, s.Compact(ctx, m.GetMetadata().GetIndex())) + g.drive(j, raftcore.CompactSnapshot{Snapshot: m}) + } + g.drain() + recoverMembership(t, g) + if r := submit(registration, cs.OK); !reflect.DeepEqual(r, first) { + t.Fatal("R17 compacted result changed", r, first) + } + pending := command(id, k, 1, cs.Mutation{Publish: &cs.PublishCell{Cell: "outside", Publish: cs.Publish{Fence: 1, Head: "forbidden"}}}) + submit(pending, cs.Unauthorized) + router, e := rafttransport.NewRouter(rafttransport.DefaultLimits()) + must(t, e) + initialView := peers() + must(t, router.Install(initialView)) + req := rafttransport.Request{Scope: initialView.Scope, Node: 4, Incarnation: 1, Generation: 1} + must(t, req.Sign(k)) + _, e = router.Exchange(req, now) + must(t, e) + owner(cs.Mutation{DrainWorker: &cs.DrainWorker{Worker: id, Incarnation: 1}}, cs.OK) + submit(command(id, k, 1, cs.Mutation{RegisterWorker: &cs.RegisterWorker{Previous: 1, Incarnation: 2}}), cs.OK) + if !peers().Members[3].Revoked { + t.Fatal("old peer survived worker restart") + } + // A real signed router exchange is denied after installing the narrowed view. + must(t, router.Install(peers())) + if _, e = router.Exchange(req, now); e == nil { + t.Fatal("R16 authenticated old transport survived incarnation") + } + owner(cs.Mutation{Revoke: &cs.DeviceID{Device: id}}, cs.OK) + submit(command(id, k, 2, cs.Mutation{Release: &cs.ReleaseCell{Cell: "selected", Fence: 1}}), cs.Revoked) + if _, _, e = raftdirectory.AuthorizedWorkerRead(ctx, g.stores[g.leader()], cluster, id, "selected", 2); e == nil { + t.Fatal("revoked worker retrieved a grant") + } +} diff --git a/internal/rafttransport/README.md b/internal/rafttransport/README.md index 5b11e4e6d1..18aba5c962 100644 --- a/internal/rafttransport/README.md +++ b/internal/rafttransport/README.md @@ -152,3 +152,11 @@ and persist router generation before admitting exchanges after inbox loss. An authenticated source's three provisioned anchor keys may bootstrap protocol traffic before registry commitment. A personal key requires committed enrollment; never accept a caller's arbitrary view as an enrollment grant. + +P15 adds `CommittedViewAt` for delegated worker learners. Commit current ingress +time and supply the applied configuration/read-set under the actor lock, then +Install before acknowledging the result. It checks the worker's explicit learner +consent, exclusive certificate expiry, revocation, rotation and current process +incarnation. The old metadata-only CommittedView marks workers revoked, so a +host that has not supplied these dependencies fails closed. Worker restarts +cannot resurrect a retired peer: remove/re-enroll with a fresh node ID. diff --git a/internal/rafttransport/registry.go b/internal/rafttransport/registry.go index 876117ffdc..257afd7e7e 100644 --- a/internal/rafttransport/registry.go +++ b/internal/rafttransport/registry.go @@ -20,3 +20,20 @@ func CommittedView(scope Scope, generation, applied uint64, conf clusterstate.Co } return v, nil } + +// CommittedViewAt must be used by hosts admitting delegated learner workers. +// Time was committed before the actor barrier; a local clock grants nothing. +func CommittedViewAt(scope Scope, generation, applied uint64, conf clusterstate.Configuration, now int64, read clusterstate.ReadRecord) (View, error) { + _, peers, err := clusterstate.MembershipViewAt(read, conf, applied, now) + if err != nil { + return View{}, err + } + if len(peers) == 0 || applied == 0 { + return View{}, ErrAuth + } + v := View{Scope: scope, Generation: generation, Revision: applied} + for _, p := range peers { + v.Members = append(v.Members, Member{Node: p.Node, Incarnation: p.Incarnation, Key: p.Key, Voter: p.Voter, Revoked: p.Revoked}) + } + return v, nil +} diff --git a/relay/hosted/src/raft/membership.js b/relay/hosted/src/raft/membership.js index e931275e2b..c3c1df361d 100644 --- a/relay/hosted/src/raft/membership.js +++ b/relay/hosted/src/raft/membership.js @@ -15,7 +15,11 @@ export function committedRegistry(runtime, store) { const raw = store.read('enrollment', 'node/' + node); if (!raw) throw new Error('raft member: enrollment is incomplete'); const m = decode(raw), device = m.device ? store.read('state', 'device/' + segment(m.device)) : null; - const r = response(runtime.call(frame(16).u64(node).u64(app.applied).blob(unb64(app.conf)).blob(header).blob(raw).blob(device ?? new Uint8Array()).finish())); + const w = frame(16).u64(node).u64(app.applied).blob(unb64(app.conf)).blob(header).blob(raw).blob(device ?? new Uint8Array()); + // Worker policy is Go's decision at committed time. JavaScript supplies + // only immutable rows, including the current incarnation after restart. + if (m.worker) w.u64(app.now).blob(store.read('state', 'identity') ?? new Uint8Array()).blob(store.read('enrollment', 'actor/' + segment(m.device)) ?? new Uint8Array()).blob(store.read('state', 'worker/' + segment(m.device)) ?? new Uint8Array()); + const r = response(runtime.call(w.finish())); const peer = decode(r.blob()); r.end(); view[peer.node] = { incarnation: peer.incarnation, key: peer.key, voter: peer.voter, revoked: peer.revoked }; } diff --git a/relay/hosted/src/raft/worker.js b/relay/hosted/src/raft/worker.js new file mode 100644 index 0000000000..1639db515e --- /dev/null +++ b/relay/hosted/src/raft/worker.js @@ -0,0 +1,14 @@ +import { frame, response } from './abi.js'; + +const segment = s => [...new TextEncoder().encode(s)].map(b => b.toString(16).padStart(2, '0')).join(''); + +// Run under the member queue AFTER committing current ingress time and passing +// a fresh current-term quorum barrier. Go alone decides scopes and expiry; +// this helper does not provide an independent local-success read door. +export function workerCell(runtime, store, cluster, worker, cell, incarnation) { + const app = store.application(), w = frame(27).str(cluster).str(worker).str(cell).u64(incarnation).u64(app.now); + const keys = [['state','identity'], ['enrollment','actor/' + segment(worker)], ['state','device/' + segment(worker)], ['state','worker/' + segment(worker)], ['state','cell/' + segment(cell)]]; + for (const [namespace,key] of keys) w.blob(store.read(namespace,key) ?? new Uint8Array()); + const r = response(runtime.call(w.finish())), raw = r.blob(); r.end(); + return raw; +} diff --git a/relay/hosted/test/raft/worker.test.js b/relay/hosted/test/raft/worker.test.js new file mode 100644 index 0000000000..82afdff39c --- /dev/null +++ b/relay/hosted/test/raft/worker.test.js @@ -0,0 +1,110 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { spawn } from 'node:child_process'; +import { DatabaseSync } from 'node:sqlite'; +import { createHash, generateKeyPairSync, sign, createCipheriv } from 'node:crypto'; +import { Reactor } from '../../src/raft/abi.js'; +import { encodeCanonical, decodeCanonical } from '../../src/raft/codec.js'; +import { MemberActor } from '../../src/raft/actor.js'; +import { directoryEvaluator } from '../../src/raft/directory.js'; +import { committedRegistry } from '../../src/raft/membership.js'; +import { workerCell } from '../../src/raft/worker.js'; + +const out = process.env.CODEAF_RAFT_PROOF_DIR ?? '/tmp/codeaf-p15'; +const enc = new TextEncoder(), dec = new TextDecoder(); +const bytes = value => enc.encode(encodeCanonical(value)); +const sha = value => createHash('sha256').update(value).digest('hex'); +const segment = value => Buffer.from(value).toString('hex'); +const caps = {os:'linux',arch:'arm64',sandbox:null,container:null,gpu:null,cow:'none'}; +const seal = text => { const cipher = createCipheriv('aes-256-gcm',Buffer.alloc(32,3),Buffer.alloc(12,7)); return Buffer.concat([Buffer.alloc(12,7),cipher.update(text),cipher.final(),cipher.getAuthTag()]).toString('base64'); }; +class Host { + constructor() { this.db = new DatabaseSync(':memory:'); this.sql = { exec: (q, ...p) => { const rows = this.db.prepare(q).all(...p); return { toArray: () => rows, [Symbol.iterator]: () => rows[Symbol.iterator]() }; } }; } + transactionSync(fn) { this.db.exec('BEGIN'); try { const result = fn(); this.db.exec('COMMIT'); return result; } catch (e) { this.db.exec('ROLLBACK'); throw e; } } + async sync() {} +} +class Native { + constructor() { + this.child = spawn(out + '/native-peer', [], { stdio: ['pipe','pipe','inherit'], env: { ...process.env, GOMEMLIMIT: '32MiB' } }); this.buffer = Buffer.alloc(0); + this.child.stdout.on('data', b => { this.buffer = Buffer.concat([this.buffer, b]); this.pump(); }); + this.child.on('exit', code => { if (this.waiter) { clearTimeout(this.waiter.timer); this.waiter.reject(new Error(`native exited ${code}`)); } }); + } + pump() { if (!this.waiter || this.buffer.length < 4) return; const n = this.buffer.readUInt32LE(0); assert(n <= 2 * 1024 * 1024); if (this.buffer.length < n + 4) return; const b = this.buffer.subarray(4, n + 4); this.buffer = this.buffer.subarray(n + 4); const w = this.waiter; this.waiter = null; clearTimeout(w.timer); w.resolve(new Uint8Array(b)); } + call(b) { assert(!this.waiter); return new Promise((resolve, reject) => { this.waiter = { resolve, reject, timer: setTimeout(() => reject(new Error('native ABI timeout')), 5000) }; const h = Buffer.alloc(4); h.writeUInt32LE(b.length); this.child.stdin.write(h); this.child.stdin.write(b); this.pump(); }); } + async close() { this.child.stdin.end(); if (this.child.exitCode === null) await new Promise(resolve => this.child.once('exit', resolve)); } +} + + +test('R3 R13 R14 R15 R16 R17 R18 delegated worker policy and selected grants match native Go', { timeout:30000 }, async () => { + const reactor = new Reactor(new WebAssembly.Module(readFileSync(out+'/core.wasm'))), native = new Native(), calls = []; + const runtime = {call(b) {const r=reactor.call(b);calls.push([b.slice(),r.slice()]);return r;}}; + const cluster='worker-p15', keys=Object.fromEntries(['owner','root','worker','x1','x2','x3'].map(id=>[id,generateKeyPairSync('ed25519')])); + const pub = id => keys[id].publicKey.export({format:'der',type:'spki'}).subarray(-32); + const identity='id_'+sha(pub('root')).slice(0,32), worker='dev_'+sha(pub('worker')).slice(0,32); + const rows=[{namespace:'state',key:'identity',data:bytes({V:1,identity,signing_key:pub('root').toString('base64')})},{namespace:'enrollment',key:'actor/'+segment('owner'),data:bytes({cluster,key:pub('owner').toString('base64'),personal:true})}]; + const actors=[],hosts=[],queue=[],results=[[],[],[]],reads=[];let serial=0; + const configs=[1,2,3].map(id=>({cluster,id:String(id),seed:String(60+id),bootstrap:['1','2','3']})); + const initialSize=rows.reduce((n,r)=>n+12+r.namespace.length+r.key.length+r.data.length,0); + const accept=e=>{queue.push(...e.send);reads.push(...e.reads);}; + const pump=async()=>{for(let n=0;queue.length;n++){assert(n<2000);const m=queue.shift();if(Number(m.to)>actors.length)continue;accept(await actors[Number(m.to)-1].input(2,m.bytes));}}; + const leader=()=>actors.findIndex(a=>a.status.role===2); + const evaluator=i=>(raw,read,app)=>{ + const base=read('state','identity')?[]:rows; + const r=directoryEvaluator(runtime,cluster,8*1024*1024)(raw,(ns,key)=>base.find(r=>r.namespace===ns&&r.key===key)?.data??read(ns,key),{...app,size:app.size+(base.length?initialSize:0)}); + results[i].push(decodeCanonical(dec.decode(r.result)));return {...r,writes:[...base,...r.writes]}; + }; + const command=(change,{signer='owner',inc=0n,now=1000}={})=>{ + const actor=signer==='worker'?worker:signer, session='req-'+(++serial), sequence='1'; + const fields={cluster,actor,session,sequence,change,...(inc?{worker_incarnation:String(inc)}:{})}; + const c={schema:1,...fields,request_digest:sha(bytes(fields)),now,signature:null}; + c.signature=sign(null,Buffer.concat([Buffer.from('codeaf.directory.command.v1\0'),bytes(c)]),keys[signer].privateKey).toString('base64');return bytes(c); + }; + const submit=async raw=>{const i=leader(),n=results[i].length;accept(await actors[i].input(3,raw));await pump();assert.equal(results[i].length,n+1);return results[i].at(-1);}; + const send=async(change,opts,code='ok')=>{const r=await submit(command(change,opts));assert.equal(r.code,code);return r;}; + const read=async cell=>{const id='read-'+(++serial),i=leader();accept(await actors[i].input(4,id));await pump();assert(reads.some(r=>r.id===id));return decodeCanonical(dec.decode(workerCell(runtime,actors[i].store,cluster,worker,cell,1n)));}; + const workrow=()=>decodeCanonical(dec.decode(actors[leader()].store.read('state','worker/'+segment(worker)))); + try { + for(let i=0;i<3;i++){const h=new Host();hosts.push(h);const a=new MemberActor(h,runtime,evaluator(i));actors.push(a);accept(await a.init(configs[i]));} + await pump();for(let n=0;n<30&&leader()<0;n++){for(const a of actors)accept(await a.input(1));await pump();}assert(leader()>=0); + const cert={v:1,identity,cluster,key:pub('worker').toString('base64'),made:1000,expires:100000,cells:['chat'],learner:true,signature:null}; + cert.signature=sign(null,Buffer.concat([Buffer.from('codeaf/worker-cert/v1\0'),bytes(cert)]),keys.root.privateKey).toString('base64'); + const grant=seal('secret selected-chat key credential /private/repo'); + await send({enroll_worker:{certificate:cert,grants:{chat:grant}}}); + const registration=command({register_worker:{previous:'0',incarnation:'1',caps,sealed_labels:seal('private worker name and scheduling requirement')}},{signer:'worker'}); + const registered=await submit(registration);assert.equal(registered.code,'ok');assert.deepEqual(await submit(registration),registered); + const beat={worker,incarnation:'1',sequence:'1',seen_at:1001,work:null,signature:null}; + beat.signature=sign(null,Buffer.concat([Buffer.from('codeaf/worker-beat/v1\0'),bytes({cluster,beat})]),keys.worker.privateKey).toString('base64'); + let r=await send({worker_beat_batch:{items:[beat]}},{now:1001});assert.equal(r.workers[0].code,'ok');assert.equal(workrow().available_until,91001); + await send({create:{v1:true,cell:'chat',init:{head:'head',class:'',size:'0',keys:null},receipt:''}}); + assert.equal((await read('outside')).code,'unauthorized');r=await read('chat');assert.equal(r.code,'ok');assert.equal(r.grant,grant);assert.equal(r.view.cell.head,'head'); + for(const change of [{rotate:{op:'freeze'}},{decide_join:{code:'pair',device:'',record:{V:0,name:'',added_by:'',revoked:false,caps:{os:'',arch:'',sandbox:null,container:null,gpu:null,cow:''}},grant:'',cert:'',deny:true}},{request_leadership:{device:worker,node:'4',revision:'1'}},{acquire:{cell:'chat',force:true,expect:{device:'owner',fence:'1'}}},{publish:{cell:'outside',publish:{fence:'1',old_head:'head',head:'bad',size:'0',class:'',pending:0},receipt:''}}])await send(change,{signer:'worker',inc:1n},'unauthorized'); + const anchors=[1,2,3].map(node=>({node:String(node),device:'',key:pub('x'+node).toString('base64'),incarnation:'1',protocol:1,anchor:true,retired:false,ready_index:'0'})); + await send({init_membership:{anchors,limit:4096}}); + const header=()=>decodeCanonical(dec.decode(actors[leader()].store.read('membership','registry'))); + let h=header();await send({enroll_member:{node:h.next_node,revision:h.revision,device:worker,protocol:1}}); + let i=leader(),e=await actors[i].membership();accept(e);await pump();e=await actors[i].membership();accept(e);await pump();assert(e.settle); + h=header();await send({settle_membership:{node:h.pending.node,revision:h.revision}}); + let registry=committedRegistry(runtime,actors[leader()].store);assert.equal(registry['4'].revoked,false);assert.equal(registry['4'].voter,false); + for(let j=0;j<3;j++)await actors[j].checkpoint(); + // Reopen a real stored follower; an active leader is never cache-evicted. + const reopened=(leader()+1)%3; + await actors[reopened].close();actors[reopened]=new MemberActor(hosts[reopened],runtime,evaluator(reopened));accept(await actors[reopened].init(configs[reopened]));await pump(); + assert.equal(decodeCanonical(dec.decode(actors[reopened].store.read('state','worker/'+segment(worker)))).grants.chat,grant); + assert.deepEqual(await submit(registration),registered);assert.equal((await read('chat')).grant,grant); + await send({drain_worker:{worker,incarnation:'1'}}); + await send({register_worker:{previous:'1',incarnation:'2',caps,sealed_labels:seal('private worker name and scheduling requirement')}},{signer:'worker',inc:1n}); + assert.equal(workrow().draining,true);assert.equal(workrow().available_until,0); + registry=committedRegistry(runtime,actors[leader()].store);assert.equal(registry['4'].revoked,true,'old peer incarnation'); + assert.equal((await submit(registration)).code,'conflict'); + await send({revoke:{device:worker}}); + await send({release:{cell:'chat',fence:'1',quiet:false}},{signer:'worker',inc:2n},'revoked'); + assert.equal((await read('chat')).code,'revoked'); + for(const a of actors){ + for(const [ns,key] of [['state','worker/'+segment(worker)],['enrollment','actor/'+segment(worker)]]){ + const b=a.store.read(ns,key);assert(b);for(const needle of ['private worker name','/private/repo','selected-chat key'])assert(!Buffer.from(b).includes(Buffer.from(needle))); + } + } + for(const [input,expected]of calls)assert.deepEqual(await native.call(input),expected); + assert(calls.length>100);assert.equal(reactor.metrics().wasi.poll_oneoff??0,0); + }finally{await native.close();for(const h of hosts)h.db.close();} +}); From 1abc35b3f0d1a12fc265f4d2b3c32e2f2114498f Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 04:27:20 -0400 Subject: [PATCH 036/103] Explain delegated worker trust and its current limits. Describe selected-chat access, explicit learner metadata disclosure and draining in the embedded manual while keeping worker setup and assignment execution absent until their client lifecycle is implemented. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../chat/continuing-a-chat-on-another-computer.md | 12 ++++++++++++ internal/manual/chat_test.go | 2 ++ 2 files changed, 14 insertions(+) diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index 2113ecb2cf..5416887c13 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -296,3 +296,15 @@ The sync connection carries presence hints and committed lease acknowledgments. ## Does syncing keep sending requests when nothing is happening? A home that has never paired sends no sync traffic. `CODEAF_SYNC_URL=off` also blocks new sync requests from clients that were opened before the setting changed. Presence pings on an open view are hints; they do not write chat ownership. On the consensus directory, a group with no work or held chats can stop its periodic coordination timer after an idle barrier has reached every voter. The first operation wakes coordination and must obtain a fresh majority response. A saved leader address cannot answer that operation by itself. Losing the majority leaves syncing unavailable and keeps local work intact. + +## Can a worker access all my chats or approve pairing? + +A delegated worker uses its own key and an expiring certificate naming selected chats. It receives only sealed grants for those chats, rather than the full exported identity. It cannot approve pairing, rotate the identity, read the identity vault, enroll other devices or request leadership. Richer capabilities, worker labels and execution credentials stay encrypted; only the existing platform and sandbox capability shape is readable by the relay. + +Worker setup and assignment execution are not available as a terminal command yet. Registration and a fresh signed heartbeat only record availability; they never start tools or give the worker ownership of a chat. Removing a worker blocks later requests after that removal commits. Expiry and removal cannot take back bytes the worker already downloaded. + +## Worker restarts, draining and learner metadata visibility + +A worker restart registers a newer process number. Delayed requests and heartbeats from its previous process cannot refresh availability or save work. A drained worker cannot receive new claims; already held work can still save and let go. Restarting does not clear draining. Expired and removed enrollment records remain retained; reaching their capacity limit refuses new enrollment instead of deleting records or saved chats. + +A scoped API worker does not join the coordination log. Joining as a learner requires explicit consent in its certificate: the full log reveals identity-wide routing and ownership metadata, while chat content, names and task instructions remain encrypted. Use scoped API access for an untrusted third-party worker. A delegated learner gets no vote and cannot become leader. Existing paired personal computers keep their shared-root trust model. diff --git a/internal/manual/chat_test.go b/internal/manual/chat_test.go index 6070d68cce..2645ca8d24 100644 --- a/internal/manual/chat_test.go +++ b/internal/manual/chat_test.go @@ -3213,6 +3213,8 @@ func TestTheChatManualAnswersTheQuestionsPeopleAsk(t *testing.T) { {"how do I take a chat back from my other computer", "continuing-a-chat-on-another-computer"}, {"what happens if two computers continue the same chat at once", "continuing-a-chat-on-another-computer"}, {"which computer coordinates cluster status cluster lead", "continuing-a-chat-on-another-computer"}, + {"can a worker access all my chats or approve pairing", "continuing-a-chat-on-another-computer"}, + {"worker restarts draining and learner metadata visibility", "continuing-a-chat-on-another-computer"}, {"why does it say this chat is arriving", "continuing-a-chat-on-another-computer"}, {"does /remember work on my other computer", "what-i-remember"}, {"are my memories shared across computers", "what-i-remember"}, From 79a1b05c687192d3bee47bdf400cb5cd90051597 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 04:55:04 -0400 Subject: [PATCH 037/103] Commit explicit worker assignments with conditional chat ownership. Keep scheduling decisions and chat fences in one durable application transaction so stale continuations, revoked workers and replaced processes cannot claim the same observed work. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterstate/apply.go | 11 +- internal/clusterstate/assignment.go | 309 ++++++++++++++++++ internal/clusterstate/assignment_test.go | 175 ++++++++++ internal/clusterstate/liveness.go | 17 + internal/clusterstate/types.go | 6 + internal/clusterstate/worker.go | 28 +- internal/raftsim/assignment_test.go | 380 ++++++++++++++++++++++ internal/raftsim/model/assignment.go | 122 +++++++ internal/raftsim/model/assignment_test.go | 28 ++ 9 files changed, 1074 insertions(+), 2 deletions(-) create mode 100644 internal/clusterstate/assignment.go create mode 100644 internal/clusterstate/assignment_test.go create mode 100644 internal/raftsim/assignment_test.go create mode 100644 internal/raftsim/model/assignment.go create mode 100644 internal/raftsim/model/assignment_test.go diff --git a/internal/clusterstate/apply.go b/internal/clusterstate/apply.go index cae16322c2..5e48703bff 100644 --- a/internal/clusterstate/apply.go +++ b/internal/clusterstate/apply.go @@ -149,6 +149,7 @@ func Dependencies(c Command) []RecordKey { } keys = append(keys, membershipDependencies(m)...) keys = append(keys, workerDependencies(c)...) + keys = append(keys, assignmentDependencies(c)...) if m.PutDevice != nil { keys = append(keys, DeviceKey(m.PutDevice.ID)) } @@ -266,6 +267,12 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er target = new(Device) case k.Namespace == "state" && len(k.Key) >= 5 && k.Key[:5] == "cell/": target = new(Cell) + case k.Namespace == "state" && len(k.Key) > 11 && k.Key[:11] == "assignment/": + target = new(Assignment) + case k.Namespace == "state" && len(k.Key) > 16 && k.Key[:16] == "assignment-slot/": + target = new(AssignmentSlot) + case k.Namespace == "state" && len(k.Key) > 12 && k.Key[:12] == "worker-hold/": + target = new(WorkerHold) case k == WorkerRegistryKey(): target = new(WorkerRegistry) case k.Namespace == "state" && len(k.Key) > 7 && k.Key[:7] == "worker/": @@ -398,7 +405,9 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er } else if service { code = applyEffect(c, out.Now, rows, put) } else { - if workerMutation(c.Change) { + if assignmentMutation(c.Change) { + code = applyAssignment(c, out.Now, identity, rows, put, &out.Result) + } else if workerMutation(c.Change) { code = applyWorker(c, out.Now, identity, rows, put, &out.Result) } else if membershipMutation(c.Change) { code = applyMembership(c, meta, rows, put, &out.Result) diff --git a/internal/clusterstate/assignment.go b/internal/clusterstate/assignment.go new file mode 100644 index 0000000000..5b884b77ff --- /dev/null +++ b/internal/clusterstate/assignment.go @@ -0,0 +1,309 @@ +package clusterstate + +// Assignment routing is public; instructions, credentials and task descriptors +// stay inside SealedPayload. A target is always selected by an authorized client. +type Assign struct { + ID string `json:"id"` + Revision uint64 `json:"revision"` + Cell string `json:"cell"` + ParentCell string `json:"parent_cell"` + Expect LeaseExpectation `json:"expect"` + Target string `json:"target"` + Incarnation uint64 `json:"incarnation"` + Expires int64 `json:"expires"` + SealedPayload []byte `json:"sealed_payload"` +} + +// AssignmentRef repeats routing dependencies so evaluation never discovers an +// undeclared row after the host has fixed its immutable application read-set. +type AssignmentRef struct { + ID string `json:"id"` + Revision uint64 `json:"revision"` + Cell string `json:"cell"` + Target string `json:"target"` + Incarnation uint64 `json:"incarnation"` +} +type ClaimAssignment struct { + Ref AssignmentRef `json:"ref"` + Expect LeaseExpectation `json:"expect"` +} +type FinishAssignment struct { + Ref AssignmentRef `json:"ref"` + Fence uint64 `json:"fence"` + Head string `json:"head"` +} +type CancelAssignment struct { + Ref AssignmentRef `json:"ref"` +} + +type Assignment struct { + Spec Assign `json:"spec"` + Status string `json:"status"` + Fence uint64 `json:"fence"` + Head string `json:"head"` +} + +func (a Assignment) Ref() AssignmentRef { + v := a.Spec + return AssignmentRef{ID: v.ID, Revision: v.Revision, Cell: v.Cell, Target: v.Target, Incarnation: v.Incarnation} +} + +// The per-cell slot bounds active scheduling independently of retained runs. +// A completed/cancelled slot is reusable, but its original run is never deleted. +type AssignmentSlot struct { + Ref AssignmentRef `json:"ref"` + Active bool `json:"active"` +} +type WorkerHold struct { + Incarnation uint64 `json:"incarnation"` + Fence uint64 `json:"fence"` +} + +func AssignmentKey(id string) RecordKey { return RecordKey{"state", "assignment/" + segment(id)} } +func AssignmentSlotKey(cell string) RecordKey { + return RecordKey{"state", "assignment-slot/" + segment(cell)} +} +func WorkerHoldKey(worker, cell string) RecordKey { + return RecordKey{"state", "worker-hold/" + segment(worker) + "/" + segment(cell)} +} +func assignmentMutation(m Mutation) bool { + return m.Assign != nil || m.Reassign != nil || m.ClaimAssignment != nil || m.FinishAssignment != nil || m.CancelAssignment != nil +} +func assignmentRef(m Mutation) AssignmentRef { + if m.Assign != nil { + return (Assignment{Spec: *m.Assign}).Ref() + } + if m.Reassign != nil { + return (Assignment{Spec: *m.Reassign}).Ref() + } + if m.ClaimAssignment != nil { + return m.ClaimAssignment.Ref + } + if m.FinishAssignment != nil { + return m.FinishAssignment.Ref + } + if m.CancelAssignment != nil { + return m.CancelAssignment.Ref + } + return AssignmentRef{} +} +func assignmentDependencies(c Command) []RecordKey { + var keys []RecordKey + if assignmentMutation(c.Change) { + r := assignmentRef(c.Change) + keys = append(keys, AssignmentKey(r.ID), AssignmentSlotKey(r.Cell), CellKey(r.Cell), WorkerKey(r.Target), AuthorityKey(r.Target), DeviceKey(r.Target), WorkerHoldKey(r.Target, r.Cell)) + v := c.Change.Assign + if v == nil { + v = c.Change.Reassign + } + if v != nil && v.ParentCell != "" { + keys = append(keys, CellKey(v.ParentCell)) + } + } + for _, id := range cellIDs(c.Change) { + keys = append(keys, WorkerHoldKey(c.Actor, id)) + } + return keys +} + +func applyAssignment(c Command, now int64, identity IdentityRec, rows map[RecordKey][]byte, put func(RecordKey, any), result *Result) Code { + m := c.Change + r := assignmentRef(m) + if r.ID == "" || r.Cell == "" || r.Target == "" || r.Revision == 0 || r.Incarnation == 0 { + return BadRequest + } + var a Assignment + var slot AssignmentSlot + var cell Cell + if rows[CellKey(r.Cell)] == nil { + return NotFound + } + if Decode(rows[CellKey(r.Cell)], &cell) != nil { + return BadRequest + } + if b := rows[AssignmentKey(r.ID)]; b != nil { + if Decode(b, &a) != nil { + return BadRequest + } + } + if b := rows[AssignmentSlotKey(r.Cell)]; b != nil { + if Decode(b, &slot) != nil { + return BadRequest + } + } + v := m.Assign + if v == nil { + v = m.Reassign + } + if v != nil { + if m.Assign != nil { + if rows[AssignmentKey(r.ID)] != nil { + return Exists + } + if r.Revision != 1 { + return Conflict + } + } else { + if rows[AssignmentKey(r.ID)] == nil { + return NotFound + } + if a.Spec.Revision == ^uint64(0) || r.Revision != a.Spec.Revision+1 || a.Spec.Cell != r.Cell || a.Spec.ParentCell != v.ParentCell || a.Status == "finished" || a.Status == "cancelled" { + return Conflict + } + } + if slot.Active && (m.Assign != nil || slot.Ref.ID != r.ID || slot.Ref.Revision != a.Spec.Revision) { + return Conflict + } + if v.ParentCell == r.Cell || cell.ParentCell != v.ParentCell { + return BadRequest + } + if v.ParentCell != "" && rows[CellKey(v.ParentCell)] == nil { + return NotFound + } + if v.Expires <= now || v.Expires > int64(MaxSafeInteger) || len(v.SealedPayload) == 0 || len(v.SealedPayload) > MaxWorkerSealedBytes { + return BadRequest + } + if cell.Lease.Device != v.Expect.Device || cell.Lease.Fence != v.Expect.Fence { + return LeaseHeld + } + auth, w, code := workerFromRows(c.Cluster, r.Target, now, identity, rows) + if code != OK { + return code + } + var d Device + if b := rows[DeviceKey(r.Target)]; b != nil { + _ = Decode(b, &d) + } + if code = WorkerCanClaim(c.Cluster, r.Target, r.Cell, r.Incarnation, now, identity, auth, d, w); code != OK { + return code + } + a = Assignment{Spec: *v, Status: "pending"} + put(AssignmentKey(r.ID), a) + put(AssignmentSlotKey(r.Cell), AssignmentSlot{Ref: r, Active: true}) + result.Assignment = &a + return OK + } + if rows[AssignmentKey(r.ID)] == nil { + return NotFound + } + if a.Ref() != r || slot.Ref != r { + return Conflict + } + if m.CancelAssignment != nil { + if a.Status == "finished" { + return Conflict + } + a.Status = "cancelled" + slot.Active = false + } else if m.ClaimAssignment != nil { + if c.Actor != r.Target || c.WorkerIncarnation != r.Incarnation { + return Unauthorized + } + if a.Status != "pending" || !slot.Active || a.Spec.Expires <= now { + return Conflict + } + auth, w, code := workerFromRows(c.Cluster, r.Target, now, identity, rows) + if code != OK { + return code + } + var d Device + if b := rows[DeviceKey(r.Target)]; b != nil { + _ = Decode(b, &d) + } + if code = WorkerCanClaim(c.Cluster, r.Target, r.Cell, r.Incarnation, now, identity, auth, d, w); code != OK { + return code + } + if m.ClaimAssignment.Expect != a.Spec.Expect { + return LeaseHeld + } + if cell.Lease.Fence == ^uint64(0) { + return Capacity + } + next, err := Acquire(cell, c.Actor, now, true, &a.Spec.Expect) + if err != nil { + return LeaseHeld + } + cell = next + a.Status = "claimed" + a.Fence = cell.Lease.Fence + a.Head = cell.Head + put(CellKey(r.Cell), cell) + put(WorkerHoldKey(r.Target, r.Cell), WorkerHold{Incarnation: r.Incarnation, Fence: a.Fence}) + result.View = &CellView{Now: now, Cell: cell} + } else if m.FinishAssignment != nil { + v := m.FinishAssignment + if c.Actor != r.Target || c.WorkerIncarnation != r.Incarnation { + return Unauthorized + } + if a.Status != "claimed" || !slot.Active { + return Conflict + } + if cell.Lease.Device != r.Target || cell.Lease.Fence != v.Fence || a.Fence != v.Fence { + return FenceStale + } + if cell.Head != v.Head || v.Head == "" { + return HeadMoved + } + a.Status = "finished" + a.Head = v.Head + slot.Active = false + } else { + return BadRequest + } + put(AssignmentKey(r.ID), a) + put(AssignmentSlotKey(r.Cell), slot) + result.Assignment = &a + return OK +} + +// AssignmentRead exposes only the selected run and its selected chat grant. +type AssignmentRead struct { + Code Code `json:"code"` + Assignment *Assignment `json:"assignment,omitempty"` + View *CellView `json:"view,omitempty"` + Grant []byte `json:"grant,omitempty"` +} + +func AssignmentReadKeys(worker string, ref AssignmentRef) []RecordKey { + return append(WorkerReadKeys(worker, ref.Cell), AssignmentKey(ref.ID)) +} + +// ReadWorkerAssignment runs at the host's current committed time and fresh +// quorum barrier. Revoked/expired targets expose no saved grant or execution +// permission, even when a raw pending row is retained for recovery. +func ReadWorkerAssignment(cluster, worker string, incarnation uint64, ref AssignmentRef, now int64, read ReadRecord) (AssignmentRead, error) { + size := 0 + bounded := func(k RecordKey) ([]byte, error) { + b, e := read(k) + size += len(k.Namespace) + len(k.Key) + len(b) + if size > MaxEvaluationBytes { + return nil, ErrDependency + } + return b, e + } + selected, err := ReadWorkerCell(cluster, worker, ref.Cell, incarnation, now, bounded) + if err != nil { + return AssignmentRead{}, err + } + if selected.Code != OK { + return AssignmentRead{Code: selected.Code}, nil + } + raw, err := bounded(AssignmentKey(ref.ID)) + if err != nil { + return AssignmentRead{}, err + } + if raw == nil { + return AssignmentRead{Code: NotFound}, nil + } + var a Assignment + if Decode(raw, &a) != nil { + return AssignmentRead{}, ErrDependency + } + if a.Ref() != ref || ref.Target != worker || ref.Incarnation != incarnation { + return AssignmentRead{Code: Conflict}, nil + } + if a.Status == "pending" && a.Spec.Expires <= now { + a.Status = "expired" + } + return AssignmentRead{Code: OK, Assignment: &a, View: selected.View, Grant: selected.Grant}, nil +} diff --git a/internal/clusterstate/assignment_test.go b/internal/clusterstate/assignment_test.go new file mode 100644 index 0000000000..069799ccef --- /dev/null +++ b/internal/clusterstate/assignment_test.go @@ -0,0 +1,175 @@ +package clusterstate_test + +import ( + "bytes" + "fmt" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "math/rand" + "reflect" + "testing" +) + +func assignmentFixture(t *testing.T) (*workerFixture, cs.Assign) { + f := workerFixtureFor(t, false) + expectWorker(t, f.register(0, 1001), cs.OK) + expectWorker(t, f.beat(f.proof(1, 1, 1002), 1002), cs.OK) + f.a.put(cs.CellKey("chat"), cs.Created(cs.CellInit{Head: "head"}, "a", 1000)) + return f, cs.Assign{ID: "opaque-run", Revision: 1, Cell: "chat", Expect: cs.LeaseExpectation{Device: "a", Fence: 1}, Target: f.id, Incarnation: 1, Expires: 10000, SealedPayload: sealedWorker(t, "private task /repo model-secret shell body")} +} +func ref(v cs.Assign) cs.AssignmentRef { return (cs.Assignment{Spec: v}).Ref() } +func assign(t *testing.T, f *workerFixture, v cs.Assign) cs.Assignment { + t.Helper() + r := f.owner(1003, cs.Mutation{Assign: &v}) + expectWorker(t, r, cs.OK) + if r.Assignment == nil { + t.Fatal(r) + } + return *r.Assignment +} +func claim(f *workerFixture, v cs.Assign) cs.Command { + return f.command(f.id, f.key, v.Incarnation, 1004, cs.Mutation{ClaimAssignment: &cs.ClaimAssignment{Ref: ref(v), Expect: v.Expect}}) +} + +func TestR18R17AssignmentConditionalClaimAndExactRetry(t *testing.T) { + f, v := assignmentFixture(t) + assign(t, f, v) + before := f.a.cell() + other := v + other.ID = "second" + expectWorker(t, f.owner(1003, cs.Mutation{Assign: &other}), cs.Conflict) + c := claim(f, v) + r := f.eval(c) + expectWorker(t, r, cs.OK) + if r.Assignment.Status != "claimed" || r.View.Cell.Lease.Fence != 2 || r.View.Cell.Head != "head" || r.View.Cell.Lease.Device != f.id { + t.Fatal(r) + } + if retry := f.eval(c); !reflect.DeepEqual(r, retry) { + t.Fatal("claim retry changed exact result", r, retry) + } + if f.a.cell().Lease.Fence != 2 || before.Head != f.a.cell().Head { + t.Fatal("claim changed head or double fenced") + } + expectWorker(t, f.owner(1005, cs.Mutation{Acquire: &cs.AcquireCell{Cell: "chat", Force: true, Expect: v.Expect}}), cs.LeaseHeld) + expectWorker(t, f.run(1, 1005, cs.Mutation{FinishAssignment: &cs.FinishAssignment{Ref: ref(v), Fence: 2, Head: "unpublished"}}), cs.HeadMoved) + expectWorker(t, f.run(1, 1005, cs.Mutation{FinishAssignment: &cs.FinishAssignment{Ref: ref(v), Fence: 2, Head: "head"}}), cs.OK) + other.Expect = cs.LeaseExpectation{Device: f.id, Fence: 2} + expectWorker(t, f.owner(1006, cs.Mutation{Assign: &other}), cs.OK) + for _, b := range f.a.rows { + for _, needle := range []string{"private task", "/repo", "model-secret", "shell body"} { + if bytes.Contains(b, []byte(needle)) { + t.Fatal("R15 assignment plaintext") + } + } + } +} +func TestR18ClaimLosesToContinueCancelReassignIncarnationDrainRevokeExpiry(t *testing.T) { + for _, event := range []string{"continue", "cancel", "reassign", "incarnation", "drain", "revoke", "expiry"} { + t.Run(event, func(t *testing.T) { + f, v := assignmentFixture(t) + assign(t, f, v) + c := claim(f, v) + want := cs.Conflict + switch event { + case "continue": + expectWorker(t, f.owner(1004, cs.Mutation{Acquire: &cs.AcquireCell{Cell: "chat", Force: true, Expect: v.Expect}}), cs.OK) + want = cs.LeaseHeld + case "cancel": + expectWorker(t, f.owner(1004, cs.Mutation{CancelAssignment: &cs.CancelAssignment{Ref: ref(v)}}), cs.OK) + case "reassign": + next := v + next.Revision++ + expectWorker(t, f.owner(1004, cs.Mutation{Reassign: &next}), cs.OK) + case "incarnation": + expectWorker(t, f.register(1, 1004), cs.OK) + case "drain": + expectWorker(t, f.owner(1004, cs.Mutation{DrainWorker: &cs.DrainWorker{Worker: f.id, Incarnation: 1}}), cs.OK) + case "revoke": + expectWorker(t, f.owner(1004, cs.Mutation{Revoke: &cs.DeviceID{Device: f.id}}), cs.OK) + want = cs.Revoked + case "expiry": + expectWorker(t, f.owner(10000, cs.Mutation{AdvanceTime: &cs.AdvanceTime{}}), cs.OK) + } + before := f.a.cell() + expectWorker(t, f.eval(c), want) + if !reflect.DeepEqual(before, f.a.cell()) { + t.Fatal("losing claim changed chat") + } + }) + } +} +func TestR18R19R21ChildAssignmentsAndParentFence(t *testing.T) { + f, v := assignmentFixture(t) + // Issue a separately scoped child delegation: parallel tasks cannot alias the + // parent's lease or obtain authority by inventing a parent in the payload. + f.a.put(cs.CellKey("parent"), cs.Created(cs.CellInit{Head: "parent-head"}, "a", 1000)) + child := f.a.cell() + child.ParentCell = "parent" + f.a.put(cs.CellKey("chat"), child) + expectWorker(t, f.owner(1003, cs.Mutation{Assign: &v}), cs.BadRequest) + v.ParentCell = "parent" + assign(t, f, v) + r := f.eval(claim(f, v)) + expectWorker(t, r, cs.OK) + expectWorker(t, f.run(1, 1004, cs.Mutation{Publish: &cs.PublishCell{Cell: "chat", Publish: cs.Publish{Fence: 2, OldHead: "head", Head: "result"}}}), cs.BadRequest) + receipt := cs.UploadReceipt{Device: f.id, Cell: "chat", WorkerIncarnation: 1, Head: "result", HeadRID: "rid", Frames: []string{"frame"}} + f.a.put(cs.UploadKey(cs.UploadID(receipt)), receipt) + expectWorker(t, f.run(1, 1005, cs.Mutation{Publish: &cs.PublishCell{Cell: "chat", Receipt: cs.UploadID(receipt), Publish: cs.Publish{Fence: 2, OldHead: "head", Head: "result", Frames: receipt.Frames}}}), cs.OK) + expectWorker(t, f.owner(1006, cs.Mutation{Acquire: &cs.AcquireCell{Cell: "parent", Force: true, Expect: cs.LeaseExpectation{Device: "a", Fence: 1}}}), cs.OK) + parent := append([]byte(nil), f.a.rows[cs.CellKey("parent")]...) + expectWorker(t, f.run(1, 1007, cs.Mutation{FinishAssignment: &cs.FinishAssignment{Ref: ref(v), Fence: 2, Head: "result"}}), cs.OK) + if !bytes.Equal(parent, f.a.rows[cs.CellKey("parent")]) { + t.Fatal("finish wrote parent") + } + expectWorker(t, f.owner(1008, cs.Mutation{Publish: &cs.PublishCell{Cell: "parent", Publish: cs.Publish{Fence: 1, OldHead: "parent-head", Head: "child-result"}}}), cs.FenceStale) + expectWorker(t, f.register(1, 1009), cs.OK) + expectWorker(t, f.run(2, 1010, cs.Mutation{Release: &cs.ReleaseCell{Cell: "chat", Fence: 2}}), cs.Conflict) +} +func TestR18R27AssignmentActivityAndRetention(t *testing.T) { + f, v := assignmentFixture(t) + a := assign(t, f, v) + b, _ := cs.Encode(a) + busy, e := cs.RecordActive(cs.AssignmentKey(v.ID), b, 1004) + if e != nil || !busy { + t.Fatal(busy, e) + } + busy, e = cs.RecordActive(cs.AssignmentKey(v.ID), b, v.Expires) + if e != nil || busy { + t.Fatal(busy, e) + } + expectWorker(t, f.owner(1005, cs.Mutation{CancelAssignment: &cs.CancelAssignment{Ref: ref(v)}}), cs.OK) + retained := append([]byte(nil), f.a.rows[cs.AssignmentKey(v.ID)]...) + var size uint64 + for k, b := range f.a.rows { + size += cs.RecordBytes(k, b) + } + f.a.meta.Ceiling = size + 2048 + v.ID = "too-many" + v.SealedPayload = bytes.Repeat([]byte{7}, cs.MaxWorkerSealedBytes) + expectWorker(t, f.owner(1006, cs.Mutation{Assign: &v}), cs.Capacity) + b = retained + v.ID = "opaque-run" + if !bytes.Equal(b, f.a.rows[cs.AssignmentKey(v.ID)]) { + t.Fatal("admission deleted retained assignment") + } +} +func TestR18SeededAssignmentOrdering(t *testing.T) { + for seed := 0; seed < seedCount(t); seed++ { + t.Run(fmt.Sprintf("seed-%04d", seed), func(t *testing.T) { + f, v := assignmentFixture(t) + assign(t, f, v) + rng := rand.New(rand.NewSource(int64(seed))) + c := claim(f, v) + take := f.command("b", actorKey("b"), 0, 1004, cs.Mutation{Acquire: &cs.AcquireCell{Cell: "chat", Force: true, Expect: v.Expect}}) + if rng.Intn(2) == 0 { + expectWorker(t, f.eval(c), cs.OK) + expectWorker(t, f.eval(take), cs.LeaseHeld) + } else { + expectWorker(t, f.eval(take), cs.OK) + expectWorker(t, f.eval(c), cs.LeaseHeld) + } + if f.a.cell().Lease.Fence != 2 { + t.Fatal("R18 multiple authoritative winners") + } + }) + } +} diff --git a/internal/clusterstate/liveness.go b/internal/clusterstate/liveness.go index 0692abf381..e913bc1d27 100644 --- a/internal/clusterstate/liveness.go +++ b/internal/clusterstate/liveness.go @@ -72,6 +72,15 @@ func renewHolds(c Command, now int64, rows map[RecordKey][]byte, put func(Record if !a.Worker.Allows(h.Cell) { return Unauthorized } + if b := rows[WorkerHoldKey(v.Device, h.Cell)]; b != nil { + var hold WorkerHold + if Decode(b, &hold) != nil { + return BadRequest + } + if hold.Incarnation != v.WorkerIncarnation { + return Conflict + } + } } } seen := map[string]bool{} @@ -115,6 +124,14 @@ func renewHolds(c Command, now int64, rows map[RecordKey][]byte, put func(Record // RecordActive is evaluated one immutable row at a time so large imported // identities never have to be materialized to decide whether they may sleep. func RecordActive(key RecordKey, raw []byte, now int64) (bool, error) { + if key.Namespace == "state" && len(key.Key) > 11 && key.Key[:11] == "assignment/" { + var a Assignment + if Decode(raw, &a) != nil { + return false, ErrDependency + } + return (a.Status == "pending" || a.Status == "claimed") && a.Spec.Expires > now, nil + } + if key.Namespace == "state" && len(key.Key) > 7 && key.Key[:7] == "worker/" { var worker WorkerRecord if Decode(raw, &worker) != nil { diff --git a/internal/clusterstate/types.go b/internal/clusterstate/types.go index ec5ad8c619..70b6e0c6ef 100644 --- a/internal/clusterstate/types.go +++ b/internal/clusterstate/types.go @@ -45,6 +45,11 @@ type Command struct { // their typed operations here; unknown fields fail decoding on older members. // There is no arbitrary JSON payload in which task text or secrets can hide. type Mutation struct { + Assign *Assign `json:"assign,omitempty"` + Reassign *Assign `json:"reassign,omitempty"` + ClaimAssignment *ClaimAssignment `json:"claim_assignment,omitempty"` + FinishAssignment *FinishAssignment `json:"finish_assignment,omitempty"` + CancelAssignment *CancelAssignment `json:"cancel_assignment,omitempty"` EnrollWorker *EnrollWorker `json:"enroll_worker,omitempty"` RegisterWorker *RegisterWorker `json:"register_worker,omitempty"` WorkerBeatBatch *WorkerBeatBatch `json:"worker_beat_batch,omitempty"` @@ -151,6 +156,7 @@ type DeviceSeen struct { // no free-form diagnostic text, so logging a refusal cannot disclose payloads. // Now and Version are application time/version, never a local clock/Raft index. type Result struct { + Assignment *Assignment `json:"assignment,omitempty"` Workers []WorkerResult `json:"workers,omitempty"` Index uint64 `json:"index,omitempty"` Membership *MembershipResult `json:"membership,omitempty"` diff --git a/internal/clusterstate/worker.go b/internal/clusterstate/worker.go index 6112e6f969..547ba07b2b 100644 --- a/internal/clusterstate/worker.go +++ b/internal/clusterstate/worker.go @@ -114,6 +114,9 @@ func workerDependencies(c Command) []RecordKey { } if v := c.Change.RenewHolds; v != nil { add(v.Device) + for _, h := range v.Items { + keys = append(keys, WorkerHoldKey(v.Device, h.Cell)) + } } if v := c.Change.EnrollMember; v != nil { add(v.Device) @@ -181,7 +184,21 @@ func authorizeWorkerCommand(c Command, now int64, identity IdentityRec, a Author } return OK } - // Ownership acquisition is reserved for P16's committed conditional claim. + if c.Change.ClaimAssignment != nil { + r := c.Change.ClaimAssignment.Ref + if r.Target != c.Actor || r.Incarnation != w.Incarnation || !a.Worker.Allows(r.Cell) { + return Unauthorized + } + return OK + } + if c.Change.FinishAssignment != nil { + r := c.Change.FinishAssignment.Ref + if r.Target != c.Actor || r.Incarnation != w.Incarnation || !a.Worker.Allows(r.Cell) { + return Unauthorized + } + return OK + } + // Ownership acquisition uses only the committed conditional claim. // Drain does not block saving/releasing a fence already held by this worker. if c.Change.Publish == nil && c.Change.Beat == nil && c.Change.BeatBatch == nil && c.Change.Release == nil { return Unauthorized @@ -190,6 +207,15 @@ func authorizeWorkerCommand(c Command, now int64, identity IdentityRec, a Author if !a.Worker.Allows(cell) { return Unauthorized } + if b := rows[WorkerHoldKey(c.Actor, cell)]; b != nil { + var hold WorkerHold + if Decode(b, &hold) != nil { + return BadRequest + } + if hold.Incarnation != w.Incarnation { + return Conflict + } + } } if c.Change.Publish != nil && c.Change.Publish.V1 { return Unauthorized diff --git a/internal/raftsim/assignment_test.go b/internal/raftsim/assignment_test.go new file mode 100644 index 0000000000..0aafd22bd0 --- /dev/null +++ b/internal/raftsim/assignment_test.go @@ -0,0 +1,380 @@ +package raftsim + +import ( + "crypto/ed25519" + "fmt" + "math/rand" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsim/model" + "github.com/anishathalye/porcupine" + "go.etcd.io/raft/v3" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +type assignmentNode struct { + core *raftcore.Core + disk raftcore.RecoveredState + rows map[cs.RecordKey][]byte + meta cs.ApplicationMeta +} +type assignmentGroup struct { + responders map[string]int + trace []string + t *testing.T + seed uint64 + nodes [3]assignmentNode + queue []raftcore.Outbound + rng *rand.Rand + replies map[string]cs.Result + events int + cut string + cutDone bool + worker string + keys map[string]ed25519.PrivateKey + spec cs.Assign +} + +func assignmentKey(id string) ed25519.PrivateKey { + var seed [32]byte + copy(seed[:], id) + return ed25519.NewKeyFromSeed(seed[:]) +} +func newAssignmentGroup(t *testing.T, seed uint64) *assignmentGroup { + g := &assignmentGroup{t: t, seed: seed, rng: rand.New(rand.NewSource(int64(seed))), replies: map[string]cs.Result{}, responders: map[string]int{}, keys: map[string]ed25519.PrivateKey{}} + for _, id := range []string{"root", "owner", "worker", "other"} { + g.keys[id] = assignmentKey(id) + } + g.worker = cs.DeviceIDOfKey(g.keys["worker"].Public().(ed25519.PublicKey)) + cert := cs.WorkerCertificate{V: 1, Identity: cs.IdentityID(g.keys["root"].Public().(ed25519.PublicKey)), Cluster: "assignment-sim", Key: g.keys["worker"].Public().(ed25519.PublicKey), Made: 1, Expires: 100000, Cells: []string{"child"}} + b, _ := cs.WorkerCertificateBytes(cert) + cert.Signature = ed25519.Sign(g.keys["root"], b) + g.spec = cs.Assign{ID: "opaque-run", Revision: 1, Cell: "child", ParentCell: "parent", Expect: cs.LeaseExpectation{Device: "owner", Fence: 1}, Target: g.worker, Incarnation: 1, Expires: 90000, SealedPayload: []byte{0x81, 0x54}} + for i := range g.nodes { + n := &g.nodes[i] + n.rows = map[cs.RecordKey][]byte{} + n.meta = cs.ApplicationMeta{Cluster: "assignment-sim", Now: 1000} + put := func(k cs.RecordKey, v any) { n.rows[k], _ = cs.Encode(v) } + put(cs.IdentityKey(), cs.IdentityRec{V: 1, Identity: cert.Identity, SigningKey: g.keys["root"].Public().(ed25519.PublicKey)}) + for _, id := range []string{"owner", "other"} { + put(cs.AuthorityKey(id), cs.Authority{Cluster: "assignment-sim", Key: g.keys[id].Public().(ed25519.PublicKey), Personal: true}) + put(cs.DeviceKey(id), cs.Device{V: 1}) + } + put(cs.AuthorityKey(g.worker), cs.Authority{Cluster: "assignment-sim", Key: cert.Key, Worker: &cert}) + put(cs.DeviceKey(g.worker), cs.Device{V: 1}) + put(cs.WorkerKey(g.worker), cs.WorkerRecord{Incarnation: 1, AvailableUntil: 91000, Grants: map[string][]byte{"child": {0x80}}}) + child := cs.Created(cs.CellInit{Head: "head"}, "owner", 1000) + child.ParentCell = "parent" + put(cs.CellKey("child"), child) + put(cs.CellKey("parent"), cs.Created(cs.CellInit{Head: "parent-head"}, "owner", 1000)) + put(cs.AssignmentKey(g.spec.ID), cs.Assignment{Spec: g.spec, Status: "pending"}) + put(cs.AssignmentSlotKey("child"), cs.AssignmentSlot{Ref: (cs.Assignment{Spec: g.spec}).Ref(), Active: true}) + var err error + n.core, err = raftcore.New(g.config(i, true), n.disk) + if err != nil { + t.Fatal(err) + } + } + for i := range g.nodes { + g.step(i, raftcore.Poll{}) + } + g.pump() + for k := 0; k < 96 && g.leader() < 0; k++ { + for i := range g.nodes { + g.step(i, raftcore.Tick{}) + } + g.pump() + } + if g.leader() < 0 { + t.Fatal("R18 no elected leader", seed) + } + return g +} +func (g *assignmentGroup) config(i int, bootstrap bool) raftcore.Config { + c := raftcore.Config{ClusterUUID: "assignment-sim", NodeID: uint64(i + 1), Seed: g.seed*7 + uint64(i+1), Protocol: 1} + if bootstrap { + c.Bootstrap = []uint64{1, 2, 3} + } + return c +} +func (g *assignmentGroup) leader() int { + for i := range g.nodes { + e, err := g.nodes[i].core.Step(raftcore.Poll{}) + if err != nil { + g.t.Fatal(err) + } + g.accept(i, e) + if e.Status.Role == raft.StateLeader { + return i + } + } + return -1 +} +func (g *assignmentGroup) step(i int, in raftcore.Input) { + g.events++ + g.trace = append(g.trace, fmt.Sprintf("%d node=%d input=%T", g.events, i, in)) + if g.events > 2000 { + g.t.Fatal("R18 event budget", g.seed) + } + e, err := g.nodes[i].core.Step(in) + if err != nil { + g.t.Fatal(g.seed, err) + } + g.accept(i, e) +} +func (g *assignmentGroup) restart(i int) { + n := &g.nodes[i] + var err error + n.core, err = raftcore.New(g.config(i, false), n.disk) + if err != nil { + g.t.Fatal(err) + } + g.step(i, raftcore.Poll{}) +} +func (g *assignmentGroup) accept(i int, e raftcore.Effects) { + n := &g.nodes[i] + g.queue = append(g.queue, e.Send...) + if e.Persist != nil { + p := e.Persist + if !g.cutDone && g.cut == "before-persist" { + g.cutDone = true + g.restart(i) + return + } + if !raft.IsEmptyHardState(p.HardState) { + n.disk.HardState = proto.Clone(p.HardState).(*pb.HardState) + } + if len(p.Entries) > 0 { + first := p.Entries[0].GetIndex() + kept := 0 + for _, v := range n.disk.Entries { + if v.GetIndex() < first { + kept++ + } + } + n.disk.Entries = n.disk.Entries[:kept] + for _, v := range p.Entries { + n.disk.Entries = append(n.disk.Entries, proto.Clone(v).(*pb.Entry)) + } + } + if !g.cutDone && g.cut == "after-persist" { + g.cutDone = true + g.restart(i) + return + } + g.step(i, raftcore.Persisted{BatchID: p.ID}) + } + if e.Apply != nil { + a := e.Apply + if !g.cutDone && g.cut == "before-apply" { + g.cutDone = true + g.restart(i) + return + } + results := map[string]cs.Result{} + for _, entry := range a.Entries { + if entry.GetType() != pb.EntryNormal || len(entry.Data) == 0 { + continue + } + n.meta.Index = entry.GetIndex() + n.meta.Bytes = 0 + for k, b := range n.rows { + n.meta.Bytes += cs.RecordBytes(k, b) + } + out, err := cs.Evaluate(entry.Data, n.meta, func(k cs.RecordKey) ([]byte, error) { return n.rows[k], nil }) + if err != nil { + g.t.Fatal(g.seed, err) + } + for _, w := range out.Writes { + n.rows[w.RecordKey] = w.Data + } + n.meta.Now = out.Now + n.meta.Version = out.Version + c, _ := cs.DecodeCommand(entry.Data) + results[c.Session] = out.Result + } + n.disk.Applied = a.Index + n.disk.ConfState = proto.Clone(a.ConfState).(*pb.ConfState) + if !g.cutDone && (g.cut == "after-apply" || g.cut == "before-reply") { + g.cutDone = true + g.restart(i) + return + } + g.step(i, raftcore.Applied{BatchID: a.ID}) + for id, r := range results { + if g.responders[id] == i { + g.replies[id] = r + } + } + } +} +func (g *assignmentGroup) pump() { + for len(g.queue) > 0 { + j := g.rng.Intn(len(g.queue)) + m := g.queue[j] + g.queue = append(g.queue[:j], g.queue[j+1:]...) + g.step(int(m.To)-1, raftcore.PeerMessage{Bytes: m.Bytes}) + } +} +func (g *assignmentGroup) command(kind string) cs.Command { + ref := (cs.Assignment{Spec: g.spec}).Ref() + m := cs.Mutation{} + actor := "owner" + inc := uint64(0) + switch kind { + case "claim": + actor = "worker" + inc = 1 + m.ClaimAssignment = &cs.ClaimAssignment{Ref: ref, Expect: g.spec.Expect} + case "acquire": + actor = "other" + m.Acquire = &cs.AcquireCell{Cell: "child", Force: true, Expect: g.spec.Expect} + case "cancel": + m.CancelAssignment = &cs.CancelAssignment{Ref: ref} + case "reassign": + v := g.spec + v.Revision = 2 + m.Reassign = &v + case "incarnation": + actor = "worker" + inc = 1 + m.RegisterWorker = &cs.RegisterWorker{Previous: 1, Incarnation: 2} + case "drain": + m.DrainWorker = &cs.DrainWorker{Worker: g.worker, Incarnation: 1} + case "revoke": + m.Revoke = &cs.DeviceID{Device: g.worker} + case "parent_take": + actor = "other" + m.Acquire = &cs.AcquireCell{Cell: "parent", Force: true, Expect: cs.LeaseExpectation{Device: "owner", Fence: 1}} + case "finish": + actor = "worker" + inc = 1 + m.FinishAssignment = &cs.FinishAssignment{Ref: ref, Fence: 2, Head: "head"} + } + id := actor + if actor == "worker" { + id = g.worker + } + c, err := cs.Sign(cs.Command{Schema: 1, Cluster: "assignment-sim", Actor: id, WorkerIncarnation: inc, Session: kind, Sequence: 1, Now: 1001, Change: m}, g.keys[actor]) + if err != nil { + g.t.Fatal(err) + } + return c +} +func (g *assignmentGroup) submit(c cs.Command) { + b, err := cs.EncodeCommand(c) + if err != nil { + g.t.Fatal(err) + } + i := g.leader() + if i < 0 { + g.t.Fatal("no leader before proposal") + } + g.responders[c.Session] = i + g.step(i, raftcore.Propose{Command: b}) +} + +// R18/R19 run the production signed evaluator behind three REAL cores and +// durable Ready/Apply completions. Seeded reordering and crash cuts preserve +// unknown operations in one independent Porcupine history, never dropping them. +func TestR18R19AssignmentCoreRaceAndCrashCampaign(t *testing.T) { + for seed := 0; seed < seedCount(t); seed++ { + t.Run(fmt.Sprintf("seed-%04d", seed), func(t *testing.T) { + g := newAssignmentGroup(t, uint64(seed)) + t.Cleanup(func() { + if t.Failed() { + t.Logf("R18 replay seed=%d flush/event trace=%v", seed, g.trace) + } + }) + kinds := []string{"acquire", "cancel", "reassign", "incarnation", "drain", "revoke", "parent_take"} + other := kinds[seed%len(kinds)] + first, second := g.command("claim"), g.command(other) + if g.rng.Intn(2) == 1 { + first, second = second, first + } + cuts := []string{"", "before-persist", "after-persist", "before-apply", "after-apply", "before-reply"} + g.cut = cuts[seed%len(cuts)] + // Both invocations precede either reply. A cut may remove a proposal, so + // bounded retransmission retries the SAME durable sequence after recovery. + g.submit(first) + g.pump() + for n := 0; n < 96; n++ { + if g.leader() >= 0 { + break + } + for i := range g.nodes { + g.step(i, raftcore.Tick{}) + } + g.pump() + } + g.submit(second) + g.pump() + for _, c := range []cs.Command{first, second} { + for n := 0; n < 96; n++ { + if _, ok := g.replies[c.Session]; ok { + break + } + for i := range g.nodes { + g.step(i, raftcore.Tick{}) + } + g.pump() + if g.leader() >= 0 { + g.submit(c) + g.pump() + } + } + } + history := []model.AssignmentOperation{} + for j, c := range []cs.Command{first, second} { + r, ok := g.replies[c.Session] + if !ok { + t.Fatal("R18 retry never resolved", seed, c.Session) + } + history = append(history, model.AssignmentOperation{ID: j, Call: 1, Return: 3, Input: model.AssignmentCall{Kind: c.Session}, Code: string(r.Code)}) + } + if other == "parent_take" { + finish := g.command("finish") + g.submit(finish) + g.pump() + r, ok := g.replies[finish.Session] + if !ok { + t.Fatal("R19 finish did not commit") + } + history = append(history, model.AssignmentOperation{ID: 2, Call: 4, Return: 5, Input: model.AssignmentCall{Kind: "finish"}, Code: string(r.Code)}) + for i := range g.nodes { + var p cs.Cell + _ = cs.Decode(g.nodes[i].rows[cs.CellKey("parent")], &p) + if p.Lease.Fence != 2 || p.Head != "parent-head" { + t.Fatal("R19 finish changed parent's head/fence") + } + } + } + initial := model.AssignmentState{Holder: "owner", Fence: 1, Status: "pending", Revision: 1, Incarnation: 1, Available: true, ParentFence: 1} + if got := model.CheckAssignments(initial, history); got != porcupine.Ok { + t.Fatalf("R18 seed %d history %s: %+v", seed, got, history) + } + // A lost successful reply remains unknown in a second checked history. + unknown := append([]model.AssignmentOperation(nil), history...) + unknown[0].Unknown = true + if model.CheckAssignments(initial, unknown) != porcupine.Ok { + t.Fatal("R18 unknown outcome dropped") + } + for i := range g.nodes { + g.restart(i) + } + g.pump() + for i := range g.nodes { + var c cs.Cell + if cs.Decode(g.nodes[i].rows[cs.CellKey("child")], &c) != nil { + t.Fatal("lost child") + } + if c.Lease.Fence > 2 { + t.Fatal("R18 duplicate claim advanced fence") + } + } + }) + } +} diff --git a/internal/raftsim/model/assignment.go b/internal/raftsim/model/assignment.go new file mode 100644 index 0000000000..6a67d40ba1 --- /dev/null +++ b/internal/raftsim/model/assignment.go @@ -0,0 +1,122 @@ +package model + +import ( + "github.com/anishathalye/porcupine" + "reflect" + "time" +) + +// AssignmentState is an independent small run/lease model. Literal protocol +// values here intentionally do not import the production rules or constants. +type AssignmentState struct { + Holder string + Fence uint64 + Status string + Revision uint64 + Incarnation uint64 + Available bool + Revoked bool + ParentFence uint64 +} +type AssignmentCall struct{ Kind string } +type AssignmentOperation struct { + ID int + Call, Return int64 + Input AssignmentCall + Code string + Unknown bool +} + +func AssignmentStep(s AssignmentState, c AssignmentCall) (AssignmentState, string) { + switch c.Kind { + case "claim": + if s.Revoked { + return s, "revoked" + } + if s.Incarnation != 1 || !s.Available || s.Revision != 1 || s.Status != "pending" { + return s, "conflict" + } + if s.Holder != "owner" || s.Fence != 1 { + return s, "lease_held" + } + s.Holder = "worker" + s.Fence++ + s.Status = "claimed" + case "acquire": + if s.Holder != "owner" || s.Fence != 1 { + return s, "lease_held" + } + s.Holder = "other" + s.Fence++ + case "cancel": + if s.Revision != 1 { + return s, "conflict" + } + s.Status = "cancelled" + case "reassign": + if s.Status == "cancelled" || s.Status == "finished" || s.Revision != 1 { + return s, "conflict" + } + if s.Holder != "owner" || s.Fence != 1 { + return s, "lease_held" + } + s.Revision++ + s.Status = "pending" + case "incarnation": + s.Incarnation++ + s.Available = false + case "drain": + s.Available = false + case "revoke": + s.Revoked = true + case "parent_take": + s.ParentFence++ + case "finish": + if s.Revoked { + return s, "revoked" + } + if s.Incarnation != 1 || s.Revision != 1 || s.Status != "claimed" { + return s, "conflict" + } + if s.Holder != "worker" || s.Fence != 2 { + return s, "fence_stale" + } + s.Status = "finished" + default: + return s, "unsupported" + } + return s, "ok" +} +func CheckAssignments(initial AssignmentState, history []AssignmentOperation) porcupine.CheckResult { + if len(history) > 100 { + return porcupine.Unknown + } + end := int64(1) + for _, o := range history { + end = max(end, o.Call+1, o.Return+1) + } + ops := make([]porcupine.Operation, 0, len(history)) + for _, o := range history { + if o.Call < 0 || !o.Unknown && o.Return <= o.Call { + return porcupine.Illegal + } + until := o.Return + if o.Unknown { + until = end + } + ops = append(ops, porcupine.Operation{ClientId: o.ID, Call: o.Call, Return: until, Input: o.Input, Output: o}) + } + m := porcupine.NondeterministicModel{Init: func() []any { return []any{initial} }, Equal: func(a, b any) bool { return reflect.DeepEqual(a, b) }, Step: func(state, input, output any) []any { + s := state.(AssignmentState) + next, code := AssignmentStep(s, input.(AssignmentCall)) + o := output.(AssignmentOperation) + if o.Unknown { + return []any{s, next} + } + if code == o.Code { + return []any{next} + } + return nil + }} + return porcupine.CheckOperationsTimeout(m.ToModel(), ops, time.Second) +} diff --git a/internal/raftsim/model/assignment_test.go b/internal/raftsim/model/assignment_test.go new file mode 100644 index 0000000000..63c69c1973 --- /dev/null +++ b/internal/raftsim/model/assignment_test.go @@ -0,0 +1,28 @@ +package model + +import ( + "github.com/anishathalye/porcupine" + "testing" +) + +func TestR18AssignmentOracleRejectsTwoSuccessfulOwners(t *testing.T) { + initial := AssignmentState{Holder: "owner", Fence: 1, Status: "pending", Revision: 1, Incarnation: 1, Available: true, ParentFence: 1} + history := []AssignmentOperation{{ID: 1, Call: 1, Return: 3, Input: AssignmentCall{Kind: "claim"}, Code: "ok"}, {ID: 2, Call: 1, Return: 3, Input: AssignmentCall{Kind: "acquire"}, Code: "ok"}} + if CheckAssignments(initial, history) != porcupine.Illegal { + t.Fatal("R18 oracle accepted two successful owners") + } + history[1].Code = "lease_held" + if CheckAssignments(initial, history) != porcupine.Ok { + t.Fatal("R18 oracle rejected legitimate race") + } + history[0].Call = 4 + history[0].Return = 5 + if CheckAssignments(initial, history) != porcupine.Illegal { + t.Fatal("R18 real-time order was dropped") + } + history[0].Call = 1 + history[0].Unknown = true + if CheckAssignments(initial, history) != porcupine.Ok { + t.Fatal("R18 unknown commit was dropped") + } +} From 7a9222075e8aaf592c6c688051831197d3c4bb2f Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 04:55:05 -0400 Subject: [PATCH 038/103] Keep worker execution behind staged handoff and durable local intent. Reuse selected content, ordinary upload and handoff dependencies while retaining original signed retries and refusing automatic tool replay after interruption or reassignment. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/raftabi/abi.go | 2 + internal/raftabi/worker.go | 30 ++ internal/raftdirectory/assignment.go | 37 ++ internal/raftdirectory/assignment_test.go | 327 ++++++++++++++++++ internal/syncsetup/worker.go | 263 ++++++++++++++ internal/syncsetup/worker_test.go | 225 ++++++++++++ relay/hosted/src/raft/worker.js | 9 + .../test/raft/assignment-workerd.test.js | 50 +++ relay/hosted/test/raft/assignment.test.js | 115 ++++++ 9 files changed, 1058 insertions(+) create mode 100644 internal/raftdirectory/assignment.go create mode 100644 internal/raftdirectory/assignment_test.go create mode 100644 internal/syncsetup/worker.go create mode 100644 internal/syncsetup/worker_test.go create mode 100644 relay/hosted/test/raft/assignment-workerd.test.js create mode 100644 relay/hosted/test/raft/assignment.test.js diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index 8450e05fad..9d1cb2a35b 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -164,6 +164,8 @@ func (r *Runtime) Call(b []byte) (out []byte) { func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { e := encoder{} switch op { + case 28: + return assignmentRead(d) case 27: return workerRead(d) case 23: diff --git a/internal/raftabi/worker.go b/internal/raftabi/worker.go index 01f5bf7b63..95318251a3 100644 --- a/internal/raftabi/worker.go +++ b/internal/raftabi/worker.go @@ -27,3 +27,33 @@ func workerRead(d *decoder) ([]byte, error) { e.blob(raw) return e.b, nil } + +func assignmentRead(d *decoder) ([]byte, error) { + cluster, worker, incarnation, now := d.str(), d.str(), d.u64(), d.u64() + var ref clusterstate.AssignmentRef + if clusterstate.Decode(d.blob(), &ref) != nil { + return nil, errFrame + } + rows := map[clusterstate.RecordKey][]byte{} + for _, k := range clusterstate.AssignmentReadKeys(worker, ref) { + b := d.blob() + if len(b) == 0 { + b = nil + } + rows[k] = b + } + if d.end() != nil || now > clusterstate.MaxSafeInteger { + return nil, errFrame + } + result, err := clusterstate.ReadWorkerAssignment(cluster, worker, incarnation, ref, int64(now), func(k clusterstate.RecordKey) ([]byte, error) { return rows[k], nil }) + if err != nil { + return nil, err + } + raw, err := clusterstate.Encode(result) + if err != nil { + return nil, err + } + e := encoder{} + e.blob(raw) + return e.b, nil +} diff --git a/internal/raftdirectory/assignment.go b/internal/raftdirectory/assignment.go new file mode 100644 index 0000000000..50aef990be --- /dev/null +++ b/internal/raftdirectory/assignment.go @@ -0,0 +1,37 @@ +package raftdirectory + +import ( + "context" + "errors" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftstore" +) + +// AuthorizedWorkerAssignment runs inside the actor queue AFTER current ingress +// time commits and a fresh quorum ReadIndex passes. This helper cannot turn a +// follower's local state into permission to execute work. +func AuthorizedWorkerAssignment(ctx context.Context, store *raftstore.SQLite, cluster, worker string, incarnation uint64, ref cs.AssignmentRef) (cs.Assignment, cs.CellView, error) { + app, _, err := store.ApplicationView(ctx) + if err != nil { + return cs.Assignment{}, cs.CellView{}, err + } + source := cs.AssignmentReadKeys(worker, ref) + keys := make([]raftstore.Key, len(source)) + for i, k := range source { + keys[i] = raftstore.Key{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key} + } + rows, _, err := store.Read(ctx, app.Applied, keys) + if err != nil { + return cs.Assignment{}, cs.CellView{}, err + } + r, err := cs.ReadWorkerAssignment(cluster, worker, incarnation, ref, app.LogicalTime, func(k cs.RecordKey) ([]byte, error) { + return rows[raftstore.Key{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key}], nil + }) + if err != nil { + return cs.Assignment{}, cs.CellView{}, err + } + if r.Code != cs.OK { + return cs.Assignment{}, cs.CellView{}, errors.New("raftdirectory: assignment unavailable") + } + return *r.Assignment, *r.View, nil +} diff --git a/internal/raftdirectory/assignment_test.go b/internal/raftdirectory/assignment_test.go new file mode 100644 index 0000000000..0c3904635d --- /dev/null +++ b/internal/raftdirectory/assignment_test.go @@ -0,0 +1,327 @@ +package raftdirectory_test + +import ( + "bytes" + "context" + "crypto/ed25519" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/blobstore" + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/cellsync" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/handoff" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +type assignmentLocal struct{} + +func (assignmentLocal) Dirty(context.Context, cell.Cell) (bool, error) { return false, nil } +func (assignmentLocal) Unpublished(cell.Cell, string) (string, uint32, error) { return "", 0, nil } +func (assignmentLocal) Seal(context.Context, cell.Cell) (string, uint32, error) { + return "", 0, errors.New("unexpected seal") +} +func (assignmentLocal) Follow(context.Context, cell.Cell, cell.Cell) error { return nil } + +type assignmentDir struct { + directory.Client + read func(context.Context, string) (directory.CellView, error) + release func(context.Context, string, uint64) error + publish func(context.Context, string, directory.Publish) (directory.CellView, error) +} + +func (d assignmentDir) Cell(ctx context.Context, id string) (directory.CellView, error) { + return d.read(ctx, id) +} +func (d assignmentDir) Release(ctx context.Context, id string, f uint64) error { + return d.release(ctx, id, f) +} +func (d assignmentDir) Publish(ctx context.Context, id string, p directory.Publish) (directory.CellView, error) { + return d.publish(ctx, id, p) +} + +// R18–R21 drive two disposable homes through a loopback coordination relay, +// three FULL-synchronous stores, the real Taker, Publisher and Fetcher. The +// tiny scripted worker never receives the identity's private/content keys. +func TestR18R19R20R21ScriptedWorkerHandoff(t *testing.T) { + now := int64(1000) + g := newGroup(t, func() time.Time { return time.UnixMilli(now) }) + serial := 0 + signed := func(actor string, k ed25519.PrivateKey, inc uint64, m cs.Mutation) cs.Command { + serial++ + c, e := cs.Sign(cs.Command{Schema: 1, Cluster: cluster, Actor: actor, WorkerIncarnation: inc, Session: hex.EncodeToString([]byte{byte(serial >> 8), byte(serial)}), Sequence: 1, Now: now, Change: m}, k) + must(t, e) + return c + } + owner := func(m cs.Mutation) cs.Result { + r, e := g.submit(signed("a", key("a"), 0, m)) + must(t, e) + if r.Code != cs.OK { + t.Fatal(r) + } + return r + } + workerKey := key("scripted-worker") + id := cs.DeviceIDOfKey(workerKey.Public().(ed25519.PublicKey)) + cert := cs.WorkerCertificate{V: 1, Identity: cs.IdentityID(key("root").Public().(ed25519.PublicKey)), Cluster: cluster, Key: workerKey.Public().(ed25519.PublicKey), Made: 1000, Expires: 100000, Cells: []string{"child-one", "child-two"}} + b, e := cs.WorkerCertificateBytes(cert) + must(t, e) + cert.Signature = ed25519.Sign(key("root"), b) + // A local credential contains only this worker's seed and PUBLIC delegation. + doc := encoded(t, struct { + V uint16 `json:"v"` + Seed string `json:"worker_seed"` + Certificate cs.WorkerCertificate `json:"certificate"` + }{1, hex.EncodeToString(workerKey.Seed()), cert}) + credential, e := identity.UnmarshalWorker(doc, key("root").Public().(ed25519.PublicKey), cluster, time.UnixMilli(now)) + must(t, e) + owner(cs.Mutation{EnrollWorker: &cs.EnrollWorker{Certificate: cert, Grants: map[string][]byte{"child-one": {0x81}, "child-two": {0x82}}}}) + r, e := g.submit(signed(id, workerKey, 0, cs.Mutation{RegisterWorker: &cs.RegisterWorker{Incarnation: 1}})) + must(t, e) + if r.Code != cs.OK { + t.Fatal(r) + } + beat := cs.WorkerHeartbeat{Worker: id, Incarnation: 1, Sequence: 1, SeenAt: now} + b, e = cs.WorkerHeartbeatBytes(cluster, beat) + must(t, e) + beat.Signature = ed25519.Sign(workerKey, b) + owner(cs.Mutation{WorkerBeatBatch: &cs.WorkerBeatBatch{Items: []cs.WorkerHeartbeat{beat}}}) + homeA, homeW := t.TempDir(), t.TempDir() + engA := cellsync.NewFakeEngine(filepath.Join(homeA, "frames")) + engW := cellsync.NewFakeEngine(filepath.Join(homeW, "frames")) + store := blobstore.NewMemory() + template, e := cell.CreateIn(t.TempDir(), cell.Options{Class: cell.FilesOnly}) + must(t, e) + meta, e := os.ReadFile(filepath.Join(template.Root, cell.MetaPath)) + must(t, e) + parent := owner(cs.Mutation{Create: &cs.CreateCell{V1: true, Cell: "parent", Init: cs.CellInit{Head: "parent-head"}}}) + var assignments []cs.Assignment + for _, cellID := range cert.Cells { + at := cell.Cell{ID: cellID, Root: filepath.Join(homeA, cellID)} + head := engA.Seal(at, map[string]string{cell.MetaPath: string(meta), "input.txt": "source files"}) + pub := cellsync.Publisher{Engine: engA, Store: store, Dir: directory.NewRaft(g).For("a")} + _, e = pub.Upload(ctx, at, head) + must(t, e) + owner(cs.Mutation{Create: &cs.CreateCell{V1: true, Cell: cellID, Init: cs.CellInit{Head: head, ParentCell: "parent"}}}) + view, e := directory.NewRaft(g).For("a").Cell(ctx, cellID) + must(t, e) + // Each parallel task uses a separate existing child and its own lease. + spec := cs.Assign{ID: "run-" + cellID, Revision: 1, Cell: cellID, ParentCell: "parent", Expect: cs.LeaseExpectation{Device: "a", Fence: view.Cell.Lease.Fence}, Target: id, Incarnation: 1, Expires: 90000, SealedPayload: []byte{0x80}} + assignments = append(assignments, *owner(cs.Mutation{Assign: &spec}).Assignment) + } + relay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, err := io.ReadAll(io.LimitReader(r.Body, cs.MaxCommandBytes+1)) + if err != nil { + http.Error(w, "read", 400) + return + } + c, err := cs.DecodeCommand(raw) + if err != nil { + http.Error(w, "command", 400) + return + } + g.mu.Lock() + result, err := g.submit(c) + g.mu.Unlock() + if err != nil { + http.Error(w, "unavailable", 503) + return + } + w.Write(encoded(t, result)) + })) + defer relay.Close() + submit := func(ctx context.Context, c cs.Command) (cs.Result, error) { + raw, e := cs.EncodeCommand(c) + if e != nil { + return cs.Result{}, e + } + req, e := http.NewRequestWithContext(ctx, http.MethodPost, relay.URL, bytes.NewReader(raw)) + if e != nil { + return cs.Result{}, e + } + res, e := relay.Client().Do(req) + if e != nil { + return cs.Result{}, e + } + defer res.Body.Close() + b, e := io.ReadAll(res.Body) + var out cs.Result + if e == nil { + e = cs.Decode(b, &out) + } + return out, e + } + current := func(ctx context.Context, ref cs.AssignmentRef) (cs.Assignment, directory.CellView, error) { + g.mu.Lock() + defer g.mu.Unlock() + i, _, e := g.barrier("a") + if e != nil { + return cs.Assignment{}, cs.CellView{}, e + } + return raftdirectory.AuthorizedWorkerAssignment(ctx, g.stores[i], cluster, id, 1, ref) + } + dir := assignmentDir{read: func(ctx context.Context, cell string) (directory.CellView, error) { + g.mu.Lock() + defer g.mu.Unlock() + i, _, e := g.barrier("a") + if e != nil { + return directory.CellView{}, e + } + v, _, e := raftdirectory.AuthorizedWorkerRead(ctx, g.stores[i], cluster, id, cell, 1) + return v, e + }, release: func(ctx context.Context, cell string, f uint64) error { + r, e := submit(ctx, signed(id, workerKey, 1, cs.Mutation{Release: &cs.ReleaseCell{Cell: cell, Fence: f}})) + if e == nil && r.Code != cs.OK { + return directory.ErrFenceStale + } + return e + }} + dir.publish = func(ctx context.Context, cell string, p directory.Publish) (directory.CellView, error) { + // Receipt creation is a restricted ingress operation after confirming all + // uploaded frames and the indexed head, never a worker-minted attestation. + loc, e := store.Locate(ctx, []string{p.Head}) + if e != nil { + return directory.CellView{}, e + } + if _, ok := loc[p.Head]; !ok { + return directory.CellView{}, fmt.Errorf("missing uploaded head") + } + for _, frame := range p.Frames { + if _, e := store.GetFrame(ctx, frame); e != nil { + return directory.CellView{}, e + } + } + receipt := cs.UploadReceipt{Device: id, Cell: cell, WorkerIncarnation: 1, Head: p.Head, HeadRID: p.Head, Frames: p.Frames} + r, e := g.submit(signed("effects", key("effects"), 0, cs.Mutation{RecordUpload: &receipt})) + if e != nil { + return directory.CellView{}, e + } + if r.Code != cs.OK { + return directory.CellView{}, directory.ErrUnauthorized + } + r, e = submit(ctx, signed(id, workerKey, 1, cs.Mutation{Publish: &cs.PublishCell{Cell: cell, Publish: p, Receipt: cs.UploadID(receipt)}})) + if e == nil && r.Code != cs.OK { + return directory.CellView{}, directory.ErrFenceStale + } + if e != nil { + return directory.CellView{}, e + } + return *r.View, nil + } + taker := handoff.Taker{Dir: dir, Fetch: &cellsync.Fetcher{Engine: engW, Store: store, Inbox: func(c cell.Cell) string { return filepath.Join(homeW, "inbox", c.ID) }}, Local: assignmentLocal{}, RootFor: func(id string) string { return filepath.Join(homeW, id) }} + worker := syncsetup.WorkerAdapter{Credential: credential, Incarnation: 1, StateDir: filepath.Join(homeW, "attempts"), Taker: taker, Submit: submit, Current: current, Now: func() int64 { return now }} + invocations := 0 + for _, a := range assignments { + must(t, worker.Run(ctx, a, func(ctx context.Context, taken handoff.Taken) (string, error) { + invocations++ + b, e := os.ReadFile(filepath.Join(taken.Cell.Root, "input.txt")) + if e != nil || string(b) != "source files" { + t.Fatal("R21 incomplete closure", string(b), e) + } + head := engW.Seal(taken.Cell, map[string]string{cell.MetaPath: string(meta), "result.txt": "scripted result"}) + d := taken.Driving + p := cellsync.Publisher{Engine: engW, Store: store, Dir: dir} + p.Store = &assignmentFailStore{Store: store} + if e := p.Publish(ctx, &d, head, cellsync.PublishInfo{}); e == nil { + t.Fatal("R21 failed upload published") + } + _, unchanged, e := current(ctx, a.Ref()) + must(t, e) + if unchanged.Cell.Head != taken.Driving.Head { + t.Fatal("R21 head exposed before upload") + } + p.Store = store + return head, p.Publish(ctx, &d, head, cellsync.PublishInfo{}) + })) + must(t, worker.Run(ctx, a, func(context.Context, handoff.Taken) (string, error) { + t.Fatal("R20 completed worker replayed") + return "", nil + })) + // A fresh home retrieves the ordinary cellsync result, not assignment text. + engResult := cellsync.NewFakeEngine(t.TempDir()) + dest := cell.Cell{ID: a.Spec.Cell, Root: filepath.Join(t.TempDir(), "chat")} + _, v, e := current(ctx, a.Ref()) + must(t, e) + f := cellsync.Fetcher{Engine: engResult, Store: store, Inbox: func(cell.Cell) string { return t.TempDir() }} + must(t, f.Fetch(ctx, dest, v.Cell.Head)) + b, e := os.ReadFile(filepath.Join(dest.Root, "result.txt")) + must(t, e) + if string(b) != "scripted result" { + t.Fatal(string(b)) + } + } + if invocations != 2 { + t.Fatal("R19 parallel child invocation count", invocations) + } + for i, store := range g.stores { + snap, e := store.CreateSnapshot(ctx) + must(t, e) + must(t, store.Compact(ctx, snap.GetMetadata().GetIndex())) + g.drive(i, raftcore.CompactSnapshot{Snapshot: snap}) + } + g.drain() + recoverMembership(t, g) + files, e := filepath.Glob(filepath.Join(homeW, "attempts", "*.json")) + must(t, e) + for _, path := range files { + if strings.HasPrefix(filepath.Base(path), "cell-") { + continue + } + raw, e := os.ReadFile(path) + must(t, e) + var saved struct{ Claim cs.Command } + must(t, json.Unmarshal(raw, &saved)) + r, e := submit(ctx, saved.Claim) + must(t, e) + if r.Code != cs.OK || r.View == nil || r.View.Cell.Lease.Fence != 2 || r.Assignment.Status != "claimed" { + t.Fatal("R3/R17 exact claim after compaction/restart", r) + } + } + after, e := directory.NewRaft(g).For("a").Cell(ctx, "parent") + must(t, e) + if after.Cell.Head != parent.View.Cell.Head || after.Cell.Lease.Fence != 1 { + t.Fatal("R19 worker wrote parent") + } +} + +// R20 retains both completed and incomplete call records without running any +// recovered call. The no-execution rule holds independently of assignment TTL. +func TestR20RetainedWorkerCallRecords(t *testing.T) { + path := filepath.Join(t.TempDir(), "calls.wal") + wal, _, e := cellstore.OpenWAL(path) + must(t, e) + in := cellstore.Intent{V: 2, Tool: "exec", ArgsHash: "opaque", Started: 1000, SideEffect: "external", Brief: "scripted call"} + must(t, wal.Begin(in)) + _, rec, e := cellstore.OpenWAL(path) + must(t, e) + if len(rec.Incomplete) != 1 || rec.Incomplete[0].Brief != "scripted call" { + t.Fatal(rec) + } + raw, e := json.Marshal(rec) + must(t, e) + if !bytes.Contains(raw, []byte("Incomplete")) { + t.Fatal("incomplete display lost") + } +} + +type assignmentFailStore struct{ blobstore.Store } + +func (s *assignmentFailStore) PutFrame(context.Context, []byte) (blobstore.FrameID, error) { + return "", errors.New("injected upload failure") +} diff --git a/internal/syncsetup/worker.go b/internal/syncsetup/worker.go new file mode 100644 index 0000000000..b38e1e0ace --- /dev/null +++ b/internal/syncsetup/worker.go @@ -0,0 +1,263 @@ +package syncsetup + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/filelock" + "github.com/Agent-Field/codeaf/internal/fsreplace" + "github.com/Agent-Field/codeaf/internal/handoff" + "github.com/Agent-Field/codeaf/internal/identity" +) + +var ErrWorkerIncomplete = errors.New("this work may be incomplete; its saved records have been kept") + +// WorkerAdapter is an internal selected-chat door. Its caller supplies scoped +// storage/keys and a quorum-backed directory; it never opens a personal home. +// Submit preserves the signed request and returns only after durable commit. +// Current passes a fresh quorum barrier and checks current worker authority, +// assignment revision/incarnation and the chat fence in the SAME applied view. +type WorkerAdapter struct { + Credential identity.Worker + Incarnation uint64 + StateDir string + Taker handoff.Taker + Submit func(context.Context, cs.Command) (cs.Result, error) + Current func(context.Context, cs.AssignmentRef) (cs.Assignment, directory.CellView, error) + Now func() int64 +} + +type workerAttempt struct { + Ref cs.AssignmentRef `json:"ref"` + Claim cs.Command `json:"claim"` + Started bool `json:"started"` + Done bool `json:"done"` + Finish *cs.Command `json:"finish,omitempty"` +} + +// Run stages through the ordinary handoff preparation/reconciliation path. +// A durable intent precedes Execute, and any restart after that point displays +// incomplete work instead of invoking it again. Execute uses the existing +// call journal and cellsync upload/publish path, returning its published head. +func (w WorkerAdapter) Run(ctx context.Context, a cs.Assignment, execute func(context.Context, handoff.Taken) (string, error)) error { + if w.Now == nil || w.Submit == nil || w.Current == nil || w.Taker.Dir == nil || w.Taker.RootFor == nil || w.Taker.Fetch == nil || w.Taker.Local == nil || execute == nil || w.StateDir == "" || a.Spec.Target != w.Credential.ID() || a.Spec.Incarnation != w.Incarnation || !w.Credential.Certificate.Allows(a.Spec.Cell) { + return directory.ErrUnauthorized + } + if err := os.MkdirAll(w.StateDir, 0700); err != nil { + return err + } + // The key includes routing identity, so neither another worker nor a new + // incarnation can accidentally adopt an old process's execution permission. + key, _ := cs.Encode(struct { + Cluster string + Ref cs.AssignmentRef + }{w.Credential.Certificate.Cluster, a.Ref()}) + sum := sha256.Sum256(key) + path := filepath.Join(w.StateDir, hex.EncodeToString(sum[:])+".json") + cellHash := sha256.Sum256([]byte(w.Credential.Certificate.Cluster + "\x00" + a.Spec.Cell)) + cellPath := filepath.Join(w.StateDir, "cell-"+hex.EncodeToString(cellHash[:])+".json") + lock, err := os.OpenFile(cellPath+".lock", os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return err + } + defer lock.Close() + if err = filelock.Lock(lock, true, true); err != nil { + return err + } + defer filelock.Unlock(lock) + if b, e := os.ReadFile(cellPath); e == nil { + var previous workerAttempt + if json.Unmarshal(b, &previous) != nil || previous.Started && !previous.Done && previous.Ref != a.Ref() { + return ErrWorkerIncomplete + } + } else if !errors.Is(e, os.ErrNotExist) { + return e + } + attempt := workerAttempt{Ref: a.Ref()} + raw, err := os.ReadFile(path) + if err == nil { + if json.Unmarshal(raw, &attempt) != nil || attempt.Ref != a.Ref() { + return ErrWorkerIncomplete + } + if attempt.Done { + return nil + } + if attempt.Started { + // A recorded finish can safely retry coordination, never tool execution. + if attempt.Finish == nil { + return ErrWorkerIncomplete + } + r, e := w.Submit(ctx, *attempt.Finish) + if e != nil { + return e + } + if e = workerResultError(r.Code); e != nil { + return e + } + attempt.Done = true + return errors.Join(saveWorkerAttempt(path, attempt), saveWorkerAttempt(cellPath, attempt)) + } + } else if !errors.Is(err, os.ErrNotExist) { + return err + } else { + attempt.Claim, err = w.command("claim", a.Ref(), cs.Mutation{ClaimAssignment: &cs.ClaimAssignment{Ref: a.Ref(), Expect: a.Spec.Expect}}) + if err != nil { + return err + } + if err = saveWorkerAttempt(path, attempt); err != nil { + return err + } + } + taker := w.Taker + taker.Dir = workerClaimDirectory{Client: taker.Dir, claim: func(ctx context.Context, id string, o directory.AcquireOpts) (directory.CellView, error) { + if id != a.Spec.Cell || o.Expect == nil { + return directory.CellView{}, directory.ErrLeaseHeld + } + r, e := w.Submit(ctx, attempt.Claim) + if e != nil { + return directory.CellView{}, e + } + if e = workerResultError(r.Code); e != nil { + return directory.CellView{}, e + } + if r.View == nil || r.Assignment == nil || r.Assignment.Ref() != a.Ref() || r.Assignment.Status != "claimed" { + return directory.CellView{}, directory.ErrUnauthorized + } + current, view, e := w.Current(ctx, a.Ref()) + if e != nil { + return directory.CellView{}, e + } + if current.Ref() != a.Ref() || current.Status != "claimed" || current.Fence != r.View.Cell.Lease.Fence || view.Cell.Lease.Device != w.Credential.ID() || view.Cell.Lease.Fence != current.Fence { + return directory.CellView{}, directory.ErrFenceStale + } + return view, nil + }} + taken, err := taker.TakeSeen(ctx, a.Spec.Cell, nil) + if err != nil { + return err + } + current, view, err := w.Current(ctx, a.Ref()) + if err != nil { + return err + } + if current.Ref() != a.Ref() || current.Status != "claimed" || current.Fence != taken.Driving.Fence || view.Cell.Lease.Device != w.Credential.ID() || view.Cell.Lease.Fence != current.Fence || view.Cell.Head != taken.Driving.Head { + return directory.ErrFenceStale + } + // Reassignment recovers existing records; it never instructs a replacement + // process to replay tools whose completion may be missing elsewhere. + if a.Spec.Revision > 1 { + return ErrWorkerIncomplete + } + attempt.Started = true + if err = saveWorkerAttempt(path, attempt); err != nil { + return err + } + if err = saveWorkerAttempt(cellPath, attempt); err != nil { + return err + } + head, err := execute(ctx, taken) + if err != nil { + return err + } + finish, err := w.command("finish", a.Ref(), cs.Mutation{FinishAssignment: &cs.FinishAssignment{Ref: a.Ref(), Fence: taken.Driving.Fence, Head: head}}) + if err != nil { + return err + } + attempt.Finish = &finish + if err = saveWorkerAttempt(path, attempt); err != nil { + return err + } + r, err := w.Submit(ctx, finish) + if err != nil { + return err + } + if err = workerResultError(r.Code); err != nil { + return err + } + attempt.Done = true + return errors.Join(saveWorkerAttempt(path, attempt), saveWorkerAttempt(cellPath, attempt)) +} +func (w WorkerAdapter) command(kind string, ref cs.AssignmentRef, m cs.Mutation) (cs.Command, error) { + b, _ := cs.Encode(ref) + h := sha256.Sum256(b) + c := cs.Command{Schema: cs.SchemaVersion, Cluster: w.Credential.Certificate.Cluster, Actor: w.Credential.ID(), WorkerIncarnation: w.Incarnation, Session: "worker-" + kind + "-" + hex.EncodeToString(h[:]), Sequence: 1, Now: w.Now(), Change: m} + var err error + c.RequestDigest, err = cs.Digest(c) + if err != nil { + return c, err + } + signed, err := cs.SigningBytes(c) + if err != nil { + return c, err + } + c.Signature = w.Credential.Sign(signed) + return c, nil +} + +type workerClaimDirectory struct { + directory.Client + claim func(context.Context, string, directory.AcquireOpts) (directory.CellView, error) +} + +func (d workerClaimDirectory) Acquire(ctx context.Context, id string, o directory.AcquireOpts) (directory.CellView, error) { + return d.claim(ctx, id, o) +} +func workerResultError(code cs.Code) error { + switch code { + case cs.OK: + return nil + case cs.LeaseHeld: + return directory.ErrLeaseHeld + case cs.FenceStale: + return directory.ErrFenceStale + case cs.HeadMoved: + return directory.ErrHeadMoved + case cs.Revoked: + return directory.ErrRevoked + case cs.Unauthorized: + return directory.ErrUnauthorized + default: + return fmt.Errorf("worker coordination: %s", code) + } +} + +// The execution intent uses a full local durability boundary, independent of +// the ordinary call-record batching window. Failure retains the prior record. +func saveWorkerAttempt(path string, v workerAttempt) error { + b, err := json.Marshal(v) + if err != nil { + return err + } + f, err := os.CreateTemp(filepath.Dir(path), ".worker-") + if err != nil { + return err + } + defer os.Remove(f.Name()) + if err = f.Chmod(0600); err == nil { + _, err = f.Write(b) + } + if err == nil { + err = f.Sync() + } + err = errors.Join(err, f.Close()) + if err != nil { + return err + } + if err = fsreplace.Rename(f.Name(), path); err != nil { + return err + } + d, err := os.Open(filepath.Dir(path)) + if err != nil { + return err + } + defer d.Close() + return d.Sync() +} diff --git a/internal/syncsetup/worker_test.go b/internal/syncsetup/worker_test.go new file mode 100644 index 0000000000..34df778688 --- /dev/null +++ b/internal/syncsetup/worker_test.go @@ -0,0 +1,225 @@ +package syncsetup + +import ( + "context" + "crypto/ed25519" + "encoding/hex" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cellstore" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/handoff" + "github.com/Agent-Field/codeaf/internal/identity" +) + +type workerTestFetch struct{ meta []byte } + +func (f workerTestFetch) Fetch(_ context.Context, c cell.Cell, head string) error { + if e := os.MkdirAll(filepath.Join(c.Root, ".cell"), 0700); e != nil { + return e + } + return os.WriteFile(filepath.Join(c.Root, cell.MetaPath), f.meta, 0600) +} +func (f workerTestFetch) Complete(ctx context.Context, c cell.Cell, head string) error { + return f.Fetch(ctx, c, head) +} + +type workerTestLocal struct{} + +func (workerTestLocal) Dirty(context.Context, cell.Cell) (bool, error) { return false, nil } +func (workerTestLocal) Unpublished(cell.Cell, string) (string, uint32, error) { return "", 0, nil } +func (workerTestLocal) Seal(context.Context, cell.Cell) (string, uint32, error) { + return "", 0, errors.New("unexpected seal") +} +func (workerTestLocal) Follow(context.Context, cell.Cell, cell.Cell) error { return nil } + +type workerTestDirectory struct { + directory.Client + read func() directory.CellView +} + +func (d workerTestDirectory) Cell(context.Context, string) (directory.CellView, error) { + return d.read(), nil +} +func (d workerTestDirectory) Release(context.Context, string, uint64) error { return nil } + +// R20/C28 kill the real worker process at each execution boundary. Only the +// pre-execution claim can resume automatically; every recorded execution intent +// is retained and never causes a second invocation, including after reassignment. +func TestR20WorkerProcessCutsNeverReplayTools(t *testing.T) { + for _, phase := range []string{"claim", "intent", "execution", "seal", "upload", "publish", "finish"} { + t.Run(phase, func(t *testing.T) { + dir := t.TempDir() + run := func(mode string) string { + cmd := exec.Command(os.Args[0], "-test.run=^TestR20WorkerProcessHelper$", "-test.v") + cmd.Env = append(os.Environ(), "CODEAF_P16_HELPER="+dir, "CODEAF_P16_PHASE="+phase, "CODEAF_P16_MODE="+mode) + out, e := cmd.CombinedOutput() + if mode == "cut" { + var exit *exec.ExitError + if !errors.As(e, &exit) || exit.ExitCode() != 41 { + t.Fatalf("cut %s: %v %s", phase, e, out) + } + } else if e != nil { + t.Fatalf("recover %s: %v %s", phase, e, out) + } + return string(out) + } + run("cut") + out := run("recover") + count, e := os.ReadFile(filepath.Join(dir, "invocations")) + if !errors.Is(e, os.ErrNotExist) && e != nil { + t.Fatal(e) + } + want := 1 + if phase == "intent" { + want = 0 + } + if len(count) != want { + t.Fatalf("R20 %s invoked %d times, want %d", phase, len(count), want) + } + if phase != "claim" && phase != "finish" && !strings.Contains(out, ErrWorkerIncomplete.Error()) { + t.Fatal("missing incomplete display", out) + } + }) + } +} +func TestR20WorkerProcessHelper(t *testing.T) { + dir := os.Getenv("CODEAF_P16_HELPER") + if dir == "" { + return + } + phase, mode := os.Getenv("CODEAF_P16_PHASE"), os.Getenv("CODEAF_P16_MODE") + cut := func(at string) { + if mode == "cut" && phase == at { + os.Exit(41) + } + } + var seed [32]byte + seed[0] = 7 + k := ed25519.NewKeyFromSeed(seed[:]) + root := ed25519.NewKeyFromSeed(make([]byte, 32)) + cert := cs.WorkerCertificate{V: 1, Identity: cs.IdentityID(root.Public().(ed25519.PublicKey)), Cluster: "cuts", Key: k.Public().(ed25519.PublicKey), Made: 1, Expires: 10000, Cells: []string{"chat"}} + raw, _ := cs.WorkerCertificateBytes(cert) + cert.Signature = ed25519.Sign(root, raw) + doc, _ := cs.Encode(struct { + V uint16 `json:"v"` + Seed string `json:"worker_seed"` + Certificate cs.WorkerCertificate `json:"certificate"` + }{1, hex.EncodeToString(seed[:]), cert}) + worker, e := identity.UnmarshalWorker(doc, root.Public().(ed25519.PublicKey), "cuts", time.UnixMilli(1000)) + if e != nil { + t.Fatal(e) + } + a := cs.Assignment{Spec: cs.Assign{ID: "opaque", Revision: 1, Cell: "chat", Target: worker.ID(), Incarnation: 1, Expires: 9000, Expect: cs.LeaseExpectation{Device: "owner", Fence: 1}, SealedPayload: []byte{0x80}}, Status: "pending"} + remotePath := filepath.Join(dir, "remote.json") + load := func() (cs.Assignment, directory.CellView) { + b, e := os.ReadFile(remotePath) + if e == nil { + if json.Unmarshal(b, &a) != nil { + t.Fatal("damaged remote fixture") + } + } + holder, fence := "owner", uint64(1) + if a.Fence != 0 { + holder, fence = worker.ID(), a.Fence + } + head := "head" + if a.Head != "" { + head = a.Head + } + return a, directory.CellView{Now: 1000, Cell: cs.Cell{Head: head, Lease: cs.Lease{Device: holder, Fence: fence, Expires: 9000}}} + } + save := func() { + b, _ := json.Marshal(a) + if e := os.WriteFile(remotePath, b, 0600); e != nil { + t.Fatal(e) + } + } + _, view := load() + template, e := cell.CreateIn(t.TempDir(), cell.Options{Class: cell.FilesOnly}) + if e != nil { + t.Fatal(e) + } + meta, e := os.ReadFile(filepath.Join(template.Root, cell.MetaPath)) + if e != nil { + t.Fatal(e) + } + adapter := WorkerAdapter{Credential: worker, Incarnation: 1, StateDir: filepath.Join(dir, "attempts"), Now: func() int64 { return 1000 }, Taker: handoff.Taker{Dir: workerTestDirectory{read: func() directory.CellView { _, v := load(); return v }}, Fetch: workerTestFetch{meta}, Local: workerTestLocal{}, RootFor: func(string) string { return filepath.Join(dir, "chat") }}, Current: func(context.Context, cs.AssignmentRef) (cs.Assignment, directory.CellView, error) { + a, v := load() + return a, v, nil + }} + adapter.Submit = func(_ context.Context, c cs.Command) (cs.Result, error) { + if c.Change.ClaimAssignment != nil { + a, _ = load() + if a.Status == "pending" { + a.Status = "claimed" + a.Fence = 2 + a.Head = "head" + save() + } + cut("claim") + a, view = load() + return cs.Result{Code: cs.OK, Assignment: &a, View: &view}, nil + } + if c.Change.FinishAssignment != nil { + a, _ = load() + a.Status = "finished" + a.Head = c.Change.FinishAssignment.Head + save() + cut("finish") + return cs.Result{Code: cs.OK, Assignment: &a}, nil + } + return cs.Result{Code: cs.BadRequest}, nil + } + err := adapter.Run(context.Background(), a, func(ctx context.Context, taken handoff.Taken) (string, error) { + cut("intent") + wal, _, e := cellstore.OpenWAL(filepath.Join(dir, "calls.wal")) + if e != nil { + return "", e + } + in := cellstore.Intent{V: 2, Tool: "exec", ArgsHash: "opaque", Started: 1000, SideEffect: "external", Brief: "scripted call"} + if e = wal.Begin(in); e != nil { + return "", e + } + f, e := os.OpenFile(filepath.Join(dir, "invocations"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if e != nil { + return "", e + } + _, e = f.Write([]byte("x")) + f.Close() + if e != nil { + return "", e + } + cut("execution") + if e = wal.Finish(in, cellstore.Executed{}); e != nil { + return "", e + } + cut("seal") + if e = os.WriteFile(filepath.Join(dir, "sealed-frame"), []byte("sealed"), 0600); e != nil { + return "", e + } + cut("upload") + a.Head = "result" + save() + cut("publish") + return "result", nil + }) + if err != nil { + if !errors.Is(err, ErrWorkerIncomplete) { + t.Fatal(err) + } + t.Log(err) + } + if mode == "cut" { + t.Fatal("did not reach requested cut") + } +} diff --git a/relay/hosted/src/raft/worker.js b/relay/hosted/src/raft/worker.js index 1639db515e..ce6cdcbdc2 100644 --- a/relay/hosted/src/raft/worker.js +++ b/relay/hosted/src/raft/worker.js @@ -1,4 +1,5 @@ import { frame, response } from './abi.js'; +import { encodeCanonical } from './codec.js'; const segment = s => [...new TextEncoder().encode(s)].map(b => b.toString(16).padStart(2, '0')).join(''); @@ -12,3 +13,11 @@ export function workerCell(runtime, store, cluster, worker, cell, incarnation) { const r = response(runtime.call(w.finish())), raw = r.blob(); r.end(); return raw; } + +// The same actor-queue/current-time/quorum prerequisite as workerCell applies. +export function workerAssignment(runtime, store, cluster, worker, incarnation, ref) { + const app = store.application(), w = frame(28).str(cluster).str(worker).u64(incarnation).u64(app.now).blob(new TextEncoder().encode(encodeCanonical(ref))); + const keys = [['state','identity'], ['enrollment','actor/' + segment(worker)], ['state','device/' + segment(worker)], ['state','worker/' + segment(worker)], ['state','cell/' + segment(ref.cell)], ['state','assignment/' + segment(ref.id)]]; + for (const [namespace,key] of keys) w.blob(store.read(namespace,key) ?? new Uint8Array()); + const r = response(runtime.call(w.finish())), raw = r.blob(); r.end(); return raw; +} diff --git a/relay/hosted/test/raft/assignment-workerd.test.js b/relay/hosted/test/raft/assignment-workerd.test.js new file mode 100644 index 0000000000..057d416684 --- /dev/null +++ b/relay/hosted/test/raft/assignment-workerd.test.js @@ -0,0 +1,50 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { build } from 'esbuild'; +import { mkdtemp, writeFile, copyFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { generateKeyPairSync, createHash, sign } from 'node:crypto'; +import { encodeCanonical } from '../../src/raft/codec.js'; +import { openWorkerd } from './workerd.js'; + +// R18/R19/R21 use actual persisted DOs and the production Go policy. An old +// authenticated claim crosses cancellation, and a killed isolate retains the +// exact claimed fence and already published result without replaying work. +test('R18 R19 R21 actual workerd assignment claim, delayed decision and restart',{timeout:45000},async()=>{ + const proof=process.env.CODEAF_RAFT_PROOF_DIR??'/tmp/codeaf-p16',out=await mkdtemp(join(tmpdir(),'p16-wasm-')),disk=await mkdtemp(join(tmpdir(),'p16-disk-')); + const bundle=await build({entryPoints:['test/raft/membership-worker.js'],bundle:true,format:'esm',target:'es2022',external:['core.wasm','cloudflare:*'],write:false}); + await writeFile(join(out,'worker.js'),bundle.outputFiles[0].contents);await copyFile(join(proof,'core.wasm'),join(out,'core.wasm'));await writeFile(join(out,'storage-worker.js'),'export class ProofStore{}'); + const keys=Object.fromEntries(['owner','root','worker','effects','1','2','3'].map(id=>[id,generateKeyPairSync('ed25519')])),pub=id=>keys[id].publicKey.export({format:'der',type:'spki'}).subarray(-32); + const cluster='p16-workerd',sha=b=>createHash('sha256').update(b).digest('hex'),bytes=v=>Buffer.from(encodeCanonical(v)),segment=s=>Buffer.from(s).toString('hex'); + const worker='dev_'+sha(pub('worker')).slice(0,32),identity='id_'+sha(pub('root')).slice(0,32),scope={identity:cluster,cluster,epoch:'1',protocol:1}; + const anchors=['1','2','3'].map(node=>({node,device:'',key:pub(node).toString('base64'),incarnation:'1',protocol:1,anchor:true,retired:false,ready_index:'0'})); + const genesis=[{namespace:'state',key:'identity',data:bytes({V:1,identity,signing_key:pub('root').toString('base64')}).toString('base64')},...['owner','effects'].map(id=>({namespace:'enrollment',key:'actor/'+segment(id),data:bytes({cluster,key:pub(id).toString('base64'),personal:id==='owner',...(id==='effects'?{effects:true}:{})}).toString('base64')}))]; + let server,transport=0,serial=0;const status={},queue=[]; + const restart=async()=>{if(server)await server.dispose();server=await openWorkerd(out,disk,true);}; + const call=async(id,op,extra={})=>{const r=await fetch(server.origin+'/'+id,{method:'POST',body:JSON.stringify({op,...extra}),signal:AbortSignal.timeout(5000)});assert.equal(r.status,200);const v=await r.json();if(v?.error)throw Error(v.error);return v;}; + const accept=(id,r)=>{if(r?.status)status[id]=r.status;queue.push(...(r?.send??[]).map(m=>({...m,from:id})));}; + const pump=async()=>{for(let n=0;queue.length;n++){assert(n<2000);const m=queue.shift();const e={scope,from:String(m.from),to:String(m.to),incarnation:'1',destination:'1',sequence:String(++transport),chunk:null,digest:sha(Buffer.from(m.bytes,'base64')),payload:m.bytes,signature:null};e.signature=sign(null,Buffer.from('codeaf/raft/envelope/1\n'+JSON.stringify(e)),keys[e.from].privateKey).toString('base64');accept(Number(m.to),await call(m.to,'peer',{value:Buffer.from(JSON.stringify(e)).toString('base64')}));}}; + const leader=()=>Number(Object.keys(status).find(id=>status[id].role===2)); + const command=(change,signer='owner',inc=0)=>{const actor=signer==='worker'?worker:signer,session='run-'+(++serial),fields={cluster,actor,session,sequence:'1',change,...(inc?{worker_incarnation:String(inc)}:{})};const c={schema:1,...fields,request_digest:sha(bytes(fields)),now:1000,signature:null};c.signature=sign(null,Buffer.concat([Buffer.from('codeaf.directory.command.v1\0'),bytes(c)]),keys[signer].privateKey).toString('base64');return c;}; + const row=async(ns,key)=>{const b=await call(leader(),'row',{namespace:ns,key});return b?JSON.parse(Buffer.from(b,'base64').toString()):null;}; + const submit=async c=>{const i=leader();accept(i,await call(i,'input',{tag:3,value:bytes(c).toString('base64')}));await pump();if(c.change.record_upload){assert(await row('outbox','upload/'+segment(sha(bytes(c.change.record_upload)))));return {code:'ok'};}const saved=await row('sessions',segment(c.actor)+'/'+segment(c.session));assert(saved,'missing committed result for '+Object.keys(c.change));return saved.result;}; + const send=async(change,signer,inc,code='ok')=>{const r=await submit(command(change,signer,inc));assert.equal(r.code,code);return r;}; + const tick=async()=>{for(const id of [1,2,3])accept(id,await call(id,'input',{tag:1}));await pump();}; + try{ + await restart();for(const id of [1,2,3])accept(id,await call(id,'init',{config:{cluster,id:String(id),seed:String(id+20),bootstrap:['1','2','3'],anchors,scope,genesis}}));await pump();for(let n=0;n<30&&!leader();n++)await tick();assert(leader()); + const cert={v:1,identity,cluster,key:pub('worker').toString('base64'),made:1,expires:100000,cells:['child'],learner:false,signature:null};cert.signature=sign(null,Buffer.concat([Buffer.from('codeaf/worker-cert/v1\0'),bytes(cert)]),keys.root.privateKey).toString('base64'); + await send({enroll_worker:{certificate:cert,grants:{child:Buffer.from([0x81]).toString('base64')}}}); + await send({register_worker:{previous:'0',incarnation:'1',caps:{os:'',arch:'',sandbox:null,container:null,gpu:null,cow:''},sealed_labels:null}},'worker'); + const beat={worker,incarnation:'1',sequence:'1',seen_at:1000,work:null,signature:null};beat.signature=sign(null,Buffer.concat([Buffer.from('codeaf/worker-beat/v1\0'),bytes({cluster,beat})]),keys.worker.privateKey).toString('base64');assert.equal((await send({worker_beat_batch:{items:[beat]}})).workers[0].code,'ok'); + for(const cell of ['parent','child'])await send({create:{v1:true,cell,init:{head:'head-'+cell,class:'',size:'0',keys:null,...(cell==='child'?{parent_cell:'parent'}:{})},receipt:''}}); + const spec={id:'run',revision:'1',cell:'child',parent_cell:'parent',expect:{device:'owner',fence:'1'},target:worker,incarnation:'1',expires:90000,sealed_payload:Buffer.from([0x81]).toString('base64')};const ref=v=>({id:v.id,revision:v.revision,cell:v.cell,target:v.target,incarnation:v.incarnation}); + await send({assign:spec});const delayed=command({claim_assignment:{ref:ref(spec),expect:spec.expect}},'worker',1);await send({cancel_assignment:{ref:ref(spec)}});assert.equal((await submit(delayed)).code,'conflict'); + spec.id='second';await send({assign:spec});const claim=command({claim_assignment:{ref:ref(spec),expect:spec.expect}},'worker',1);const claimed=await submit(claim);assert.equal(claimed.code,'ok');assert.equal(claimed.view.cell.lease.fence,'2'); + const publish={cell:'child',publish:{fence:'2',old_head:'head-child',head:'result',size:'0',class:'',pending:0},receipt:''};await send({publish},'worker',1,'bad_request');assert.equal((await row('state','cell/'+segment('child'))).head,'head-child'); + const receipt={device:worker,cell:'child',worker_incarnation:'1',head:'result',head_rid:'rid'};await send({record_upload:receipt},'effects');publish.receipt=sha(bytes(receipt));await send({publish},'worker',1); + await send({acquire:{cell:'parent',force:true,expect:{device:'owner',fence:'1'}}});await send({finish_assignment:{ref:ref(spec),fence:'2',head:'result'}},'worker',1); + for(const id of [1,2,3])await call(id,'checkpoint');await restart();for(const id of [1,2,3])accept(id,await call(id,'status'));await pump();for(let n=0;n<30&&!leader();n++)await tick();assert(leader()); + assert.deepEqual(await submit(claim),claimed);assert.equal((await row('state','assignment/'+segment(spec.id))).status,'finished');assert.equal((await row('state','cell/'+segment('parent'))).head,'head-parent');assert.equal((await row('state','cell/'+segment('parent'))).lease.fence,'2'); + }finally{if(server)await server.dispose();await rm(out,{recursive:true,force:true});await rm(disk,{recursive:true,force:true});} +}); diff --git a/relay/hosted/test/raft/assignment.test.js b/relay/hosted/test/raft/assignment.test.js new file mode 100644 index 0000000000..8570419641 --- /dev/null +++ b/relay/hosted/test/raft/assignment.test.js @@ -0,0 +1,115 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { spawn } from 'node:child_process'; +import { DatabaseSync } from 'node:sqlite'; +import { createHash, generateKeyPairSync, sign, createCipheriv } from 'node:crypto'; +import { Reactor } from '../../src/raft/abi.js'; +import { encodeCanonical, decodeCanonical } from '../../src/raft/codec.js'; +import { MemberActor } from '../../src/raft/actor.js'; +import { directoryEvaluator } from '../../src/raft/directory.js'; +import { committedRegistry } from '../../src/raft/membership.js'; +import { workerCell, workerAssignment } from '../../src/raft/worker.js'; + +const out = process.env.CODEAF_RAFT_PROOF_DIR ?? '/tmp/codeaf-p16'; +const enc = new TextEncoder(), dec = new TextDecoder(); +const bytes = value => enc.encode(encodeCanonical(value)); +const sha = value => createHash('sha256').update(value).digest('hex'); +const segment = value => Buffer.from(value).toString('hex'); +const caps = {os:'linux',arch:'arm64',sandbox:null,container:null,gpu:null,cow:'none'}; +const seal = text => { const cipher = createCipheriv('aes-256-gcm',Buffer.alloc(32,3),Buffer.alloc(12,7)); return Buffer.concat([Buffer.alloc(12,7),cipher.update(text),cipher.final(),cipher.getAuthTag()]).toString('base64'); }; +class Host { + constructor() { this.db = new DatabaseSync(':memory:'); this.sql = { exec: (q, ...p) => { const rows = this.db.prepare(q).all(...p); return { toArray: () => rows, [Symbol.iterator]: () => rows[Symbol.iterator]() }; } }; } + transactionSync(fn) { this.db.exec('BEGIN'); try { const result = fn(); this.db.exec('COMMIT'); return result; } catch (e) { this.db.exec('ROLLBACK'); throw e; } } + async sync() {} +} +class Native { + constructor() { + this.child = spawn(out + '/native-peer', [], { stdio: ['pipe','pipe','inherit'], env: { ...process.env, GOMEMLIMIT: '32MiB' } }); this.buffer = Buffer.alloc(0); + this.child.stdout.on('data', b => { this.buffer = Buffer.concat([this.buffer, b]); this.pump(); }); + this.child.on('exit', code => { if (this.waiter) { clearTimeout(this.waiter.timer); this.waiter.reject(new Error(`native exited ${code}`)); } }); + } + pump() { if (!this.waiter || this.buffer.length < 4) return; const n = this.buffer.readUInt32LE(0); assert(n <= 2 * 1024 * 1024); if (this.buffer.length < n + 4) return; const b = this.buffer.subarray(4, n + 4); this.buffer = this.buffer.subarray(n + 4); const w = this.waiter; this.waiter = null; clearTimeout(w.timer); w.resolve(new Uint8Array(b)); } + call(b) { assert(!this.waiter); return new Promise((resolve, reject) => { this.waiter = { resolve, reject, timer: setTimeout(() => reject(new Error('native ABI timeout')), 5000) }; const h = Buffer.alloc(4); h.writeUInt32LE(b.length); this.child.stdin.write(h); this.child.stdin.write(b); this.pump(); }); } + async close() { this.child.stdin.end(); if (this.child.exitCode === null) await new Promise(resolve => this.child.once('exit', resolve)); } +} + + +test('R18 R19 R21 assignment claims, delayed decisions and uploaded heads match native Go', { timeout:30000 }, async () => { + const reactor = new Reactor(new WebAssembly.Module(readFileSync(out+'/core.wasm'))), native = new Native(), calls = []; + const runtime = {call(b) {const r=reactor.call(b);calls.push([b.slice(),r.slice()]);return r;}}; + const cluster='worker-p15', keys=Object.fromEntries(['owner','root','worker','effects','x1','x2','x3'].map(id=>[id,generateKeyPairSync('ed25519')])); + const pub = id => keys[id].publicKey.export({format:'der',type:'spki'}).subarray(-32); + const identity='id_'+sha(pub('root')).slice(0,32), worker='dev_'+sha(pub('worker')).slice(0,32); + const rows=[{namespace:'state',key:'identity',data:bytes({V:1,identity,signing_key:pub('root').toString('base64')})},{namespace:'enrollment',key:'actor/'+segment('owner'),data:bytes({cluster,key:pub('owner').toString('base64'),personal:true})}]; + rows.push({namespace:'enrollment',key:'actor/'+segment('effects'),data:bytes({cluster,key:pub('effects').toString('base64'),personal:false,effects:true})}); + const actors=[],hosts=[],queue=[],results=[[],[],[]],reads=[];let serial=0; + const configs=[1,2,3].map(id=>({cluster,id:String(id),seed:String(60+id),bootstrap:['1','2','3']})); + const initialSize=rows.reduce((n,r)=>n+12+r.namespace.length+r.key.length+r.data.length,0); + const accept=e=>{queue.push(...e.send);reads.push(...e.reads);}; + const pump=async()=>{for(let n=0;queue.length;n++){assert(n<2000);const m=queue.shift();if(Number(m.to)>actors.length)continue;accept(await actors[Number(m.to)-1].input(2,m.bytes));}}; + const leader=()=>actors.findIndex(a=>a.status.role===2); + const evaluator=i=>(raw,read,app)=>{ + const base=read('state','identity')?[]:rows; + const r=directoryEvaluator(runtime,cluster,8*1024*1024)(raw,(ns,key)=>base.find(r=>r.namespace===ns&&r.key===key)?.data??read(ns,key),{...app,size:app.size+(base.length?initialSize:0)}); + results[i].push(decodeCanonical(dec.decode(r.result)));return {...r,writes:[...base,...r.writes]}; + }; + const command=(change,{signer='owner',inc=0n,now=1000}={})=>{ + const actor=signer==='worker'?worker:signer, session='req-'+(++serial), sequence='1'; + const fields={cluster,actor,session,sequence,change,...(inc?{worker_incarnation:String(inc)}:{})}; + const c={schema:1,...fields,request_digest:sha(bytes(fields)),now,signature:null}; + c.signature=sign(null,Buffer.concat([Buffer.from('codeaf.directory.command.v1\0'),bytes(c)]),keys[signer].privateKey).toString('base64');return bytes(c); + }; + const submit=async raw=>{const i=leader(),n=results[i].length;try {accept(await actors[i].input(3,raw));await pump();} catch(e) {throw new Error('assignment fixture operation '+Object.keys(decodeCanonical(dec.decode(raw)).change).join(',')+': '+e.message);}assert.equal(results[i].length,n+1);return results[i].at(-1);}; + const send=async(change,opts,code='ok')=>{const r=await submit(command(change,opts));assert.equal(r.code,code);return r;}; + const read=async cell=>{const id='read-'+(++serial),i=leader();accept(await actors[i].input(4,id));await pump();assert(reads.some(r=>r.id===id));return decodeCanonical(dec.decode(workerCell(runtime,actors[i].store,cluster,worker,cell,1n)));}; + const workrow=()=>decodeCanonical(dec.decode(actors[leader()].store.read('state','worker/'+segment(worker)))); + try { + for(let i=0;i<3;i++){const h=new Host();hosts.push(h);const a=new MemberActor(h,runtime,evaluator(i));actors.push(a);accept(await a.init(configs[i]));} + await pump();for(let n=0;n<30&&leader()<0;n++){for(const a of actors)accept(await a.input(1));await pump();}assert(leader()>=0); + const cert={v:1,identity,cluster,key:pub('worker').toString('base64'),made:1000,expires:100000,cells:['child-one','child-two'],learner:false,signature:null}; + cert.signature=sign(null,Buffer.concat([Buffer.from('codeaf/worker-cert/v1\0'),bytes(cert)]),keys.root.privateKey).toString('base64'); + await send({enroll_worker:{certificate:cert,grants:{'child-one':seal('selected key one'),'child-two':seal('selected key two')}}}); + await send({register_worker:{previous:'0',incarnation:'1',caps,sealed_labels:seal('private name')}},{signer:'worker'}); + const beat={worker,incarnation:'1',sequence:'1',seen_at:1000,work:null,signature:null}; + beat.signature=sign(null,Buffer.concat([Buffer.from('codeaf/worker-beat/v1\0'),bytes({cluster,beat})]),keys.worker.privateKey).toString('base64'); + assert.equal((await send({worker_beat_batch:{items:[beat]}})).workers[0].code,'ok'); + await send({create:{v1:true,cell:'parent',init:{head:'parent-head',class:'',size:'0',keys:null},receipt:''}}); + const spec=cell=>({id:'run-'+cell,revision:'1',cell,parent_cell:'parent',expect:{device:'owner',fence:'1'},target:worker,incarnation:'1',expires:90000,sealed_payload:seal('private task path /private/repo shell prompt credential')}); + const ref=v=>({id:v.id,revision:v.revision,cell:v.cell,target:v.target,incarnation:v.incarnation}); + const assigned=[]; + for(const cell of cert.cells){ + await send({create:{v1:true,cell,init:{head:'head-'+cell,parent_cell:'parent',class:'',size:'0',keys:null},receipt:''}}); + const v=spec(cell);await send({assign:v});assigned.push(v); + } + // A decision authenticated before cancellation must still lose at APPLY. + const delayed=command({claim_assignment:{ref:ref(assigned[1]),expect:assigned[1].expect}},{signer:'worker',inc:1n}); + await send({cancel_assignment:{ref:ref(assigned[1])}}); + assert.equal((await submit(delayed)).code,'conflict'); + const first=assigned[0],claim=command({claim_assignment:{ref:ref(first),expect:first.expect}},{signer:'worker',inc:1n}); + let r=await submit(claim);assert.equal(r.code,'ok');assert.equal(r.view.cell.lease.fence,'2');const claimed=r; + assert.deepEqual(await submit(claim),claimed); + const readRun=async()=>{const id='run-read-'+(++serial),i=leader();accept(await actors[i].input(4,id));await pump();assert(reads.some(r=>r.id===id));return decodeCanonical(dec.decode(workerAssignment(runtime,actors[i].store,cluster,worker,1n,ref(first))));}; + r=await readRun();assert.equal(r.code,'ok');assert.equal(r.assignment.status,'claimed');assert(r.grant); + const publish={cell:first.cell,publish:{fence:'2',old_head:'head-'+first.cell,head:'result',size:'0',class:'',pending:0},receipt:''}; + await send({publish},{signer:'worker',inc:1n},'bad_request'); + // The missing blob's receipt is absent; a retry after durable upload uses + // the SAME Go receipt check. R2 failure itself has its existing targeted gate. + const receipt={device:worker,cell:first.cell,worker_incarnation:'1',head:'result',head_rid:'rid'}; + const receiptID=sha(bytes(receipt)); + await send({record_upload:receipt},{signer:'effects'}); + publish.receipt=receiptID;await send({publish},{signer:'worker',inc:1n}); + await send({acquire:{cell:'parent',force:true,expect:{device:'owner',fence:'1'}}}); + await send({finish_assignment:{ref:ref(first),fence:'2',head:'result'}},{signer:'worker',inc:1n}); + const parent=actors[leader()].store.read('state','cell/'+segment('parent')); + const p=decodeCanonical(dec.decode(parent));assert.equal(p.head,'parent-head');assert.equal(p.lease.fence,'2'); + for(const a of actors)await a.checkpoint(); + const reopened=(leader()+1)%3;await actors[reopened].close();actors[reopened]=new MemberActor(hosts[reopened],runtime,evaluator(reopened));accept(await actors[reopened].init(configs[reopened]));await pump(); + assert.deepEqual(await submit(claim),claimed); + assert.equal((await readRun()).assignment.status,'finished'); + await send({revoke:{device:worker}});assert.equal((await readRun()).code,'revoked'); + for(const a of actors){const raw=a.store.read('state','assignment/'+segment(first.id));for(const sentinel of ['private task','/private/repo','shell prompt','credential'])assert(!Buffer.from(raw).includes(Buffer.from(sentinel)));} + for(const[input,expected]of calls)assert.deepEqual(await native.call(input),expected); + assert(calls.length>100);assert.equal(reactor.metrics().wasi.poll_oneoff??0,0); + }finally{await native.close();for(const h of hosts)h.db.close();} +}); From 391b214565ba53a407d2d7563d807873b455b354 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 04:55:05 -0400 Subject: [PATCH 039/103] Describe delegated claims and interrupted worker recovery. Keep the manual and adapter contracts accurate about conditional claims, child results, recovery limits and the remaining public setup work. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../continuing-a-chat-on-another-computer.md | 15 +++++++++++ internal/manual/chat_test.go | 2 ++ internal/raftabi/README.md | 8 ++++++ internal/raftdirectory/README.md | 27 +++++++++++++++++++ 4 files changed, 52 insertions(+) diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index 5416887c13..5742ebacf7 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -308,3 +308,18 @@ Worker setup and assignment execution are not available as a terminal command ye A worker restart registers a newer process number. Delayed requests and heartbeats from its previous process cannot refresh availability or save work. A drained worker cannot receive new claims; already held work can still save and let go. Restarting does not clear draining. Expired and removed enrollment records remain retained; reaching their capacity limit refuses new enrollment instead of deleting records or saved chats. A scoped API worker does not join the coordination log. Joining as a learner requires explicit consent in its certificate: the full log reveals identity-wide routing and ownership metadata, while chat content, names and task instructions remain encrypted. Use scoped API access for an untrusted third-party worker. A delegated learner gets no vote and cannot become leader. Existing paired personal computers keep their shared-root trust model. + + +## How does a worker claim an assignment or run parallel tasks? + +The coordination service accepts an explicit selected chat and target worker. Instructions and credentials stay encrypted. Registration and availability alone cannot start work: the worker first downloads the saved chat, then commits a conditional claim for the assigned chat's observed holder and fence. Continuing that same observed chat elsewhere competes with the claim; only one can win. A changed assignment, expired delegation, drained worker or newer worker process can refuse the claim. A claim returns the committed head, which the worker finishes downloading before opening it. + +Parallel tasks use distinct child chats, each with its own holder. Finishing a child records its already published result; incorporating that result into the parent still requires the parent's current holder and fence. The worker uploads the complete saved result before publishing its head. Leadership needs neither those files nor their keys. + +Worker setup and assignment execution are not available as a terminal command yet. The internal worker adapter supports a small scripted worker; phone controls and creating remote machines come later. + +## Will an interrupted worker replay tools after a crash or reassignment? + +No. The worker saves its claim request before sending it and records an execution intent before invoking tools. A lost coordination reply can retry that same request, retaining its original fence. Once execution may have started, reopening keeps the records and reports `this work may be incomplete; its saved records have been kept`. It never invokes that attempt again automatically. A recorded finish can retry its coordination acknowledgment without running tools. + +Reassignment changes the run revision and requires a new conditional claim. The replacement can recover saved records, but the scripted adapter refuses automatic execution of a reassigned run. Missing or incomplete calls are not instructions to execute them. Cancellation prevents future claims; stopping a claimed assignment does not cancel an already running shell process. Another take changes the fence and blocks the displaced holder from publishing. Already downloaded bytes and uploaded frames stay retained. diff --git a/internal/manual/chat_test.go b/internal/manual/chat_test.go index 2645ca8d24..e04069f563 100644 --- a/internal/manual/chat_test.go +++ b/internal/manual/chat_test.go @@ -3213,6 +3213,8 @@ func TestTheChatManualAnswersTheQuestionsPeopleAsk(t *testing.T) { {"how do I take a chat back from my other computer", "continuing-a-chat-on-another-computer"}, {"what happens if two computers continue the same chat at once", "continuing-a-chat-on-another-computer"}, {"which computer coordinates cluster status cluster lead", "continuing-a-chat-on-another-computer"}, + {"how does a delegated worker claim an assignment or run parallel tasks", "continuing-a-chat-on-another-computer"}, + {"will an interrupted worker replay tools after a crash or reassignment", "continuing-a-chat-on-another-computer"}, {"can a worker access all my chats or approve pairing", "continuing-a-chat-on-another-computer"}, {"worker restarts draining and learner metadata visibility", "continuing-a-chat-on-another-computer"}, {"why does it say this chat is arriving", "continuing-a-chat-on-another-computer"}, diff --git a/internal/raftabi/README.md b/internal/raftabi/README.md index af029da776..eb666f068e 100644 --- a/internal/raftabi/README.md +++ b/internal/raftabi/README.md @@ -191,3 +191,11 @@ barrier before returning these bytes; the ABI supplies policy, not consensus. Operation 16 accepts an optional tail: committed time uint64, then identity, authority and worker-record blobs. It is required for delegated peers; the old metadata-only form returns them revoked. Personal/anchor encodings are unchanged. + +P16 adds operation 28: cluster/worker strings, incarnation/committed-time uint64s, +canonical AssignmentRef blob, then six blobs in AssignmentReadKeys order. It +returns one canonical AssignmentRead blob. Like operation 27, it runs under the +member queue after current ingress time commits and a fresh quorum barrier +passes; the ABI never supplies a local success path. Assignments, per-cell active +slots and incarnation-bound holds use the existing state namespace and survive +ordinary native/hosted snapshots. No storage format or old ABI tag changes. diff --git a/internal/raftdirectory/README.md b/internal/raftdirectory/README.md index 930bd9dd5c..c51bdee08c 100644 --- a/internal/raftdirectory/README.md +++ b/internal/raftdirectory/README.md @@ -95,3 +95,30 @@ pages on activation/wake; no periodic network loop starts here. Delivery of a grant approved before revocation cannot restore authority after it: every subsequent operation still checks the committed device record. Already delivered sealed bytes cannot be recalled, as with existing personal pairing. + +P16 supplies Assign/Reassign/ClaimAssignment/CancelAssignment/FinishAssignment in +the shared Go evaluator. AssignmentRef declares every routing dependency before +application. Only an explicit current scoped worker/incarnation with recent +committed availability can claim; claim and ordinary fenced acquire commit in +one application transaction. Per-cell slots exclude overlapping active runs. +Finishing references an already published ordinary head and never writes a +parent. Assign/Reassign bind the existing child's ParentCell and refuse aliases. +Opaque payloads share the existing 4 KiB worker sealed-data limit. Admission +refuses growth at the persisted identity ceiling without deleting runs. + +AuthorizedWorkerAssignment and hosted workerAssignment have the SAME +actor-queue/current-committed-time/fresh-quorum prerequisite as the selected chat +read. Recheck current target authorization at that view before execution; a saved +claim response alone cannot reopen a displaced or superseded run. A revoked +pending run is permanently unclaimable and its raw history remains retained. +An owner can explicitly cancel it to reuse the per-cell slot. + +syncsetup.WorkerAdapter accepts scoped content dependencies and a selected +quorum-backed directory. It uses handoff.Taker for stage/claim/reconcile/install, +saves original signed claim/finish requests and durably records execution intent +before its callback. Completed attempts return locally; a recorded finish retries +only coordination. Interrupted attempts and reassigned revisions return incomplete +without invoking tools. Execute must use the ordinary call recorder and +cellsync.Publisher; it returns only an already published head. The adapter never +loads a personal identity, discovers a relay, provisions a box or adds a public +command. P17/P18 retain public client assembly and operator journeys. From 0bee2c49ad8c7fd2635ba087b6e301b5b5233605 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 05:38:52 -0400 Subject: [PATCH 040/103] Retain original device proofs when adapting v1 requests to committed commands. Check the original HTTP certificate, route, body and signature at application so forwarding cannot manufacture personal authority or a new observed holder. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterctl/control.go | 4 +- internal/clusterctl/provenance_test.go | 67 ++++ internal/clusterstate/apply.go | 14 +- internal/clusterstate/types.go | 23 +- internal/clusterstate/v1.go | 350 ++++++++++++++++++ internal/directory/handler.go | 15 +- internal/raftabi/abi.go | 2 + internal/raftabi/v1.go | 28 ++ internal/raftdirectory/README.md | 37 +- .../raftdirectory/testdata/v1-requests.json | 260 +++++++++++++ internal/raftdirectory/v1.go | 189 ++++++++++ internal/raftdirectory/v1_journey_test.go | 192 ++++++++++ internal/raftdirectory/v1_test.go | 303 +++++++++++++++ internal/reqsign/reqsign.go | 18 +- relay/hosted/src/raft/v1.js | 11 + relay/hosted/test/raft/v1.test.js | 62 ++++ 16 files changed, 1545 insertions(+), 30 deletions(-) create mode 100644 internal/clusterctl/provenance_test.go create mode 100644 internal/clusterstate/v1.go create mode 100644 internal/raftabi/v1.go create mode 100644 internal/raftdirectory/testdata/v1-requests.json create mode 100644 internal/raftdirectory/v1.go create mode 100644 internal/raftdirectory/v1_journey_test.go create mode 100644 internal/raftdirectory/v1_test.go create mode 100644 relay/hosted/src/raft/v1.js create mode 100644 relay/hosted/test/raft/v1.test.js diff --git a/internal/clusterctl/control.go b/internal/clusterctl/control.go index 2da0f84881..eba51f127f 100644 --- a/internal/clusterctl/control.go +++ b/internal/clusterctl/control.go @@ -76,7 +76,7 @@ func Handler(auth wireauth.Authenticate, open func(string) (Backend, error)) htt } else { var c cs.Command c, err = cs.DecodeCommand(body) - if err != nil || c.Actor != device || c.Change.RequestLeadership == nil { + if err != nil || cs.ClientCommand(c) != nil || c.Actor != device || c.Change.RequestLeadership == nil { http.Error(w, "refresh cluster status before requesting leadership", 400) return } @@ -158,7 +158,7 @@ func (c Client) Status(ctx context.Context) (s Status, err error) { return } func (c Client) Lead(ctx context.Context, command cs.Command) (r cs.LeadershipReceipt, err error) { - if command.Change.RequestLeadership == nil { + if command.Change.RequestLeadership == nil || cs.ClientCommand(command) != nil { return r, errors.New("a leadership request is required") } b, err := cs.EncodeCommand(command) diff --git a/internal/clusterctl/provenance_test.go b/internal/clusterctl/provenance_test.go new file mode 100644 index 0000000000..f817cca26d --- /dev/null +++ b/internal/clusterctl/provenance_test.go @@ -0,0 +1,67 @@ +package clusterctl_test + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + ctl "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +type provenanceBackend struct{ calls atomic.Int32 } + +func (b *provenanceBackend) Status(context.Context, string) (ctl.Status, error) { + return ctl.Status{}, ctl.ErrUnavailable +} +func (b *provenanceBackend) Lead(context.Context, cs.Command) (cs.LeadershipReceipt, error) { + b.calls.Add(1) + return cs.LeadershipReceipt{}, ctl.ErrUnavailable +} + +// R15/R25 require the public v2 door to reject compatibility provenance BEFORE +// proposal. A signed v2 body must not hide arbitrary bytes in an old HTTP proof. +func TestR15R25V2IngressRejectsV1ProvenanceBeforeDispatch(t *testing.T) { + b := &provenanceBackend{} + var requests atomic.Int32 + h := ctl.Handler(func(*http.Request, []byte) (string, string, error) { return "identity", "device", nil }, func(string) (ctl.Backend, error) { return b, nil }) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); h.ServeHTTP(w, r) })) + defer srv.Close() + c := cs.Command{Schema: 1, Cluster: "cluster", Actor: "device", Session: "leadership", Sequence: 1, RequestDigest: "digest", Change: cs.Mutation{RequestLeadership: &cs.RequestLeadership{Revision: 1, Node: 4, Device: "device"}}} + good, err := cs.EncodeCommand(c) + if err != nil { + t.Fatal(err) + } + res, err := srv.Client().Post(srv.URL+ctl.Path, "application/json", bytes.NewReader(good)) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if b.calls.Load() != 1 { + t.Fatal("ordinary command never reached dispatch") + } + c.V1 = &cs.V1Request{Body: []byte("private bytes from an invalid v2 request")} + raw, err := cs.EncodeCommand(c) + if err != nil { + t.Fatal(err) + } + res, err = srv.Client().Post(srv.URL+ctl.Path, "application/json", bytes.NewReader(raw)) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if res.StatusCode != http.StatusBadRequest || b.calls.Load() != 1 { + t.Fatal("v2 provenance entered consensus dispatch", res.StatusCode, b.calls.Load()) + } + before := requests.Load() + client := ctl.Client{Base: srv.URL, Sign: func(*http.Request, []byte) {}, HTTP: srv.Client()} + if _, err = client.Lead(context.Background(), c); err == nil { + t.Fatal("client sent v1 provenance") + } + if requests.Load() != before { + t.Fatal("invalid client command reached the wire") + } +} diff --git a/internal/clusterstate/apply.go b/internal/clusterstate/apply.go index 5e48703bff..6f1ec21734 100644 --- a/internal/clusterstate/apply.go +++ b/internal/clusterstate/apply.go @@ -327,7 +327,11 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er } digest, _ := Digest(c) signed, e := SigningBytes(c) - if e != nil || c.Cluster != meta.Cluster || auth.Cluster != c.Cluster || !auth.ValidRole() || len(auth.Key) != ed25519.PublicKeySize || !ed25519.Verify(auth.Key, signed, c.Signature) || digest != c.RequestDigest { + authenticated := e == nil && len(auth.Key) == ed25519.PublicKeySize && ed25519.Verify(ed25519.PublicKey(auth.Key), signed, c.Signature) + if c.V1 != nil { + auth, authenticated = V1AuthorityAt(c, identity, meta.Now) + } + if !authenticated || c.Cluster != meta.Cluster || auth.Cluster != c.Cluster || !auth.ValidRole() || len(auth.Key) != ed25519.PublicKeySize || digest != c.RequestDigest { return out, nil } // Revocation precedes dedup: a saved old response cannot authorize a revoked @@ -357,7 +361,7 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er if decode(SessionKey(c.Actor, c.Session), &last) != nil || decode(SessionsKey(), &counts) != nil { return out, ErrDependency } - known := rows[SessionKey(c.Actor, c.Session)] != nil + known := c.V1 == nil && rows[SessionKey(c.Actor, c.Session)] != nil refuse := func(code Code) (Evaluation, error) { out.Result.Code = code; return out, nil } if known && (last.Closed || c.Sequence < last.Sequence || c.Sequence == last.Sequence && c.RequestDigest != last.Digest) { return refuse(Conflict) @@ -366,10 +370,10 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er out.Result = last.Result return out, nil } - if !service && (!known && (c.Sequence != 1 || c.Ack != 0) || known && (last.Sequence == ^uint64(0) || c.Sequence != last.Sequence+1 || c.Ack != last.Sequence)) { + if !service && c.V1 == nil && (!known && (c.Sequence != 1 || c.Ack != 0) || known && (last.Sequence == ^uint64(0) || c.Sequence != last.Sequence+1 || c.Ack != last.Sequence)) { return refuse(Conflict) } - if !service && !known && (counts.Live >= MaxLiveSessions || counts.Retained >= MaxRetainedSessions) { + if !service && c.V1 == nil && !known && (counts.Live >= MaxLiveSessions || counts.Retained >= MaxRetainedSessions) { return refuse(Capacity) } if c.Now < 0 || c.Now > int64(MaxSafeInteger)-LeaseTTLMillis { @@ -452,7 +456,7 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er counts.Live-- } save := func() { - if service { + if service || c.V1 != nil { return } put(SessionKey(c.Actor, c.Session), session{Sequence: c.Sequence, Digest: c.RequestDigest, Closed: c.Change.CloseSession != nil, Result: out.Result}) diff --git a/internal/clusterstate/types.go b/internal/clusterstate/types.go index 70b6e0c6ef..19da648e82 100644 --- a/internal/clusterstate/types.go +++ b/internal/clusterstate/types.go @@ -28,17 +28,18 @@ var ( // client sequence. Signature authentication and committed authorization belong // to ingress and apply respectively; decoding alone grants no authority. type Command struct { - WorkerIncarnation uint64 `json:"worker_incarnation,omitempty"` - Schema uint16 `json:"schema"` - Cluster string `json:"cluster"` - Actor string `json:"actor"` - Session string `json:"session"` - Sequence uint64 `json:"sequence"` - RequestDigest string `json:"request_digest"` - Ack uint64 `json:"ack,omitempty"` - Now int64 `json:"now"` - Change Mutation `json:"change"` - Signature []byte `json:"signature"` + V1 *V1Request `json:"v1_request,omitempty"` + WorkerIncarnation uint64 `json:"worker_incarnation,omitempty"` + Schema uint16 `json:"schema"` + Cluster string `json:"cluster"` + Actor string `json:"actor"` + Session string `json:"session"` + Sequence uint64 `json:"sequence"` + RequestDigest string `json:"request_digest"` + Ack uint64 `json:"ack,omitempty"` + Now int64 `json:"now"` + Change Mutation `json:"change"` + Signature []byte `json:"signature"` } // Mutation is a closed union: exactly one field is present. Future packets add diff --git a/internal/clusterstate/v1.go b/internal/clusterstate/v1.go new file mode 100644 index 0000000000..43fd6f26ed --- /dev/null +++ b/internal/clusterstate/v1.go @@ -0,0 +1,350 @@ +package clusterstate + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "io" + "reflect" + "strconv" + "strings" +) + +// V1Request retains the original device proof through forwarding. Its body is +// admitted only after the closed legacy decoder has checked every field, so +// ignored JSON cannot smuggle private text into a replicated command. +type V1Request struct { + Method string `json:"method"` + URI string `json:"uri"` + IdentityKey []byte `json:"identity_key"` + Certificate string `json:"certificate"` + At string `json:"at"` + Signature []byte `json:"signature"` + Body []byte `json:"body"` +} + +type v1Certificate struct { + V uint16 `json:"V"` + Device string `json:"device"` + Created int64 `json:"created"` + Signature string `json:"sig"` +} + +const V1SkewMillis int64 = 5 * 60 * 1000 + +// V1Command translates only the request the device actually signed. It never +// reads a new holder to manufacture an expectation or signs on the device's behalf. +func V1Command(cluster string, now int64, p V1Request) (Command, error) { + m, err := p.Mutation() + if err != nil { + return Command{}, err + } + var cert v1Certificate + raw, err := base64.RawURLEncoding.DecodeString(p.Certificate) + if err != nil || len(raw) > 1024 || legacyBody(raw, &cert, false) != nil { + return Command{}, ErrCodec + } + pub, err := hex.DecodeString(cert.Device) + if err != nil || len(pub) != ed25519.PublicKeySize { + return Command{}, ErrCodec + } + c := Command{V1: &p, Schema: SchemaVersion, Cluster: cluster, Actor: DeviceIDOfKey(pub), Session: "v1", Sequence: 1, Now: now, Change: m} + c.RequestDigest, err = Digest(c) + return c, err +} + +// V1Authority checks the root, certificate, exact HTTP signing domain and the +// translated operation at the committed position. Ingress authentication alone +// cannot authorize a request delayed across revocation. +func V1Authority(c Command, identity IdentityRec) (Authority, bool) { + return V1AuthorityAt(c, identity, c.Now) +} + +// V1AuthorityAt additionally rejects a forwarded old proof after committed +// time has advanced. A sender cannot make stale ingress evidence fresh again +// by placing its original timestamp in an unsigned host field. +func V1AuthorityAt(c Command, identity IdentityRec, committedNow int64) (Authority, bool) { + p := c.V1 + if p == nil || c.WorkerIncarnation != 0 || c.Session != "v1" || c.Sequence != 1 || c.Ack != 0 || len(c.Signature) != 0 || !bytes.Equal(p.IdentityKey, identity.SigningKey) { + return Authority{}, false + } + ms, err := strconv.ParseInt(p.At, 10, 64) + now := max(c.Now, committedNow) + if err != nil || ms < 0 || ms > int64(MaxSafeInteger) || c.Now < 0 || now > int64(MaxSafeInteger) || ms < now-V1SkewMillis || ms > now+V1SkewMillis { + return Authority{}, false + } + raw, err := base64.RawURLEncoding.DecodeString(p.Certificate) + var cert v1Certificate + if err != nil || len(raw) > 1024 || legacyBody(raw, &cert, false) != nil { + return Authority{}, false + } + pub, err := hex.DecodeString(cert.Device) + if err != nil || DeviceIDOfKey(pub) != c.Actor { + return Authority{}, false + } + pub, err = JoinCertificate(identity, JoinRequest{Device: c.Actor, PublicKey: pub}, p.Certificate) + if err != nil { + return Authority{}, false + } + msg := V1SigningMessage(p.Method, p.URI, p.At, p.Body) + m, err := p.Mutation() + if err != nil || !reflect.DeepEqual(m, c.Change) || m.PutDevice != nil && m.PutDevice.ID != c.Actor || !ed25519.Verify(pub, msg, p.Signature) { + return Authority{}, false + } + return Authority{Cluster: c.Cluster, Key: pub, Personal: true}, true +} + +// V1SigningMessage is the existing HTTP signing domain, shared by the sender, +// ingress and deterministic committed verifier to keep their bytes identical. +func V1SigningMessage(method, uri, at string, body []byte) []byte { + h := sha256.Sum256(body) + return []byte("codeaf-req-v1\n" + method + "\n" + uri + "\n" + at + "\n" + hex.EncodeToString(h[:])) +} + +// ClientCommand refuses protocol provenance on the public v2 door. Only an +// original v1 HTTP proof may select the compatibility rules at that ingress. +func ClientCommand(c Command) error { + if c.V1 != nil || c.Change.Create != nil && c.Change.Create.V1 || c.Change.Publish != nil && c.Change.Publish.V1 || c.Change.Acquire != nil && (c.Change.Acquire.V1 || c.Change.Acquire.Legacy) { + return ErrCodec + } + return nil +} + +func legacyBody(raw []byte, v any, optional bool) error { + if optional && len(raw) == 0 { + return nil + } + if len(raw) == 0 || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return ErrCodec + } + if legacyJSONKeys(raw) != nil { + return ErrCodec + } + d := json.NewDecoder(bytes.NewReader(raw)) + d.DisallowUnknownFields() + if d.Decode(v) != nil { + return ErrCodec + } + var extra any + if d.Decode(&extra) != io.EOF { + return ErrCodec + } + return nil +} + +// Duplicate fields otherwise disappear during typed decoding while their first +// value remains in the signed body replicated by the proof. +func legacyJSONKeys(raw []byte) error { + d := json.NewDecoder(bytes.NewReader(raw)) + d.UseNumber() + var value func(int) error + value = func(depth int) error { + if depth > 64 { + return ErrCodec + } + t, err := d.Token() + if err != nil { + return err + } + switch t { + case json.Delim('{'): + seen := map[string]bool{} + for d.More() { + k, e := d.Token() + if e != nil { + return e + } + s, ok := k.(string) + if !ok || seen[s] { + return ErrCodec + } + seen[s] = true + if e = value(depth + 1); e != nil { + return e + } + } + end, e := d.Token() + if e != nil || end != json.Delim('}') { + return ErrCodec + } + case json.Delim('['): + for d.More() { + if e := value(depth + 1); e != nil { + return e + } + } + end, e := d.Token() + if e != nil || end != json.Delim(']') { + return ErrCodec + } + case json.Delim('}'), json.Delim(']'): + return ErrCodec + } + return nil + } + if err := value(0); err != nil { + return err + } + if _, err := d.Token(); err != io.EOF { + return ErrCodec + } + return nil +} + +// Mutation is shared by native and WASI ingress. V1 numeric fences keep their +// original bytes and precision; v2's decimal-string codec never parses them. +func (p V1Request) Mutation() (Mutation, error) { + bad := func() (Mutation, error) { return Mutation{}, ErrCodec } + path := strings.TrimPrefix(p.URI, "/fabric") + if strings.ContainsAny(path, "?#") || !strings.HasPrefix(path, "/v1/") { + return bad() + } + if p.Method == "GET" && (path == "/v1/dir/list" || path == "/v1/dir/presence" || path == "/v1/identity/rotation" || strings.HasPrefix(path, "/v1/dir/cells/") && len(strings.Split(strings.TrimPrefix(path, "/v1/dir/cells/"), "/")) == 1) { + if len(p.Body) != 0 { + return bad() + } + return Mutation{AdvanceTime: &AdvanceTime{}}, nil + } + decode := func(v any, optional bool) bool { return legacyBody(p.Body, v, optional) == nil } + if path == "/v1/dir/vault" && p.Method == "POST" { + var v VaultCAS + if !decode(&v, false) { + return bad() + } + return Mutation{SetVault: &v}, nil + } + if path == "/v1/identity/rotation" && p.Method == "POST" { + var v RotateRequest + if !decode(&v, false) { + return bad() + } + return Mutation{Rotate: &v}, nil + } + s := strings.Split(strings.TrimPrefix(path, "/v1/dir/"), "/") + if len(s) < 2 || len(s) > 3 || path == strings.TrimPrefix(path, "/v1/dir/") { + return bad() + } + id, err := v1Segment(s[1]) + if err != nil || id == "" { + return bad() + } + verb := "" + if len(s) == 3 { + verb = s[2] + } + if s[0] == "devices" { + if p.Method == "PUT" && verb == "" { + var v Device + if !decode(&v, false) { + return bad() + } + return Mutation{PutDevice: &PutDevice{ID: id, Device: v}}, nil + } + if p.Method == "POST" && verb == "revoke" && len(p.Body) == 0 { + return Mutation{Revoke: &DeviceID{Device: id}}, nil + } + } + if s[0] == "requests" && p.Method == "POST" { + code := strings.ToLower(strings.TrimSpace(id)) + if verb == "deny" && len(p.Body) == 0 { + return Mutation{DecideJoin: &JoinDecision{Code: code, Deny: true}}, nil + } + if verb == "approve" { + var a struct { + Device Device `json:"device"` + Cert string `json:"cert"` + Grant string `json:"grant"` + } + if !decode(&a, false) { + return bad() + } + var cert v1Certificate + raw, e := base64.RawURLEncoding.DecodeString(a.Cert) + if e != nil || len(raw) > 1024 || legacyBody(raw, &cert, false) != nil { + return bad() + } + pub, e := hex.DecodeString(cert.Device) + if e != nil || len(pub) != ed25519.PublicKeySize { + return bad() + } + return Mutation{DecideJoin: &JoinDecision{Code: code, Device: DeviceIDOfKey(pub), Record: a.Device, Cert: a.Cert, Grant: a.Grant}}, nil + } + } + if s[0] != "cells" || p.Method != "POST" { + return bad() + } + switch verb { + case "": + var v CellInit + if !decode(&v, false) { + return bad() + } + return Mutation{Create: &CreateCell{Cell: id, Init: v, V1: true}}, nil + case "acquire": + var v struct { + Force bool `json:"force,omitempty"` + Expect *LeaseExpectation `json:"expect,omitempty"` + } + if !decode(&v, true) { + return bad() + } + a := AcquireCell{Cell: id, Force: v.Force, V1: true, Legacy: v.Expect == nil} + if v.Expect != nil { + a.Expect = *v.Expect + if a.Expect.Fence > MaxSafeInteger { + return bad() + } + } + return Mutation{Acquire: &a}, nil + case "heartbeat": + var v Beat + if !decode(&v, false) || v.Fence > MaxSafeInteger { + return bad() + } + return Mutation{Beat: &BeatCell{Cell: id, Beat: v}}, nil + case "publish": + var v Publish + if !decode(&v, false) || v.Fence > MaxSafeInteger { + return bad() + } + return Mutation{Publish: &PublishCell{Cell: id, Publish: v, V1: true}}, nil + case "release": + var v struct { + Quiet bool `json:"quiet,omitempty"` + Fence uint64 `json:"fence"` + } + if !decode(&v, false) || v.Fence > MaxSafeInteger { + return bad() + } + return Mutation{Release: &ReleaseCell{Cell: id, Fence: v.Fence, Quiet: v.Quiet}}, nil + case "archive": + if len(p.Body) != 0 { + return bad() + } + return Mutation{Archive: &CellID{Cell: id}}, nil + } + return bad() +} + +func v1Segment(s string) (string, error) { + var out []byte + for i := 0; i < len(s); i++ { + if s[i] != '%' { + out = append(out, s[i]) + continue + } + if i+2 >= len(s) { + return "", ErrCodec + } + b, e := hex.DecodeString(s[i+1 : i+3]) + if e != nil { + return "", ErrCodec + } + out = append(out, b[0]) + i += 2 + } + return string(out), nil +} diff --git a/internal/directory/handler.go b/internal/directory/handler.go index 171f362fd8..e30d714a55 100644 --- a/internal/directory/handler.go +++ b/internal/directory/handler.go @@ -34,6 +34,13 @@ func (c call) code() string { return c.r.PathValue("code") } // route answers one call. A nil result is an empty 204. type route func(call) (any, error) +// RequestDirectory binds the exact authenticated v1 request to its client. +// Consensus adapters retain this proof rather than mint a device signature. +// Ordinary in-process directories need only implement Directory. +type RequestDirectory interface { + ForRequest(device string, request *http.Request, body []byte) Client +} + // Handler serves the directory wire. Each request is authenticated first, and // then answered by open(identity).For(device), both taken from auth, so a body // can never name the acting device or reach another identity's records. @@ -83,7 +90,13 @@ func (h *handler) admit(w http.ResponseWriter, r *http.Request) (call, error) { if err != nil { return call{}, err } - return call{dir: dir, cl: dir.For(device), device: device, r: r, body: body}, nil + var cl Client + if bound, ok := dir.(RequestDirectory); ok { + cl = bound.ForRequest(device, r, body) + } else { + cl = dir.For(device) + } + return call{dir: dir, cl: cl, device: device, r: r, body: body}, nil } // denial names a refusal the way the wire does: skew and revoked have their diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index 9d1cb2a35b..47a303ac89 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -164,6 +164,8 @@ func (r *Runtime) Call(b []byte) (out []byte) { func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { e := encoder{} switch op { + case 29: + return v1Command(d) case 28: return assignmentRead(d) case 27: diff --git a/internal/raftabi/v1.go b/internal/raftabi/v1.go new file mode 100644 index 0000000000..84bd3183be --- /dev/null +++ b/internal/raftabi/v1.go @@ -0,0 +1,28 @@ +package raftabi + +import "github.com/Agent-Field/codeaf/internal/clusterstate" + +func v1Command(d *decoder) ([]byte, error) { + cluster, now, raw := d.str(), d.u64(), d.blob() + if d.end() != nil || now > clusterstate.MaxSafeInteger { + return nil, errFrame + } + var proof clusterstate.V1Request + if err := clusterstate.Decode(raw, &proof); err != nil { + return nil, err + } + c, err := clusterstate.V1Command(cluster, int64(now), proof) + if err != nil { + return nil, err + } + command, err := clusterstate.EncodeCommand(c) + if err != nil { + return nil, err + } + if len(command) > clusterstate.MaxCommandBytes { + return nil, errFrame + } + e := encoder{} + e.blob(command) + return e.b, nil +} diff --git a/internal/raftdirectory/README.md b/internal/raftdirectory/README.md index c51bdee08c..08c4871b7a 100644 --- a/internal/raftdirectory/README.md +++ b/internal/raftdirectory/README.md @@ -47,8 +47,8 @@ and hosted tests provision genesis/request dependencies explicitly. The effects external request/certificate admission, authorization installation, outbox delivery, blob receipts and v2 upload-before-head. P11 owns actual genesis/import; P12 owns peer registry; P14 owns watch presence and committed renewal scheduling. No socket can -extend a lease through this evaluator. P17 supplies original v1 authentication proof -and production client wiring. These boundaries are internal, not new manual claims. +extend a lease through this evaluator. The v1 adapter below retains original HTTP authentication proof; production +member/client routing still requires the complete namespace assembly. These boundaries are internal, not new manual claims. P10 adds `Effects` and the equivalent hosted `RaftEffectsFacade`. Provision a separate `Authority{Effects:true}` protocol key; it must not be personal. It can @@ -69,10 +69,10 @@ stale fences/heads before missing upload evidence. The existing Client facade ex already identifies v1 Acquire. Old clients cannot supply the encrypted head RID; they retain the existing upload-before-publish client ordering and their existing safe API semantics. This is protocol compatibility in the same evaluator, never -a second writer. P17 must select this marker only from original authenticated v1 -ingress and reject compatibility markers in v2 requests. A v2 body cannot become -v1 merely by requesting it. P17 owns that ingress/client assembly; the production -writer is unchanged until P11 migrates it. Never derive an encrypted remote +a second writer. V1Directory selects this marker only from the original authenticated v1 +ingress; ClientCommand rejects compatibility markers at a public v2 boundary. A v2 body cannot become +v1 merely by requesting it. The production writer remains unchanged until namespace migration and host +assembly install the adapter. No internal facade activates that route by itself. Never derive an encrypted remote object ID from a local turn ID. Receipts admit at most 512 declared frame hints; a frame plan remains optional and partial. @@ -122,3 +122,28 @@ without invoking tools. Execute must use the ordinary call recorder and cellsync.Publisher; it returns only an already published head. The adapter never loads a personal identity, discovers a relay, provisions a box or adds a public command. P17/P18 retain public client assembly and operator journeys. + +The native `V1Directory` implements `directory.RequestDirectory`. The HTTP +handler first authenticates, then binds the exact request to its client. The +relay has no personal signing key. `clusterstate.V1Command` translates the closed +v1 method/URI/body shapes; apply verifies the original certificate and HTTP +signature against the committed identity and rechecks revocation/rotation there. +Unsigned host time cannot freshen a request after committed time has advanced. +V1 uses its existing reconciliation behavior and does not consume v2 sessions. +ABI 29 and `src/raft/v1.js` expose that SAME translation to the WASI host. + +`V1Host` must hold the actor queue across committed application and read +callbacks. GET first commits its ingress time; every read passes a new +current-term ReadIndex and durable applied index. A POST followed by its +rotation view passes another barrier without repeating the mutation. The +adapter checks current root/device authority inside that callback. Missing +request proof is refused; no in-process `For` call can bypass authentication. + +This adapter is an integration boundary, not the production cutover. It does +not yet supply watches, personal-member lifecycle, worker/v2 HTTP ingress, or +the wake/outbox coordinator. Encoded commands remain bounded at 64 KiB, so +large legacy HTTP bodies need a supported representation before full legacy +compatibility can be claimed. The production relay contract test intentionally +remains red until persistent namespace assembly routes its real HTTP doors +through consensus. Do not enable a second writer or weaken a quorum to make +that test pass. diff --git a/internal/raftdirectory/testdata/v1-requests.json b/internal/raftdirectory/testdata/v1-requests.json new file mode 100644 index 0000000000..70f8ff48b9 --- /dev/null +++ b/internal/raftdirectory/testdata/v1-requests.json @@ -0,0 +1,260 @@ +{ + "provenance": "Frozen v1 wire shapes, independently signed using Node Ed25519. Blind force and empty acquire are the pre-conditional API. No archived executable was provided with this packet.", + "now": 1000000, + "actor_a": "dev_bb4bec189888e15d6eea072608a0a58c", + "actor_b": "dev_bb17c3863d2842e4aabd8191179338e5", + "cases": [ + { + "name": "put-a", + "method": "PUT", + "uri": "/v1/dir/devices/dev_bb4bec189888e15d6eea072608a0a58c", + "body": "{\"V\":1,\"name\":\"c2VhbGVkLWE=\",\"platform\":\"linux\"}", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "7MfAqV-QOPrno8oAvjFShsnq9HfOgui87UOWgYOIXujd-8IvD16qc9ABDePAXk4MmD6oNrx_LurnjYkmVRnTDQ" + } + }, + { + "name": "put-b", + "method": "PUT", + "uri": "/v1/dir/devices/dev_bb17c3863d2842e4aabd8191179338e5", + "body": "{\"V\":1,\"name\":\"c2VhbGVkLWI=\",\"platform\":\"linux\"}", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiI4NTBhM2ZkZTc4MTg4OTMwZWQ2ODcyNDEwZDA0YWI4NGI0MjMyYTQ3OWY1NDkxYTZiZDliZjFkYjM0MGY3Nzk4IiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNjNmZmFjZDQ2MmYyYWZiNjIyYTRkN2M4YjQ0MjllMDc0ZjJjYTAzMjBjZTFlZmY1NjFiYzY2M2E1NGVkNzQ4NTRiOTY0YmE1NTRjNDMzMTAzNGE1NzlmMGZmYTUxYWUxZjFjNzI1ODJlYzA0ODE1MTI5N2U4MmRjZjQyY2ViMDkifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "MRCa6KdHK_9zIFfcXP6zNBxnoxa65zJvsYOc1za1ek0V1_cozlMma6FUPMUagzTpNkmLU_y3XUAZsu0Q14ZRDw" + } + }, + { + "name": "list", + "method": "GET", + "uri": "/v1/dir/list", + "body": "", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "RHVWdqksJs-VXP3MGBEwNr8eRvc3OuPlfu95dJO30CO-3d-0PF6YbwJggwNWlWyjN9LUWuYOoSNYJShSsroVAw" + } + }, + { + "name": "presence", + "method": "GET", + "uri": "/v1/dir/presence", + "body": "", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "h2FYp4AjMQAWcytDfFY2HwYm3rv6J9fb2luBdWSb-VI5XtzzF5XaOYUVa4XyLHJWkAwoHcB8PUl8_YzSEqEDBg" + } + }, + { + "name": "create", + "method": "POST", + "uri": "/v1/dir/cells/chat", + "body": "{\"head\":\"head-one\",\"class\":\"host-bound\"}", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "Jebb_9Ba74QBhI9mdP1Tv7F9H_0E4s9mY5E6xgoWAKazgN5B2WtuEpj3UQg3mJAgXF5xHaURKQn4r8VP-NVMBg" + } + }, + { + "name": "cell", + "method": "GET", + "uri": "/v1/dir/cells/chat", + "body": "", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiI4NTBhM2ZkZTc4MTg4OTMwZWQ2ODcyNDEwZDA0YWI4NGI0MjMyYTQ3OWY1NDkxYTZiZDliZjFkYjM0MGY3Nzk4IiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNjNmZmFjZDQ2MmYyYWZiNjIyYTRkN2M4YjQ0MjllMDc0ZjJjYTAzMjBjZTFlZmY1NjFiYzY2M2E1NGVkNzQ4NTRiOTY0YmE1NTRjNDMzMTAzNGE1NzlmMGZmYTUxYWUxZjFjNzI1ODJlYzA0ODE1MTI5N2U4MmRjZjQyY2ViMDkifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "V18tyh-wwXl6YDhN8nD_TXH9oDr6aYtJmUaYCQ4TWs7jDyC-cD1-pz9kmtJIAdqUZdzpcK5KrB4xXVxAIQcpDA" + } + }, + { + "name": "blind-force", + "method": "POST", + "uri": "/v1/dir/cells/chat/acquire", + "body": "{\"force\":true}", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiI4NTBhM2ZkZTc4MTg4OTMwZWQ2ODcyNDEwZDA0YWI4NGI0MjMyYTQ3OWY1NDkxYTZiZDliZjFkYjM0MGY3Nzk4IiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNjNmZmFjZDQ2MmYyYWZiNjIyYTRkN2M4YjQ0MjllMDc0ZjJjYTAzMjBjZTFlZmY1NjFiYzY2M2E1NGVkNzQ4NTRiOTY0YmE1NTRjNDMzMTAzNGE1NzlmMGZmYTUxYWUxZjFjNzI1ODJlYzA0ODE1MTI5N2U4MmRjZjQyY2ViMDkifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "X84Ue0CMtd-WV5daWsJ8aKd866dTT8IIyo295ijEHuR52WXnb3f-K7HDJF-iDbZVXeuhM-Anxd3iFOlkuO-ABA" + } + }, + { + "name": "conditional", + "method": "POST", + "uri": "/v1/dir/cells/chat/acquire", + "body": "{\"force\":true,\"expect\":{\"device\":\"dev_bb4bec189888e15d6eea072608a0a58c\",\"fence\":1}}", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiI4NTBhM2ZkZTc4MTg4OTMwZWQ2ODcyNDEwZDA0YWI4NGI0MjMyYTQ3OWY1NDkxYTZiZDliZjFkYjM0MGY3Nzk4IiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNjNmZmFjZDQ2MmYyYWZiNjIyYTRkN2M4YjQ0MjllMDc0ZjJjYTAzMjBjZTFlZmY1NjFiYzY2M2E1NGVkNzQ4NTRiOTY0YmE1NTRjNDMzMTAzNGE1NzlmMGZmYTUxYWUxZjFjNzI1ODJlYzA0ODE1MTI5N2U4MmRjZjQyY2ViMDkifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "Z31P0iLeYoM0Sz2aoYZTWu8H13-7jZjXzrEtPaM1tEx0AV4UxzCskGAmvAS5tvgfvRkMCFZpc0MOGRHCvWvdDg" + } + }, + { + "name": "beat", + "method": "POST", + "uri": "/v1/dir/cells/chat/heartbeat", + "body": "{\"fence\":2,\"pending\":3}", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiI4NTBhM2ZkZTc4MTg4OTMwZWQ2ODcyNDEwZDA0YWI4NGI0MjMyYTQ3OWY1NDkxYTZiZDliZjFkYjM0MGY3Nzk4IiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNjNmZmFjZDQ2MmYyYWZiNjIyYTRkN2M4YjQ0MjllMDc0ZjJjYTAzMjBjZTFlZmY1NjFiYzY2M2E1NGVkNzQ4NTRiOTY0YmE1NTRjNDMzMTAzNGE1NzlmMGZmYTUxYWUxZjFjNzI1ODJlYzA0ODE1MTI5N2U4MmRjZjQyY2ViMDkifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "7CIOO1ZOIma11M84YAVMFhnhvlWnpwZc-yrWoDFVTHSoriw9_ceVWMXYA4NHx37gGoCj6FrufDn1d2QRCrNrBw" + } + }, + { + "name": "publish", + "method": "POST", + "uri": "/v1/dir/cells/chat/publish", + "body": "{\"fence\":2,\"old_head\":\"head-one\",\"head\":\"head-two\"}", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiI4NTBhM2ZkZTc4MTg4OTMwZWQ2ODcyNDEwZDA0YWI4NGI0MjMyYTQ3OWY1NDkxYTZiZDliZjFkYjM0MGY3Nzk4IiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNjNmZmFjZDQ2MmYyYWZiNjIyYTRkN2M4YjQ0MjllMDc0ZjJjYTAzMjBjZTFlZmY1NjFiYzY2M2E1NGVkNzQ4NTRiOTY0YmE1NTRjNDMzMTAzNGE1NzlmMGZmYTUxYWUxZjFjNzI1ODJlYzA0ODE1MTI5N2U4MmRjZjQyY2ViMDkifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "XYuCL0chwwRO7eh5znlEzWta6B7nUwx0oZniuCVXcrjq_O3vSVlNcxFJUzEoUCx9VT_8TmBabIgA14K1pq-lBA" + } + }, + { + "name": "release", + "method": "POST", + "uri": "/v1/dir/cells/chat/release", + "body": "{\"fence\":2}", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiI4NTBhM2ZkZTc4MTg4OTMwZWQ2ODcyNDEwZDA0YWI4NGI0MjMyYTQ3OWY1NDkxYTZiZDliZjFkYjM0MGY3Nzk4IiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNjNmZmFjZDQ2MmYyYWZiNjIyYTRkN2M4YjQ0MjllMDc0ZjJjYTAzMjBjZTFlZmY1NjFiYzY2M2E1NGVkNzQ4NTRiOTY0YmE1NTRjNDMzMTAzNGE1NzlmMGZmYTUxYWUxZjFjNzI1ODJlYzA0ODE1MTI5N2U4MmRjZjQyY2ViMDkifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "wU9o_-1XrFeqvGk-7mCvWi1Dg39GQ1eBDjTPs6Ma42UyKtgXwFNB1P7OCDLkgaZLL5h05bMV2QBWWMISvIOdBg" + } + }, + { + "name": "resume", + "method": "POST", + "uri": "/v1/dir/cells/chat/acquire", + "body": "", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "lX-wuDmLiq9hSfCbWnwcI2qPc9sQ5AuLxNF8E8RIuzVm1HOMA0xMl_lZtFtNY4FS_INt9h77Tl4w-uuG0Ag4CQ" + } + }, + { + "name": "quiet-release", + "method": "POST", + "uri": "/v1/dir/cells/chat/release", + "body": "{\"fence\":3,\"quiet\":true}", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "Ku1XASmPohUBdYL58mxC8DyHkxDQ3ODGR1BMJjGzplZQuxgK5oA-siy6gZJsyXQ4hcTj0-92xRHLBqaS5VJOBQ" + } + }, + { + "name": "archive", + "method": "POST", + "uri": "/v1/dir/cells/chat/archive", + "body": "", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "25efFhtsDqrPrBU4KWEc61t3-5kyvASC00gCbNK52kjxInGq3f_yfXDizCPDmlqKl-9aBn8TXxERUkFw9goiAw" + } + }, + { + "name": "vault", + "method": "POST", + "uri": "/v1/dir/vault", + "body": "{\"old\":\"\",\"new\":\"sealed-vault\"}", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "gEHkIBsmg8-ah2aD8_IiPG-g0OScXZLV3BKCeduHsXmVVmAH430f-yvtJgNA3nX1xFEog-KkHPi0Iu5CuVb6CQ" + } + }, + { + "name": "rotation-read", + "method": "GET", + "uri": "/v1/identity/rotation", + "body": "", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "Tu9zjTl-E2d93XoICZSS4NFvw8qsfANoGVdWHib3HVtl6o5A4XDCfH4sXbjNF1Z440JP51Un_-fgX9k6W0g4Dg" + } + }, + { + "name": "freeze", + "method": "POST", + "uri": "/v1/identity/rotation", + "body": "{\"op\":\"freeze\"}", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "d_6d21AxHDkcvv4B8GV3tHvuMeku8I7wFtMLOCYzzlqdoPjr4bYTNWhSzEtqEYv5acJoNhOZ-W5oyhJJammIDg" + } + }, + { + "name": "thaw", + "method": "POST", + "uri": "/v1/identity/rotation", + "body": "{\"op\":\"thaw\"}", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "I493ScAkF8j0LWRCaKRCzEV-00FE_-p8K-R_ZNzOQHn1xEMR8OtH1XKfMlJnpKA4ByXzsvBEMhanwbWug5BJDw" + } + }, + { + "name": "deny", + "method": "POST", + "uri": "/v1/dir/requests/1234/deny", + "body": "", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "-7RwQH-feX8A6Jk0n0ssILkgs4WH6ctzshdd0wHlrBob1_WYS1ehqJ2jASeZHhE-smjjpkK853wd-R9LdzLZAg" + } + }, + { + "name": "approve", + "method": "POST", + "uri": "/v1/dir/requests/5678/approve", + "body": "{\"device\":{\"V\":1,\"name\":\"c2VhbGVkLWI=\",\"platform\":\"linux\"},\"cert\":\"eyJWIjoxLCJkZXZpY2UiOiI4NTBhM2ZkZTc4MTg4OTMwZWQ2ODcyNDEwZDA0YWI4NGI0MjMyYTQ3OWY1NDkxYTZiZDliZjFkYjM0MGY3Nzk4IiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNjNmZmFjZDQ2MmYyYWZiNjIyYTRkN2M4YjQ0MjllMDc0ZjJjYTAzMjBjZTFlZmY1NjFiYzY2M2E1NGVkNzQ4NTRiOTY0YmE1NTRjNDMzMTAzNGE1NzlmMGZmYTUxYWUxZjFjNzI1ODJlYzA0ODE1MTI5N2U4MmRjZjQyY2ViMDkifQ\",\"grant\":\"Z3JhbnQ\"}", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "wVQj2iOWX4EEejh4oN8CmbV_i0pk65ReAe_A_wm1FoIjoZR8X38PBXhSS65ePZ8QymX-J-hZt11PkH6gw8KgBA" + } + }, + { + "name": "revoke", + "method": "POST", + "uri": "/v1/dir/devices/dev_bb17c3863d2842e4aabd8191179338e5/revoke", + "body": "", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "i3LBlgUH3KeQ2LC1z4kpAhcM18dk12aJj4d52M2XaGaf-zR6s0L0P1fq9QbENnHcxFGuxFEY_K15dDIzguJwAQ" + } + } + ] +} diff --git a/internal/raftdirectory/v1.go b/internal/raftdirectory/v1.go new file mode 100644 index 0000000000..49e4855dc8 --- /dev/null +++ b/internal/raftdirectory/v1.go @@ -0,0 +1,189 @@ +package raftdirectory + +import ( + "context" + "encoding/hex" + "net/http" + "reflect" + "strings" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/reqsign" +) + +// V1Host holds its actor queue through commit and through the read callback. +// Commit returns only a durable applied result. Read commits the GET command's +// ingress time, obtains a fresh current-term ReadIndex and waits for durable +// application before calling inspect. A POST already committed by this request +// uses that time and a NEW barrier without repeating its mutation. +type V1Host interface { + CommitV1(context.Context, clusterstate.Command) (clusterstate.Result, error) + ReadV1(context.Context, clusterstate.Command, bool, func(*raftstore.SQLite, raftstore.Application) error) error +} + +// V1Directory is the authenticated legacy HTTP adapter for one frozen namespace. +// It has no key with which to impersonate a personal device. ForRequest receives +// exactly the proof admitted by directory.Handler, including archived bodies. +type V1Directory struct { + Host V1Host + Cluster string + Now func() time.Time +} + +var _ directory.RequestDirectory = (*V1Directory)(nil) + +func (d *V1Directory) For(actor string) directory.Client { + return directory.NewRaft(&v1Backend{owner: d, actor: actor, err: directory.ErrUnauthorized}).For(actor) +} +func (d *V1Directory) ForRequest(actor string, r *http.Request, raw []byte) directory.Client { + p, err := reqsign.Capture(r, raw) + b := &v1Backend{owner: d, actor: actor, err: err} + if err == nil { + now := time.Now() + if d.Now != nil { + now = d.Now() + } + b.command, b.err = clusterstate.V1Command(d.Cluster, now.UnixMilli(), p) + if b.err != nil { + b.err = directory.ErrBadRequest + } + if b.err == nil { + encoded, e := clusterstate.EncodeCommand(b.command) + if e != nil { + b.err = directory.ErrBadRequest + } else if len(encoded) > clusterstate.MaxCommandBytes { + b.err = directory.ErrTooBig + } + } + if b.err == nil && b.command.Actor != actor { + b.err = directory.ErrUnauthorized + } + } + return directory.NewRaft(b).For(actor) +} + +type v1Backend struct { + owner *V1Directory + actor string + command clusterstate.Command + committed bool + err error +} + +func (b *v1Backend) Change(ctx context.Context, actor string, m clusterstate.Mutation) (clusterstate.Result, error) { + if b.err != nil { + return clusterstate.Result{}, b.err + } + if b.owner.Host == nil || actor != b.actor || !reflect.DeepEqual(m, b.command.Change) { + return clusterstate.Result{}, directory.ErrUnauthorized + } + r, err := b.owner.Host.CommitV1(ctx, b.command) + b.committed = err == nil + return r, err +} + +func (b *v1Backend) read(ctx context.Context, actor string, fn func(*raftstore.SQLite, raftstore.Application) error) error { + if b.err != nil { + return b.err + } + if b.owner.Host == nil || actor != b.actor { + return directory.ErrUnauthorized + } + return b.owner.Host.ReadV1(ctx, b.command, b.committed, func(s *raftstore.SQLite, app raftstore.Application) error { + if err := authorizeV1Read(ctx, s, app, b.command); err != nil { + return err + } + return fn(s, app) + }) +} + +func authorizeV1Read(ctx context.Context, s *raftstore.SQLite, app raftstore.Application, c clusterstate.Command) error { + keys := []raftstore.Key{{Namespace: raftstore.State, Key: clusterstate.IdentityKey().Key}, {Namespace: raftstore.State, Key: clusterstate.DeviceKey(c.Actor).Key}} + rows, _, err := s.Read(ctx, app.Applied, keys) + if err != nil { + return err + } + var identity clusterstate.IdentityRec + if clusterstate.Decode(rows[keys[0]], &identity) != nil { + return clusterstate.ErrDependency + } + if _, ok := clusterstate.V1AuthorityAt(c, identity, app.LogicalTime); !ok { + return directory.ErrUnauthorized + } + if rows[keys[1]] != nil { + var device clusterstate.Device + if clusterstate.Decode(rows[keys[1]], &device) != nil { + return clusterstate.ErrDependency + } + if device.Revoked { + return directory.ErrRevoked + } + } + return nil +} + +func (b *v1Backend) Listing(ctx context.Context, actor string) (directory.Listing, error) { + l := directory.Listing{Devices: map[string]directory.Device{}, Cells: map[string]directory.Cell{}} + err := b.read(ctx, actor, func(s *raftstore.SQLite, app raftstore.Application) error { + l.Now, l.Version = app.LogicalTime, app.VisibleVersion + after := "" + for { + rows, _, err := s.ScanPage(ctx, app.Applied, raftstore.State, after) + if err != nil { + return err + } + if len(rows) == 0 { + return nil + } + for _, r := range rows { + after = r.Key.Key + switch { + case after == "identity": + if err := clusterstate.Decode(r.Value, &l.Identity); err != nil { + return err + } + case strings.HasPrefix(after, "device/"): + id, e := hex.DecodeString(strings.TrimPrefix(after, "device/")) + if e != nil { + return clusterstate.ErrDependency + } + var v directory.Device + if e = clusterstate.Decode(r.Value, &v); e != nil { + return e + } + l.Devices[string(id)] = v + case strings.HasPrefix(after, "cell/"): + id, e := hex.DecodeString(strings.TrimPrefix(after, "cell/")) + if e != nil { + return clusterstate.ErrDependency + } + var v directory.Cell + if e = clusterstate.Decode(r.Value, &v); e != nil { + return e + } + l.Cells[string(id)] = v + } + } + } + }) + return l, err +} +func (b *v1Backend) ReadCell(ctx context.Context, actor, id string) (directory.CellView, error) { + var v directory.CellView + err := b.read(ctx, actor, func(s *raftstore.SQLite, app raftstore.Application) error { + k := raftstore.Key{Namespace: raftstore.State, Key: clusterstate.CellKey(id).Key} + rows, _, err := s.Read(ctx, app.Applied, []raftstore.Key{k}) + if err != nil { + return err + } + if rows[k] == nil { + return directory.ErrNotFound + } + v.Now = app.LogicalTime + return clusterstate.Decode(rows[k], &v.Cell) + }) + return v, err +} diff --git a/internal/raftdirectory/v1_journey_test.go b/internal/raftdirectory/v1_journey_test.go new file mode 100644 index 0000000000..85f63151fb --- /dev/null +++ b/internal/raftdirectory/v1_journey_test.go @@ -0,0 +1,192 @@ +package raftdirectory_test + +import ( + "context" + "encoding/hex" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/blobstore" + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/cellsync" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/handoff" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/reqsign" + "github.com/Agent-Field/codeaf/internal/syncsetup" + "github.com/Agent-Field/codeaf/internal/wireauth" +) + +type v1Local struct { + assignmentLocal + head string +} + +func (l v1Local) Unpublished(cell.Cell, string) (string, uint32, error) { + if l.head != "" { + return l.head, 1, nil + } + return "", 0, nil +} + +// R1/R2/R23/R25/R29 use two saved homes and ordinary Sync clients against a +// loopback relay with three FULL stores. Taker/Fetcher/Publisher are real; the +// deterministic frame engine keeps this protocol journey bounded and keyless. +func TestR1R2R23R25R29V1TwoHomesContinueTakeBackQuitOffRevoke(t *testing.T) { + celltest.Available(t) + g := newGroup(t, time.Now) + d := &raftdirectory.V1Directory{Host: g, Cluster: cluster, Now: time.Now} + disk, e := blobstore.NewDisk(t.TempDir()) + must(t, e) + var hits atomic.Int64 + auth := func(r *http.Request, raw []byte) (string, string, error) { + id, actor, e := reqsign.Verify(r, raw, time.Now()) + if e != nil { + return "", "", e + } + g.mu.Lock() + defer g.mu.Unlock() + i, app, e := g.barrier("a") + if e != nil { + return "", "", e + } + k := raftstore.Key{Namespace: raftstore.State, Key: cs.DeviceKey(actor).Key} + rows, _, e := g.stores[i].Read(ctx, app.Applied, []raftstore.Key{k}) + if e != nil { + return "", "", e + } + if rows[k] != nil { + var device cs.Device + if e = cs.Decode(rows[k], &device); e != nil { + return "", "", e + } + if device.Revoked { + return "", "", wireauth.ErrRevoked + } + } + return id, actor, nil + } + mux := http.NewServeMux() + mux.Handle("/v1/dir/", directory.Handler(reqsign.AuthenticateAt(time.Now), func(string) (directory.Directory, error) { return d, nil })) + mux.Handle("/v1/store/", blobstore.Handler(auth, func(string) (blobstore.Store, error) { return disk, nil })) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1); mux.ServeHTTP(w, r) })) + defer srv.Close() + t.Setenv(syncsetup.URLVar, srv.URL) + id, e := identity.Unmarshal([]byte(`{"V":1,"signing_seed":"` + hex.EncodeToString(key("root").Seed()) + `","dedup_secret":"` + hex.EncodeToString(key("dedup").Seed()) + `","cell_key":"` + hex.EncodeToString(key("content").Seed()) + `"}`)) + must(t, e) + open := func() *syncsetup.Sync { + h := t.TempDir() + must(t, identity.Save(h, id)) + _, e := identity.Device(h) + must(t, e) + must(t, identity.EndSolo(h)) + s, ok, e := syncsetup.Open(h) + must(t, e) + if !ok { + t.Fatal("sync unavailable") + } + return s + } + a, b := open(), open() + must(t, a.Dir.PutDevice(ctx, a.Device.ID(), directory.Device{V: 1})) + must(t, b.Dir.PutDevice(ctx, b.Device.ID(), directory.Device{V: 1})) + engA, engB := cellsync.NewFakeEngine(filepath.Join(a.Home, "frames")), cellsync.NewFakeEngine(filepath.Join(b.Home, "frames")) + c, e := cell.CreateIn(filepath.Join(a.Home, "chats"), cell.Options{Class: cell.FilesOnly}) + must(t, e) + meta, e := os.ReadFile(filepath.Join(c.Root, cell.MetaPath)) + must(t, e) + files := map[string]string{cell.MetaPath: string(meta), "work.txt": "first files", "transcript.jsonl": "first saved answer\n"} + head := engA.Seal(c, files) + pubA := &cellsync.Publisher{Engine: engA, Store: a.Store, Dir: a.Dir} + _, e = pubA.Upload(ctx, c, head) + must(t, e) + v, e := a.Dir.Create(ctx, c.ID, directory.CellInit{Head: head}) + must(t, e) + seen := directory.LeaseExpectation{Device: v.Cell.Lease.Device, Fence: v.Cell.Lease.Fence} + taker := func(s *syncsetup.Sync, eng *cellsync.FakeEngine, root string, local handoff.Local) handoff.Taker { + return handoff.Taker{Dir: s.Dir, Fetch: &cellsync.Fetcher{Engine: eng, Store: s.Store, Inbox: func(c cell.Cell) string { return filepath.Join(s.Home, "inbox", c.ID) }}, Local: local, RootFor: func(string) string { return root }} + } + takeB := taker(b, engB, filepath.Join(b.Home, "chat"), v1Local{}) + got, e := takeB.TakeSeen(ctx, c.ID, &seen) + must(t, e) + if _, e = takeB.TakeSeen(ctx, c.ID, &seen); !errors.Is(e, directory.ErrLeaseHeld) { + t.Fatalf("observed loser %v", e) + } + if _, e = a.Dir.Publish(ctx, c.ID, directory.Publish{Fence: 1, OldHead: "wrong", Head: "stale"}); !errors.Is(e, directory.ErrFenceStale) { + t.Fatalf("stale publisher %v", e) + } + raw, e := os.ReadFile(filepath.Join(got.Cell.Root, "work.txt")) + must(t, e) + if string(raw) != "first files" { + t.Fatal(string(raw)) + } + files["work.txt"] = "B saved result" + next := engB.Seal(got.Cell, files) + pubB := &cellsync.Publisher{Engine: engB, Store: b.Store, Dir: b.Dir} + must(t, pubB.Publish(ctx, &got.Driving, next, cellsync.PublishInfo{})) + files["work.txt"] = "A unpublished work" + unsent := engA.Seal(c, files) + back := taker(a, engA, c.Root, v1Local{head: unsent}) + branch := cellsync.Brancher{Dir: a.Dir, Publisher: pubA, NewID: func() (string, error) { return "retained-branch", nil }} + back.Branch = func(ctx context.Context, from *cellsync.Driving, head string, turns uint32) (string, error) { + return branch.Branch(ctx, from, head, turns, cellsync.PublishInfo{}) + } + returned, e := back.Take(ctx, c.ID) + must(t, e) + if returned.Kept != "retained-branch" { + t.Fatal("unsent work not retained") + } + raw, e = os.ReadFile(filepath.Join(c.Root, "work.txt")) + must(t, e) + if string(raw) != "B saved result" { + t.Fatal("take-back lost B result") + } + branchView, e := a.Dir.Cell(ctx, "retained-branch") + must(t, e) + if branchView.Cell.Head != unsent || branchView.Cell.ParentCell != c.ID { + t.Fatal("branch missing") + } + files["work.txt"] = "final quit files" + final := engA.Seal(c, files) + batch := &cellsync.Batcher{Publisher: pubA, Driving: &returned.Driving} + batch.Note(cellstore.Turn{ID: final}) + must(t, batch.Close(ctx)) + v, e = b.Dir.Cell(ctx, c.ID) + must(t, e) + if v.Cell.Head != final || v.Cell.Lease.Expires != 0 || v.Cell.Lease.ReleasedAt == 0 { + t.Fatal("quit did not publish before release", v) + } + got, e = takeB.Take(ctx, c.ID) + must(t, e) + raw, e = os.ReadFile(filepath.Join(got.Cell.Root, "work.txt")) + must(t, e) + if string(raw) != "final quit files" { + t.Fatal("quit files missing") + } + must(t, a.Dir.Revoke(ctx, b.Device.ID())) + if _, e = b.Dir.Cell(ctx, c.ID); !errors.Is(e, directory.ErrRevoked) { + t.Fatalf("revoked request %v", e) + } + before := hits.Load() + t.Setenv(syncsetup.URLVar, "off") + _, _ = a.Dir.List(ctx) + _, _ = a.Store.Has(ctx, []string{head}) + if hits.Load() != before { + t.Fatal("retained clients sent after off") + } + // The saved homes and their transcript/workspace/branch bytes still exist. + if _, e = os.Stat(filepath.Join(c.Root, "work.txt")); e != nil { + t.Fatal(e) + } +} diff --git a/internal/raftdirectory/v1_test.go b/internal/raftdirectory/v1_test.go new file mode 100644 index 0000000000..6f87030768 --- /dev/null +++ b/internal/raftdirectory/v1_test.go @@ -0,0 +1,303 @@ +package raftdirectory_test + +import ( + "bytes" + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/reqsign" +) + +func (g *group) CommitV1(_ context.Context, c cs.Command) (cs.Result, error) { + g.mu.Lock() + defer g.mu.Unlock() + return g.submit(c) +} +func (g *group) ReadV1(ctx context.Context, c cs.Command, committed bool, inspect func(*raftstore.SQLite, raftstore.Application) error) error { + g.mu.Lock() + defer g.mu.Unlock() + if !committed { + if c.Change.AdvanceTime == nil { + return directory.ErrUnauthorized + } + r, e := g.submit(c) + if e != nil { + return e + } + if r.Code == cs.Revoked { + return directory.ErrRevoked + } + if r.Code != cs.OK { + return directory.ErrUnauthorized + } + } + i := g.leader() + if i < 0 { + return directory.ErrUnreachable + } + g.readID++ + id := "v1-read-" + time.Unix(int64(g.readID), 0).Format(time.RFC3339Nano) + g.drive(i, raftcore.Read{RequestID: id}) + g.drain() + if !g.reads[id] { + return directory.ErrUnreachable + } + app, _, e := g.stores[i].ApplicationView(ctx) + if e != nil { + return e + } + return inspect(g.stores[i], app) +} + +type v1Fixture struct { + Name, Method, URI, Body string + Headers map[string]string +} +type v1Fixtures struct { + Now int64 + ActorA string `json:"actor_a"` + ActorB string `json:"actor_b"` + Cases []v1Fixture +} + +func frozenV1(t *testing.T) v1Fixtures { + t.Helper() + raw, e := os.ReadFile("testdata/v1-requests.json") + must(t, e) + var f v1Fixtures + must(t, json.Unmarshal(raw, &f)) + return f +} +func frozenRequest(t *testing.T, f v1Fixture, base string) *http.Request { + t.Helper() + r, e := http.NewRequest(f.Method, base+f.URI, strings.NewReader(f.Body)) + must(t, e) + for k, v := range f.Headers { + r.Header.Set(k, v) + } + return r +} + +// R25 uses frozen independently signed v1 requests at the actual HTTP door. +// No test-only device signing or synthetic expectation replaces these bytes. +func TestR25V1FrozenRequestsThroughCommittedHTTP(t *testing.T) { + f := frozenV1(t) + clock := func() time.Time { return time.UnixMilli(f.Now) } + var extra []raftstore.Write + for _, code := range []string{"1234", "5678"} { + k := cs.JoinKey(code) + extra = append(extra, raftstore.Write{Key: raftstore.Key{Namespace: raftstore.Outbox, Key: k.Key}, Value: encoded(t, cs.JoinRequest{Device: f.ActorB, PublicKey: key("p17-b").Public().(ed25519.PublicKey), Expires: f.Now + 90000})}) + } + g := newGroup(t, clock, extra...) + d := &raftdirectory.V1Directory{Host: g, Cluster: cluster, Now: clock} + srv := httptest.NewServer(directory.Handler(reqsign.AuthenticateAt(clock), func(string) (directory.Directory, error) { return d, nil })) + defer srv.Close() + for _, one := range f.Cases { + t.Run(one.Name, func(t *testing.T) { + r := frozenRequest(t, one, srv.URL) + resp, e := srv.Client().Do(r) + must(t, e) + defer resp.Body.Close() + raw, e := io.ReadAll(resp.Body) + must(t, e) + want := http.StatusOK + if one.Name == "blind-force" { + want = http.StatusConflict + } + if resp.StatusCode >= 300 && resp.StatusCode != want { + t.Fatalf("%s: status %d: %s", one.Name, resp.StatusCode, raw) + } + if one.Name == "blind-force" && (resp.StatusCode != want || !bytes.Contains(raw, []byte(`"err":"lease_held"`))) { + t.Fatalf("legacy refusal %d %s", resp.StatusCode, raw) + } + if one.Name == "conditional" || one.Name == "publish" || one.Name == "resume" { + var v directory.CellView + must(t, json.Unmarshal(raw, &v)) + if one.Name == "resume" && v.Cell.Lease.Fence != 3 { + t.Fatal(v) + } + if one.Name == "conditional" && v.Cell.Lease.Fence != 2 { + t.Fatal(v) + } + if one.Name == "publish" && v.Cell.Head != "head-two" { + t.Fatal(v) + } + } + if resp.Header.Get(directory.ConditionalTakeHeader) != "1" && one.Name != "blind-force" { + t.Fatal("conditional promise absent") + } + }) + } + // An old successful request arriving after revoke cannot regain authority. + for _, one := range f.Cases { + if one.Name == "cell" { + resp, e := srv.Client().Do(frozenRequest(t, one, srv.URL)) + must(t, e) + defer resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("revoked replay status %d", resp.StatusCode) + } + } + } + for i := range 3 { + app, _, e := g.stores[i].ApplicationView(ctx) + must(t, e) + rows, _, e := g.stores[i].ScanPage(ctx, app.Applied, raftstore.Sessions, "") + must(t, e) + if len(rows) != 0 { + t.Fatal("v1 request consumed durable v2 sessions") + } + } +} + +// R16/R25 prove that neither the relay nor another device can rewrite an old +// authenticated proof into a different command after admission. +func TestR16R25V1OriginalProofCannotBeRewritten(t *testing.T) { + f := frozenV1(t) + var create v1Fixture + for _, one := range f.Cases { + if one.Name == "create" { + create = one + } + } + p, e := reqsign.Capture(frozenRequest(t, create, "http://localhost"), []byte(create.Body)) + must(t, e) + c, e := cs.V1Command(cluster, f.Now, p) + must(t, e) + id := cs.IdentityRec{V: 1, Identity: cs.IdentityID(key("root").Public().(ed25519.PublicKey)), SigningKey: key("root").Public().(ed25519.PublicKey)} + if _, ok := cs.V1Authority(c, id); !ok { + t.Fatal("original request refused") + } + if _, ok := cs.V1AuthorityAt(c, id, f.Now+cs.V1SkewMillis+1); ok { + t.Fatal("stale forwarded timestamp became fresh") + } + for _, name := range []string{"mutation", "actor", "identity", "signature", "body", "route", "time", "session"} { + t.Run(name, func(t *testing.T) { + raw := encoded(t, c) + var bad cs.Command + must(t, cs.Decode(raw, &bad)) + switch name { + case "mutation": + bad.Change.Create.Init.Head = "forged" + case "actor": + bad.Actor = f.ActorB + case "identity": + bad.V1.IdentityKey = key("other").Public().(ed25519.PublicKey) + case "signature": + bad.V1.Signature[0] ^= 1 + case "body": + bad.V1.Body = []byte(`{"head":"forged"}`) + case "route": + bad.V1.URI = "/v1/dir/cells/other" + case "time": + bad.Now += cs.V1SkewMillis + 1 + case "session": + bad.Session = "v2" + } + if _, ok := cs.V1Authority(bad, id); ok { + t.Fatal("rewritten proof authorized") + } + }) + } + if cs.ClientCommand(c) == nil { + t.Fatal("v2 accepted v1 proof") + } + for _, m := range []cs.Mutation{{Create: &cs.CreateCell{V1: true}}, {Publish: &cs.PublishCell{V1: true}}, {Acquire: &cs.AcquireCell{Legacy: true}}, {Acquire: &cs.AcquireCell{V1: true}}} { + if cs.ClientCommand(cs.Command{Change: m}) == nil { + t.Fatal("v2 compatibility marker accepted") + } + } + for _, body := range []string{`{"head":"h","private_path":"/home/private"}`, `null`, `{"head":"h"} {}`, `{"head":"private","head":"h"}`} { + p.Body = []byte(body) + if _, e := p.Mutation(); e == nil { + t.Fatal("nonrouting body admitted") + } + } +} + +type fixtureSigner struct { + private ed25519.PrivateKey + certificate identity.Cert +} + +func (s fixtureSigner) IdentityKey() ed25519.PublicKey { + return key("root").Public().(ed25519.PublicKey) +} +func (s fixtureSigner) Cert() identity.Cert { return s.certificate } +func (s fixtureSigner) Sign(raw []byte) []byte { return ed25519.Sign(s.private, raw) } + +// R1/R25 check concurrent current HTTP clients against the same frozen holder +// observation. The losing wire error is the existing sentinel consumed by the +// real continuation screen, whose sentence is pinned by its focused gate. +func TestR1R25V1ConcurrentOriginalObservationsHaveOneWinner(t *testing.T) { + f := frozenV1(t) + clock := func() time.Time { return time.UnixMilli(f.Now) } + g := newGroup(t, clock) + d := &raftdirectory.V1Directory{Host: g, Cluster: cluster, Now: clock} + srv := httptest.NewServer(directory.Handler(reqsign.AuthenticateAt(clock), func(string) (directory.Directory, error) { return d, nil })) + defer srv.Close() + makeClient := func(name, label string) *directory.HTTP { + var fixture v1Fixture + for _, one := range f.Cases { + if one.Name == name { + fixture = one + } + } + p, e := reqsign.Capture(frozenRequest(t, fixture, srv.URL), []byte(fixture.Body)) + must(t, e) + raw, e := base64.RawURLEncoding.DecodeString(p.Certificate) + must(t, e) + var cert identity.Cert + must(t, json.Unmarshal(raw, &cert)) + return directory.NewHTTP(srv.URL, reqsign.SignFor(fixtureSigner{key(label), cert}, clock), srv.Client()) + } + a, b := makeClient("put-a", "p17-a"), makeClient("put-b", "p17-b") + _, e := a.Create(ctx, "chat", directory.CellInit{Head: "saved"}) + must(t, e) + seen := directory.LeaseExpectation{Device: f.ActorA, Fence: 1} + start := make(chan struct{}) + out := make(chan error, 2) + var wg sync.WaitGroup + for _, cl := range []directory.Client{a, b} { + wg.Add(1) + go func(cl directory.Client) { + defer wg.Done() + <-start + _, e := cl.Acquire(ctx, "chat", directory.AcquireOpts{Force: true, Expect: &seen}) + out <- e + }(cl) + } + close(start) + wg.Wait() + close(out) + won, lost := 0, 0 + for err := range out { + if err == nil { + won++ + } else if errors.Is(err, directory.ErrLeaseHeld) { + lost++ + } else { + t.Fatal(err) + } + } + if won != 1 || lost != 1 { + t.Fatalf("winners %d losers %d", won, lost) + } +} diff --git a/internal/reqsign/reqsign.go b/internal/reqsign/reqsign.go index 5a664680ab..c3063f06d8 100644 --- a/internal/reqsign/reqsign.go +++ b/internal/reqsign/reqsign.go @@ -15,12 +15,13 @@ import ( "strconv" "time" + "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/identity" "github.com/Agent-Field/codeaf/internal/wireauth" ) // Skew is how far a request's time may be from the receiver's clock. -const Skew = 5 * time.Minute +const Skew = time.Duration(clusterstate.V1SkewMillis) * time.Millisecond // The four headers of a signed request. const ( @@ -30,8 +31,6 @@ const ( HeaderSig = "Codeaf-Sig" ) -const msgLabel = "codeaf-req-v1\n" - // Signer is the sending side: a device and the identity key that vouches for it. type Signer interface { IdentityKey() ed25519.PublicKey // verifies the cert @@ -68,8 +67,7 @@ func IDOf(identityKey ed25519.PublicKey) string { // message is what the device signs: the method, the path with its query, the // time and the body's hash, so none of them can change in flight. func message(r *http.Request, at string, body []byte) []byte { - sum := sha256.Sum256(body) - return []byte(msgLabel + r.Method + "\n" + r.URL.RequestURI() + "\n" + at + "\n" + hex.EncodeToString(sum[:])) + return clusterstate.V1SigningMessage(r.Method, r.URL.RequestURI(), at, body) } // Sign stamps r with the four headers, timed at now. @@ -153,3 +151,13 @@ func SignFor(s Signer, now func() time.Time) wireauth.Sign { func AuthenticateAt(now func() time.Time) wireauth.Authenticate { return func(r *http.Request, body []byte) (string, string, error) { return Verify(r, body, now()) } } + +// Capture retains only the original public proof and body. The caller first +// authenticates ingress; committed application independently checks this copy. +func Capture(r *http.Request, body []byte) (clusterstate.V1Request, error) { + p, err := readProof(r) + if err != nil { + return clusterstate.V1Request{}, err + } + return clusterstate.V1Request{Method: r.Method, URI: r.URL.RequestURI(), IdentityKey: append([]byte(nil), p.identityKey...), Certificate: r.Header.Get(HeaderCert), At: p.at, Signature: append([]byte(nil), p.sig...), Body: append([]byte(nil), body...)}, nil +} diff --git a/relay/hosted/src/raft/v1.js b/relay/hosted/src/raft/v1.js new file mode 100644 index 0000000000..6fd6c465d4 --- /dev/null +++ b/relay/hosted/src/raft/v1.js @@ -0,0 +1,11 @@ +import { frame, response } from './abi.js'; +import { encodeCanonical } from './codec.js'; + +// Preserve the original public proof; Go alone translates legacy bodies and +// applies the compatibility rule. The surrounding ingress authenticates first, +// binds the committed identity, and supplies its own current clock sample. +export function v1Command(runtime, cluster, now, request) { + const proof = new TextEncoder().encode(encodeCanonical(request)); + const r = response(runtime.call(frame(29).str(cluster).u64(now).blob(proof).finish())); + const command = r.blob(); r.end(); return command; +} diff --git a/relay/hosted/test/raft/v1.test.js b/relay/hosted/test/raft/v1.test.js new file mode 100644 index 0000000000..64bbeb3e4c --- /dev/null +++ b/relay/hosted/test/raft/v1.test.js @@ -0,0 +1,62 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { spawn } from 'node:child_process'; +import { DatabaseSync } from 'node:sqlite'; +import { Reactor } from '../../src/raft/abi.js'; +import { encodeCanonical, decodeCanonical } from '../../src/raft/codec.js'; +import { MemberActor } from '../../src/raft/actor.js'; +import { directoryEvaluator } from '../../src/raft/directory.js'; +import { v1Command } from '../../src/raft/v1.js'; + +const out = process.env.CODEAF_RAFT_PROOF_DIR; +const enc = new TextEncoder(), dec = new TextDecoder(); +const bytes = v => enc.encode(encodeCanonical(v)); +const segment = s => Buffer.from(s).toString('hex'); + +// R14/R16/R25 run the SAME frozen independently signed legacy requests through +// real WASI members and compare every actual Go ABI call with the native core. +test('R14 R16 R25 original v1 requests commit through WASI with native-identical proofs', { timeout: 30000 }, async () => { + assert(out, 'build the proof artifacts first'); + const fixture = JSON.parse(readFileSync(new URL('../../../../internal/raftdirectory/testdata/v1-requests.json', import.meta.url))); + const runtime = new Reactor(new WebAssembly.Module(readFileSync(out + '/core.wasm'))); + const calls = [], traced = { call(input) { const result = runtime.call(input); calls.push([input.slice(), result.slice()]); return result; } }; + const child = spawn(out + '/native-peer', [], { stdio: ['pipe','pipe','inherit'], env: { ...process.env, GOMEMLIMIT: '32MiB' } }); + let buffer = Buffer.alloc(0), waiter; + const pumpNative = () => { if (!waiter || buffer.length < 4) return; const n = buffer.readUInt32LE(); assert(n <= 2 * 1024 * 1024); if (buffer.length < n + 4) return; const b = buffer.subarray(4,n+4); buffer = buffer.subarray(n+4); const w = waiter; waiter = null; clearTimeout(w.timer); w.resolve(new Uint8Array(b)); }; + child.stdout.on('data', b => { buffer = Buffer.concat([buffer,b]); pumpNative(); }); + const native = input => new Promise((resolve,reject) => { assert(!waiter); waiter = {resolve,reject,timer:setTimeout(()=>reject(new Error('native ABI timeout')),5000)}; const h = Buffer.alloc(4); h.writeUInt32LE(input.length); child.stdin.write(h); child.stdin.write(input); pumpNative(); }); + const stores = [], actors = [], queue = [], results = [[],[],[]], cluster = 'p17-v1'; + const first = fixture.cases[0], root = Buffer.from(first.headers['Codeaf-Identity'],'base64url'); + const initial = [{ namespace:'state', key:'identity', data:bytes({ V:1, identity:'id_' + (await import('node:crypto')).createHash('sha256').update(root).digest('hex').slice(0,32), signing_key:root.toString('base64') }) }]; + const b = fixture.cases.find(c => c.name === 'put-b'), cert = JSON.parse(Buffer.from(b.headers['Codeaf-Cert'],'base64url')); + for (const code of ['1234','5678']) initial.push({namespace:'outbox',key:'join/'+segment(code),data:bytes({device:fixture.actor_b,expires:fixture.now+90000,public_key:Buffer.from(cert.device,'hex').toString('base64')})}); + const size = initial.reduce((n,r)=>n+12+r.namespace.length+r.key.length+r.data.length,0); + const accept = effects => queue.push(...effects.send); + const pump = async () => { for (let n=0;queue.length;n++) { assert(n<3000); const m=queue.shift();accept(await actors[Number(m.to)-1].input(2,m.bytes)); } }; + const leader = () => actors.findIndex(a => a.status.role === 2); + try { + for (let i=0;i<3;i++) { + const db = new DatabaseSync(':memory:'); stores.push(db); + const storage = {sql:{exec(q,...p){const rows=db.prepare(q).all(...p);return {toArray:()=>rows,[Symbol.iterator]:()=>rows[Symbol.iterator]()};}},transactionSync(fn){db.exec('BEGIN');try{const v=fn();db.exec('COMMIT');return v;}catch(e){db.exec('ROLLBACK');throw e;}},async sync(){}}; + const evaluator = (raw,read,app) => { + const base = read('state','identity') ? [] : initial; + const result = directoryEvaluator(traced,cluster,8*1024*1024)(raw,(ns,k)=>base.find(r=>r.namespace===ns&&r.key===k)?.data??read(ns,k),{...app,size:app.size+(base.length?size:0)}); + results[i].push(decodeCanonical(dec.decode(result.result)));return {...result,writes:[...base,...result.writes]}; + }; + const actor = new MemberActor(storage,runtime,evaluator);actors.push(actor);accept(await actor.init({cluster,id:String(i+1),seed:String(i+20),bootstrap:['1','2','3']})); + } + await pump();for(let n=0;n<30&&leader()<0;n++){for(const actor of actors)accept(await actor.input(1));await pump();}assert(leader()>=0); + for(const c of fixture.cases){ + const h=c.headers, proof={method:c.method,uri:c.uri,identity_key:Buffer.from(h['Codeaf-Identity'],'base64url').toString('base64'),certificate:h['Codeaf-Cert'],at:h['Codeaf-Time'],signature:Buffer.from(h['Codeaf-Sig'],'base64url').toString('base64'),body:Buffer.from(c.body).toString('base64')}; + const raw=v1Command(traced,cluster,fixture.now,proof), l=leader(), before=results[l].length; + accept(await actors[l].input(3,raw));await pump();assert.equal(results[l].length,before+1,c.name); + assert.equal(results[l].at(-1).code,c.name==='blind-force'?'lease_held':'ok',c.name); + } + for(const [input,expected] of calls)assert.deepEqual(await native(input),expected); + assert(calls.length>100); + for(const actor of actors){const row=decodeCanonical(dec.decode(actor.store.read('state','device/'+segment(fixture.actor_b))));assert.equal(row.revoked,true);assert.equal(actor.store.one("SELECT count(*) AS n FROM records WHERE namespace='sessions'").n,0);} + } finally { + child.stdin.end();if(child.exitCode===null)await new Promise(resolve=>child.once('exit',resolve));for(const db of stores)db.close(); + } +}); From 5b72f3819978c66114f5cca7cf403e3e23c24809 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 05:38:53 -0400 Subject: [PATCH 041/103] Pin the production compatibility door and explain older-client refusals. Retain a failing real-relay contract gate until namespace migration and member assembly replace the legacy writer, and keep the manual accurate about safe continuation and incomplete saves. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../continuing-a-chat-on-another-computer.md | 8 +- internal/manual/chat_test.go | 3 + internal/relayserve/raft_cutover_test.go | 92 +++++++++++++++++++ 3 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 internal/relayserve/raft_cutover_test.go diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index 5742ebacf7..120bd8508c 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -137,7 +137,7 @@ Moving chats and pairing are available by default on Linux, macOS and native Win A chat has one holder at a time. To take a chat back from the computer that took it, continue it there the same way: press enter on its row on that computer and choose `continue here`. Nothing has to be returned or switched off first. A computer with the chat open when another takes it says ` continued this chat; this window now only shows it`, stops taking new turns, and keeps what it had not yet sent as a branch (` turns from : open separately / discard`). A tool call the model tries there answers with that same line. Taking the chat back leaves a `node_modules` or other install folder that computer already has alone. -The choice covers the holder and turn of ownership shown on the card. If another computer continues the chat before you confirm or while the copy is arriving, your choice cannot take it from that new holder. The slower one sees `another device continued this chat first`, its local copy is not replaced, and its row goes back to `running on `. Older clients that do not send this condition keep their earlier behavior. No turn is lost on either side: whatever the earlier holder had not sent is kept for you as the branch above. This build offers discard, which archives a branch, and opening the branch as a separate chat; merging it into the original chat is unavailable. Pressing `2` (`leave it there`) on the question, or `esc`, changes nothing and leaves the chat where it is. +The choice covers the holder and turn of ownership shown on the card. If another computer continues the chat before you confirm or while the copy is arriving, your choice cannot take it from that new holder. The slower one sees `another device continued this chat first`, its local copy is not replaced, and its row goes back to `running on `. No turn is lost on either side: whatever the earlier holder had not sent is kept for you as the branch above. This build offers discard, which archives a branch, and opening the branch as a separate chat; merging it into the original chat is unavailable. Pressing `2` (`leave it there`) on the question, or `esc`, changes nothing and leaves the chat where it is. ## I reopened an old local chat after the other computer let go — where are the newer turns @@ -246,7 +246,7 @@ An older chat whose folder is a symbolic link, or whose saved conversation state A saved chat reaches your other computers only after its encrypted objects have been uploaded and its new saved head has been accepted. If an upload fails, the other computer keeps seeing the previous saved head. Retry when the connection -returns. Uploaded objects are kept for reuse if another device continued the +returns. If the final save fails during quit, the work stays local and that computer does not announce a clean release; another computer does not automatically open that incomplete save. Uploaded objects are kept for reuse if another device continued the chat first; this does not discard the work saved on your computer. ## Relay directory upgrade pauses and saved chats @@ -323,3 +323,7 @@ Worker setup and assignment execution are not available as a terminal command ye No. The worker saves its claim request before sending it and records an execution intent before invoking tools. A lost coordination reply can retry that same request, retaining its original fence. Once execution may have started, reopening keeps the records and reports `this work may be incomplete; its saved records have been kept`. It never invokes that attempt again automatically. A recorded finish can retry its coordination acknowledgment without running tools. Reassignment changes the run revision and requires a new conditional claim. The replacement can recover saved records, but the scripted adapter refuses automatic execution of a reassigned run. Missing or incomplete calls are not instructions to execute them. Cancellation prevents future claims; stopping a claimed assignment does not cancel an already running shell process. Another take changes the fence and blocks the displaced holder from publishing. Already downloaded bytes and uploaded frames stay retained. + +## Why can an older computer list chats but not continue a chat that is still running? + +Older builds can still list, save, publish and release chats through a compatible relay. A relay using quorum-backed coordination refuses an older request that tries to take a live chat without naming its observed holder and ownership number. It uses the existing refusal, `another device continued this chat first`; an old build does not necessarily show an extra update hint. Quit on the computer holding the chat before continuing it elsewhere, or update codeaf on the computer trying to continue. A released or expired chat can still be taken. A conditional choice never silently changes to a newer holder. diff --git a/internal/manual/chat_test.go b/internal/manual/chat_test.go index e04069f563..203529e795 100644 --- a/internal/manual/chat_test.go +++ b/internal/manual/chat_test.go @@ -33,6 +33,9 @@ func TestTheChatManualAnswersTheQuestionsPeopleAsk(t *testing.T) { question string page string }{ + // R25/R29 keep older-client refusals and failed quit saves reachable. + {"why can an older computer list chats but not continue a chat that is still running", "continuing-a-chat-on-another-computer"}, + {"what happens if the final save fails during quit", "continuing-a-chat-on-another-computer"}, // C29.4/C30.2/C31.5 must be reachable in the person's own words. {"can I rewind an old chat on my new laptop", "continuing-a-chat-on-another-computer"}, {"I quit on my laptop and opened my desktop", "continuing-a-chat-on-another-computer"}, diff --git a/internal/relayserve/raft_cutover_test.go b/internal/relayserve/raft_cutover_test.go new file mode 100644 index 0000000000..1e05f71101 --- /dev/null +++ b/internal/relayserve/raft_cutover_test.go @@ -0,0 +1,92 @@ +package relayserve_test + +import ( + "database/sql" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/relayserve" + "github.com/Agent-Field/codeaf/internal/reqsign" +) + +// R25/R30 pin the real persistent relay door, independently of an adapter or +// fixture backend. This gate must stay red until the sole production route +// actually commits requests through the migrated namespace. +func TestR25R30ProductionRelayRefusesLegacyBlindForce(t *testing.T) { + raw, err := os.ReadFile("../raftdirectory/testdata/v1-requests.json") + if err != nil { + t.Fatal(err) + } + var f struct { + Now int64 + Cases []struct { + Name, Method, URI, Body string + Headers map[string]string + } + } + if err = json.Unmarshal(raw, &f); err != nil { + t.Fatal(err) + } + store := t.TempDir() + svc := relayserve.New(relayserve.Config{Store: store, Now: func() time.Time { return time.UnixMilli(f.Now) }}) + defer svc.Close() + srv := httptest.NewServer(svc.Handler) + defer srv.Close() + for _, one := range f.Cases { + if one.Name != "put-a" && one.Name != "put-b" && one.Name != "create" && one.Name != "blind-force" { + continue + } + r, err := http.NewRequest(one.Method, srv.URL+one.URI, strings.NewReader(one.Body)) + if err != nil { + t.Fatal(err) + } + for k, v := range one.Headers { + r.Header.Set(k, v) + } + resp, err := srv.Client().Do(r) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + t.Fatal(err) + } + if one.Name == "blind-force" { + if resp.StatusCode != http.StatusConflict || !strings.Contains(string(body), `"err":"lease_held"`) { + t.Fatalf("R25/R30 persistent production relay accepted blind force or lost the legacy error: status=%d body=%s; production migration/member assembly is required", resp.StatusCode, body) + } + } else if resp.StatusCode >= 300 { + t.Fatalf("%s: status=%d body=%s", one.Name, resp.StatusCode, body) + } + } + // A legacy writer could be taught only the blind-force refusal. The durable + // source ownership receipt independently requires the actual P11 cutover. + root, err := base64.RawURLEncoding.DecodeString(f.Cases[0].Headers[reqsign.HeaderIdentity]) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", "file:"+filepath.Join(store, "directory", reqsign.IDOf(root)+".db")+"?mode=ro") + if err != nil { + t.Fatal(err) + } + defer db.Close() + var gateRaw string + if err = db.QueryRow("SELECT rec FROM legacy_gate WHERE id=1").Scan(&gateRaw); err != nil { + t.Fatalf("R30 production namespace has no durable ownership receipt: %v", err) + } + var gate struct { + Stage string `json:"stage"` + } + if json.Unmarshal([]byte(gateRaw), &gate) != nil || gate.Stage != "raft-owned" { + t.Fatalf("R30 production namespace still has no committed owner: %s", gateRaw) + } +} From 6e72473646ea8c7306bd206f67288b6018b3177f Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 07:05:22 -0400 Subject: [PATCH 042/103] Preserve certified coordination and portable snapshots at durable boundaries. Keep returning holders usable at their imported capacity without enrolling extra rows. Bind operator policy and anchor keys to the source manifest, retain retryable authority failures, and preserve the exact portable snapshot encoding after the library fills default fields. These shared boundaries let the production relay use the same deterministic policy without weakening its storage checks. Give committed read callbacks the actor context so losing a client response cannot cancel durability. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/blobstore/wire.go | 1 + internal/clusterstate/apply.go | 21 +++++- internal/clusterstate/legacy.go | 51 ++++++++++--- internal/clusterstate/liveness.go | 21 +++++- internal/clusterstate/liveness_test.go | 73 +++++++++++++++++++ internal/clusterstate/records.go | 17 ++++- .../directory/directorytest/conditional.go | 8 +- internal/directory/directorytest/lease.go | 4 +- internal/directory/feed.go | 50 ++++++++++--- internal/directory/raft.go | 18 +++-- internal/directory/watch.go | 9 ++- internal/directory/watchpresence.go | 4 + internal/directory/wire.go | 2 + internal/raftcore/README.md | 7 ++ internal/raftcore/core.go | 48 ++++++++++++ internal/raftcore/snapshot_test.go | 48 ++++++++++++ internal/raftcore/types.go | 6 ++ internal/raftdirectory/v1.go | 23 ++++-- internal/raftdirectory/v1_test.go | 5 +- internal/wireauth/wireauth.go | 10 ++- 20 files changed, 375 insertions(+), 51 deletions(-) diff --git a/internal/blobstore/wire.go b/internal/blobstore/wire.go index a5962b049c..07202dbd42 100644 --- a/internal/blobstore/wire.go +++ b/internal/blobstore/wire.go @@ -25,6 +25,7 @@ var wireErrors = []struct { {"full", http.StatusInsufficientStorage, ErrFull}, {"damaged", http.StatusInternalServerError, ErrDamaged}, {"unreachable", http.StatusServiceUnavailable, ErrUnreachable}, + {"unreachable", http.StatusServiceUnavailable, wireauth.ErrUnavailable}, {"rate_limited", http.StatusTooManyRequests, wireauth.ErrRateLimited}, {"too_many_identities", http.StatusTooManyRequests, wireauth.ErrTooManyIdentities}, {"skew", http.StatusUnauthorized, wireauth.ErrSkew}, diff --git a/internal/clusterstate/apply.go b/internal/clusterstate/apply.go index 6f1ec21734..5d87e85371 100644 --- a/internal/clusterstate/apply.go +++ b/internal/clusterstate/apply.go @@ -349,7 +349,7 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er return out, nil } } - service := c.Change.RenewHolds != nil || c.Change.RecordUpload != nil || c.Change.AdmitJoin != nil || c.Change.CompleteJoin != nil || auth.Effects && c.Change.WorkerBeatBatch != nil + service := c.Change.RenewHolds != nil || c.Change.RecordUpload != nil || c.Change.AdmitJoin != nil || c.Change.CompleteJoin != nil || auth.Effects && (c.Change.WorkerBeatBatch != nil || c.Change.DeviceSeen != nil) if service != auth.Effects || auth.Personal && auth.Effects || auth.Anchor && (auth.Personal || auth.Effects || c.Change.PrepareQuiescence == nil && c.Change.SettleMembership == nil && c.Change.SettleLeadership == nil && c.Change.CompleteLeadership == nil && c.Change.RestoreAnchors == nil && c.Change.CloseSession == nil) { return out, nil } @@ -406,6 +406,17 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er out.Result.Index = meta.Index } else if c.Change.WorkerBeatBatch != nil { code = applyWorker(c, out.Now, identity, rows, put, &out.Result) + } else if service && c.Change.DeviceSeen != nil { + if d := rows[DeviceKey(c.Change.DeviceSeen.Device)]; d != nil { + var seen Device + if Decode(d, &seen) != nil { + return out, ErrDependency + } + if !seen.Revoked { + seen.LastSeen = out.Now + put(DeviceKey(c.Change.DeviceSeen.Device), seen) + } + } } else if service { code = applyEffect(c, out.Now, rows, put) } else { @@ -581,11 +592,15 @@ func applyMutation(c Command, now int64, identity IdentityRec, rows map[RecordKe return Rotated } if r.State != "retired" { + policy := GracePolicy{Min: MinGraceMillis, Max: MaxGraceMillis, Default: DefaultGraceMillis} + if identity.Grace != nil { + policy = *identity.Grace + } g := v.GraceMS if g == 0 { - g = DefaultGraceMillis + g = policy.Default } - if g < MinGraceMillis || g > MaxGraceMillis || now > int64(MaxSafeInteger)-g { + if g < policy.Min || g > policy.Max || now > int64(MaxSafeInteger)-g { return BadGrace } r = &Rotation{State: "retired", By: r.By, At: r.At, RetireAt: now + g} diff --git a/internal/clusterstate/legacy.go b/internal/clusterstate/legacy.go index e346e580c2..30c5bb0319 100644 --- a/internal/clusterstate/legacy.go +++ b/internal/clusterstate/legacy.go @@ -16,17 +16,22 @@ var ErrMigration = errors.New("codeaf: directory upgrade is incomplete; retry sh // LegacyConfig is allocated once at the source gate. Only public protocol keys // belong here; content keys and private signing keys never enter an import. type LegacyConfig struct { - Cluster string `json:"cluster"` - Generation uint64 `json:"generation"` - Identity string `json:"identity"` - SigningKey []byte `json:"signing_key"` - Anchors []uint64 `json:"anchors"` - EffectsKey []byte `json:"effects_key"` - Now int64 `json:"now"` - Certificates []string `json:"certificates,omitempty"` + Grace *GracePolicy `json:"grace,omitempty"` + Cluster string `json:"cluster"` + Generation uint64 `json:"generation"` + Identity string `json:"identity"` + SigningKey []byte `json:"signing_key"` + Anchors []uint64 `json:"anchors"` + AnchorKeys [][]byte `json:"anchor_keys,omitempty"` + EffectsKey []byte `json:"effects_key"` + Now int64 `json:"now"` + Certificates []string `json:"certificates,omitempty"` } func (c LegacyConfig) Validate() error { + if c.Grace != nil && (c.Grace.Min <= 0 || c.Grace.Max < c.Grace.Min || c.Grace.Default < c.Grace.Min || c.Grace.Default > c.Grace.Max || c.Grace.Max > int64(MaxSafeInteger)) { + return ErrMigration + } if c.Cluster == "" || c.Generation != 1 || len(c.SigningKey) != ed25519.PublicKeySize || IdentityID(c.SigningKey) != c.Identity || len(c.EffectsKey) != ed25519.PublicKeySize || bytes.Equal(c.EffectsKey, c.SigningKey) || len(c.Anchors) != 3 || c.Now < 0 || c.Now > int64(MaxSafeInteger) { return ErrMigration } @@ -35,6 +40,18 @@ func (c LegacyConfig) Validate() error { return ErrMigration } } + if len(c.AnchorKeys) != 0 { + if len(c.AnchorKeys) != len(c.Anchors) { + return ErrMigration + } + seen := map[string]bool{} + for _, key := range c.AnchorKeys { + if len(key) != ed25519.PublicKeySize || seen[string(key)] || bytes.Equal(key, c.SigningKey) || bytes.Equal(key, c.EffectsKey) { + return ErrMigration + } + seen[string(key)] = true + } + } if len(c.Certificates) > MaxLiveSessions { return ErrMigration } @@ -50,11 +67,9 @@ func (c LegacyConfig) Validate() error { } func legacyCertificate(c LegacyConfig, text string) ([]byte, error) { - var cert struct { - Device string `json:"device"` - } + var cert v1Certificate raw, e := base64.RawURLEncoding.DecodeString(text) - if e != nil || len(raw) > 1024 || json.Unmarshal(raw, &cert) != nil { + if e != nil || len(raw) > 1024 || legacyBody(raw, &cert, false) != nil { return nil, ErrMigration } public, e := hex.DecodeString(cert.Device) @@ -105,6 +120,7 @@ func LegacyRow(c LegacyConfig, kind, id string, raw []byte, holdUntil int64) (Re if legacyDecode(raw, &v) != nil || v.V != 1 || (v.Identity != "" && v.Identity != c.Identity) || (len(v.SigningKey) > 0 && !bytes.Equal(v.SigningKey, c.SigningKey)) { return RecordWrite{}, ErrMigration } + v.Grace = c.Grace v.Identity = c.Identity v.SigningKey = c.SigningKey key = IdentityKey() @@ -192,3 +208,14 @@ func LegacyAuthorities(c LegacyConfig) ([]RecordWrite, error) { sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) return out, nil } + +// PersonalCertificate checks only the existing closed public certificate +// vocabulary. Hosts call it before retaining a certificate in an effects log, +// where ignored or duplicate fields could otherwise carry private plaintext. +func PersonalCertificate(identity IdentityRec, device, text string) error { + key, err := legacyCertificate(LegacyConfig{Identity: identity.Identity, SigningKey: identity.SigningKey}, text) + if err != nil || DeviceIDOfKey(key) != device { + return ErrCodec + } + return nil +} diff --git a/internal/clusterstate/liveness.go b/internal/clusterstate/liveness.go index e913bc1d27..c57e62458f 100644 --- a/internal/clusterstate/liveness.go +++ b/internal/clusterstate/liveness.go @@ -14,6 +14,10 @@ const ( // authenticating fresh socket evidence. Evidence time fixes the expiry: replay // can never turn an old ping into another ninety seconds of ownership. type RenewHolds struct { + // Certificate carries an original root-signed personal certificate for a + // legacy holder whose public authority was never retained in the old store. + // It authorizes this evidence only and never creates an enrollment record. + Certificate string `json:"certificate,omitempty"` WorkerIncarnation uint64 `json:"worker_incarnation,omitempty"` Device string `json:"device"` SeenAt int64 `json:"seen_at"` @@ -47,7 +51,22 @@ func renewHolds(c Command, now int64, rows map[RecordKey][]byte, put func(Record } var actor Device var auth Authority - if Decode(rows[AuthorityKey(v.Device)], &auth) != nil || !auth.ValidRole() || auth.Cluster != c.Cluster || !auth.Personal && auth.Worker == nil { + if rows[AuthorityKey(v.Device)] == nil && v.Certificate != "" { + var identity IdentityRec + if Decode(rows[IdentityKey()], &identity) != nil { + return BadRequest + } + key, err := legacyCertificate(LegacyConfig{Identity: identity.Identity, SigningKey: identity.SigningKey}, v.Certificate) + if err != nil || DeviceIDOfKey(key) != v.Device { + return Unauthorized + } + auth = Authority{Cluster: c.Cluster, Key: key, Personal: true} + } else if raw := rows[AuthorityKey(v.Device)]; raw != nil { + if Decode(raw, &auth) != nil { + return BadRequest + } + } + if !auth.ValidRole() || auth.Cluster != c.Cluster || !auth.Personal && auth.Worker == nil { return Unauthorized } if raw := rows[DeviceKey(v.Device)]; raw != nil && Decode(raw, &actor) != nil { diff --git a/internal/clusterstate/liveness_test.go b/internal/clusterstate/liveness_test.go index eee7860769..b145496a3d 100644 --- a/internal/clusterstate/liveness_test.go +++ b/internal/clusterstate/liveness_test.go @@ -2,6 +2,7 @@ package clusterstate_test import ( "crypto/ed25519" + "encoding/base64" "fmt" "math/rand" "testing" @@ -142,3 +143,75 @@ func TestR26R27CommittedOutboxPreventsSleep(t *testing.T) { } } } + +// R16/R22 require an ingress presence stamp to remain advisory and scoped. A +// watch may precede metadata; absence is neither corrupt state nor enrollment. +func TestR16R22IngressPresenceDoesNotGrantDirectoryAuthority(t *testing.T) { + a := newApplication(t) + a.put(cs.AuthorityKey("ingress"), cs.Authority{Cluster: "test", Key: actorKey("ingress").Public().(ed25519.PublicKey), Effects: true}) + for _, device := range []string{"missing", "a"} { + r := a.run("ingress", 1001, cs.Mutation{DeviceSeen: &cs.DeviceSeen{Device: device}}) + if r.Code != cs.OK { + t.Fatal("presence refused without a metadata row", r) + } + } + for _, m := range []cs.Mutation{{Create: &cs.CreateCell{Cell: "chat", Init: cs.CellInit{Head: "sealed"}}}, {SetVault: &cs.VaultCAS{New: "forged"}}, {PutDevice: &cs.PutDevice{ID: "missing", Device: cs.Device{V: 1}}}} { + r := a.run("ingress", 1002, m) + if r.Code != cs.Unauthorized { + t.Fatal("presence service gained personal authority", r) + } + } + if a.meta.Version != 0 { + t.Fatal("advisory presence moved the visible directory") + } +} + +// R16/R22/R24 use a returning holder's original public certificate without +// growing an imported identity's retained rows or granting peer membership. +func TestR16R22R24LegacyHoldCertificateDoesNotConsumeEnrollmentCapacity(t *testing.T) { + a := newApplication(t) + public := actorKey("returning").Public().(ed25519.PublicKey) + device := cs.DeviceIDOfKey(public) + a.put(cs.DeviceKey(device), cs.Device{V: 1}) + a.put(cs.CellKey("chat"), cs.Cell{V: 1, Head: "sealed", Lease: cs.Lease{Device: device, Fence: 1, Expires: 91000}}) + before := len(a.rows) + evidence := cs.RenewHolds{Certificate: joinCert(t, "returning"), Device: device, SeenAt: 2000, Items: []cs.HoldFence{{Cell: "chat", Fence: 1}}} + r := service(a, 2000, cs.Mutation{RenewHolds: &evidence}) + if r.Code != cs.OK || len(r.Items) != 1 || r.Items[0].Code != cs.OK || a.cell().Lease.Expires != 92000 { + t.Fatal(r, a.cell()) + } + // service adds its fixture authority once; the holder adds no retained row. + if len(a.rows) != before+1 || a.rows[cs.AuthorityKey(device)] != nil { + t.Fatal("legacy renewal enrolled authority or grew history") + } + evidence.Certificate = joinCert(t, "other") + if r = service(a, 2001, cs.Mutation{RenewHolds: &evidence}); r.Code != cs.Unauthorized { + t.Fatal("another certificate renewed the holder", r) + } + evidence.Certificate = joinCert(t, "returning") + a.put(cs.DeviceKey(device), cs.Device{V: 1, Revoked: true}) + if r = service(a, 2002, cs.Mutation{RenewHolds: &evidence}); r.Code != cs.Revoked { + t.Fatal("retained certificate bypassed revocation", r) + } +} + +// R15 requires discarded certificate fields to stay out of a retained watch +// witness; a valid signature over known fields cannot authorize extra text. +func TestR15LegacyWatchCertificateRejectsHiddenFieldsBeforeRetention(t *testing.T) { + cert := joinCert(t, "returning") + identity := cs.IdentityRec{V: 1, Identity: cs.IdentityID(actorKey("root").Public().(ed25519.PublicKey)), SigningKey: actorKey("root").Public().(ed25519.PublicKey)} + device := cs.DeviceIDOfKey(actorKey("returning").Public().(ed25519.PublicKey)) + if e := cs.PersonalCertificate(identity, device, cert); e != nil { + t.Fatal(e) + } + raw, e := base64.RawURLEncoding.DecodeString(cert) + if e != nil { + t.Fatal(e) + } + for _, injected := range []string{`"private":"sentinel",`, `"device":"sentinel",`} { + bad := base64.RawURLEncoding.EncodeToString(append([]byte("{"+injected), raw[1:]...)) + if e := cs.PersonalCertificate(identity, device, bad); e == nil { + t.Fatal("ignored or duplicate certificate text may enter the log") + } + } +} diff --git a/internal/clusterstate/records.go b/internal/clusterstate/records.go index 77737bf131..46028c7d88 100644 --- a/internal/clusterstate/records.go +++ b/internal/clusterstate/records.go @@ -4,11 +4,20 @@ package clusterstate // IdentityRec is the per-identity root record. +// GracePolicy is an operator-provisioned rotation policy preserved by import. +// Requests cannot change it or choose a weaker policy during evaluation. +type GracePolicy struct { + Min int64 `json:"min"` + Max int64 `json:"max"` + Default int64 `json:"default"` +} + type IdentityRec struct { - V uint16 `json:"V"` - Identity string `json:"identity"` // id_…, derived from the signing key - SigningKey []byte `json:"signing_key,omitempty"` // Public identity key for committed certificate checks. - Vault string `json:"vault,omitempty"` // rid of the current vault object + Grace *GracePolicy `json:"grace,omitempty"` + V uint16 `json:"V"` + Identity string `json:"identity"` // id_…, derived from the signing key + SigningKey []byte `json:"signing_key,omitempty"` // Public identity key for committed certificate checks. + Vault string `json:"vault,omitempty"` // rid of the current vault object // Rotation is set once the identity has been replaced by a rotation: it // takes no writes from then on. Nil means live. Rotation *Rotation `json:"rotation,omitempty"` diff --git a/internal/directory/directorytest/conditional.go b/internal/directory/directorytest/conditional.go index f7c158979d..26fe767508 100644 --- a/internal/directory/directorytest/conditional.go +++ b/internal/directory/directorytest/conditional.go @@ -42,10 +42,14 @@ func conditionalForcedTakeOneWins(t *testing.T, e env) { // RunConsensus uses every existing verb case, with the R25 compatibility // contract at the one place where blind legacy force deliberately changes. func RunConsensus(t *testing.T, factory Factory) { + RunConsensusRigs(t, func(t *testing.T) Rig { clock := NewFakeClock(); return Rig{Clock: clock, Devices: factory(t, clock)} }) +} + +// RunConsensusRigs retains every wire case while asserting R25 at blind force. +func RunConsensusRigs(t *testing.T, factory RigFactory) { for name, fn := range cases { t.Run(name, func(t *testing.T) { - clock := NewFakeClock() - e := envOf(Rig{Clock: clock, Devices: factory(t, clock)}) + e := envOf(factory(t)) if name == "C7ConditionalForcedTakeOneWins" { conditionalForcedTakeRace(t, e) _, err := e.as(devA).Acquire(ctx, cell, directory.AcquireOpts{Force: true}) diff --git a/internal/directory/directorytest/lease.go b/internal/directory/directorytest/lease.go index af79a13ac0..4f24340135 100644 --- a/internal/directory/directorytest/lease.go +++ b/internal/directory/directorytest/lease.go @@ -211,7 +211,9 @@ func takeoverSilencesOldSocket(w watchEnv) { tl := w.timeline() tl.to(leaseAge) old.ping() - _, err := w.dirAs(devB).Acquire(ctx, cell, directory.AcquireOpts{Force: true}) + seen, err := w.dirAs(devB).Cell(ctx, cell) + must(w.t, err) + _, err = w.dirAs(devB).Acquire(ctx, cell, directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: seen.Cell.Lease.Device, Fence: seen.Cell.Lease.Fence}}) must(w.t, err) tl.to(leasePast) old.ping() diff --git a/internal/directory/feed.go b/internal/directory/feed.go index b469b680ac..3edf052c58 100644 --- a/internal/directory/feed.go +++ b/internal/directory/feed.go @@ -150,12 +150,27 @@ func (f *Feed) Subscribe(device string, holds []Hold) (*Sub, error) { // SubscribeBeat is Subscribe for a watcher that promised to ping every beat: it // counts as online for Window(beat) after its last sign of life. -func (f *Feed) SubscribeBeat(device string, holds []Hold, beat time.Duration) (*Sub, error) { +// SubscribeBeatWithEvidence reserves a watch before asking a consensus host +// to commit its authenticated liveness. The callback belongs to this socket, +// so its original public certificate disappears with the subscription. +func (f *Feed) SubscribeBeatWithEvidence(device string, holds []Hold, beat time.Duration, evidence func(string, []Hold, int64) error) (*Sub, error) { s, err := f.subscribe(device, holds, beat) - if err == nil { - f.pres.opened(s) + if err != nil { + return nil, err + } + s.evidence = evidence + if evidence != nil { + if err = evidence(device, holds, f.clock().UnixMilli()); err != nil { + s.leave() + return nil, err + } } - return s, err + f.pres.opened(s) + return s, nil +} + +func (f *Feed) SubscribeBeat(device string, holds []Hold, beat time.Duration) (*Sub, error) { + return f.SubscribeBeatWithEvidence(device, holds, beat, nil) } func (f *Feed) subscribe(device string, holds []Hold, beat time.Duration) (*Sub, error) { @@ -234,10 +249,11 @@ func (f *Feed) differs(id string, old, next any, now int64) bool { // Sub is one watcher. It keeps no queue: it always reads the newest Status, so // a slow watcher skips intermediate versions and never sees one go down. type Sub struct { - f *Feed - poke chan struct{} - sent uint64 - started bool + evidence func(string, []Hold, int64) error + f *Feed + poke chan struct{} + sent uint64 + started bool device string shut atomic.Bool @@ -270,7 +286,23 @@ func (s *Sub) lapse() { // Ping records a sign of life at the directory's time. It changes no version // and wakes nobody: evidence is not a change a person can see. -func (s *Sub) Ping() { s.alive.Store(s.f.clock().UnixMilli()) } +func (s *Sub) Ping() { _ = s.PingCommitted() } + +// PingCommitted withholds a pong when a consensus renewal cannot commit. +func (s *Sub) PingCommitted() error { + at := s.f.clock().UnixMilli() + if s.evidence != nil { + holds := make([]Hold, len(s.keys)) + for i, k := range s.keys { + holds[i] = Hold{Cell: k.cell, Fence: k.fence} + } + if err := s.evidence(s.device, holds, at); err != nil { + return err + } + } + s.alive.Store(at) + return nil +} func (s *Sub) wake() { select { diff --git a/internal/directory/raft.go b/internal/directory/raft.go index 842a4f7e00..668427c411 100644 --- a/internal/directory/raft.go +++ b/internal/directory/raft.go @@ -19,9 +19,16 @@ type RaftBackend interface { ReadCell(context.Context, string, string) (CellView, error) } -type Raft struct{ backend RaftBackend } +type Raft struct { + backend RaftBackend + grace GraceBounds +} + +func NewRaft(backend RaftBackend) *Raft { return &Raft{backend: backend, grace: DefaultGraceBounds} } + +// SetGraceBounds installs the durable namespace policy before serving requests. +func (r *Raft) SetGraceBounds(g GraceBounds) { r.grace = g } -func NewRaft(backend RaftBackend) *Raft { return &Raft{backend: backend} } func (r *Raft) For(device string) Client { return &raftClient{r: r, device: device} } type raftClient struct { @@ -37,7 +44,7 @@ func (c *raftClient) change(ctx context.Context, m clusterstate.Mutation) (CellV if err != nil { return CellView{}, err } - if err = resultError(r.Code); err != nil { + if err = ResultError(r.Code); err != nil { return CellView{}, err } if r.View != nil { @@ -110,7 +117,7 @@ func (c *raftClient) Rotate(ctx context.Context, req RotationReq) (RotationView, } func (c *raftClient) Rotation(ctx context.Context) (RotationView, error) { l, e := c.List(ctx) - return viewOf(l.Identity, l.Now, DefaultGraceBounds), e + return viewOf(l.Identity, l.Now, c.r.grace), e } // Pairing commits a sealed decision dependency; delivery follows through the @@ -130,7 +137,8 @@ func (c *raftClient) DenyRequest(ctx context.Context, code string) error { var errRaftConflict = errors.New("directory: request sequence conflict") -func resultError(code clusterstate.Code) error { +// ResultError preserves the existing wire refusals at host read boundaries. +func ResultError(code clusterstate.Code) error { switch code { case clusterstate.OK: return nil diff --git a/internal/directory/watch.go b/internal/directory/watch.go index 92f03a794c..73a5905578 100644 --- a/internal/directory/watch.go +++ b/internal/directory/watch.go @@ -95,6 +95,11 @@ func subscribe(c call) (*Sub, error) { if err != nil { return nil, err } + if authenticated, ok := c.dir.(interface { + SubscribeWatch(string, *http.Request, []Hold, time.Duration) (*Sub, error) + }); ok { + return authenticated.SubscribeWatch(c.device, c.r, holds, beat) + } return feed.Feed().SubscribeBeat(c.device, holds, beat) } @@ -203,7 +208,9 @@ func answerPings(ctx context.Context, conn *websocket.Conn, sub *Sub) { return } if kind == websocket.MessageText && string(msg) == "ping" { - sub.Ping() + if sub.PingCommitted() != nil { + return + } if sendText(ctx, conn, "pong") != nil { return } diff --git a/internal/directory/watchpresence.go b/internal/directory/watchpresence.go index 650d24ea23..43573bd8d5 100644 --- a/internal/directory/watchpresence.go +++ b/internal/directory/watchpresence.go @@ -317,6 +317,10 @@ func (f *Feed) announceJoin(j Joined) { f.pres.announce(dirwatch.JoinedEvent(j.Device, j.Name, j.Platform, j.At), j.Device) } +// AnnounceJoin publishes an advisory event only after a consensus host has +// committed the device and completed its pairing delivery. +func (f *Feed) AnnounceJoin(j Joined) { f.announceJoin(j) } + // OnSeen registers fn to stamp a device's last_seen: it is called with the // directory time (unix ms) at each watch connect and when a device closes its // last socket, never per ping and never while the feed holds a lock. A diff --git a/internal/directory/wire.go b/internal/directory/wire.go index d195b08eeb..9596075ed2 100644 --- a/internal/directory/wire.go +++ b/internal/directory/wire.go @@ -19,6 +19,8 @@ type wireError struct { // wireErrors is the single place a sentinel and its code meet, so the handler // and the client cannot drift apart. Order matters only for errors.Is chains. var wireErrors = []wireError{ + {ErrUnreachable, "unreachable", http.StatusServiceUnavailable}, + {wireauth.ErrUnavailable, "unreachable", http.StatusServiceUnavailable}, {clusterstate.ErrMigration, "unreachable", http.StatusServiceUnavailable}, {errFeedClosed, "unreachable", http.StatusServiceUnavailable}, {ErrNotFound, "not_found", http.StatusNotFound}, diff --git a/internal/raftcore/README.md b/internal/raftcore/README.md index 7d6db33b2b..84c527e964 100644 --- a/internal/raftcore/README.md +++ b/internal/raftcore/README.md @@ -240,3 +240,10 @@ commit completion and obtain a new receipt. Returning completed intents never regain leadership automatically. `QuiescenceCoordinator` sequences the idle part without adding another coordination writer. P17 supplies the sole product host and the outbound router assembly; no deployment or rollout switch exists. + +Portable snapshot metadata keeps the manifest's exact protobuf field presence. +The pinned library fills omitted default fields such as `auto_leave:false` in +memory. The wrapper checks the complete effective configuration, index and term, +then restores the manifest's canonical presence on storage reads and incoming +Ready snapshots. This prevents a real transfer from disagreeing with its durable +chunk manifest without changing any voter, learner or joint-consensus value. diff --git a/internal/raftcore/core.go b/internal/raftcore/core.go index 3198dade4d..7b480c1c79 100644 --- a/internal/raftcore/core.go +++ b/internal/raftcore/core.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "errors" + "reflect" "slices" "github.com/Agent-Field/codeaf/internal/clusterstate" @@ -28,6 +29,41 @@ func (s *storage) InitialState() (*pb.HardState, *pb.ConfState, error) { return h, s.conf, err } +// Snapshot returns the exact portable configuration encoding. The library +// fills optional zero fields in memory; that presence must not change the +// manifest-bound bytes which the durable store and another runtime compare. +func (s *storage) Snapshot() (*pb.Snapshot, error) { + p, err := s.MemoryStorage.Snapshot() + if err != nil { + return nil, err + } + return canonicalSnapshot(p) +} + +// canonicalSnapshot accepts only equal effective configurations and restores +// the manifest's exact protobuf presence. It never changes a voter, learner, +// joint field, term or index, and rejects unknown snapshot metadata. +func canonicalSnapshot(p *pb.Snapshot) (*pb.Snapshot, error) { + if raft.IsEmptySnap(p) { + return p, nil + } + m, err := raftsnapshot.Decode(p.GetData()) + if err != nil { + return nil, ErrSnapshot + } + conf, err := m.Configuration() + if err != nil { + return nil, ErrSnapshot + } + meta := p.GetMetadata() + if meta == nil || len(p.ProtoReflect().GetUnknown()) != 0 || len(meta.ProtoReflect().GetUnknown()) != 0 || meta.GetConfState() == nil || len(meta.GetConfState().ProtoReflect().GetUnknown()) != 0 || meta.GetIndex() != m.Index || meta.GetTerm() != m.Term || !reflect.DeepEqual(Configuration(meta.GetConfState()), Configuration(conf)) { + return nil, ErrSnapshot + } + p = clone(p) + p.Metadata.ConfState = conf + return p, nil +} + type pending struct { id uint64 ready raft.Ready @@ -318,6 +354,10 @@ func (c *Core) Step(input Input) (effects Effects, err error) { return effects, ErrSnapshot } } + case Campaign: + if err = c.node.Campaign(); err != nil { + return effects, err + } case Poll: case Tick: if !c.schedule.sleeping { @@ -486,6 +526,14 @@ func (c *Core) collect(e *Effects) error { } c.serial++ rd := c.node.Ready() + if !raft.IsEmptySnap(rd.Snapshot) { + var err error + rd.Snapshot, err = canonicalSnapshot(rd.Snapshot) + if err != nil { + c.stopped = true + return err + } + } c.pending = &pending{id: c.serial, ready: rd} e.Persist = &PersistBatch{ID: c.serial, HardState: clone(rd.HardState), Entries: copyEntries(rd.Entries), Snapshot: clone(rd.Snapshot)} return nil diff --git a/internal/raftcore/snapshot_test.go b/internal/raftcore/snapshot_test.go index 2ac639eb44..f79ec91f58 100644 --- a/internal/raftcore/snapshot_test.go +++ b/internal/raftcore/snapshot_test.go @@ -77,3 +77,51 @@ func TestR11R12R13SnapshotHintAndDurableCompactInput(t *testing.T) { t.Fatal("R13 snapshot retained caller aliases") } } + +// R13/R14 preserve the portable manifest when the library fills an omitted +// default ConfState field. Every effective configuration field remains bound. +func TestR13R14PortableSnapshotDefaultPresenceThroughRealCore(t *testing.T) { + conf := &pb.ConfState{Voters: []uint64{1, 2, 3}} + pointer := func(index, term uint64) *pb.Snapshot { + encoded, e := raftsnapshot.Configuration(conf) + if e != nil { + t.Fatal(e) + } + m := raftsnapshot.Manifest{Version: 1, Cluster: "snapshot-default", Protocol: 1, Index: index, Term: term, Conf: encoded, RecordsHash: strings.Repeat("0", 64), Digest: raftsnapshot.ID(nil)} + data, e := m.Encode() + if e != nil { + t.Fatal(e) + } + return &pb.Snapshot{Data: data, Metadata: &pb.SnapshotMetadata{Index: proto.Uint64(index), Term: proto.Uint64(term), ConfState: clone(conf)}} + } + saved := pointer(5, 1) + c, e := New(Config{ClusterUUID: "snapshot-default", NodeID: 1, Seed: 1, Protocol: 1, Anchors: []uint64{1, 2, 3}}, RecoveredState{HardState: &pb.HardState{Term: proto.Uint64(1), Commit: proto.Uint64(5)}, ConfState: conf, Snapshot: saved, Applied: 5}) + if e != nil { + t.Fatal(e) + } + retained, e := c.store.Snapshot() + if e != nil || !proto.Equal(retained, saved) { + t.Fatal("library default changed durable snapshot", retained.GetMetadata(), e) + } + incoming := pointer(7, 2) + raw, e := wire.Marshal(&pb.Message{Type: pb.MsgSnap.Enum(), From: proto.Uint64(2), To: proto.Uint64(1), Term: proto.Uint64(2), Snapshot: incoming}) + if e != nil { + t.Fatal(e) + } + effects, e := c.Step(PeerMessage{Bytes: raw}) + if e != nil || effects.Persist == nil || !proto.Equal(effects.Persist.Snapshot, incoming) { + t.Fatal("received snapshot lost its manifest-bound presence", effects, e) + } + effects, e = c.Step(Persisted{BatchID: effects.Persist.ID}) + if e != nil || effects.Apply == nil || !proto.Equal(effects.Apply.Snapshot, incoming) || !proto.Equal(effects.Apply.ConfState, conf) { + t.Fatal("apply changed portable snapshot", effects, e) + } + if _, e = c.Step(Applied{BatchID: effects.Apply.ID}); e != nil { + t.Fatal(e) + } + bad := clone(incoming) + bad.Metadata.ConfState.AutoLeave = proto.Bool(true) + if _, e = canonicalSnapshot(bad); !errors.Is(e, ErrSnapshot) { + t.Fatal("changed effective joint policy accepted", e) + } +} diff --git a/internal/raftcore/types.go b/internal/raftcore/types.go index 3498802797..cea6dedca8 100644 --- a/internal/raftcore/types.go +++ b/internal/raftcore/types.go @@ -86,6 +86,12 @@ type SnapshotPoint struct { Term uint64 } +// Campaign starts an explicit host-requested election without inventing elapsed ticks. +// It is a local input and can never arrive through authenticated peer bytes. +type Campaign struct{} + +func (Campaign) coreInput() {} + type Poll struct{} type Tick struct{} type PeerMessage struct{ Bytes []byte } diff --git a/internal/raftdirectory/v1.go b/internal/raftdirectory/v1.go index 49e4855dc8..cb4e1e0765 100644 --- a/internal/raftdirectory/v1.go +++ b/internal/raftdirectory/v1.go @@ -18,16 +18,19 @@ import ( // Commit returns only a durable applied result. Read commits the GET command's // ingress time, obtains a fresh current-term ReadIndex and waits for durable // application before calling inspect. A POST already committed by this request -// uses that time and a NEW barrier without repeating its mutation. +// uses that time and a NEW barrier without repeating its mutation. Callbacks +// receive the actor-owned context: a caller losing its response cannot cancel +// durability or poison a healthy member while committed work is completing. type V1Host interface { CommitV1(context.Context, clusterstate.Command) (clusterstate.Result, error) - ReadV1(context.Context, clusterstate.Command, bool, func(*raftstore.SQLite, raftstore.Application) error) error + ReadV1(context.Context, clusterstate.Command, bool, func(context.Context, *raftstore.SQLite, raftstore.Application) error) error } // V1Directory is the authenticated legacy HTTP adapter for one frozen namespace. // It has no key with which to impersonate a personal device. ForRequest receives // exactly the proof admitted by directory.Handler, including archived bodies. type V1Directory struct { + Grace directory.GraceBounds Host V1Host Cluster string Now func() time.Time @@ -62,7 +65,11 @@ func (d *V1Directory) ForRequest(actor string, r *http.Request, raw []byte) dire b.err = directory.ErrUnauthorized } } - return directory.NewRaft(b).For(actor) + adapter := directory.NewRaft(b) + if d.Grace != (directory.GraceBounds{}) { + adapter.SetGraceBounds(d.Grace) + } + return adapter.For(actor) } type v1Backend struct { @@ -85,18 +92,18 @@ func (b *v1Backend) Change(ctx context.Context, actor string, m clusterstate.Mut return r, err } -func (b *v1Backend) read(ctx context.Context, actor string, fn func(*raftstore.SQLite, raftstore.Application) error) error { +func (b *v1Backend) read(ctx context.Context, actor string, fn func(context.Context, *raftstore.SQLite, raftstore.Application) error) error { if b.err != nil { return b.err } if b.owner.Host == nil || actor != b.actor { return directory.ErrUnauthorized } - return b.owner.Host.ReadV1(ctx, b.command, b.committed, func(s *raftstore.SQLite, app raftstore.Application) error { + return b.owner.Host.ReadV1(ctx, b.command, b.committed, func(ctx context.Context, s *raftstore.SQLite, app raftstore.Application) error { if err := authorizeV1Read(ctx, s, app, b.command); err != nil { return err } - return fn(s, app) + return fn(ctx, s, app) }) } @@ -127,7 +134,7 @@ func authorizeV1Read(ctx context.Context, s *raftstore.SQLite, app raftstore.App func (b *v1Backend) Listing(ctx context.Context, actor string) (directory.Listing, error) { l := directory.Listing{Devices: map[string]directory.Device{}, Cells: map[string]directory.Cell{}} - err := b.read(ctx, actor, func(s *raftstore.SQLite, app raftstore.Application) error { + err := b.read(ctx, actor, func(ctx context.Context, s *raftstore.SQLite, app raftstore.Application) error { l.Now, l.Version = app.LogicalTime, app.VisibleVersion after := "" for { @@ -173,7 +180,7 @@ func (b *v1Backend) Listing(ctx context.Context, actor string) (directory.Listin } func (b *v1Backend) ReadCell(ctx context.Context, actor, id string) (directory.CellView, error) { var v directory.CellView - err := b.read(ctx, actor, func(s *raftstore.SQLite, app raftstore.Application) error { + err := b.read(ctx, actor, func(ctx context.Context, s *raftstore.SQLite, app raftstore.Application) error { k := raftstore.Key{Namespace: raftstore.State, Key: clusterstate.CellKey(id).Key} rows, _, err := s.Read(ctx, app.Applied, []raftstore.Key{k}) if err != nil { diff --git a/internal/raftdirectory/v1_test.go b/internal/raftdirectory/v1_test.go index 6f87030768..900a20b658 100644 --- a/internal/raftdirectory/v1_test.go +++ b/internal/raftdirectory/v1_test.go @@ -30,7 +30,8 @@ func (g *group) CommitV1(_ context.Context, c cs.Command) (cs.Result, error) { defer g.mu.Unlock() return g.submit(c) } -func (g *group) ReadV1(ctx context.Context, c cs.Command, committed bool, inspect func(*raftstore.SQLite, raftstore.Application) error) error { +func (g *group) ReadV1(ctx context.Context, c cs.Command, committed bool, inspect func(context.Context, *raftstore.SQLite, raftstore.Application) error) error { + ctx = context.WithoutCancel(ctx) g.mu.Lock() defer g.mu.Unlock() if !committed { @@ -63,7 +64,7 @@ func (g *group) ReadV1(ctx context.Context, c cs.Command, committed bool, inspec if e != nil { return e } - return inspect(g.stores[i], app) + return inspect(ctx, g.stores[i], app) } type v1Fixture struct { diff --git a/internal/wireauth/wireauth.go b/internal/wireauth/wireauth.go index 3dd0e2dbcb..3c15748a7c 100644 --- a/internal/wireauth/wireauth.go +++ b/internal/wireauth/wireauth.go @@ -40,11 +40,15 @@ var ErrRevoked = errors.New("wireauth: device revoked") // other refusal: only pairing with the new identity brings a machine back. var ErrGone = errors.New("wireauth: this identity was replaced and deleted") -// Narrow reduces any authentication failure to the four the wires name: skew, -// revoked and gone, which the person can act on, and the rest. Both wires call it, +// ErrUnavailable preserves a retryable failure after signature authentication +// when the authoritative namespace cannot complete its quorum read or import. +var ErrUnavailable = errors.New("wireauth: coordination unavailable") + +// Narrow preserves named authentication and coordination failures, then reduces +// all other errors to unauthorized. Both wires call it, // so they cannot disagree about what a refusal is. func Narrow(err error) error { - for _, named := range []error{ErrSkew, ErrRevoked, ErrGone} { + for _, named := range []error{ErrSkew, ErrRevoked, ErrGone, ErrUnavailable} { if errors.Is(err, named) { return named } From 64f44959c4efba9d227d97ff45d1d12e91412f37 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 07:05:23 -0400 Subject: [PATCH 043/103] Replace the persistent relay writer with three durable local anchors. Freeze and preserve the original namespace before one resumable three-store import, then require quorum commit and fresh reads for directory, rotation and blob authority. Keep pairing and watches working through committed effects, recover returning stores through checked snapshot chunks, and refuse to recreate lost voter identities or re-import a published voter without its own receipt. Keep disconnected clients from poisoning shared actors. Document the native topology and limits, include the modified Raft notices, and record the measured runtime cost in the size budget. Co-Authored-By: Claude Opus 5.5 (1M context) --- PERF.md | 6 + SIZE-BUDGET | 2 +- THIRD-PARTY-NOTICES.md | 278 ++++++++++ cmd/codeaf-notices/notices.go | 13 +- docs/relay.md | 36 +- internal/manual/chat/choosing-a-folder.md | 6 + .../continuing-a-chat-on-another-computer.md | 21 + internal/raftdirectory/native.go | 482 ++++++++++++++++++ internal/raftdirectory/native_host_test.go | 57 +++ internal/relayserve/namespaces.go | 75 +-- internal/relayserve/native.go | 442 ++++++++++++++++ internal/relayserve/native_test.go | 339 ++++++++++++ internal/relayserve/retire.go | 29 +- internal/relayserve/revoked_test.go | 2 +- internal/relayserve/service.go | 7 +- 15 files changed, 1725 insertions(+), 70 deletions(-) create mode 100644 internal/raftdirectory/native.go create mode 100644 internal/raftdirectory/native_host_test.go create mode 100644 internal/relayserve/native.go create mode 100644 internal/relayserve/native_test.go diff --git a/PERF.md b/PERF.md index 01c56095a2..eeed16d8ad 100644 --- a/PERF.md +++ b/PERF.md @@ -108,3 +108,9 @@ artifact. The reference target is darwin/amd64 with its own pinned artifact stag other platforms can have different absolute sizes. Reduce unnecessary code, data or dependencies first. A justified cap adjustment belongs in the same change as the feature and its measurements; keep `SIZE-BUDGET` explicit and reviewable. + +Every build links the Raft coordination core: the modified Raft copy, protobuf, +the signed state machine and its SQLite snapshot host. The relay and devices +always coordinate through it, so no build leaves it out. On linux/amd64 it added +2,519,040 bytes to the stripped binary (75,268,361 to 77,787,401), and the cap +moved by the same amount, from 75,811,848 to 78,330,888. diff --git a/SIZE-BUDGET b/SIZE-BUDGET index 8bab5c3102..7f9f69cc47 100644 --- a/SIZE-BUDGET +++ b/SIZE-BUDGET @@ -1 +1 @@ -75811848 +78330888 diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index b20901dd1e..b5b4b7598d 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -2873,6 +2873,220 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ``` +## go.etcd.io/raft/v3 + +- Version: v3.7.0 +- Licence: Apache-2.0 + +The Apache-2.0 Raft source is a modified copy in `third_party/raft`: codeaf injects per-node election randomness for deterministic replay. The pinned upstream version, patch and licence are retained there. + +### LICENSE + +```text + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` + ## go.yaml.in/yaml/v4 - Version: v4.0.0-rc.2 @@ -3436,6 +3650,70 @@ rights granted to you under this License for this implementation of Go shall terminate as of the date such litigation is filed. ``` +## google.golang.org/protobuf + +- Version: v1.36.11 +- Licence: BSD-3-Clause with the Go PATENTS grant + +### LICENSE + +```text +Copyright (c) 2018 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` + +### PATENTS + +```text +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. +``` + ## gopkg.in/yaml.v3 - Version: v3.0.1 diff --git a/cmd/codeaf-notices/notices.go b/cmd/codeaf-notices/notices.go index abf545f600..34bd9de478 100644 --- a/cmd/codeaf-notices/notices.go +++ b/cmd/codeaf-notices/notices.go @@ -68,10 +68,15 @@ func collectComponents(root string, environment []string) ([]component, error) { if err != nil { return nil, fmt.Errorf("read licences for %s: %w", module.Path, err) } + description := "" + if module.Path == "go.etcd.io/raft/v3" { + description = "The Apache-2.0 Raft source is a modified copy in `third_party/raft`: codeaf injects per-node election randomness for deterministic replay. The pinned upstream version, patch and licence are retained there." + } components = append(components, component{ - Identifier: module.Path, - Version: module.Version, - Licences: licences, + Description: description, + Identifier: module.Path, + Version: module.Version, + Licences: licences, }) } @@ -123,7 +128,7 @@ func collectComponents(root string, environment []string) ([]component, error) { func listCompiledModules(root string, environment []string) ([]moduleVersion, error) { const format = `{{if and (not .Standard) .Module}}{{if not .Module.Main}}{{.Module.Path}} {{.Module.Version}}{{end}}{{end}}` - out, err := runGo(root, environment, "list", "-deps", "-f", format, "./cmd/codeaf") + out, err := runGo(root, environment, "list", "-p=1", "-deps", "-f", format, "./cmd/codeaf") if err != nil { return nil, fmt.Errorf("list modules compiled into cmd/codeaf: %w", err) } diff --git a/docs/relay.md b/docs/relay.md index eb30455dbc..1019c5e101 100644 --- a/docs/relay.md +++ b/docs/relay.md @@ -25,7 +25,9 @@ Point each computer at it with `CODEAF_SYNC_URL=https://relay.example.com`, or p The relay has no TLS and no allow-list. Put a TLS-terminating proxy in front of it, and keep it on a private network or behind a proxy that checks who calls: any valid signature may make -an identity and store on it. It has no quota of its own, so the disk is the limit. +an identity and store on it. Coordination rows have an 8 MiB default admission ceiling; +a larger imported identity keeps its measured ceiling. Reaching that ceiling refuses +growth without deleting saved records. Sealed blobs still need operator disk capacity. It logs the first 8 characters of an identity id, the kind of request, the status and two byte counts. Never a body. @@ -34,22 +36,40 @@ byte counts. Never a body. Everything is under `--store`: -- `directory/.db`: one SQLite file for each identity (chat records, device records, leases); +- `raft//{1,2,3}.db`: three independent FULL-synchronous WAL stores for coordination; +- `raft//provision.json`: the immutable cluster ID and private protocol keys; +- `directory/.db`: the preserved frozen source and migration ownership receipt; - `blobs//`: that identity's chunks and the pointers that find an object inside one. Stop the relay (SIGTERM; requests under way finish) and copy the whole folder, or copy a filesystem snapshot. Copying the `.db` files while the relay runs can give a torn copy. -Restore by putting the folder back -and starting the relay. A restore to an older copy loses turns sent after it; a computer that -still holds them sends them again at its next sync (not tested here). +Restart using the same complete store. Never replace a surviving member with a fresh +import under its old ID. Two actual surviving anchor stores can coordinate; fewer than +two refuse service. Keep damaged or unavailable stores for diagnosis and recovery. + +A stale backup is not an ordinary restart. Do not start it beside a live cluster or +reactivate its frozen source. Disaster recovery needs a new epoch and exclusion of the +old cluster; automatic stale-backup recovery is not offered by this relay. The relay cannot decrypt what it keeps, so its backup is ciphertext too. It holds no key that -could recover a lost identity. +could recover a lost identity. Protocol keys cannot decrypt chats, but keep the store +private and back up its provisioning file together with its databases. ## Upgrade -Stop it, replace the `codeaf` binary, start it with the same `--store`. There is no migration command, and this page does not promise that every version reads an older store: back up first. A client newer than the -relay by a pairing change sees `that relay is too old for pairing`. +Stop it, replace the `codeaf` binary, start it with the same `--store`. Back up first. +There is no migration command or feature switch. First use freezes an existing identity, +imports the same rows into three local anchors, checks a fresh majority and publishes +one durable owner. Interrupted imports resume with the same UUID, keys and ceiling; +they never reopen the legacy writer. The original source stays as a backup until an +explicit owner cleanup or retirement. + +All directory, rotation and blob authority requests use consensus. Three local voters +still share one host and disk: losing that host loses service. This setup does not +claim independent host failure tolerance. The current v1 adapter bounds encoded +commands at 64 KiB; a legal older request with a very large frame plan can still be +refused before proposal. Larger v1 requests are not supported yet. +A client newer than the relay by a pairing change sees `that relay is too old for pairing`. ## The hosted relay diff --git a/internal/manual/chat/choosing-a-folder.md b/internal/manual/chat/choosing-a-folder.md index 5a0447baf1..25186042f2 100644 --- a/internal/manual/chat/choosing-a-folder.md +++ b/internal/manual/chat/choosing-a-folder.md @@ -1,5 +1,11 @@ # Choosing a folder +## How do I change directory? + +Use `/folder`, `/place` or `/dir` to choose the folder this conversation is about. +They open the add-context sheet. Browsing attaches nothing; choose a folder with +`alt+m` or its action row. Over `--host`, `/folder` refuses because a folder cannot travel. + ## /folder — choose a folder, pick a directory, say which project you mean `/folder` opens the **add context** sheet — a framed window over the middle of the screen, diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index 120bd8508c..504fb3ce34 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -327,3 +327,24 @@ Reassignment changes the run revision and requires a new conditional claim. The ## Why can an older computer list chats but not continue a chat that is still running? Older builds can still list, save, publish and release chats through a compatible relay. A relay using quorum-backed coordination refuses an older request that tries to take a live chat without naming its observed holder and ownership number. It uses the existing refusal, `another device continued this chat first`; an old build does not necessarily show an extra update hint. Quit on the computer holding the chat before continuing it elsewhere, or update codeaf on the computer trying to continue. A released or expired chat can still be taken. A conditional choice never silently changes to a newer holder. + + +## Running a relay on one persistent computer + +`codeaf relay --store ` serves pairing, encrypted files and the chat directory +with one command. Each identity uses three local persistent coordination copies, +with separate databases under `/raft/`. Changes and authority reads require +a fresh majority response. All three copies still depend on this computer and +its disk; this setup does not survive losing the relay host. Without `--store`, +the relay is only a transport and pairing mailbox. + +On first use, an existing directory is frozen, copied into the three stores and +checked through a majority before requests resume. The original directory stays +as a frozen backup under `/directory/`. An interrupted upgrade resumes with +the same identity and stores; an incomplete or damaged import refuses requests +instead of reopening the old writer. Keep the whole store directory when +restarting or backing up the relay. Chat content stays encrypted in the existing +blob store. No switch selects the former directory writer. + +Encoded directory commands currently have a 64 KiB limit; a larger older request +can be refused before it changes ownership. Larger imported records stay retained. diff --git a/internal/raftdirectory/native.go b/internal/raftdirectory/native.go new file mode 100644 index 0000000000..edc94f11f5 --- /dev/null +++ b/internal/raftdirectory/native.go @@ -0,0 +1,482 @@ +package raftdirectory + +import ( + "context" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + "github.com/Agent-Field/codeaf/internal/raftstore" + "go.etcd.io/raft/v3" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// Native owns three independent durable actors on one host. The queue covers +// each persistence completion, authenticated local delivery and read callback; +// no caller can inspect a replica between its barrier and its application read. +// All peer bytes originate inside this queue, never at a public HTTP door. +type Native struct { + mu sync.Mutex + manifest clusterstate.LegacyManifest + stores [3]*raftstore.SQLite + nodes [3]*raftcore.Core + status [3]raftcore.Status + queue []raftcore.Outbound + reads map[string]bool + results [3][]clusterstate.Result + serial uint64 + failed error + closed bool + // Changed projects durable rows into advisory notifications under the queue. + Changed func(*raftstore.SQLite, raftstore.Application) error +} + +// OpenNative resumes the SAME imported genesis on every anchor. A partial or +// corrupt import refuses service and retains its source; it never bootstraps a +// singleton or recovers by deleting a member database. +func OpenNative(ctx context.Context, path string, source *directory.SQLite, m clusterstate.LegacyManifest) (_ *Native, err error) { + if m.Validate() != nil { + return nil, clusterstate.ErrMigration + } + g := &Native{manifest: m, reads: map[string]bool{}} + defer func() { + if err != nil { + _ = g.Close() + } + }() + gate, e := source.LegacyGate(ctx) + if e != nil { + return nil, e + } + owned := gate != nil && gate.Stage == "raft-owned" + live := 0 + for i, id := range m.Config.Anchors { + memberPath := filepath.Join(path, fmt.Sprintf("%d.db", id)) + // A published member with lost storage is unavailable, never a fresh copy + // under its retired durable identity. Only actual surviving stores vote. + if owned { + if _, e := os.Stat(memberPath); errors.Is(e, os.ErrNotExist) { + continue + } else if e != nil { + return nil, e + } + } + g.stores[i], err = raftstore.Open(memberPath, raftstore.Identity{ClusterUUID: m.Config.Cluster, NodeID: id, Protocol: raftcore.ProtocolVersion}) + if err != nil { + if owned && ctx.Err() == nil { + g.stores[i] = nil + continue + } + return nil, err + } + // Once ownership is published, even a valid empty file is not a returning + // voter. Its own durable import receipt must predate this activation. + if owned { + imported, e := g.stores[i].ImportedLegacy(ctx) + if e != nil && ctx.Err() != nil { + return nil, e + } + if e != nil || imported == nil { + _ = g.stores[i].Close() + g.stores[i] = nil + continue + } + } + if err = g.stores[i].ImportLegacy(ctx, m, func(visit func(clusterstate.RecordWrite) error) error { + _, e := source.ExportLegacy(ctx, visit) + return e + }); err != nil { + if owned && ctx.Err() == nil { + _ = g.stores[i].Close() + g.stores[i] = nil + continue + } + return nil, err + } + recovered, e := g.stores[i].Recover(ctx) + if e != nil { + if owned && ctx.Err() == nil { + _ = g.stores[i].Close() + g.stores[i] = nil + continue + } + return nil, e + } + var seed [8]byte + if _, err = rand.Read(seed[:]); err != nil { + return nil, err + } + g.nodes[i], err = raftcore.New(raftcore.Config{ClusterUUID: m.Config.Cluster, NodeID: id, Protocol: raftcore.ProtocolVersion, Seed: binary.LittleEndian.Uint64(seed[:]), Anchors: m.Config.Anchors}, recovered.Core) + if err != nil { + return nil, err + } + live++ + } + if live < 2 { + return nil, directory.ErrUnreachable + } + for i := range g.nodes { + if g.nodes[i] == nil { + continue + } + if err = g.drive(ctx, i, raftcore.Poll{}); err != nil { + return nil, err + } + } + if err = g.drain(ctx); err != nil { + return nil, err + } + // A surviving anchor may have an older suffix. Request a real campaign + // from each available anchor until a current majority elects one of them. + for i, node := range g.nodes { + if node == nil { + continue + } + if err = g.drive(ctx, i, raftcore.Campaign{}); err != nil { + return nil, err + } + if err = g.drain(ctx); err != nil { + return nil, err + } + if g.leader() >= 0 { + break + } + } + if _, _, err = g.barrier(ctx); err != nil { + return nil, err + } + receipts := []directory.LegacyReceipt{} + for i, s := range g.stores { + if s == nil { + continue + } + got, e := s.ImportedLegacy(ctx) + if e != nil { + return nil, e + } + if got == nil { + return nil, clusterstate.ErrMigration + } + app, _, e := s.ApplicationView(ctx) + if e != nil { + return nil, e + } + receipts = append(receipts, directory.LegacyReceipt{Node: m.Config.Anchors[i], Manifest: *got, Applied: app.Applied}) + } + if err = source.OwnLegacy(ctx, receipts); err != nil { + return nil, err + } + return g, nil +} + +func (g *Native) Close() error { + g.mu.Lock() + defer g.mu.Unlock() + g.closed = true + var err error + for _, s := range g.stores { + if s != nil { + err = errors.Join(err, s.Close()) + } + } + return err +} +func (g *Native) leader() int { + if g.closed || g.failed != nil { + return -1 + } + for i, s := range g.status { + if s.Role == raft.StateLeader && !s.Stopped { + return i + } + } + return -1 +} +func (g *Native) drive(ctx context.Context, i int, input raftcore.Input) error { + for n := 0; n < 10000; n++ { + e, err := g.nodes[i].Step(input) + if err != nil { + return err + } + g.status[i] = e.Status + g.queue = append(g.queue, e.Send...) + for _, r := range e.Reads { + app, _, err := g.stores[i].ApplicationView(ctx) + if err != nil { + return err + } + if app.Applied < r.Index { + return directory.ErrUnreachable + } + g.reads[r.RequestID] = true + } + if e.Persist != nil { + if err = g.stores[i].Persist(ctx, *e.Persist); err != nil { + _, _ = g.nodes[i].Step(raftcore.StorageFailed{BatchID: e.Persist.ID}) + return fmt.Errorf("raftdirectory: persist member %d snapshot %d: %w", g.manifest.Config.Anchors[i], e.Persist.Snapshot.GetMetadata().GetIndex(), err) + } + input = raftcore.Persisted{BatchID: e.Persist.ID} + } else if e.Apply != nil { + results, err := Apply(ctx, g.stores[i], *e.Apply, g.manifest.Config.Cluster, g.manifest.Ceiling) + if err != nil { + _, _ = g.nodes[i].Step(raftcore.StorageFailed{BatchID: e.Apply.ID}) + return fmt.Errorf("raftdirectory: apply member %d index %d: %w", g.manifest.Config.Anchors[i], e.Apply.Index, err) + } + g.results[i] = append(g.results[i], results...) + input = raftcore.Applied{BatchID: e.Apply.ID} + } else { + return nil + } + } + return directory.ErrUnreachable +} +func (g *Native) drain(ctx context.Context) error { + for n := 0; len(g.queue) > 0; n++ { + if n >= 10000 { + return directory.ErrUnreachable + } + m := g.queue[0] + g.queue = g.queue[1:] + found := false + for i, id := range g.manifest.Config.Anchors { + if m.To == id { + found = true + if g.nodes[i] == nil { + break + } + message := &pb.Message{} + if proto.Unmarshal(m.Bytes, message) != nil { + return raftcore.ErrPeer + } + if message.GetType() == pb.MsgSnap { + if err := g.stageSnapshot(ctx, i, message); err != nil { + return err + } + } + if err := g.drive(ctx, i, raftcore.PeerMessage{Bytes: m.Bytes}); err != nil { + return err + } + break + } + } + if !found { + return directory.ErrUnauthorized + } + } + return nil +} + +// stageSnapshot streams the SAME portable generation before RawNode may +// persist its pointer. Neither a copied chunk nor a completed upload is an +// append acknowledgment; the target's subsequent Ready installation is. +func (g *Native) stageSnapshot(ctx context.Context, target int, message *pb.Message) error { + pointer := message.GetSnapshot() + app, _, err := g.stores[target].ApplicationView(ctx) + if err != nil { + return err + } + if pointer.GetMetadata().GetIndex() <= app.Applied { + return nil + } + manifest, err := raftsnapshot.Decode(pointer.GetData()) + if err != nil { + return err + } + var source *raftstore.SQLite + for i, id := range g.manifest.Config.Anchors { + if id == message.GetFrom() { + source = g.stores[i] + break + } + } + if source == nil { + return directory.ErrUnreachable + } + id, err := g.stores[target].BeginSnapshot(ctx, pointer.GetData()) + if err != nil { + return fmt.Errorf("raftdirectory: begin snapshot %d on member %d: %w", manifest.Index, g.manifest.Config.Anchors[target], err) + } + for offset := uint64(0); offset < manifest.Chunks; offset++ { + chunk, err := source.ReadSnapshotChunk(ctx, id, offset) + if err != nil { + return err + } + if err = g.stores[target].PutSnapshotChunk(ctx, id, offset, chunk, raftsnapshot.ID(chunk)); err != nil { + return err + } + } + complete, err := g.stores[target].CompleteSnapshot(ctx, id) + if err != nil { + return fmt.Errorf("raftdirectory: complete snapshot %d on member %d: %w", manifest.Index, g.manifest.Config.Anchors[target], err) + } + if !proto.Equal(complete, pointer) { + return fmt.Errorf("raftdirectory: snapshot pointer mismatch on member %d: %w", g.manifest.Config.Anchors[target], raftstore.ErrSnapshot) + } + return nil +} + +func (g *Native) submit(ctx context.Context, c clusterstate.Command) (clusterstate.Result, error) { + i := g.leader() + if i < 0 { + return clusterstate.Result{}, directory.ErrUnreachable + } + raw, err := clusterstate.EncodeCommand(c) + if err != nil { + return clusterstate.Result{}, err + } + for j := range g.results { + g.results[j] = nil + } + if err = g.drive(ctx, i, raftcore.Propose{Command: raw}); err == nil { + err = g.drain(ctx) + } + if err != nil { + g.failed = err + return clusterstate.Result{}, err + } + if len(g.results[i]) != 1 { + return clusterstate.Result{}, directory.ErrUnreachable + } + r := g.results[i][0] + // Compaction follows complete local delivery so every local voter has the + // checkpoint's prefix. It removes only redundant consensus history. + for j, s := range g.stores { + if s == nil { + continue + } + due, e := s.SnapshotDue(ctx) + if e != nil { + g.failed = e + return r, e + } + if due { + p, e := s.CreateSnapshot(ctx) + if e != nil { + g.failed = e + return r, e + } + if e = s.Compact(ctx, p.GetMetadata().GetIndex()); e != nil { + g.failed = e + return r, e + } + if e = g.drive(ctx, j, raftcore.CompactSnapshot{Snapshot: p}); e != nil { + g.failed = e + return r, e + } + } + } + if g.Changed != nil { + app, _, e := g.stores[i].ApplicationView(ctx) + if e != nil { + return r, e + } + if e = g.Changed(g.stores[i], app); e != nil { + return r, e + } + } + return r, nil +} +func (g *Native) barrier(ctx context.Context) (int, raftstore.Application, error) { + i := g.leader() + if i < 0 { + return i, raftstore.Application{}, directory.ErrUnreachable + } + g.serial++ + id := fmt.Sprintf("native-%d", g.serial) + if err := g.drive(ctx, i, raftcore.Read{RequestID: id}); err != nil { + return i, raftstore.Application{}, err + } + if err := g.drain(ctx); err != nil { + g.failed = err + return i, raftstore.Application{}, err + } + ok := g.reads[id] + delete(g.reads, id) + if !ok { + return i, raftstore.Application{}, directory.ErrUnreachable + } + app, _, err := g.stores[i].ApplicationView(ctx) + return i, app, err +} +func (g *Native) CommitV1(ctx context.Context, c clusterstate.Command) (clusterstate.Result, error) { + ctx = context.WithoutCancel(ctx) + g.mu.Lock() + defer g.mu.Unlock() + return g.submit(ctx, c) +} +func (g *Native) ReadV1(ctx context.Context, c clusterstate.Command, committed bool, inspect func(context.Context, *raftstore.SQLite, raftstore.Application) error) error { + ctx = context.WithoutCancel(ctx) + g.mu.Lock() + defer g.mu.Unlock() + if !committed { + if c.Change.AdvanceTime == nil { + return directory.ErrUnauthorized + } + r, e := g.submit(ctx, c) + if e != nil { + return e + } + if e = directory.ResultError(r.Code); e != nil { + return e + } + } + i, app, err := g.barrier(ctx) + if err != nil { + return err + } + return inspect(ctx, g.stores[i], app) +} +func (g *Native) CommitEffect(ctx context.Context, c clusterstate.Command) (clusterstate.Result, error) { + ctx = context.WithoutCancel(ctx) + g.mu.Lock() + defer g.mu.Unlock() + return g.submit(ctx, c) +} + +// Inspect is an internal service read. It always passes a new current-term +// quorum barrier and gives the callback its actor-owned context. A disconnected +// HTTP request cannot turn a healthy durable read into a storage failure. +func (g *Native) Inspect(ctx context.Context, inspect func(context.Context, *raftstore.SQLite, raftstore.Application) error) error { + ctx = context.WithoutCancel(ctx) + g.mu.Lock() + defer g.mu.Unlock() + i, app, err := g.barrier(ctx) + if err != nil { + return err + } + return inspect(ctx, g.stores[i], app) +} +func (g *Native) ReadEffect(ctx context.Context, k clusterstate.RecordKey) (raw []byte, err error) { + err = g.Inspect(ctx, func(ctx context.Context, s *raftstore.SQLite, a raftstore.Application) error { + if _, e := AuthorizedEffectRead(ctx, s, g.manifest.Config.Cluster, clusterstate.DeviceIDOfKey(g.manifest.Config.EffectsKey)); e != nil { + return e + } + key := raftstore.Key{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key} + rows, _, e := s.Read(ctx, a.Applied, []raftstore.Key{key}) + raw = rows[key] + return e + }) + return +} +func (g *Native) ScanEffects(ctx context.Context, after string, limit int) (rows []clusterstate.RecordWrite, err error) { + err = g.Inspect(ctx, func(ctx context.Context, s *raftstore.SQLite, a raftstore.Application) error { + if _, e := AuthorizedEffectRead(ctx, s, g.manifest.Config.Cluster, clusterstate.DeviceIDOfKey(g.manifest.Config.EffectsKey)); e != nil { + return e + } + page, _, e := s.ScanPage(ctx, a.Applied, raftstore.Outbox, after) + for _, r := range page[:min(len(page), limit)] { + rows = append(rows, clusterstate.RecordWrite{RecordKey: clusterstate.RecordKey{Namespace: string(r.Namespace), Key: r.Key.Key}, Data: r.Value}) + } + return e + }) + return +} diff --git a/internal/raftdirectory/native_host_test.go b/internal/raftdirectory/native_host_test.go new file mode 100644 index 0000000000..a0db74d7e6 --- /dev/null +++ b/internal/raftdirectory/native_host_test.go @@ -0,0 +1,57 @@ +package raftdirectory_test + +import ( + "context" + "crypto/ed25519" + "path/filepath" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/reqsign" +) + +// R3/R4 require a disconnected caller to leave healthy durable actors usable. +// Client cancellation is an unknown response, never a disk-failure completion. +func TestR3R4ClientCancellationDoesNotStopNativeActors(t *testing.T) { + f := frozenV1(t) + clock := func() time.Time { return time.UnixMilli(f.Now) } + source, e := directory.OpenSQLite(filepath.Join(t.TempDir(), "source.db"), clock) + must(t, e) + defer source.Close() + root := key("root").Public().(ed25519.PublicKey) + cfg := cs.LegacyConfig{Cluster: cluster, Generation: 1, Identity: cs.IdentityID(root), SigningKey: root, EffectsKey: key("effects").Public().(ed25519.PublicKey), Anchors: []uint64{1, 2, 3}, Now: f.Now} + must(t, source.FreezeLegacy(ctx, cfg)) + manifest, e := source.InventoryLegacy(ctx) + must(t, e) + host, e := raftdirectory.OpenNative(ctx, t.TempDir(), source, manifest) + must(t, e) + defer host.Close() + var command cs.Command + for _, one := range f.Cases { + if one.Name == "list" { + proof, e := reqsign.Capture(frozenRequest(t, one, "http://localhost"), []byte(one.Body)) + must(t, e) + command, e = cs.V1Command(cluster, f.Now, proof) + must(t, e) + break + } + } + if command.Change.AdvanceTime == nil { + t.Fatal("no original signed read fixture") + } + cancelled, cancel := context.WithCancel(ctx) + cancel() + // The first call may lose its response. The next real quorum read MUST work. + _ = host.ReadV1(cancelled, command, false, func(_ context.Context, _ *raftstore.SQLite, _ raftstore.Application) error { return nil }) + must(t, host.ReadV1(ctx, command, false, func(owned context.Context, s *raftstore.SQLite, app raftstore.Application) error { + if owned.Err() != nil { + t.Fatal("actor inherited a disconnected client's cancellation") + } + _, _, e := s.ApplicationView(owned) + return e + })) +} diff --git a/internal/relayserve/namespaces.go b/internal/relayserve/namespaces.go index c48d38d504..5be92ce58c 100644 --- a/internal/relayserve/namespaces.go +++ b/internal/relayserve/namespaces.go @@ -27,8 +27,8 @@ func checkedID(id string) error { return nil } -// namespaces gives each identity its own directory file and its own blob -// tree under one root, so an identity can only reach what its id names. +// namespaces isolates each identity's frozen source, three durable anchors +// and encrypted blob tree under the identity proven by its request. type namespaces struct { root string now func() time.Time @@ -39,24 +39,26 @@ type namespaces struct { links directory.Decider // where an approve decides a pending request; set once by New - mu sync.Mutex - dirs map[string]*directory.SQLite + activation sync.Mutex + groups map[string]*nativeNamespace + mu sync.Mutex + dirs map[string]*directory.SQLite } func newNamespaces(root string, now func() time.Time, grace directory.GraceBounds, watchers int) *namespaces { if grace == (directory.GraceBounds{}) { grace = directory.DefaultGraceBounds } - return &namespaces{root: root, now: now, grace: grace, watchers: watchers, dirs: map[string]*directory.SQLite{}} + return &namespaces{root: root, now: now, grace: grace, watchers: watchers, dirs: map[string]*directory.SQLite{}, groups: map[string]*nativeNamespace{}} } -// directory opens (once) the SQLite directory of one identity. A SQLite file -// is one identity's directory, so the file is per identity: /directory/.db. +// directory resolves only the irrevocably published consensus owner. Source +// access stays inside activation and never becomes a fallback request writer. func (n *namespaces) directory(id string) (directory.Directory, error) { - return n.sqlite(id) + return n.activate(context.Background(), id, nil) } -// sqlite is directory with its concrete type, for what only a relay may ask. +// sqlite opens the preserved migration source, never a live request backend. func (n *namespaces) sqlite(id string) (*directory.SQLite, error) { if err := checkedID(id); err != nil { return nil, err @@ -92,7 +94,7 @@ func (n *namespaces) sqlite(id string) (*directory.SQLite, error) { // blobs opens one identity's disk store at /blobs//. A replaced // identity's store takes no frame: reads stay open for the grace period. func (n *namespaces) blobs(id string) (blobstore.Store, error) { - d, err := n.sqlite(id) + d, err := n.activate(context.Background(), id, nil) if err != nil { return nil, err } @@ -100,21 +102,13 @@ func (n *namespaces) blobs(id string) (blobstore.Store, error) { if err != nil { return nil, err } - return readOnlyOnceReplaced{Store: disk, rotation: d.Rotated}, nil -} - -// readOnlyOnceReplaced is a store that refuses a put when its identity has been -// replaced by a rotation, and is the store unchanged otherwise. -type readOnlyOnceReplaced struct { - blobstore.Store - rotation func() *directory.Rotation -} - -func (r readOnlyOnceReplaced) PutFrame(ctx context.Context, frame []byte) (blobstore.FrameID, error) { - if r.rotation() != nil { - return "", wireauth.ErrRotated - } - return r.Store.PutFrame(ctx, frame) + return CommittedBlobStore{Store: disk, Authorize: func(ctx context.Context) error { + actor, _ := ctx.Value(blobActorKey{}).(string) + if actor == "" { + return wireauth.ErrUnauthorized + } + return d.authority(ctx, actor, true) + }}, nil } // Close releases every directory file this relay opened. @@ -122,6 +116,12 @@ func (n *namespaces) Close() error { n.mu.Lock() defer n.mu.Unlock() var first error + for id, d := range n.groups { + if err := d.Close(); err != nil && first == nil { + first = err + } + delete(n.groups, id) + } for id, d := range n.dirs { if err := d.Close(); err != nil && first == nil { first = err @@ -131,22 +131,11 @@ func (n *namespaces) Close() error { return first } -// revoked says whether device has been stopped by its identity. It opens the -// identity's directory (once) and then answers from memory, so the check costs -// no query per request. -func (n *namespaces) revoked(identity, device string) (bool, error) { - d, err := n.sqlite(identity) - if err != nil { - return false, err - } - return d.Revoked(device), nil -} - // CommittedBlobStore checks current committed authority on both sides of an // external put. A revocation or freeze during disk/R2 I/O leaves the bytes for // reuse but returns no successful upload to that caller. The receipt command // rechecks authority again at its own log position before any head may move. -// P11 assembles this only after the namespace's old writer is frozen. +// Namespace activation installs it only after the old writer is frozen. type CommittedBlobStore struct { blobstore.Store Authorize func(context.Context) error @@ -183,15 +172,3 @@ func (s CommittedBlobStore) ConfirmReferences(ctx context.Context, head string, } return s.Authorize(ctx) } -func (r readOnlyOnceReplaced) ConfirmReferences(ctx context.Context, head string, frames []string) error { - if r.rotation() != nil { - return wireauth.ErrRotated - } - if err := blobstore.CheckReferences(ctx, r.Store, head, frames); err != nil { - return err - } - if r.rotation() != nil { - return wireauth.ErrRotated - } - return nil -} diff --git a/internal/relayserve/native.go b/internal/relayserve/native.go new file mode 100644 index 0000000000..2f66b29825 --- /dev/null +++ b/internal/relayserve/native.go @@ -0,0 +1,442 @@ +package relayserve + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/reqsign" + "github.com/Agent-Field/codeaf/internal/wireauth" +) + +// nativeProvision is durable before freeze so retrying an interrupted import +// reuses its source UUID and protocol keys. No content or device private key is +// present: these keys authorize only their provisioned service roles. +type nativeProvision struct { + Cluster string `json:"cluster"` + Effects []byte `json:"effects"` + Anchors [3][]byte `json:"anchors"` +} +type nativeNamespace struct { + *raftdirectory.V1Directory + host *raftdirectory.Native + feed *directory.Feed + effects raftdirectory.Effects + links *directory.Links + now func() time.Time + grace directory.GraceBounds + identity clusterstate.IdentityRec +} + +func (d *nativeNamespace) Feed() *directory.Feed { return d.feed } +func (d *nativeNamespace) Close() error { d.feed.Close(); return d.host.Close() } + +// activate runs outside every legacy admission lock. The frozen source drains +// admitted writes itself and can never become a fallback directory afterward. +func (n *namespaces) activate(ctx context.Context, id string, root []byte) (*nativeNamespace, error) { + if err := checkedID(id); err != nil { + return nil, err + } + if n.gone(id) { + return nil, wireauth.ErrGone + } + n.activation.Lock() + defer n.activation.Unlock() + n.mu.Lock() + d := n.groups[id] + n.mu.Unlock() + if d != nil { + return d, nil + } + source, err := n.sqlite(id) + if err != nil { + return nil, err + } + gate, err := source.LegacyGate(ctx) + if err != nil { + return nil, err + } + if gate != nil { + root = gate.Config.SigningKey + } + if clusterstate.IdentityID(root) != id { + return nil, wireauth.ErrUnauthorized + } + path := filepath.Join(n.root, "raft", id) + if err = os.MkdirAll(path, 0700); err != nil { + return nil, err + } + provisionPath := filepath.Join(path, "provision.json") + raw, err := os.ReadFile(provisionPath) + var provision nativeProvision + if errors.Is(err, os.ErrNotExist) { + if gate != nil { + return nil, clusterstate.ErrMigration + } + var uuid [16]byte + if _, err = rand.Read(uuid[:]); err != nil { + return nil, err + } + provision.Cluster = hex.EncodeToString(uuid[:]) + _, key, e := ed25519.GenerateKey(rand.Reader) + if e != nil { + return nil, e + } + provision.Effects = key + for i := range provision.Anchors { + _, key, e := ed25519.GenerateKey(rand.Reader) + if e != nil { + return nil, e + } + provision.Anchors[i] = key + } + raw, err = json.Marshal(provision) + if err != nil { + return nil, err + } + // The final filename is published only after its complete contents survive + // power loss. A killed temporary file is never authoritative on restart. + f, e := os.CreateTemp(path, ".provision-") + if e != nil { + return nil, e + } + tmp := f.Name() + defer os.Remove(tmp) + _, e = f.Write(raw) + e = errors.Join(e, f.Sync(), f.Close()) + if e != nil { + return nil, e + } + if e = os.Rename(tmp, provisionPath); e != nil { + return nil, e + } + // Newly created ancestors must survive before the source UUID freezes. + for _, dir := range []string{path, filepath.Dir(path), n.root, filepath.Dir(n.root)} { + parent, e := os.Open(dir) + if e != nil { + return nil, e + } + e = errors.Join(parent.Sync(), parent.Close()) + if e != nil { + return nil, e + } + } + } else if err != nil { + return nil, err + } + if json.Unmarshal(raw, &provision) != nil || len(provision.Effects) != ed25519.PrivateKeySize || provision.Cluster == "" { + return nil, clusterstate.ErrMigration + } + for _, key := range append(provision.Anchors[:], provision.Effects) { + if len(key) != ed25519.PrivateKeySize || !bytes.Equal(ed25519.NewKeyFromSeed(key[:ed25519.SeedSize]), key) { + return nil, clusterstate.ErrMigration + } + } + cfg := clusterstate.LegacyConfig{Cluster: provision.Cluster, Generation: 1, Identity: id, SigningKey: root, EffectsKey: ed25519.PrivateKey(provision.Effects).Public().(ed25519.PublicKey), Anchors: []uint64{1, 2, 3}, Now: n.now().UnixMilli()} + for _, private := range provision.Anchors { + cfg.AnchorKeys = append(cfg.AnchorKeys, ed25519.PrivateKey(private).Public().(ed25519.PublicKey)) + } + if n.grace != directory.DefaultGraceBounds { + cfg.Grace = &clusterstate.GracePolicy{Min: n.grace.Min.Milliseconds(), Max: n.grace.Max.Milliseconds(), Default: n.grace.Default.Milliseconds()} + } + if gate != nil { + if gate.Config.Cluster != cfg.Cluster || !bytes.Equal(gate.Config.EffectsKey, cfg.EffectsKey) { + return nil, clusterstate.ErrMigration + } + if len(gate.Config.AnchorKeys) != 0 { + for i, key := range gate.Config.AnchorKeys { + if i >= len(cfg.AnchorKeys) || !bytes.Equal(key, cfg.AnchorKeys[i]) { + return nil, clusterstate.ErrMigration + } + } + } + cfg = gate.Config + } + if err = source.FreezeLegacy(ctx, cfg); err != nil { + return nil, err + } + manifest, err := source.InventoryLegacy(ctx) + if err != nil { + return nil, err + } + host, err := raftdirectory.OpenNative(ctx, path, source, manifest) + if err != nil { + return nil, err + } + feed := directory.NewFeed(n.now) + if n.watchers > 0 { + feed.SetMaxWatchers(n.watchers) + } + if n.after != nil { + feed.SetAfter(n.after) + } + grace := directory.DefaultGraceBounds + if cfg.Grace != nil { + grace = directory.GraceBounds{Min: time.Duration(cfg.Grace.Min) * time.Millisecond, Max: time.Duration(cfg.Grace.Max) * time.Millisecond, Default: time.Duration(cfg.Grace.Default) * time.Millisecond} + } + d = &nativeNamespace{V1Directory: &raftdirectory.V1Directory{Host: host, Cluster: cfg.Cluster, Now: n.now, Grace: grace}, identity: clusterstate.IdentityRec{V: 1, Identity: id, SigningKey: cfg.SigningKey}, host: host, feed: feed, links: n.links.(*directory.Links), now: n.now, grace: grace} + d.effects = raftdirectory.Effects{Host: host, Cluster: cfg.Cluster, Actor: clusterstate.DeviceIDOfKey(cfg.EffectsKey), Key: ed25519.PrivateKey(provision.Effects)} + host.Changed = d.project + feed.OnSeen(func(device string, at int64) { _ = d.seen(device, at) }) + if err = host.Inspect(ctx, func(ctx context.Context, s *raftstore.SQLite, a raftstore.Application) error { return d.project(s, a) }); err != nil { + d.Close() + return nil, err + } + // Repair committed decisions before returning a public namespace. Restarted + // in-memory public requests may have expired; CompleteJoin records that fact. + if _, err = d.effects.Replay(ctx, n.now().UnixMilli(), d.links, ""); err != nil { + d.Close() + return nil, err + } + n.mu.Lock() + n.groups[id] = d + n.mu.Unlock() + return d, nil +} +func (d *nativeNamespace) project(s *raftstore.SQLite, a raftstore.Application) error { + ctx := context.Background() + key := raftstore.Key{Namespace: raftstore.State, Key: clusterstate.IdentityKey().Key} + rows, _, err := s.Read(ctx, a.Applied, []raftstore.Key{key}) + if err != nil { + return err + } + var identity clusterstate.IdentityRec + if err = clusterstate.Decode(rows[key], &identity); err != nil { + return err + } + st := directory.Status{Version: a.VisibleVersion, Stopped: map[string]bool{}, Rotation: identity.Rotation} + // Status needs devices and the identity only. Start at the device prefix so + // a large retained cell inventory never becomes watch-notification work. + after := "device/" + for { + page, _, err := s.ScanPage(ctx, a.Applied, raftstore.State, after) + if err != nil { + return err + } + if len(page) == 0 { + break + } + for _, r := range page { + after = r.Key.Key + if !strings.HasPrefix(after, "device/") { + d.feed.Publish(st) + return nil + } + var dev clusterstate.Device + if err = clusterstate.Decode(r.Value, &dev); err != nil { + return err + } + if dev.Revoked { + id, err := hex.DecodeString(strings.TrimPrefix(after, "device/")) + if err != nil { + return err + } + st.Stopped[string(id)] = true + } + } + } + d.feed.Publish(st) + return nil +} + +func (d *nativeNamespace) authority(ctx context.Context, actor string, write bool) error { + err := d.host.Inspect(ctx, func(ctx context.Context, s *raftstore.SQLite, a raftstore.Application) error { + keys := []raftstore.Key{{Namespace: raftstore.State, Key: clusterstate.IdentityKey().Key}, {Namespace: raftstore.State, Key: clusterstate.DeviceKey(actor).Key}} + rows, _, err := s.Read(ctx, a.Applied, keys) + if err != nil { + return err + } + var identity clusterstate.IdentityRec + if err = clusterstate.Decode(rows[keys[0]], &identity); err != nil { + return err + } + if raw := rows[keys[1]]; raw != nil { + var dev clusterstate.Device + if err = clusterstate.Decode(raw, &dev); err != nil { + return err + } + if dev.Revoked { + return wireauth.ErrRevoked + } + } + if write && identity.Rotation != nil { + return wireauth.ErrRotated + } + return nil + }) + if err != nil && !errors.Is(err, wireauth.ErrRevoked) && !errors.Is(err, wireauth.ErrRotated) { + return errors.Join(wireauth.ErrUnavailable, err) + } + return err +} + +// SubscribeWatch retains the certificate from this exact authenticated socket, +// rather than enrolling a row that would consume grandfathered capacity. +func (d *nativeNamespace) SubscribeWatch(device string, r *http.Request, holds []directory.Hold, beat time.Duration) (*directory.Sub, error) { + certificate := r.Header.Get(reqsign.HeaderCert) + if clusterstate.PersonalCertificate(d.identity, device, certificate) != nil { + return nil, directory.ErrBadRequest + } + return d.feed.SubscribeBeatWithEvidence(device, holds, beat, func(device string, holds []directory.Hold, at int64) error { + return d.renew(device, holds, at, certificate) + }) +} +func (d *nativeNamespace) renew(device string, holds []directory.Hold, at int64, certificate string) error { + ctx := context.Background() + if err := d.authority(ctx, device, false); err != nil { + return err + } + // Old sockets may name zero, missing or several historical fences. Only + // their one current held tuple is evidence; none of the others can renew it. + items := []clusterstate.HoldFence{} + seen := map[string]bool{} + for _, h := range holds { + if h.Fence == 0 || seen[h.Cell] { + continue + } + raw, err := d.host.ReadEffect(ctx, clusterstate.CellKey(h.Cell)) + if err != nil { + return err + } + if raw == nil { + continue + } + var cell clusterstate.Cell + if err = clusterstate.Decode(raw, &cell); err != nil { + return err + } + if cell.Lease.Device != device || cell.Lease.Fence != h.Fence || cell.Lease.Expires == 0 { + continue + } + seen[h.Cell] = true + items = append(items, clusterstate.HoldFence{Cell: h.Cell, Fence: h.Fence}) + } + if len(items) == 0 { + return nil + } + _, err := d.effects.Renew(ctx, at, clusterstate.RenewHolds{Certificate: certificate, Device: device, SeenAt: at, Items: items}) + return err +} + +func (d *nativeNamespace) seen(device string, at int64) error { + c, e := clusterstate.Sign(clusterstate.Command{Schema: clusterstate.SchemaVersion, Cluster: d.Cluster, Actor: d.effects.Actor, Session: "presence", Sequence: 1, Now: at, Change: clusterstate.Mutation{DeviceSeen: &clusterstate.DeviceSeen{Device: device}}}, d.effects.Key) + if e != nil { + return e + } + r, e := d.host.CommitEffect(context.Background(), c) + if e != nil { + return e + } + return directory.ResultError(r.Code) +} + +// Every signature is checked before allocating storage or looking up authority. +// The actor then comes from that proof for both the initial and post-I/O check. +func (n *namespaces) authenticate(r *http.Request, body []byte) (string, string, error) { + id, actor, err := reqsign.Verify(r, body, n.now()) + if err != nil { + return "", "", err + } + // Signature verification already consumed the body. This lookup needs only + // public headers; do not copy an encrypted frame into a coordination proof. + p, err := reqsign.Capture(r, nil) + if err != nil { + return "", "", err + } + d, err := n.activate(r.Context(), id, p.IdentityKey) + if err != nil { + if errors.Is(err, wireauth.ErrGone) { + return "", "", err + } + return "", "", errors.Join(wireauth.ErrUnavailable, err) + } + if err = d.authority(r.Context(), actor, false); err != nil { + if errors.Is(err, wireauth.ErrRevoked) { + return "", "", err + } + return "", "", errors.Join(wireauth.ErrUnavailable, err) + } + *r = *r.WithContext(context.WithValue(r.Context(), blobActorKey{}, actor)) + return id, actor, nil +} + +type blobActorKey struct{} + +func (d *nativeNamespace) ForRequest(actor string, r *http.Request, raw []byte) directory.Client { + return &nativeClient{Client: d.V1Directory.ForRequest(actor, r, raw), owner: d} +} + +type nativeClient struct { + directory.Client + owner *nativeNamespace +} + +func (c *nativeClient) Presence(ctx context.Context) (directory.PresenceView, error) { + p, e := c.Client.Presence(ctx) + if e != nil { + return p, e + } + online := c.owner.feed.OnlineSet() + for id, v := range p.Devices { + v.Online = online[id] + if v.Online { + v.LastSeen = p.Now + } + p.Devices[id] = v + } + return p, nil +} +func (c *nativeClient) ApproveRequest(ctx context.Context, code string, a directory.Approval) error { + if err := c.owner.effects.Admit(ctx, c.owner.now().UnixMilli(), c.owner.links, code); err != nil { + return err + } + if err := c.Client.ApproveRequest(ctx, code, a); err != nil { + return err + } + raw, err := c.owner.host.ReadEffect(ctx, clusterstate.JoinKey(directory.NormalizeCode(code))) + if err != nil { + return err + } + var request clusterstate.JoinRequest + if err = clusterstate.Decode(raw, &request); err != nil { + return err + } + if err = c.owner.effects.Deliver(ctx, c.owner.now().UnixMilli(), c.owner.links, code); err != nil { + return err + } + if request.Outcome == "" && request.Decision != nil && !request.Decision.Deny { + raw, err = c.owner.host.ReadEffect(ctx, clusterstate.DeviceKey(request.Decision.Device)) + if err != nil { + return err + } + var device clusterstate.Device + if err = clusterstate.Decode(raw, &device); err != nil { + return err + } + c.owner.feed.AnnounceJoin(directory.Joined{Device: request.Decision.Device, Name: device.Name, Platform: device.Platform, At: device.Created}) + } + return nil +} + +func (c *nativeClient) DenyRequest(ctx context.Context, code string) error { + if err := c.owner.effects.Admit(ctx, c.owner.now().UnixMilli(), c.owner.links, code); err != nil { + return err + } + if err := c.Client.DenyRequest(ctx, code); err != nil { + return err + } + return c.owner.effects.Deliver(ctx, c.owner.now().UnixMilli(), c.owner.links, code) +} diff --git a/internal/relayserve/native_test.go b/internal/relayserve/native_test.go new file mode 100644 index 0000000000..65173dabea --- /dev/null +++ b/internal/relayserve/native_test.go @@ -0,0 +1,339 @@ +package relayserve_test + +import ( + "bytes" + "crypto/ed25519" + "database/sql" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/directory/directorytest" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/raftstore" +) + +// R3/R24/R28 prove the real first-use relay path imports a preexisting source, +// preserves its exact original bytes, and resumes the SAME three stores after +// restart. Neither API hints nor a taught legacy refusal can satisfy this test. +func TestR3R24R28NativeProductionMigratesAndRestartsAllAnchors(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + r.stop() + path := filepath.Join(r.store, "directory", id.ID()+".db") + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + source, err := directory.OpenSQLite(path, r.clock.Now) + if err != nil { + t.Fatal(err) + } + client := source.For(a.dev.ID()) + if err = client.PutDevice(bg, a.dev.ID(), directory.Device{V: 1, Name: "c2VhbGVk"}); err != nil { + t.Fatal(err) + } + if _, err = client.Create(bg, cell, directory.CellInit{Head: head1, Frames: []string{"old-sealed-frame"}}); err != nil { + t.Fatal(err) + } + if err = source.Close(); err != nil { + t.Fatal(err) + } + original := func() string { + t.Helper() + db, e := sql.Open("sqlite", "file:"+path+"?mode=ro") + if e != nil { + t.Fatal(e) + } + defer db.Close() + var raw string + if e = db.QueryRow("SELECT rec FROM cells WHERE id=?", cell).Scan(&raw); e != nil { + t.Fatal(e) + } + return raw + } + before := original() + r.start() + // R22/R24: the first use of an old held chat may be a signed watch, + // before a returning device sends any metadata or ordinary directory call. + watch, _, err := directorytest.DialWatchHolding(bg, r.srv.URL, r.signer(a), []directory.Hold{{Cell: cell, Fence: 1}}) + if err != nil { + t.Fatal(err) + } + defer watch.CloseNow() + if _, _, err = watch.Read(bg); err != nil { + t.Fatal(err) + } + got, err := r.dir(a).Cell(bg, cell) + if err != nil { + t.Fatal(err) + } + if got.Cell.Head != head1 || got.Cell.Lease.Fence != 1 || len(got.Cell.Frames) != 1 { + t.Fatal(got) + } + provisionPath := filepath.Join(r.store, "raft", id.ID(), "provision.json") + provision, err := os.ReadFile(provisionPath) + if err != nil { + t.Fatal(err) + } + // At each externally observed successful write, all three local voters have + // applied the same state through independent FULL-synchronous databases. + check := func(want uint64) { + t.Helper() + var p struct{ Cluster string } + if json.Unmarshal(provision, &p) != nil { + t.Fatal("provision") + } + for i := uint64(1); i <= 3; i++ { + s, e := raftstore.Open(filepath.Join(r.store, "raft", id.ID(), string(rune('0'+i))+".db"), raftstore.Identity{ClusterUUID: p.Cluster, NodeID: i, Protocol: 1}) + if e != nil { + t.Fatal(e) + } + app, _, e := s.ApplicationView(bg) + if e != nil { + t.Fatal(e) + } + key := raftstore.Key{Namespace: raftstore.State, Key: clusterstate.CellKey(cell).Key} + rows, _, e := s.Read(bg, app.Applied, []raftstore.Key{key}) + if e != nil { + t.Fatal(e) + } + var c directory.Cell + if e = clusterstate.Decode(rows[key], &c); e != nil { + t.Fatal(e) + } + if c.Lease.Fence != want || c.Head != head1 || app.Applied <= 1 { + t.Fatalf("anchor %d lost state: %+v", i, c) + } + if e = s.Close(); e != nil { + t.Fatal(e) + } + } + } + check(1) + b := newDevice(t, id, t.TempDir()) + expect := directory.LeaseExpectation{Device: a.dev.ID(), Fence: 1} + if _, err = r.dir(b).Acquire(bg, cell, directory.AcquireOpts{Force: true, Expect: &expect}); err != nil { + t.Fatal(err) + } + check(2) + r.restart() + if _, err = r.dir(a).Acquire(bg, cell, directory.AcquireOpts{Force: true, Expect: &expect}); !errors.Is(err, directory.ErrLeaseHeld) { + t.Fatal(err) + } + check(2) + after, e := os.ReadFile(provisionPath) + if e != nil { + t.Fatal(e) + } + if !bytes.Equal(provision, after) || original() != before { + t.Fatal("migration/restart rewrote its source or provision") + } +} + +// R24/R28 require incomplete migration to refuse rather than silently reset a +// damaged provision or resume the old writer. Restoring the original bytes +// resumes the same group without changing saved ownership. +func TestR24R28NativeProductionRefusesDamagedProvisionAndResumes(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + if _, err := r.dir(a).Create(bg, cell, directory.CellInit{Head: head1}); err != nil { + t.Fatal(err) + } + r.stop() + path := filepath.Join(r.store, "raft", id.ID(), "provision.json") + original, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err = os.WriteFile(path, []byte("damaged"), 0600); err != nil { + t.Fatal(err) + } + r.start() + if _, err = r.dir(a).Cell(bg, cell); !errors.Is(err, directory.ErrUnreachable) { + t.Fatalf("damaged authoritative provision must be retryable: %v", err) + } + r.stop() + // R24 also refuses a different, internally valid anchor key. Its public + // bytes are anchored in the SAME durable source manifest, not just JSON length. + var provision map[string]json.RawMessage + if err = json.Unmarshal(original, &provision); err != nil { + t.Fatal(err) + } + var anchors [][]byte + if err = json.Unmarshal(provision["anchors"], &anchors); err != nil { + t.Fatal(err) + } + anchors[0] = ed25519.NewKeyFromSeed(bytes.Repeat([]byte{7}, ed25519.SeedSize)) + provision["anchors"], err = json.Marshal(anchors) + if err != nil { + t.Fatal(err) + } + changed, err := json.Marshal(provision) + if err != nil { + t.Fatal(err) + } + if err = os.WriteFile(path, changed, 0600); err != nil { + t.Fatal(err) + } + r.start() + if _, err = r.dir(a).Cell(bg, cell); !errors.Is(err, directory.ErrUnreachable) { + t.Fatal("source manifest accepted changed anchor authority", err) + } + r.stop() + if err = os.WriteFile(path, original, 0600); err != nil { + t.Fatal(err) + } + r.start() + got, err := r.dir(a).Cell(bg, cell) + if err != nil || got.Cell.Head != head1 || got.Cell.Lease.Fence != 1 { + t.Fatal(got, err) + } +} + +// R22/R30 preserve ordinary directory requests while a real watch is open. +func TestR22R30NativeWatchDoesNotChangeRequestAuthority(t *testing.T) { + r := newRig(t) + _, a := newIdentity(t) + conn, _, err := directorytest.DialWatch(bg, r.srv.URL, r.signer(a)) + if err != nil { + t.Fatal(err) + } + defer conn.CloseNow() + if _, _, err = conn.Read(bg); err != nil { + t.Fatal(err) + } + if err = r.dir(a).PutDevice(bg, a.dev.ID(), directory.Device{V: 1, Name: "sealed"}); err != nil { + t.Fatalf("put after watch: %v; safe request log=%v", err, r.logs) + } +} + +// R3/R8/R24 prohibit re-importing a wiped published voter under its old ID. +// Two actual surviving stores coordinate; one cannot recreate a majority. +func TestR3R8R24NativeProductionUsesOnlySurvivingAnchorStores(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + if _, err := r.dir(a).Create(bg, cell, directory.CellInit{Head: head1}); err != nil { + t.Fatal(err) + } + r.stop() + first := filepath.Join(r.store, "raft", id.ID(), "1.db") + second := filepath.Join(r.store, "raft", id.ID(), "2.db") + if err := os.Rename(first, first+".offline"); err != nil { + t.Fatal(err) + } + r.start() + b := newDevice(t, id, t.TempDir()) + if _, err := r.dir(b).Acquire(bg, cell, directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: a.dev.ID(), Fence: 1}}); err != nil { + t.Fatal("two surviving anchors could not coordinate", err) + } + if _, err := os.Stat(first); !errors.Is(err, os.ErrNotExist) { + t.Fatal("lost voter was re-imported under its old ID", err) + } + r.stop() + if err := os.Rename(second, second+".offline"); err != nil { + t.Fatal(err) + } + r.start() + if _, err := r.dir(a).Cell(bg, cell); !errors.Is(err, directory.ErrUnreachable) { + t.Fatal("one surviving voter must stay unavailable", err) + } + for _, path := range []string{first, second} { + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatal("minority recreated missing authority", path, err) + } + } + r.stop() + for _, path := range []string{first, second} { + if err := os.Rename(path+".offline", path); err != nil { + t.Fatal(err) + } + } + r.start() + v, err := r.dir(a).Cell(bg, cell) + if err != nil || v.Cell.Lease.Fence != 2 || v.Cell.Lease.Device != b.dev.ID() { + source, e := directory.OpenSQLite(filepath.Join(r.store, "directory", id.ID()+".db"), r.clock.Now) + if e == nil { + gate, e := source.LegacyGate(bg) + if e == nil && gate != nil && gate.Manifest != nil { + host, e := raftdirectory.OpenNative(bg, filepath.Join(r.store, "raft", id.ID()), source, *gate.Manifest) + t.Logf("R3/R8/R24 recovery cause: %v", e) + if host != nil { + host.Close() + } + } + source.Close() + } + t.Fatal("returning actual stores lost committed ownership", v, err) + } +} + +// R3/R8/R24 require a published voter to retain its own import provenance. +// A fresh database with the old identity is not a returning durable member. +func TestR3R8R24NativeProductionRefusesUnclaimedReplacement(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + if _, err := r.dir(a).Create(bg, cell, directory.CellInit{Head: head1}); err != nil { + t.Fatal(err) + } + r.stop() + base := filepath.Join(r.store, "raft", id.ID()) + raw, err := os.ReadFile(filepath.Join(base, "provision.json")) + if err != nil { + t.Fatal(err) + } + var provision struct{ Cluster string } + if err = json.Unmarshal(raw, &provision); err != nil { + t.Fatal(err) + } + path := filepath.Join(base, "1.db") + if err = os.Rename(path, path+".actual"); err != nil { + t.Fatal(err) + } + member := raftstore.Identity{ClusterUUID: provision.Cluster, NodeID: 1, Protocol: 1} + fresh, err := raftstore.Open(path, member) + if err != nil { + t.Fatal(err) + } + if err = fresh.Close(); err != nil { + t.Fatal(err) + } + r.start() + b := newDevice(t, id, t.TempDir()) + if _, err = r.dir(b).Acquire(bg, cell, directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: a.dev.ID(), Fence: 1}}); err != nil { + t.Fatal("actual majority must remain available", err) + } + r.stop() + fresh, err = raftstore.Open(path, member) + if err != nil { + t.Fatal(err) + } + imported, err := fresh.ImportedLegacy(bg) + if err != nil { + t.Fatal(err) + } + app, _, err := fresh.ApplicationView(bg) + if err != nil { + t.Fatal(err) + } + if err = fresh.Close(); err != nil { + t.Fatal(err) + } + if imported != nil || app.Applied != 0 { + t.Fatal("published identity was silently re-imported", app.Applied) + } + if err = os.Remove(path); err != nil { + t.Fatal(err) + } + if err = os.Rename(path+".actual", path); err != nil { + t.Fatal(err) + } + r.start() + view, err := r.dir(a).Cell(bg, cell) + if err != nil || view.Cell.Lease.Fence != 2 || view.Cell.Lease.Device != b.dev.ID() { + t.Fatal("actual returning store lost majority work", view, err) + } +} diff --git a/internal/relayserve/retire.go b/internal/relayserve/retire.go index c308f9ddf2..b6f4f9c6fd 100644 --- a/internal/relayserve/retire.go +++ b/internal/relayserve/retire.go @@ -1,7 +1,10 @@ package relayserve import ( + "context" "errors" + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftstore" "io/fs" "os" "path/filepath" @@ -52,11 +55,27 @@ func (n *namespaces) sweepOne(id string) error { // due says whether the identity's retirement time has come by the relay's clock. func (n *namespaces) due(id string) (bool, error) { - d, err := n.sqlite(id) + d, err := n.activate(context.Background(), id, nil) + if err != nil { + return false, err + } + var r *directory.Rotation + err = d.host.Inspect(context.Background(), func(ctx context.Context, s *raftstore.SQLite, a raftstore.Application) error { + key := raftstore.Key{Namespace: raftstore.State, Key: clusterstate.IdentityKey().Key} + rows, _, e := s.Read(ctx, a.Applied, []raftstore.Key{key}) + if e != nil { + return e + } + var identity clusterstate.IdentityRec + if e = clusterstate.Decode(rows[key], &identity); e != nil { + return e + } + r = identity.Rotation + return nil + }) if err != nil { return false, err } - r := d.Rotated() return r != nil && r.State == directory.StateRetired && n.now().UnixMilli() >= r.RetireAt, nil } @@ -82,13 +101,17 @@ func (n *namespaces) erase(id string) error { failed = errors.Join(failed, err) } } - return errors.Join(failed, os.RemoveAll(filepath.Join(n.root, "blobs", id))) + return errors.Join(failed, os.RemoveAll(filepath.Join(n.root, "blobs", id)), os.RemoveAll(filepath.Join(n.root, "raft", id))) } // forget closes the identity's open directory, if it has one. func (n *namespaces) forget(id string) { n.mu.Lock() defer n.mu.Unlock() + if d, ok := n.groups[id]; ok { + _ = d.Close() + delete(n.groups, id) + } if d, ok := n.dirs[id]; ok { _ = d.Close() delete(n.dirs, id) diff --git a/internal/relayserve/revoked_test.go b/internal/relayserve/revoked_test.go index 46e998db52..b814a1dafa 100644 --- a/internal/relayserve/revoked_test.go +++ b/internal/relayserve/revoked_test.go @@ -159,7 +159,7 @@ func TestRevokeUnderConcurrentRequests(t *testing.T) { // real signed HTTP against the relay. Each case gets a fresh relay and a fresh // identity; a device is made under it the first time a case names it. func TestRelayPassesDirectoryConformance(t *testing.T) { - directorytest.RunRigs(t, func(t *testing.T) directorytest.Rig { + directorytest.RunConsensusRigs(t, func(t *testing.T) directorytest.Rig { r := newRig(t) id, first := newIdentity(t) var mu sync.Mutex diff --git a/internal/relayserve/service.go b/internal/relayserve/service.go index c87ff7c2c6..94c1b84904 100644 --- a/internal/relayserve/service.go +++ b/internal/relayserve/service.go @@ -14,7 +14,6 @@ import ( "github.com/Agent-Field/codeaf/internal/directory" "github.com/Agent-Field/codeaf/internal/pairbox" "github.com/Agent-Field/codeaf/internal/relay" - "github.com/Agent-Field/codeaf/internal/reqsign" ) // Config is everything a relay is told. The zero Store is the blind pipe. @@ -99,8 +98,8 @@ func (s *Service) sweepLoop(every time.Duration, logf func(string, ...any)) { // New builds the relay. Every relay is the blind pipe and the pairing mailbox, // which needs no disk and no identity. Given a store it also serves the -// directory and store wires, every request authenticated by the real signature -// check against the relay's own clock. +// directory and store wires through three persistent local anchors per identity. +// First use migrates the source; every request proves its original signature. func New(cfg Config) *Service { now := cfg.Now if now == nil { @@ -118,7 +117,7 @@ func New(cfg Config) *Service { svc.ns = newNamespaces(cfg.Store, now, cfg.Grace, cfg.MaxWatchers) svc.ns.after = cfg.After svc.ns.links = links - auth := noting(turningRevokedAway(reqsign.AuthenticateAt(now), svc.ns.revoked)) + auth := noting(svc.ns.authenticate) dirs := logged(cfg.Logf, directory.Handler(auth, svc.ns.directory)) mux.Handle("/v1/dir/", dirs) mux.Handle("/v1/identity/", dirs) From 266e8bca41988475ea7b41dbd1d38d3492767306 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 07:57:51 -0400 Subject: [PATCH 044/103] Keep original v1 proofs stable across typed hosted boundaries. Normalize empty frame hints and accept the original safe rotation schema marker so canonical forwarding retains the signed operation. Prepare restricted hosted effects, typed legacy projections and committed authority checks in the shared Go ABI. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterstate/v1.go | 18 +++++- internal/clusterstate/v1_test.go | 43 +++++++++++++ internal/raftabi/abi.go | 8 +++ internal/raftabi/hosted.go | 104 +++++++++++++++++++++++++++++++ internal/raftabi/hosted_test.go | 92 +++++++++++++++++++++++++++ 5 files changed, 262 insertions(+), 3 deletions(-) create mode 100644 internal/clusterstate/v1_test.go create mode 100644 internal/raftabi/hosted.go create mode 100644 internal/raftabi/hosted_test.go diff --git a/internal/clusterstate/v1.go b/internal/clusterstate/v1.go index 43fd6f26ed..4ce872df0b 100644 --- a/internal/clusterstate/v1.go +++ b/internal/clusterstate/v1.go @@ -217,11 +217,15 @@ func (p V1Request) Mutation() (Mutation, error) { return Mutation{SetVault: &v}, nil } if path == "/v1/identity/rotation" && p.Method == "POST" { - var v RotateRequest - if !decode(&v, false) { + var v struct { + V uint16 `json:"V,omitempty"` + Op string `json:"op"` + GraceMS int64 `json:"grace_ms,omitempty"` + } + if !decode(&v, false) || v.V > 1 { return bad() } - return Mutation{Rotate: &v}, nil + return Mutation{Rotate: &RotateRequest{Op: v.Op, GraceMS: v.GraceMS}}, nil } s := strings.Split(strings.TrimPrefix(path, "/v1/dir/"), "/") if len(s) < 2 || len(s) > 3 || path == strings.TrimPrefix(path, "/v1/dir/") { @@ -282,6 +286,11 @@ func (p V1Request) Mutation() (Mutation, error) { if !decode(&v, false) { return bad() } + // Empty and absent frame plans both replace the previous hint. Normalize + // before signing so canonical omitempty survives original-proof replay. + if len(v.Frames) == 0 { + v.Frames = nil + } return Mutation{Create: &CreateCell{Cell: id, Init: v, V1: true}}, nil case "acquire": var v struct { @@ -310,6 +319,9 @@ func (p V1Request) Mutation() (Mutation, error) { if !decode(&v, false) || v.Fence > MaxSafeInteger { return bad() } + if len(v.Frames) == 0 { + v.Frames = nil + } return Mutation{Publish: &PublishCell{Cell: id, Publish: v, V1: true}}, nil case "release": var v struct { diff --git a/internal/clusterstate/v1_test.go b/internal/clusterstate/v1_test.go new file mode 100644 index 0000000000..f06789bec6 --- /dev/null +++ b/internal/clusterstate/v1_test.go @@ -0,0 +1,43 @@ +package clusterstate_test + +import ( + "crypto/ed25519" + "strconv" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +// R25 requires legal legacy empty frame plans to survive canonical forwarding +// without invalidating the original signature or retaining an old frame hint. +func TestR25V1EmptyFramePlansKeepOriginalAuthority(t *testing.T) { + root := actorKey("root") + device := actorKey("a") + identity := cs.IdentityRec{V: 1, Identity: cs.IdentityID(root.Public().(ed25519.PublicKey)), SigningKey: root.Public().(ed25519.PublicKey)} + for _, path := range []string{"/v1/dir/cells/chat", "/v1/dir/cells/chat/publish"} { + body := []byte(`{"head":"new","frames":[]}`) + if path == "/v1/dir/cells/chat/publish" { + body = []byte(`{"fence":1,"old_head":"old","head":"new","frames":[]}`) + } + proof := cs.V1Request{Method: "POST", URI: path, IdentityKey: identity.SigningKey, Certificate: joinCert(t, "a"), At: strconv.Itoa(1000), Body: body} + proof.Signature = ed25519.Sign(device, cs.V1SigningMessage(proof.Method, proof.URI, proof.At, body)) + command, err := cs.V1Command("test", 1000, proof) + if err != nil { + t.Fatal(err) + } + raw, err := cs.EncodeCommand(command) + if err != nil { + t.Fatal(err) + } + forwarded, err := cs.DecodeCommand(raw) + if err != nil { + t.Fatal(err) + } + if _, ok := cs.V1AuthorityAt(forwarded, identity, 1000); !ok { + t.Fatal("R25 original empty frame request lost its authority", path) + } + if forwarded.Change.Create != nil && forwarded.Change.Create.Init.Frames != nil || forwarded.Change.Publish != nil && forwarded.Change.Publish.Publish.Frames != nil { + t.Fatal("empty frame plan was not normalized") + } + } +} diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index 47a303ac89..66092e9ec3 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -164,6 +164,12 @@ func (r *Runtime) Call(b []byte) (out []byte) { func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { e := encoder{} switch op { + case 30: + return hostedEffect(d) + case 31: + return hostedProjection(d) + case 32: + return hostedAuthority(d) case 29: return v1Command(d) case 28: @@ -286,6 +292,8 @@ func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { return legacyAuthorities(d) case 13: return legacyGenesis(d) + case 16: + in = raftcore.Campaign{} case 15: in = raftcore.IdleRead{RequestID: d.str()} case 12: diff --git a/internal/raftabi/hosted.go b/internal/raftabi/hosted.go new file mode 100644 index 0000000000..59b27db031 --- /dev/null +++ b/internal/raftabi/hosted.go @@ -0,0 +1,104 @@ +package raftabi + +import ( + "bytes" + "crypto/ed25519" + "encoding/json" + "io" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +// Hosted effects use only a source-provisioned protocol key. The key is an ABI +// input, never a command field, snapshot record or diagnostic value. +func hostedEffect(d *decoder) ([]byte, error) { + cluster, now, seed, raw := d.str(), d.u64(), d.blob(), d.blob() + if d.end() != nil || len(seed) != ed25519.SeedSize || now > cs.MaxSafeInteger { + return nil, errFrame + } + var change cs.Mutation + j := json.NewDecoder(bytes.NewReader(raw)) + j.DisallowUnknownFields() + if j.Decode(&change) != nil || j.Decode(new(any)) != io.EOF { + return nil, errFrame + } + if change.DeviceSeen == nil && change.RenewHolds == nil && change.AdmitJoin == nil && change.CompleteJoin == nil && change.RecordUpload == nil { + return nil, errFrame + } + key := ed25519.NewKeyFromSeed(seed) + command, err := cs.Sign(cs.Command{Schema: cs.SchemaVersion, Cluster: cluster, Actor: cs.DeviceIDOfKey(key.Public().(ed25519.PublicKey)), Session: "effects", Sequence: 1, Now: int64(now), Change: change}, key) + if err != nil { + return nil, err + } + raw, err = cs.EncodeCommand(command) + if err != nil || len(raw) > cs.MaxCommandBytes { + return nil, errFrame + } + e := encoder{} + e.blob(raw) + return e.b, nil +} + +// Projection uses the declared Go shape so sealed map values named fence or +// size remain opaque and v1 integer fields keep their original numeric wire. +func hostedProjection(d *decoder) ([]byte, error) { + kind, raw := d.str(), d.blob() + if d.end() != nil { + return nil, errFrame + } + var value any + switch kind { + case "identity": + value = new(cs.IdentityRec) + case "device": + value = new(cs.Device) + case "cell", "list_cell": + value = new(cs.Cell) + case "result", "view": + value = new(cs.Result) + default: + return nil, errFrame + } + if err := cs.Decode(raw, value); err != nil { + return nil, err + } + if kind == "list_cell" { + value.(*cs.Cell).Frames = nil + } + if kind == "view" { + value = value.(*cs.Result).View + } + wire, err := json.Marshal(value) + if err != nil { + return nil, err + } + e := encoder{} + e.blob(wire) + return e.b, nil +} + +// Ingress proves the original signature before external I/O. Every later +// authorization uses a fresh committed view, including the root certificate, +// revocation and rotation; socket presence supplies none of that authority. +func hostedAuthority(d *decoder) ([]byte, error) { + identity, device, actor, cert, write := d.blob(), d.blob(), d.str(), d.str(), d.u32() + if d.end() != nil || write > 1 { + return nil, errFrame + } + var root cs.IdentityRec + var rec cs.Device + if cs.Decode(identity, &root) != nil || len(device) > 0 && cs.Decode(device, &rec) != nil { + return nil, cs.ErrDependency + } + code := cs.OK + if cs.PersonalCertificate(root, actor, cert) != nil { + code = cs.Unauthorized + } else if rec.Revoked { + code = cs.Revoked + } else if write == 1 && root.Rotation != nil { + code = cs.Rotated + } + e := encoder{} + e.str(string(code)) + return e.b, nil +} diff --git a/internal/raftabi/hosted_test.go b/internal/raftabi/hosted_test.go new file mode 100644 index 0000000000..0547482e73 --- /dev/null +++ b/internal/raftabi/hosted_test.go @@ -0,0 +1,92 @@ +package raftabi + +import ( + "bytes" + "crypto/ed25519" + "encoding/base64" + "encoding/hex" + "encoding/json" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +// R15/R16/R25 keep effect keys out of commands, preserve opaque map values and +// check the committed certificate/revocation/rotation at each authorization. +func TestR15R16R25HostedBoundary(t *testing.T) { + runtime := New() + seed := bytes.Repeat([]byte{17}, 32) + key := ed25519.NewKeyFromSeed(seed) + e := request(30) + e.str("hosted") + e.u64(1000) + e.blob(seed) + e.blob([]byte(`{"device_seen":{"device":"a"}}`)) + d := answer(t, runtime.Call(e.b)) + raw := d.blob() + if d.end() != nil { + t.Fatal("trailing effect") + } + command, err := cs.DecodeCommand(raw) + if err != nil { + t.Fatal(err) + } + signed, _ := cs.SigningBytes(command) + if !ed25519.Verify(key.Public().(ed25519.PublicKey), signed, command.Signature) || bytes.Contains(raw, []byte(base64.StdEncoding.EncodeToString(seed))) { + t.Fatal("R15 effect key boundary") + } + bad := request(30) + bad.str("hosted") + bad.u64(1000) + bad.blob(seed) + bad.blob([]byte(`{"revoke":{"device":"a"}}`)) + r := decoder{b: runtime.Call(bad.b)} + r.u32() + if r.str() == "" { + t.Fatal("effect signer admitted owner mutation") + } + rootKey := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{33}, 32)) + pub := key.Public().(ed25519.PublicKey) + actor := cs.DeviceIDOfKey(pub) + cert := struct { + V uint16 `json:"V"` + Device string `json:"device"` + Created int64 `json:"created"` + Sig string `json:"sig"` + }{1, hex.EncodeToString(pub), 1, ""} + certRaw, _ := json.Marshal(cert) + cert.Sig = hex.EncodeToString(ed25519.Sign(rootKey, append([]byte("codeaf/device-cert/v1\n"), certRaw...))) + certRaw, _ = json.Marshal(cert) + identity := cs.IdentityRec{V: 1, Identity: cs.IdentityID(rootKey.Public().(ed25519.PublicKey)), SigningKey: rootKey.Public().(ed25519.PublicKey)} + for _, test := range []struct { + revoked, write, frozen bool + want cs.Code + }{{false, false, false, cs.OK}, {true, false, false, cs.Revoked}, {false, true, true, cs.Rotated}, {false, false, true, cs.OK}} { + if test.frozen { + identity.Rotation = &cs.Rotation{State: "frozen", By: actor, At: 1000} + } else { + identity.Rotation = nil + } + root, _ := cs.Encode(identity) + device, _ := cs.Encode(cs.Device{V: 1, Revoked: test.revoked}) + e = request(32) + e.blob(root) + e.blob(device) + e.str(actor) + e.str(base64.RawURLEncoding.EncodeToString(certRaw)) + e.flag(test.write) + if got := answer(t, runtime.Call(e.b)).str(); got != string(test.want) { + t.Fatalf("R16 got %s want %s", got, test.want) + } + } + cell := cs.Cell{V: 1, Size: 7, Lease: cs.Lease{Fence: 9}, Keys: map[string]map[string]string{"fence": {"size": "opaque"}}} + raw, _ = cs.Encode(cell) + e = request(31) + e.str("cell") + e.blob(raw) + projected := answer(t, runtime.Call(e.b)).blob() + var v cs.Cell + if json.Unmarshal(projected, &v) != nil || v.Lease.Fence != 9 || v.Keys["fence"]["size"] != "opaque" { + t.Fatal("R25 typed v1 projection changed sealed bytes") + } +} From e45f7704ff35e3d0ffa4fbf4936fa5cac2597a86 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 07:57:52 -0400 Subject: [PATCH 045/103] Route hosted identities through the same durable three-anchor namespace. Freeze and resume first-use imports with one manifest before publishing quorum-backed ownership. Keep blob I/O outside the coordination queue, recheck authority after it, exclude unclaimed replacement stores and retain resumable all-replica retirement. Distinguish the local platform name table from unknown retained application data, and finish KV transport cleanup after an interrupted retirement. Co-Authored-By: Claude Opus 5.5 (1M context) --- relay/hosted/src/erasure.js | 7 +- relay/hosted/src/identity.js | 78 ++++++----- relay/hosted/src/raft/actor.js | 7 + relay/hosted/src/raft/hosted.js | 231 +++++++++++++++++++++++++++++++ relay/hosted/src/raft/member.js | 93 ++++++++++++- relay/hosted/src/raft/storage.js | 3 +- relay/hosted/src/sql-schema.js | 4 + relay/hosted/src/tenant.js | 2 +- 8 files changed, 386 insertions(+), 39 deletions(-) create mode 100644 relay/hosted/src/raft/hosted.js create mode 100644 relay/hosted/src/sql-schema.js diff --git a/relay/hosted/src/erasure.js b/relay/hosted/src/erasure.js index 0b9901be4b..18bc7b92c4 100644 --- a/relay/hosted/src/erasure.js +++ b/relay/hosted/src/erasure.js @@ -1,3 +1,5 @@ +import { APPLICATION_TABLES } from './sql-schema.js'; + // What retiring an identity deletes once its grace has run out. The frames are deleted a page at a // time, so one call never outgrows a Durable Object turn, and the caller asks again until a page // comes back empty. Both halves are idempotent: a second run over what is already gone changes nothing. @@ -10,10 +12,7 @@ export async function erasePage(bucket, prefix, pageSize) { return true; } -// The platform's own tables (`_cf_KV` holds the object's key-value storage) are not ours to drop. -const OURS = "type = 'table' AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\' AND name NOT LIKE '\\_cf\\_%' ESCAPE '\\'"; - /** dropTables deletes every table the relay made in the identity's SQLite: directory, index, quota, counts. */ export function dropTables(sql) { - for (const { name } of sql.exec(`SELECT name FROM sqlite_master WHERE ${OURS}`).toArray()) sql.exec(`DROP TABLE "${name}"`); + for (const { name } of sql.exec(`SELECT name FROM sqlite_master WHERE ${APPLICATION_TABLES}`).toArray()) sql.exec(`DROP TABLE "${name}"`); } diff --git a/relay/hosted/src/identity.js b/relay/hosted/src/identity.js index 1fa4b3d300..7ac99526d9 100644 --- a/relay/hosted/src/identity.js +++ b/relay/hosted/src/identity.js @@ -1,13 +1,9 @@ -// IdentityDO: one Durable Object per identity. It is the single writer of that identity's -// directory (the compare-and-swap the lease rules need), the counter of its puts in flight, and -// the place its caps are kept, because it already sees every request the identity makes. -// -// The front Worker forwards a request here on the identity named in the unsigned header, before -// the body is read. The order inside is then: prove the caller holds a cert of this identity -// (needs no body), count a put as in flight, read the body under its cap, verify the request -// signature over it, and only then spend the caller's rate budget and serve. +// IdentityDO retains the source backup, blob index and socket ingress. Its +// irreversible migration gate routes coordination to three durable members; +// none of its legacy directory methods are a production writer. import { DurableObject } from 'cloudflare:workers'; import { LegacyAdmission } from './raft/legacy.js'; +import { HostedIdentity } from './raft/hosted.js'; import { signedUri } from './base.js'; import { readBody } from './body.js'; import { dropTables, erasePage } from './erasure.js'; @@ -16,9 +12,9 @@ import { Lazy } from './lazy.js'; import { limitsOf, RateLimit } from './limits.js'; import { matchRoute } from './routes.js'; import { policyOf } from './rules.js'; -import { Tenant } from './tenant.js'; +import { Tenant, HAS_WAIT_MS } from './tenant.js'; import { finishClose, Watchers } from './watch.js'; -import { checkCert, checkRequest, Refusal } from './verify.js'; +import { checkCert, checkRequest } from './verify.js'; import { gone, ipOf, rateLimited, respond, unwrap } from './wire.js'; const SIGNED = { identity: 'codeaf-identity', cert: 'codeaf-cert', time: 'codeaf-time', sig: 'codeaf-sig' }; @@ -41,15 +37,16 @@ export class IdentityDO extends DurableObject { this.limits.maxWatchers, this.policy.ttlMs, (at) => this.ctx.waitUntil(this.#arm(at)), - // A sweep stamps last_seen through the tenant, which an alarm wake has built just before it (see alarm). - (device, at) => this.#tenant?.dir.seen(device, at), + // Old sockets contribute only their frozen import evidence; every later + // last-seen update belongs to the committed effects path. + () => {}, ); this.identityRate = new RateLimit(this.limits.requestsPerMinute); this.deviceRate = new RateLimit(this.limits.requestsPerMinutePerDevice); } fetch(request) { - return respond(() => this.legacyAdmission.run(() => this.#serve(request))); + return respond(() => this.#serve(request)); } /** webSocketMessage runs only for a client frame that is not `ping` (the platform answers that without waking us); such a frame is ignored (contract 21.3). */ @@ -71,7 +68,7 @@ export class IdentityDO extends DurableObject { async #noteLeft(ws) { const device = this.watchers.left(ws); const identity = device && (await this.ctx.storage.get('identity')); - if (identity && !(await this.#isGone())) this.#tenantOf(identity).dir.seen(device, Date.now()); + if (identity && !(await this.#isGone()) && this.host?.ready) await this.host.serial(() => this.host.commitEffect({device_seen:{device}},Date.now())); } async #serve(request) { @@ -86,8 +83,9 @@ export class IdentityDO extends DurableObject { await this.#admitted.get(ipOf(request, this.env)); const tenant = this.#tenantOf(who.identity); this.#admit(tenant, who.device); - if (route.write) tenant.assertWritable(); - return await route.handler({ tenant, device: who.device, body, request, env: this.env }, route.args); + this.host ??= new HostedIdentity(this.ctx,this.env,tenant,this.legacyAdmission); + const reply=await this.host.dispatch({tenant,device:who.device,body,request,env:this.env},route); + this.watchers=this.host.watchers;return reply; } finally { leave(); } @@ -126,17 +124,26 @@ export class IdentityDO extends DurableObject { * may have stood in front of it. */ async alarm() { - const identity = await this.ctx.storage.get('identity'); - const retired = identity && (await this.#isGone()); - if (identity && !retired) this.#tenantOf(identity); - this.watchers.expire(); - if (!identity) return; - if (retired) return this.#erase(identity); - const tenant = this.#tenantOf(identity); - tenant.stats.flush(); - const at = tenant.retireAt(); - if (at === undefined) return; - return at <= Date.now() ? this.#erase(identity) : this.#arm(at); + const identity=await this.ctx.storage.get('identity');if(!identity)return; + if(await this.#isGone())return this.#erase(identity); + const provision=await this.ctx.storage.get('raft-provision'); + if(!provision){ + // An old source may know only the root hash. Its next authenticated + // request supplies the public root; an alarm cannot invent that key. + this.watchers.closeAll(1012,'reconnecting');return; + } + const tenant=this.#tenantOf(identity);this.host ??= new HostedIdentity(this.ctx,this.env,tenant,this.legacyAdmission); + const erase=await this.host.serial(async()=>{ + await this.host.ensure(Uint8Array.from(atob(provision.config.signing_key),c=>c.charCodeAt(0))); + this.watchers=this.host.watchers;this.watchers.expire();tenant.stats.flush(); + const raw=await this.host.barrierRecord({namespace:'state',key:'identity'}); + const record=this.host.project('identity',raw),at=record.rotation?.retire_at; + if(at && at<=Date.now())return true; + if(at)await this.#arm(at);return false; + }); + // Uploads must finish their post-I/O authority check before leaving + // Flight. Waiting for them while owning that queue would deadlock erasure. + if(erase)return this.#erase(identity); } /** @@ -171,20 +178,29 @@ export class IdentityDO extends DurableObject { async #erase(identity) { await this.ctx.storage.put('gone', Date.now()); this.#gone = true; + if(!await this.flight.idle(HAS_WAIT_MS))return this.#arm(Date.now()+this.limits.statsFlushMs); this.#tenant = null; if (await erasePage(this.env.FRAMES, `${identity}/`, this.limits.sweepPageSize)) return this.#arm(Date.now()); unwrap(await this.#global().forget(identity)); + const sourceGate=this.ctx.storage.sql.exec("SELECT name FROM sqlite_master WHERE type='table' AND name='legacy_gate'").toArray().length && this.ctx.storage.sql.exec('SELECT doc FROM legacy_gate WHERE one=1').toArray()[0]; + let erasing=await this.ctx.storage.get('raft-erasure'); + if(!erasing&&sourceGate){erasing=JSON.parse(sourceGate.doc).manifest;await this.ctx.storage.put('raft-erasure',erasing);await this.ctx.storage.sync();} + if(erasing){ + try{for(const id of erasing.config.anchors)await this.env.RAFT_MEMBERS.get(this.env.RAFT_MEMBERS.idFromName(erasing.config.cluster+'/'+id)).eraseRetired(erasing);} + catch(error){await this.#arm(Date.now()+this.limits.statsFlushMs);throw error;} + } dropTables(this.ctx.storage.sql); - await this.ctx.storage.delete(['identity', 'admitted']); + await this.ctx.storage.delete(['identity', 'admitted','raft-provision']); + await this.ctx.storage.sync();this.host=null; } /** - * #admit turns away a revoked device, then charges the request to the device's rate and the - * identity's. The device goes first, so one noisy device that is refused spends none of the + * #admit charges the authenticated request to the device's rate and the + * identity's. Committed authorization follows inside the coordinator. The device goes first, + * so one noisy device that is refused spends none of the * identity's share, which its other devices still need. */ #admit(tenant, device) { - if (tenant.dir.revoked(device)) throw new Refusal('revoked', 'device revoked'); const now = Date.now(); this.deviceRate.admit(device, now); this.identityRate.admit('', now); diff --git a/relay/hosted/src/raft/actor.js b/relay/hosted/src/raft/actor.js index e7b3993b34..0afe3f2364 100644 --- a/relay/hosted/src/raft/actor.js +++ b/relay/hosted/src/raft/actor.js @@ -169,6 +169,13 @@ export class MemberActor { begin(data) { data = data.slice(); return this.serial(() => this.store.beginSnapshot(data)); } chunk(id, offset, data, hash) { data = data.slice(); return this.serial(() => this.store.putChunk(id, offset, data, hash)); } complete(id) { return this.serial(() => this.store.completeSnapshot(id)); } + // Explicit owner erasure has already durably fenced every ingress. Dropping + // this handle reclaims memory; it is never a pressure eviction of a leader. + disposeErased() { return this.serial(() => { + this.stopped=true; + if(this.handle){const r=response(this.reactor.call(frame(2).u32(this.handle).finish()));r.end();} + this.handle=0;active.delete(this); + }); } close() { return this.serial(() => { if (this.status?.role === 2 || this.pending > 1) throw new Error('raft member: active member cannot be evicted'); if (this.handle) { const r = response(this.reactor.call(frame(2).u32(this.handle).finish())); r.end(); } diff --git a/relay/hosted/src/raft/hosted.js b/relay/hosted/src/raft/hosted.js new file mode 100644 index 0000000000..a68b302d1e --- /dev/null +++ b/relay/hosted/src/raft/hosted.js @@ -0,0 +1,231 @@ +import { frame, response, HostCodec } from './abi.js'; +import { sharedRuntime } from './actor.js'; +import { encodeCanonical, decodeCanonical } from './codec.js'; +import { b64, unb64, bytes } from './storage.js'; +import { v1Command } from './v1.js'; +import { RaftEffectsFacade } from './effects.js'; +import { Wire, empty, json, unwrap } from '../wire.js'; +import { signedUri } from '../base.js'; +import { fromBase64Url } from '../codec.js'; +import { presenceOf, parseBeat } from '../presence.js'; +import { parseHolds } from '../vouch.js'; +import { rotationView } from '../rotation.js'; +import { CommittedWatchers, CLOSE_REVOKED, CLOSE_ROTATED } from '../watch.js'; + +const enc=new TextEncoder(),dec=new TextDecoder('utf-8',{fatal:true}); +const segment=s=>Array.from(enc.encode(s),b=>b.toString(16).padStart(2,'0')).join(''); +const unsegment=s=>dec.decode(Uint8Array.from(s.match(/../g)??[],h=>parseInt(h,16))); +const data=v=>enc.encode(encodeCanonical(v)); +const MIGRATION_PAGE=64; +const unavailable=()=>new Wire('unreachable',503,1); +const status={not_found:404,unauthorized:401,revoked:401,rotated:410,bad_request:400,bad_grace:400,self_revoke:400,capacity:507,full:507}; +function requireOK(code){if(code!=='ok')throw new Wire(code,status[code]??409);} +async function keypair(){ + const pair=await crypto.subtle.generateKey('Ed25519',true,['sign','verify']); + const privateBytes=new Uint8Array(await crypto.subtle.exportKey('pkcs8',pair.privateKey)); + return {public:b64(new Uint8Array(await crypto.subtle.exportKey('raw',pair.publicKey))),private:b64(privateBytes),seed:b64(privateBytes.slice(-32))}; +} + +// The source remains the namespace router and blob index, but its irreversible +// gate prevents it ever becoming a second directory writer. All coordination +// replies below depend on the members' actual durable application and quorum. +export class HostedIdentity { + constructor(ctx,env,tenant,admission){ + this.ctx=ctx;this.env=env;this.tenant=tenant;this.runtime=sharedRuntime();this.codec=new HostCodec(this.runtime); + this.source=tenant.migration(ctx.storage,this.runtime);this.tail=Promise.resolve();this.pending=0;this.members=new Map();this.states=new Map();this.queue=[];this.reads=[]; + } + serial(fn){ + if(this.pending>=64)return Promise.reject(unavailable());this.pending++; + const task=this.tail.then(fn);this.tail=task.catch(()=>{});return task.finally(()=>this.pending--); + } + member(id){return this.env.RAFT_MEMBERS.get(this.env.RAFT_MEMBERS.idFromName(this.manifest.config.cluster+'/'+id));} + config(id){ + const c=this.manifest.config; + return {cluster:c.cluster,id:String(id),seed:String(BigInt(id)+100n),bootstrap:c.anchors,ceiling:Number(this.manifest.ceiling),scope:{identity:c.identity,cluster:c.cluster,epoch:c.generation,protocol:1},anchors:c.anchors.map((node,i)=>({node,device:'',key:c.anchor_keys[i],incarnation:'1',protocol:1,anchor:true,retired:false,ready_index:'0'}))}; + } + async provision(identityKey){ + let p=await this.ctx.storage.get('raft-provision'); + if(p)return p; + // A frozen source must never receive a different UUID or replacement keys. + if(this.source.gate())throw unavailable(); + const anchors=[];for(let i=0;i<3;i++)anchors.push(await keypair()); + const effects=await keypair(),limits=this.tenant.limits; + p={config:{cluster:crypto.randomUUID(),generation:'1',identity:this.tenant.identity,signing_key:b64(identityKey),anchors:['1','2','3'],anchor_keys:anchors.map(a=>a.public),effects_key:effects.public,now:Date.now(),grace:{min:limits.minGraceMs,max:limits.maxGraceMs,default:limits.defaultGraceMs}},anchors:anchors.map(a=>a.private),effects:effects.seed}; + await this.ctx.storage.put('raft-provision',p);await this.ctx.storage.sync();return p; + } + async ensure(identityKey){ + if(this.ready)return; + this.provisioning=await this.provision(identityKey); + if(!this.source.gate())await this.source.freeze(this.provisioning.config); + this.manifest=await this.source.inventory(); + if(encodeCanonical(this.manifest.config)!==encodeCanonical(this.provisioning.config))throw unavailable(); + const owned=this.source.gate().stage==='raft-owned'; + if(!owned){ + // Each retry advances a bounded page; the cursor is durable only after + // all three copies accepted identical bytes. A cut merely repeats a row. + const imported=[]; + for(const id of this.manifest.config.anchors)if(await this.member(id).importReceipt(this.manifest))imported.push(id); + if(imported.length<3){ + let cursor=await this.ctx.storage.get('raft-import-cursor')??{namespace:'',key:''},count=0; + for(const row of this.ctx.storage.sql.exec(`SELECT namespace,key,data FROM legacy_rows WHERE namespace>? OR (namespace=? AND key>?) ORDER BY namespace,key LIMIT ${MIGRATION_PAGE+1}`,cursor.namespace,cursor.namespace,cursor.key)){ + if(count++===MIGRATION_PAGE)throw unavailable(); + const copied={namespace:row.namespace,key:row.key,data:bytes(row.data)}; + for(const id of this.manifest.config.anchors){ + if(!imported.includes(id)){await this.member(id).legacyImport(this.config(id),this.manifest,copied);} + } + cursor={namespace:row.namespace,key:row.key};await this.ctx.storage.put('raft-import-cursor',cursor);await this.ctx.storage.sync(); + } + for(const id of this.manifest.config.anchors){ + if(!imported.includes(id)){await this.member(id).legacyImport(this.config(id),this.manifest,null,true);} + } + } + } + // AFTER publication, absent or unclaimed stores are excluded, never + // imported again under old voter IDs. Two actual survivors are required. + const receipts=[]; + for(const id of this.manifest.config.anchors){ + try{ + const receipt=await this.member(id).importReceipt(this.manifest);if(!receipt)continue; + const member=this.member(id);this.members.set(id,member);receipts.push(receipt); + this.accept(id,await member.initialize(this.config(id))); + this.accept(id,await member.drainRecovery()); + }catch(error){this.members.delete(id);this.states.delete(id);if(!owned)throw error;} + } + if(this.members.size<2)throw unavailable(); + await this.pump();await this.elect();await this.barrier(); + if(!owned)await this.source.own(receipts); + this.source.route(()=>true); + this.watchers=new CommittedWatchers(this.ctx,this.tenant.limits.maxWatchers,this.tenant.watchers.ttlMs,at=>this.ctx.waitUntil(this.ctx.storage.setAlarm(at))); + this.tenant.watchers=this.watchers;this.ready=true; + this.effects=new RaftEffectsFacade(this,this.manifest.config.cluster); + } + accept(id,e){if(!e)return;if(e.status)this.states.set(String(id),e.status);this.queue.push(...(e.send??[]).map(m=>({...m,from:String(id),to:String(m.to)})));this.reads.push(...(e.reads??[]).map(r=>({...r,node:String(id)})));} + async input(id,tag,value){const result=await this.members.get(String(id)).hostInput(tag,value);this.accept(id,result);return result;} + async envelope(from,to,payload,chunk=null){ + const next=BigInt(await this.ctx.storage.get('raft-transport-sequence')??'0')+1n; + await this.ctx.storage.put('raft-transport-sequence',String(next));await this.ctx.storage.sync(); + const e={scope:this.config(from).scope,from,to,incarnation:'1',destination:'1',sequence:String(next),chunk,digest:Array.from(this.codec.hash(payload).digest,b=>b.toString(16).padStart(2,'0')).join(''),payload:b64(payload),signature:null}; + const rank=this.manifest.config.anchors.indexOf(from); + const key=await crypto.subtle.importKey('pkcs8',unb64(this.provisioning.anchors[rank]),'Ed25519',false,['sign']); + e.signature=b64(new Uint8Array(await crypto.subtle.sign('Ed25519',key,enc.encode('codeaf/raft/envelope/1\n'+JSON.stringify(e))))); + return enc.encode(JSON.stringify(e)); + } + async pump(){ + for(let n=0;this.queue.length;n++){ + if(n>=2048){this.queue=[];throw unavailable();} + const m=this.queue.shift(),to=this.members.get(m.to);if(!to)continue; + const snapshot=this.codec.inspect(4,m.bytes).snapshot; + try{ + if(snapshot.length){ + const meta=this.codec.inspect(3,snapshot),current=await to.hostInput(0);this.accept(m.to,current); + // Delayed snapshots below the durable applied index go directly to + // Raft for its ordinary stale reply; they must not stage over a newer + // checkpoint or exclude a healthy member from the coordinator. + if(meta.index>current.status.applied){ + const manifest=JSON.parse(dec.decode(meta.data)),id=await to.hostSnapshot('begin',meta.data); + for(let offset=0;offsetb.toString(16).padStart(2,'0')).join('')})); + } + await to.hostSnapshot('complete',id); + } + } + const result=await to.peer(await this.envelope(m.from,m.to,m.bytes));this.accept(m.to,result); + if(snapshot.length)await this.input(m.from,12,{peer:m.to,snapshot:true,success:result.installed===true}); + }catch(error){ + if(snapshot.length)await this.input(m.from,12,{peer:m.to,snapshot:true,success:false}); + this.members.delete(m.to);this.states.delete(m.to); + } + } + } + leader(){return [...this.states].find(([id,s])=>this.members.has(id)&&s.role===2)?.[0];} + async elect(){ + if(this.leader())return; + for(const id of this.members.keys()){await this.input(id,16);await this.pump();if(this.leader())return;} + throw unavailable(); + } + async barrier(){ + await this.elect();const leader=this.leader(),id=crypto.randomUUID();this.reads=[]; + await this.input(leader,4,id);await this.pump(); + const read=this.reads.find(r=>r.node===leader&&r.id===id);this.reads=[]; + if(!read) {await this.input(leader,10,id);throw unavailable();} + return {node:leader,index:read.index}; + } + async commit(raw){ + await this.elect();const id=this.leader(),proposal=await this.input(id,3,raw);await this.pump(); + const result=await this.members.get(id)?.hostResult(raw,proposal.proposalAfter);if(!result)throw unavailable(); + return {raw:result.raw,value:decodeCanonical(dec.decode(result.raw)),index:result.index}; + } + async commitEffect(change,now=Date.now()){ + const r=response(this.runtime.call(frame(30).str(this.manifest.config.cluster).u64(now).blob(unb64(this.provisioning.effects)).blob(enc.encode(JSON.stringify(change))).finish())); + const raw=r.blob();r.end();return (await this.commit(raw)).value; + } + async barrierRecord({namespace,key}){const b=await this.barrier();return (await this.members.get(b.node).hostView(b.index,namespace,key)).raw;} + async barrierOutbox(after,limit){const b=await this.barrier();return (await this.members.get(b.node).hostView(b.index,'outbox','',after)).rows.slice(0,limit);} + projectJSON(kind,raw){const r=response(this.runtime.call(frame(31).str(kind).blob(raw).finish())),value=dec.decode(r.blob());r.end();return value;} + project(kind,raw){return JSON.parse(this.projectJSON(kind,raw));} + async authorize(device,certificate,write=false){ + requireOK((await this.commitEffect({device_seen:{device}},Date.now())).code); + const b=await this.barrier(),member=this.members.get(b.node),root=await member.hostView(b.index,'state','identity'),actor=await member.hostView(b.index,'state','device/'+segment(device)); + const r=response(this.runtime.call(frame(32).blob(root.raw).blob(actor.raw??new Uint8Array()).str(device).str(certificate).u32(write?1:0).finish()));requireOK(r.str());r.end();return {barrier:b,app:root.app,identity:this.project('identity',root.raw)}; + } + async listingJSON(b,app){ + const devices=[],cells=[];let identity='null',after=''; + for(;;){const page=await this.members.get(b.node).hostView(b.index,'state','',after);if(!page.rows.length)break; + // The coordinator holds the only public mutation queue throughout pages. + for(const row of page.rows){ + after=row.key; + if(after==='identity')identity=this.projectJSON('identity',row.data); + else if(after.startsWith('device/'))devices.push(JSON.stringify(unsegment(after.slice(7)))+':'+this.projectJSON('device',row.data)); + else if(after.startsWith('cell/'))cells.push(JSON.stringify(unsegment(after.slice(5)))+':'+this.projectJSON('list_cell',row.data)); + } + } + return '{"now":'+app.now+',"identity":'+identity+',"devices":{'+devices.join(',')+'},"cells":{'+cells.join(',')+'}}'; + } + jsonBytes(raw){return new Response(raw,{headers:{'content-type':'application/json'}});} + proof(request,body){return {method:request.method,uri:signedUri(this.env,new URL(request.url)),identity_key:b64(fromBase64Url(request.headers.get('codeaf-identity'))),certificate:request.headers.get('codeaf-cert'),at:request.headers.get('codeaf-time'),signature:b64(fromBase64Url(request.headers.get('codeaf-sig'))),body:b64(body)};} + async dispatch(c,route){ + if(!new URL(c.request.url).pathname.startsWith('/v1/store/'))return this.serial(()=>this.serve(c,route)); + const cert=c.request.headers.get('codeaf-cert'); + await this.serial(async()=>{ + await this.ensure(fromBase64Url(c.request.headers.get('codeaf-identity'))); + await this.authorize(c.device,cert,route.write); + }); + // Blob waits cannot hold the coordination queue: revoke/rotation must be + // able to commit during I/O, and Has must not block queued frame puts. + const result=await route.handler(c,route.args);await this.ctx.storage.sync(); + await this.serial(()=>this.authorize(c.device,cert,route.write));return result; + } + async serve(c,route){ + const {request,device,body}=c,url=new URL(request.url),path=url.pathname,cert=request.headers.get('codeaf-cert'); + await this.ensure(fromBase64Url(request.headers.get('codeaf-identity'))); + if(path==='/v1/dir/watch'){ + if(request.headers.get('upgrade')?.toLowerCase()!=='websocket')throw new Wire('upgrade_required',426); + const holds=parseHolds(url.searchParams.getAll('hold'),this.tenant.limits.maxHolds),beat=parseBeat(url.searchParams.getAll('beat')); + const a=await this.authorize(device,cert,true); + return this.watchers.accept(device,Number(a.app.version),Date.now(),{holds,beat,events:url.searchParams.get('events')==='1',receipts:url.searchParams.get('receipts')==='1',certificate:cert}); + } + const join=/^\/v1\/dir\/requests\/([^/]+)\/(approve|deny)$/.exec(path); + if(join){await this.authorize(device,cert,true);const code=decodeURIComponent(join[1]).trim().toLowerCase();const box=this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(code));await this.effects.admit(box,code,Date.now());} + let raw;try{raw=v1Command(this.runtime,this.manifest.config.cluster,Date.now(),this.proof(request,body));}catch{throw new Wire('bad_request',400);} + const result=await this.commit(raw);requireOK(result.value.code); + const b=await this.barrier(),snapshot=await this.members.get(b.node).hostView(b.index,'state','identity'),identity=this.project('identity',snapshot.raw),version=snapshot.app.version; + this.watchers.broadcast(version); + if(join){const code=decodeURIComponent(join[1]).trim().toLowerCase();await this.effects.deliver(this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(code)),code,Date.now());unwrap(await this.env.LINK_GATE.get(this.env.LINK_GATE.idFromName('gate')).decided(code));} + if(request.method==='GET'){ + if(path==='/v1/identity/rotation')return json(rotationView(identity.rotation,Number(snapshot.app.now),this.tenant.limits)); + if(path==='/v1/dir/list'||path==='/v1/dir/presence'){ + const raw=await this.listingJSON(b,snapshot.app),reply=path.endsWith('/presence')?json(presenceOf(JSON.parse(raw).devices,this.watchers.online(),Number(snapshot.app.now))):this.jsonBytes(raw);reply.headers.set('Codeaf-Dir-Version',version);return reply; + } + const row=await this.members.get(b.node).hostView(b.index,'state','cell/'+segment(decodeURIComponent(route.args[0])));if(!row.raw)throw new Wire('not_found',404); + const reply=this.jsonBytes('{"now":'+row.app.now+',"cell":'+this.projectJSON('cell',row.raw)+'}');reply.headers.set('codeaf-conditional-takes','1');return reply; + } + if(path==='/v1/identity/rotation'){ + this.watchers.closeAll(CLOSE_ROTATED,'rotated');if(identity.rotation?.retire_at)await this.ctx.storage.setAlarm(identity.rotation.retire_at); + return json(rotationView(identity.rotation,Number(snapshot.app.now),this.tenant.limits)); + } + if(path.endsWith('/revoke'))this.watchers.closeDevice(decodeURIComponent(route.args[0]),CLOSE_REVOKED,'revoked'); + if(path.endsWith('/release')||path.endsWith('/archive'))return empty(); + return result.value.view?this.jsonBytes(this.projectJSON('view',result.raw)):empty(); + } +} diff --git a/relay/hosted/src/raft/member.js b/relay/hosted/src/raft/member.js index 4c79632a56..2c5692c684 100644 --- a/relay/hosted/src/raft/member.js +++ b/relay/hosted/src/raft/member.js @@ -4,17 +4,105 @@ import { MemberInbox, INBOX_BYTES } from './inbox.js'; import { directoryEvaluator } from './directory.js'; import { DEFAULT_ROWS_BYTES, MemberStorage, b64 } from './storage.js'; import { HostCodec } from './abi.js'; +import { encodeCanonical, decodeCanonical } from './codec.js'; +import { dropTables } from '../erasure.js'; import { committedRegistry, provisionedRegistry } from './membership.js'; import { MemberAlarms, coordinationPolicy } from './liveness.js'; import { stageLegacyImport, finishLegacyImport } from './legacy.js'; export { installRaftModule, runtimeMetrics } from './actor.js'; // Only internal namespace calls can initialize a provisioned member. The -// public Worker exports this class but does not route directory traffic here. +// public Worker routes identity coordination through its authenticated source. // Enrollment is derived from the durable Go registry; P09 supplies evaluation. export class RaftMemberDO extends DurableObject { constructor(ctx, env) { super(ctx, env); this.actor = null; this.loading = null; this.ctx = ctx; this.env = env; } - evaluator(config) { return directoryEvaluator(sharedRuntime(), config.cluster, config.ceiling ?? DEFAULT_ROWS_BYTES); } + evaluator(config) { + const evaluate=directoryEvaluator(sharedRuntime(),config.cluster,config.ceiling??DEFAULT_ROWS_BYTES); + this.results=new Map();this.resultBytes=0; + return (raw,read,app)=>{ + const result=evaluate(raw,read,app),hash=new HostCodec(sharedRuntime()).hash(raw).digest; + const key=b64(hash),previous=this.results.get(key); + if(previous){this.resultBytes-=previous.raw.length;this.results.delete(key);} + this.results.set(key,{index:BigInt(app.index),raw:result.result});this.resultBytes+=result.result.length; + while(this.results.size>1 && (this.results.size>32||this.resultBytes>1024*1024)){ + const first=this.results.keys().next().value;this.resultBytes-=this.results.get(first).raw.length;this.results.delete(first); + } + return result; + }; + } + // These RPC methods have no public fetch route. The authenticated source + // coordinator alone provisions stores and drives the same durable actor. + async importReceipt(manifest) { + if(await this.ctx.storage.get('raft-erased'))throw new Error('raft member: identity retired'); + const config=await this.ctx.storage.get('raft-config'); + if(!config)return null; + const store=new MemberStorage(this.ctx.storage,new HostCodec(sharedRuntime())); + if(!store.one("SELECT name FROM sqlite_master WHERE type='table' AND name='meta'"))return null; + await store.open({cluster:config.cluster,id:config.id,protocol:1},config.ceiling); + const raw=store.read('membership','legacy-import'); + if(!raw)return null; + if(new TextDecoder().decode(raw)!==encodeCanonical(manifest))throw new Error('raft member: import provenance differs'); + return {node:config.id,applied:store.application().applied,manifest:decodeCanonical(new TextDecoder().decode(raw))}; + } + async eraseRetired(manifest) { + const gone=await this.ctx.storage.get('raft-erased'); + if(gone && gone!==manifest.config.cluster)throw new Error('raft member: wrong erased cluster'); + if(!gone){ + if(!await this.importReceipt(manifest))throw new Error('raft member: import provenance absent'); + await this.load(); + await this.actor.serial(async()=>{ + const raw=this.actor.store.read('state','identity'),record=raw&&decodeCanonical(new TextDecoder().decode(raw)); + if(record?.rotation?.state!=='retired'||record.rotation.retire_at>Date.now())throw new Error('raft member: erasure not authorized'); + await this.ctx.storage.put('raft-erased',manifest.config.cluster);await this.ctx.storage.sync();this.erased=true; + }); + } + if(this.actor?.handle)await this.actor.disposeErased(); + dropTables(this.ctx.storage.sql); + // A retry after the tombstone must finish KV transport cleanup too. Parts + // may outlive a lost manifest reply; none are an owner-retained backup. + const keys=[...(await this.ctx.storage.list({prefix:'raft-'})).keys()].filter(key=>key!=='raft-erased'); + if(keys.length)await this.ctx.storage.delete(keys); + await this.ctx.storage.delete('legacy-pending');await this.ctx.storage.sync(); + } + async hostInput(tag,value) { + if(![0,1,3,4,10,12,16].includes(tag))throw new Error('raft member: unsupported coordinator input'); + await this.load(); + if(tag===3){ + const raw=value.slice();return this.actor.serial(async()=>{ + const proposalAfter=this.actor.status.applied; + return {...await this.actor.drive(tag,raw),proposalAfter}; + }); + } + return this.actor.input(tag,value); + } + async hostView(index,namespace,key,after=null) { + await this.load();return this.actor.serial(()=>{ + if(this.actor.stopped||this.actor.store.stopped||this.actor.status.applied? ORDER BY key LIMIT 64',namespace,after)){ + const raw=this.actor.store.read(namespace,row.key);if(rows.length&&size+raw.length>1024*1024)break; + rows.push({key:row.key,data:raw});size+=raw.length; + } + return {app,rows}; + }); + } + async hostResult(raw,after=0n) { + await this.load();return this.actor.serial(()=>{ + if(this.actor.stopped||this.actor.store.stopped)throw new Error('raft member: stopped'); + const value=this.results.get(b64(this.actor.codec.hash(raw).digest)); + return value&&value.index>BigInt(after)&&value.index<=this.actor.status.applied?value:null; + }); + } + async hostSnapshot(op,...args) { + await this.load(); + if(op==='begin')return this.actor.begin(...args); + if(op==='chunk')return this.actor.chunk(...args); + if(op==='complete')return this.actor.complete(...args); + if(op==='read')return this.actor.serial(()=>this.actor.store.chunk(...args)); + throw new Error('raft member: unsupported snapshot operation'); + } // Source migration feeds bounded immutable rows before a core can activate. // The durable pending marker fences initialize/load across process death. legacyImport(config, manifest, row = null, finish = false) { @@ -64,6 +152,7 @@ export class RaftMemberDO extends DurableObject { } async load() { + if(this.erased || await this.ctx.storage.get('raft-erased'))throw new Error('raft member: identity retired'); if (this.actor) return; if(await this.ctx.storage.get('legacy-pending')||this.importPending)throw new Error('raft member: import incomplete; retry'); if (this.loading) return this.loading; diff --git a/relay/hosted/src/raft/storage.js b/relay/hosted/src/raft/storage.js index b6b68e0529..1c8e7860d2 100644 --- a/relay/hosted/src/raft/storage.js +++ b/relay/hosted/src/raft/storage.js @@ -1,3 +1,4 @@ +import { APPLICATION_TABLES } from '../sql-schema.js'; import { Writer } from './abi.js'; export const CHUNK_BYTES = 256 * 1024; @@ -45,7 +46,7 @@ export class MemberStorage { async open(identity, ceiling = DEFAULT_ROWS_BYTES) { const exists = this.one("SELECT name FROM sqlite_master WHERE type='table' AND name='meta'"); if (!exists) { - check(this.one("SELECT count(*) AS n FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' AND name NOT LIKE '_cf_%'").n === 0); + check(this.one(`SELECT count(*) AS n FROM sqlite_master WHERE ${APPLICATION_TABLES}`).n === 0); await this.transaction(() => { this.rows('CREATE TABLE meta (name TEXT PRIMARY KEY, data BLOB NOT NULL, hash TEXT NOT NULL)'); this.rows('CREATE TABLE journal (idx TEXT PRIMARY KEY, data BLOB NOT NULL, hash TEXT NOT NULL)'); diff --git a/relay/hosted/src/sql-schema.js b/relay/hosted/src/sql-schema.js new file mode 100644 index 0000000000..87beff4711 --- /dev/null +++ b/relay/hosted/src/sql-schema.js @@ -0,0 +1,4 @@ +// Platform bookkeeping is neither application authority nor owner data to +// erase. Miniflare's local explorer adds exactly this name table before the +// first RPC; unknown application tables still make a fresh store refuse. +export const APPLICATION_TABLES = "type = 'table' AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\' AND name NOT LIKE '\\_cf\\_%' ESCAPE '\\' AND name <> '__miniflare_do_name'"; diff --git a/relay/hosted/src/tenant.js b/relay/hosted/src/tenant.js index fb95fbeb7d..4d9ea4ca48 100644 --- a/relay/hosted/src/tenant.js +++ b/relay/hosted/src/tenant.js @@ -13,7 +13,7 @@ import { Wire } from './wire.js'; // How long Has waits for the puts in flight before it fails: a stuck put must not hang Has, // and must not make it lie (a "no" would make the client send a frame that is about to land). -const HAS_WAIT_MS = 30_000; +export const HAS_WAIT_MS = 30_000; // The cell key id of the frame that answers a batch get: the answer belongs to no cell, so all zeros. const MANY_KEY_ID = '0'.repeat(32); From baa5e4c4087dd94f9ec1cd04e247dcedef46d2a1 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 07:57:52 -0400 Subject: [PATCH 046/103] Prove the public hosted compatibility and recovery doors. Exercise migration cuts, majority loss, original v1 takes and pairing, retained wide numbers, stale snapshots, result provenance and rotation through the exported Worker. Reuse the real two-home Go journey and pin the actual lost-race screen sentence while measuring local runtime budgets. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/raftdirectory/v1_journey_test.go | 28 ++- internal/tui3/homepanel_continue_test.go | 5 + relay/hosted/test/raft/actor-unit.test.js | 14 ++ relay/hosted/test/raft/hosted-fixture.js | 47 +++++ relay/hosted/test/raft/hosted-go.test.js | 17 ++ .../hosted/test/raft/hosted-migration.test.js | 32 ++++ .../hosted/test/raft/hosted-resources.test.js | 46 +++++ relay/hosted/test/raft/hosted-worker.js | 101 +++++++++++ relay/hosted/test/raft/hosted.test.js | 162 ++++++++++++++++++ relay/hosted/test/rotation.test.js | 9 + 10 files changed, 460 insertions(+), 1 deletion(-) create mode 100644 relay/hosted/test/raft/hosted-fixture.js create mode 100644 relay/hosted/test/raft/hosted-go.test.js create mode 100644 relay/hosted/test/raft/hosted-migration.test.js create mode 100644 relay/hosted/test/raft/hosted-resources.test.js create mode 100644 relay/hosted/test/raft/hosted-worker.js create mode 100644 relay/hosted/test/raft/hosted.test.js diff --git a/internal/raftdirectory/v1_journey_test.go b/internal/raftdirectory/v1_journey_test.go index 85f63151fb..0026a36f75 100644 --- a/internal/raftdirectory/v1_journey_test.go +++ b/internal/raftdirectory/v1_journey_test.go @@ -6,6 +6,8 @@ import ( "errors" "net/http" "net/http/httptest" + "net/http/httputil" + "net/url" "os" "path/filepath" "sync/atomic" @@ -82,7 +84,31 @@ func TestR1R2R23R25R29V1TwoHomesContinueTakeBackQuitOffRevoke(t *testing.T) { mux.Handle("/v1/store/", blobstore.Handler(auth, func(string) (blobstore.Store, error) { return disk, nil })) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1); mux.ServeHTTP(w, r) })) defer srv.Close() - t.Setenv(syncsetup.URLVar, srv.URL) + runV1TwoHomeJourney(t, srv.URL, &hits) +} + +// R1/R2/R23/R25/R29 repeat the same real handoff, files, quit, off and +// revocation assertions through a local Worker's public HTTP door. +func TestR1R2R23R25R29HostedTwoHomes(t *testing.T) { + endpoint := os.Getenv("CODEAF_HOSTED_PROOF_URL") + if endpoint == "" { + t.Skip("local hosted proof runner only") + } + target, err := url.Parse(endpoint) + if err != nil || target.Scheme != "http" || target.Hostname() != "127.0.0.1" { + t.Fatal("hosted proof requires loopback HTTP") + } + celltest.Available(t) + proxy := httputil.NewSingleHostReverseProxy(target) + var hits atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1); proxy.ServeHTTP(w, r) })) + defer server.Close() + runV1TwoHomeJourney(t, server.URL, &hits) +} + +func runV1TwoHomeJourney(t *testing.T, endpoint string, hits *atomic.Int64) { + t.Helper() + t.Setenv(syncsetup.URLVar, endpoint) id, e := identity.Unmarshal([]byte(`{"V":1,"signing_seed":"` + hex.EncodeToString(key("root").Seed()) + `","dedup_secret":"` + hex.EncodeToString(key("dedup").Seed()) + `","cell_key":"` + hex.EncodeToString(key("content").Seed()) + `"}`)) must(t, e) open := func() *syncsetup.Sync { diff --git a/internal/tui3/homepanel_continue_test.go b/internal/tui3/homepanel_continue_test.go index e84325dc43..66bb423163 100644 --- a/internal/tui3/homepanel_continue_test.go +++ b/internal/tui3/homepanel_continue_test.go @@ -199,12 +199,17 @@ func (w *winnerTakes) Rows(context.Context) ([]chatlist.Row, error) { return rows, nil } +// R25 keeps the old client's actual visible refusal, independently of the +// internal directory error's diagnostic spelling. func TestTakeoverScreenLostRace(t *testing.T) { taker := &fakeTaker{err: fmt.Errorf("take: %w", directory.ErrLeaseHeld)} a := continueHome(t, taker, &winnerTakes{}, 200) standOn(t, a, "c-off") confirm(t, a) text := homeText(a) + if !strings.Contains(text, "another device continued this chat first") { + t.Fatalf("R25 visible refusal changed:\n%s", text) + } if !strings.Contains(text, "Port the picker running on desk") || !strings.Contains(text, lostRaceWord) { t.Fatalf("the row did not come back as running on the winner, with one line saying so:\n%s", text) } diff --git a/relay/hosted/test/raft/actor-unit.test.js b/relay/hosted/test/raft/actor-unit.test.js index 4b5ec0b52b..dbea105c3d 100644 --- a/relay/hosted/test/raft/actor-unit.test.js +++ b/relay/hosted/test/raft/actor-unit.test.js @@ -146,3 +146,17 @@ test('R27 isolate activation refuses before allocating a new member store and re for (const actor of actors) assert.equal((await actor.input(0)).status.applied, 3n); } finally { for (const actor of actors) await actor.close(); for (const host of hosts) host.db.close(); } }); + +// R3/R27 distinguish the local platform's name index from unknown retained +// application data. An explorer-created table cannot block first activation, +// and it never licenses resetting an unrelated authoritative table. +test('R3 R27 a fresh hosted store accepts platform name metadata and refuses unknown tables',async()=>{ + const host=new Host();host.db.exec('CREATE TABLE __miniflare_do_name(name TEXT); INSERT INTO __miniflare_do_name VALUES(\'anchor-a\')'); + try{ + const store=new MemberStorage(host,codec);const recovered=await store.open(identity(1));assert.equal(recovered.applied,0n); + assert.equal(host.db.prepare('SELECT name FROM __miniflare_do_name').get().name,'anchor-a'); + assert.equal(store.application().count,0); + }finally{host.db.close();} + const unknown=new Host();unknown.db.exec('CREATE TABLE evidence(body TEXT);INSERT INTO evidence VALUES(\'retained\')'); + try{await assert.rejects(new MemberStorage(unknown,codec).open(identity(2)),/invalid|damaged/);assert.equal(unknown.db.prepare('SELECT body FROM evidence').get().body,'retained');}finally{unknown.db.close();} +}); diff --git a/relay/hosted/test/raft/hosted-fixture.js b/relay/hosted/test/raft/hosted-fixture.js new file mode 100644 index 0000000000..8c775c7f36 --- /dev/null +++ b/relay/hosted/test/raft/hosted-fixture.js @@ -0,0 +1,47 @@ +import assert from 'node:assert/strict'; +import { prefixOf } from '../../src/base.js'; +import { Miniflare, convertV4MiniflareOptions } from 'miniflare'; +import { build } from 'esbuild'; +import { mkdtemp, copyFile, rm, readFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { generateKeyPairSync, createHash, sign } from 'node:crypto'; + +const sha=b=>createHash('sha256').update(b).digest('hex'); +export async function hostedFixture({fixture=false,base='',go=false}={}) { + const prefix=prefixOf({BASE_PATH:base}); + const out=process.env.CODEAF_RAFT_PROOF_DIR;assert(out,'build the native/WASI proof first'); + const dir=await mkdtemp(join(tmpdir(),'p17b-')); + const root=generateKeyPairSync('ed25519'),rootKey=root.publicKey.export({format:'der',type:'spki'}).subarray(-32); + const identity='id_'+sha(rootKey).slice(0,32); + const client=()=>{ + const key=generateKeyPairSync('ed25519'),pub=key.publicKey.export({format:'der',type:'spki'}).subarray(-32); + const cert={V:1,device:pub.toString('hex'),created:1000,sig:''}; + cert.sig=sign(null,Buffer.from('codeaf/device-cert/v1\n'+JSON.stringify(cert)),root.privateKey).toString('hex'); + return {key,cert,device:'dev_'+sha(pub).slice(0,32)}; + }; + await copyFile(join(out,'core.wasm'),join(dir,'core.wasm')); + let mf; + const compile=async(cut)=>{ + const plugins=[{name:'local-wasm',setup(b){b.onResolve({filter:/\.wasm$/},()=>({path:'core.wasm',external:true}));}}]; + if(cut)plugins.push({name:'process-cut',setup(b){b.onLoad({filter:/\/src\/raft\/hosted.js$/},async args=>{ + let text=await readFile(args.path,'utf8');const needle=cut.needle;assert.equal(text.split(needle).length,2,'cut must identify exactly one boundary'); + text=text.replace(needle,needle+(cut.after??"throw new Error('H2 injected crash after durable boundary');"));return {contents:text,loader:'js'}; + });}}); + await build({entryPoints:[fixture?'test/raft/hosted-worker.js':'src/raft-worker.js'],bundle:true,format:'esm',target:'es2022',external:['cloudflare:*'],plugins,outfile:join(dir,'worker.js')}); + }; + const start=async(cut)=>{ + if(mf)await mf.dispose();await compile(cut); + const worker={name:'hosted-proof',modulesRoot:dir,modules:[{type:'ESModule',path:join(dir,'worker.js')},{type:'CompiledWasm',path:join(dir,'core.wasm')}],compatibilityDate:'2026-09-01',durableObjects:Object.fromEntries(Object.entries({IDENTITY:'IdentityDO',RAFT_MEMBERS:'RaftMemberDO',NEWCOMERS:'NewcomerGate',PAIR_GATE:'PairGate',MAILBOX:'Mailbox',LINK_GATE:'LinkGate',LINK_REQUEST:'LinkRequest'}).map(([binding,className])=>[binding,{className,useSQLite:true}])),r2Buckets:['FRAMES'],bindings:{BASE_PATH:base,LEASE_POLICY:'amended',CAF_LIMITS:JSON.stringify({requestsPerMinute:100000,requestsPerMinutePerDevice:100000,minGraceMs:2000})},outboundService:()=>{throw Error('H4 outbound network forbidden');}}; + mf=new Miniflare(convertV4MiniflareOptions({host:'127.0.0.1',inspectorPort:0,resourcePersistencePath:join(dir,'persist'),telemetry:{enabled:false},cf:false,workers:[worker]})); + await mf.ready;return mf; + }; + const request=async(c,method,path,body)=>{ + const raw=body===undefined?'':body instanceof Uint8Array?body:JSON.stringify(body),at=String(Date.now()); + const headers={'Codeaf-Identity':rootKey.toString('base64url'),'Codeaf-Cert':Buffer.from(JSON.stringify(c.cert)).toString('base64url'),'Codeaf-Time':at,'Codeaf-Sig':sign(null,Buffer.from('codeaf-req-v1\n'+method+'\n'+prefix+path+'\n'+at+'\n'+sha(raw)),c.key.privateKey).toString('base64url')}; + return mf.dispatchFetch('http://local'+prefix+path,{method,headers,...(method!=='GET'&&raw?{body:raw}:{})}); + }; + const call=async(...args)=>{const response=await request(...args),text=await response.text();return {status:response.status,body:text?JSON.parse(text):null};}; + const control=async value=>{assert(fixture);const r=await mf.dispatchFetch('http://local/fixture',{method:'POST',body:JSON.stringify({identity,...value})});assert.equal(r.status,200,await r.clone().text());return r.json();}; + return {identity,client,dir,start,request,call,control,get mf(){return mf;},async close(){if(mf)await mf.dispose();await rm(dir,{recursive:true,force:true});}}; +} diff --git a/relay/hosted/test/raft/hosted-go.test.js b/relay/hosted/test/raft/hosted-go.test.js new file mode 100644 index 0000000000..14cf2c244c --- /dev/null +++ b/relay/hosted/test/raft/hosted-go.test.js @@ -0,0 +1,17 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { hostedFixture } from './hosted-fixture.js'; + +// H4/R1/R2/R23/R25/R29 run ordinary Go Sync clients and the unchanged real +// handoff assertions through the public Worker, with two disposable homes. +test('H4 R1 R2 R23 R25 R29 two Go homes continue, take back, quit, switch off and revoke through workerd',{timeout:180000},async()=>{ + const h=await hostedFixture(); + try{ + await h.start();const url=await h.mf.ready;assert.equal(url.hostname,'127.0.0.1'); + const child=spawn('go',['test','-p=1','./internal/raftdirectory','-run','^TestR1R2R23R25R29HostedTwoHomes$','-count=1','-timeout=120s','-v'],{cwd:new URL('../../../../',import.meta.url),env:{...process.env,CODEAF_HOSTED_PROOF_URL:url.origin,GOMEMLIMIT:'192MiB'},stdio:['ignore','pipe','pipe']}); + let output='';child.stdout.on('data',b=>{output+=b;});child.stderr.on('data',b=>{output+=b;}); + const code=await new Promise((resolve,reject)=>{child.once('error',reject);child.once('exit',resolve);}); + console.log(output);assert.equal(code,0,output);assert.match(output,/--- PASS: TestR1R2R23R25R29HostedTwoHomes/);assert.doesNotMatch(output,/--- SKIP:/); + }finally{await h.close();} +}); diff --git a/relay/hosted/test/raft/hosted-migration.test.js b/relay/hosted/test/raft/hosted-migration.test.js new file mode 100644 index 0000000000..70d52c659c --- /dev/null +++ b/relay/hosted/test/raft/hosted-migration.test.js @@ -0,0 +1,32 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { hostedFixture } from './hosted-fixture.js'; + +// H2/R24/R28 cut the actual coordinator after every durable migration stage, +// including partial imports at each rank. Every resumed call uses public HTTP. +test('H2 R3 R24 R28 hosted first-use migration resumes every cut with one UUID and unchanged originals',{timeout:120000},async()=>{ + const cuts=[ + {name:'freeze',needle:'if(!this.source.gate())await this.source.freeze(this.provisioning.config);'}, + {name:'inventory',needle:'this.manifest=await this.source.inventory();'}, + ...['1','2','3'].map(id=>({name:'stage-'+id,needle:'await this.member(id).legacyImport(this.config(id),this.manifest,copied);',after:`if(id==='${id}')throw new Error('H2 injected process cut');`})), + ...['1','2','3'].map(id=>({name:'install-'+id,needle:'await this.member(id).legacyImport(this.config(id),this.manifest,null,true);',after:`if(id==='${id}')throw new Error('H2 injected process cut');`})), + {name:'readback',needle:'await this.pump();await this.elect();await this.barrier();'}, + {name:'ownership',needle:'if(!owned)await this.source.own(receipts);'}, + ]; + for(const cut of cuts){ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start(); + const cell={V:1,head:'saved-original',durable_at:1000,class:'work',size:17,title:'c2VhbGVk',parent_cell:'parent',keys:{key:{[h.identity]:'wrapped'}},frames:['sealed-frame'],lease:{device:a.device,fence:7,expires:0,pending:0,released_at:900}}; + await h.control({op:'seed',records:[{kind:'identity',id:'',doc:{V:1,identity:h.identity,vault:'saved-vault'}},{kind:'devices',id:a.device,doc:{V:1,name:'c2VhbGVk',added_by:h.identity,revoked:false,caps:{os:'linux',arch:'amd64',sandbox:null,container:null,gpu:null,cow:'none'}}},{kind:'cells',id:'chat',doc:cell}]}); + const before=await h.control({op:'source'}); + await h.start(cut);const interrupted=await h.call(a,'GET','/v1/dir/list');assert.equal(interrupted.status,500,cut.name); + const frozen=await h.control({op:'source'}),gate=JSON.parse(frozen.gate);assert.equal(gate.config.generation,'1');assert.deepEqual(frozen.originals,before.originals); + await h.start();const list=await h.call(a,'GET','/v1/dir/list');assert.equal(list.status,200,cut.name+JSON.stringify(list)); + assert.equal(list.body.identity.vault,'saved-vault');assert.equal(list.body.cells.chat.lease.fence,7);assert.equal(list.body.cells.chat.lease.released_at,900); + const full=await h.call(a,'GET','/v1/dir/cells/chat');assert.deepEqual(full.body.cell,cell); + const after=await h.control({op:'source'}),owned=JSON.parse(after.gate);assert.equal(owned.stage,'raft-owned');assert.deepEqual(owned.config,gate.config);assert.deepEqual(after.originals,before.originals); + for(const node of ['1','2','3']){const member=await h.control({op:'member',cluster:owned.config.cluster,node});assert.deepEqual(JSON.parse(member.provenance),owned.manifest);assert(BigInt(member.app.applied)>1n);} + }finally{await h.close();} + } +}); diff --git a/relay/hosted/test/raft/hosted-resources.test.js b/relay/hosted/test/raft/hosted-resources.test.js new file mode 100644 index 0000000000..90de29a838 --- /dev/null +++ b/relay/hosted/test/raft/hosted-resources.test.js @@ -0,0 +1,46 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFile, writeFile } from 'node:fs/promises'; +import { execFileSync } from 'node:child_process'; +import { hostedFixture } from './hosted-fixture.js'; + +const ticks=Number(execFileSync('getconf',['CLK_TCK'],{encoding:'utf8'})); +async function sample(){ + const ids=[process.pid,...(await readFile(`/proc/${process.pid}/task/${process.pid}/children`,'utf8')).trim().split(/\s+/).filter(Boolean).map(Number)],rows=[]; + for(const pid of ids){try{const status=await readFile(`/proc/${pid}/status`,'utf8'),stat=(await readFile(`/proc/${pid}/stat`,'utf8')).split(') ').slice(1).join(') ').split(' ');rows.push({pid,cpu:(Number(stat[11])+Number(stat[12]))/ticks,rss:Number(status.match(/^VmRSS:\s+(\d+)/m)?.[1]??0)*1024});}catch{}} + return rows; +} +async function inspect(mf){ + const base=await mf.getInspectorURL();base.protocol='http:'; + const targets=await (await fetch(new URL('/json/list',base))).json(),target=targets.find(t=>t.title?.includes('hosted-proof'))??targets.find(t=>t.title==='Cloudflare Worker');assert(target); + const ws=new WebSocket(target.webSocketDebuggerUrl);await new Promise((resolve,reject)=>{ws.onopen=resolve;ws.onerror=reject;});let seq=0;const waiting=new Map(); + ws.onmessage=e=>{const m=JSON.parse(e.data),p=waiting.get(m.id);if(!p)return;waiting.delete(m.id);clearTimeout(p.timer);m.error?p.reject(Error(JSON.stringify(m.error))):p.resolve(m.result);}; + return {close:()=>ws.close(),call:method=>new Promise((resolve,reject)=>{const id=++seq,timer=setTimeout(()=>reject(Error('inspector timeout')),5000);waiting.set(id,{resolve,reject,timer});ws.send(JSON.stringify({id,method}));})}; +} + +// H4/R27 measures the actual public coordinator and all three co-resident +// anchors. The average CPU budget includes source, peers and storage work; +// endpoint wall latency includes loopback waits and is reported separately. +test('H4 R27 public Worker stays within isolate and ordinary-turn CPU budgets',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(),report={calls:[],heaps:[],samples:[]};let inspector; + try{ + await h.start();inspector=await inspect(h.mf); + const begin=performance.now();assert.equal((await h.call(a,'POST','/v1/dir/cells/chat',{head:'saved',class:'work',size:0,keys:{}})).status,200);report.coldMs=performance.now()-begin; + await h.control({op:'reset-metrics'});const before=await sample();report.samples.push(before); + for(let i=0;i<24;i++){ + const start=performance.now(),r=await h.call(a,'GET','/v1/dir/cells/chat');assert.equal(r.status,200);report.calls.push(performance.now()-start); + if(i%4===0){report.samples.push(await sample());report.heaps.push(await inspector.call('Runtime.getHeapUsage'));} + } + const after=await sample(),metrics=await h.control({op:'metrics'});report.samples.push(after);report.metrics=metrics; + const prior=new Map(before.map(p=>[p.pid,p.cpu]));report.cpuMs=after.reduce((n,p)=>n+(p.cpu-(prior.get(p.pid)??p.cpu))*1000,0);report.cpuMsPerTurn=report.cpuMs/metrics.turns; + report.combinedRSS=Math.max(...report.samples.map(s=>s.reduce((n,p)=>n+p.rss,0))); + report.isolateBytes=Math.max(...report.heaps.map(h=>h.totalSize+(h.embedderHeapUsedSize??0)+(h.backingStorageSize??0)))+metrics.peakMemory; + report.calls.sort((a,b)=>a-b);report.endpointWall={p50:report.calls[Math.floor(report.calls.length*.5)],p95:report.calls[Math.floor(report.calls.length*.95)],p99:report.calls.at(-1)}; + const walls=metrics.wall.slice().sort((a,b)=>a-b);report.turnWall={p50:walls[Math.floor(walls.length*.5)],p95:walls[Math.floor(walls.length*.95)],p99:walls.at(-1)}; + console.log(JSON.stringify({cpuMsPerTurn:report.cpuMsPerTurn,isolateBytes:report.isolateBytes,combinedRSS:report.combinedRSS,endpointWall:report.endpointWall,turnWall:report.turnWall})); + await writeFile(process.env.CODEAF_RAFT_PROOF_DIR+'/hosted-runtime.json',JSON.stringify(report,null,2)+'\n'); + assert.equal(metrics.active,3);assert(report.isolateBytes<=96*1024*1024);assert(metrics.peakMemory<=64*1024*1024);assert(report.cpuMsPerTurn<=5); + // This is a workerd CPU/isolate proof. Miniflare's SDK/coordinator RSS is + // reported, while the separate native/workerd gate enforces summed 256 MiB. + }finally{inspector?.close();await h.close();} +}); diff --git a/relay/hosted/test/raft/hosted-worker.js b/relay/hosted/test/raft/hosted-worker.js new file mode 100644 index 0000000000..247edd21ac --- /dev/null +++ b/relay/hosted/test/raft/hosted-worker.js @@ -0,0 +1,101 @@ +// Only this loopback fixture exposes source seeding and storage inspection. +// Public requests use the production Worker and the actual identity/member DOs. +export * from '../../src/raft-worker.js'; +import production from '../../src/raft-worker.js'; +import { IdentityDO as Source } from '../../src/identity.js'; +import { RaftMemberDO as Member } from '../../src/raft/member.js'; +import { Directory } from '../../src/directory.js'; +import { unb64 } from '../../src/raft/storage.js'; +import { runtimeMetrics } from '../../src/raft/actor.js'; +const measurements={turns:0,wall:[]}; +async function measured(fn){const start=performance.now();measurements.turns++;try{return await fn();}finally{measurements.wall.push(performance.now()-start);}} +import { policyOf } from '../../src/rules.js'; + +export class IdentityDO extends Source { + async fetch(request){ + const result=await measured(()=>super.fetch(request)); + if(this.host&&!this.capturing){ + const commit=this.host.commit.bind(this.host);this.host.commit=async raw=>{this.lastCommand=raw.slice();return commit(raw);};this.capturing=true; + } + return result; + } + holdUpload(){ + const put=this.host.tenant.store.put.bind(this.host.tenant.store); + this.pauseUpload=new Promise(resolve=>{this.releaseUpload=resolve;}); + this.host.tenant.store.put=async(...args)=>{this.uploadStarted=true;await this.pauseUpload;return put(...args);}; + } + releaseHeldUpload(){this.releaseUpload?.();} + async retryWithoutQuorum(){ + return this.host.serial(async()=>{ + const leader=this.host.leader();for(const id of [...this.host.members.keys()])if(id!==leader)this.host.members.delete(id); + this.host.queue=[]; + try{await this.host.commit(this.lastCommand);return 'acknowledged';}catch{return 'unavailable';} + }); + } + async seed(identity,records) { + const dir=new Directory(this.ctx.storage.sql,identity,Date.now,policyOf()); + for(const r of records){ + if(r.raw){dir.assertLegacy();this.ctx.storage.sql.exec('INSERT OR REPLACE INTO dir VALUES(?,?,?)',r.kind,r.id,r.raw);} + else dir.write(r.kind,r.id,r.doc); + } + await this.ctx.storage.put({identity,admitted:true});await this.ctx.storage.sync(); + } + async replayStale() { + return this.host.serial(async()=>{ + const from=this.host.leader(),to=this.host.manifest.config.anchors.find(id=>id!==from); + const member=this.host.member(to),snapshot=await member.oldSnapshotAndCheckpoint(); + const uint=v=>{let n=BigInt(v),b=[];do{const x=Number(n&127n);n>>=7n;b.push(x|(n?128:0));}while(n);return b;}; + const field=(n,v)=>[...uint(n*8),...uint(v)]; + const raw=Uint8Array.from([...field(1,7),...field(2,to),...field(3,from),...field(4,this.host.states.get(from).term),...uint(9*8+2),...uint(snapshot.length),...snapshot]); + this.host.queue.push({from,to,bytes:raw});await this.host.pump();return this.host.members.size; + }); + } + inspect() { + return {gate:this.ctx.storage.sql.exec('SELECT doc FROM legacy_gate WHERE one=1').toArray()[0]?.doc, + originals:this.ctx.storage.sql.exec('SELECT kind,id,doc FROM dir ORDER BY kind,id').toArray()}; + } +} +export class RaftMemberDO extends Member { + hostInput(...args){return measured(()=>super.hostInput(...args));} + peer(...args){return measured(()=>super.peer(...args));} + hostView(...args){return measured(()=>super.hostView(...args));} + hostResult(...args){return measured(()=>super.hostResult(...args));} + async seedTransportTail(){await this.load();await this.ctx.storage.put('raft-scheduled-part/interrupted/0',new Uint8Array([1,2,3]));await this.ctx.storage.sync();} + async retainedControl(){return [...(await this.ctx.storage.list({prefix:'raft-'})).keys()].filter(key=>key!=='raft-erased');} + async oldSnapshotAndCheckpoint(){ + await this.load();const old=await this.actor.serial(()=>unb64(this.actor.store.get('snapshot')));await this.actor.checkpoint();return old; + } + async discardStore() { + // The next fixture action kills the runtime. This models lost durable + // bytes while retaining the source's published identity and provenance. + for(const {name} of this.ctx.storage.sql.exec("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '_cf_%'").toArray())this.ctx.storage.sql.exec(`DROP TABLE "${name}"`); + await this.ctx.storage.sync(); + } + hasStore() {return this.ctx.storage.sql.exec("SELECT name FROM sqlite_master WHERE type='table' AND name='meta'").toArray().length>0;} + async inspect() { + await this.load();return this.actor.serial(()=>({app:this.actor.store.application(),identity:this.actor.store.identity,provenance:new TextDecoder().decode(this.actor.store.read('membership','legacy-import'))})); + } +} +export default { + async fetch(request,env) { + const url=new URL(request.url); + if(url.pathname==='/fixture'){ + const {op,identity,records,node,cluster}=await request.json(); + if(op==='metrics')return Response.json({...runtimeMetrics(),...measurements}); + if(op==='reset-metrics'){measurements.turns=0;measurements.wall=[];return Response.json(true);} + if(op==='seed'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).seed(identity,records);return Response.json(true);} + if(op==='hold-upload'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).holdUpload();return Response.json(true);} + if(op==='release-upload'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).releaseHeldUpload();return Response.json(true);} + if(op==='retry-without-quorum')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).retryWithoutQuorum()); + if(op==='stale-snapshot')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).replayStale()); + if(op==='source')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).inspect()); + if(op==='seed-transport-tail'){await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).seedTransportTail();return Response.json(true);} + if(op==='retained-control')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).retainedControl()); + if(op==='discard-member'){await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).discardStore();return Response.json(true);} + if(op==='has-member')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).hasStore()); + if(op==='member')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).inspect()); + return new Response(null,{status:404}); + } + return production.fetch(request,env); + } +}; diff --git a/relay/hosted/test/raft/hosted.test.js b/relay/hosted/test/raft/hosted.test.js new file mode 100644 index 0000000000..204833225a --- /dev/null +++ b/relay/hosted/test/raft/hosted.test.js @@ -0,0 +1,162 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { randomBytes } from 'node:crypto'; +import { encode } from '../../src/frame.js'; +import { setTimeout as delay } from 'node:timers/promises'; +import { hostedFixture } from './hosted-fixture.js'; +// H1/H3/R25 drive the exported Worker rather than internal fixture controls. +// The two independently certified clients never possess an anchor/effects key. +test('H1 H3 R1 R16 R25 public Worker commits v1 compatibility through three anchors',{timeout:60000},async()=>{ + const h=await hostedFixture();const {identity,client,call}=h;const a=client(),b=client(); + try{ + await h.start(); + const device={V:1,name:'c2VhbGVk',added_by:identity,revoked:false,caps:{os:'linux',arch:'amd64',sandbox:null,container:null,gpu:null,cow:'none'}}; + for(const c of [a,b]){const r=await call(c,'PUT','/v1/dir/devices/'+c.device,device);assert.equal(r.status,204,JSON.stringify(r));} + const init={head:'saved',class:'work',size:7,keys:{key:{[identity]:'wrapped'}},frames:['sealed-frame']}; + const created=await call(a,'POST','/v1/dir/cells/chat',init);assert.equal(created.status,200,JSON.stringify(created));assert.equal(created.body.cell.lease.fence,1); + const blind=await call(b,'POST','/v1/dir/cells/chat/acquire',{force:true});assert.deepEqual(blind,{status:409,body:{err:'lease_held'}}); + const observed={device:a.device,fence:1},raced=await Promise.all([b,a].map(c=>call(c,'POST','/v1/dir/cells/chat/acquire',{force:true,expect:observed}))); + assert.deepEqual(raced.map(r=>r.status).sort(),[200,409]);assert.equal(raced.find(r=>r.status===409).body.err,'lease_held'); + const owner=raced[0].status===200?b:a,winner=raced.find(r=>r.status===200);assert.equal(winner.body.cell.lease.fence,2); + assert.equal((await call(a,'POST','/v1/dir/cells/chat/publish',{fence:1,old_head:'saved',head:'stale',size:8})).status,409); + {const r=await call(owner,'POST','/v1/dir/cells/chat/publish',{fence:2,old_head:'saved',head:'winner',size:8,frames:[]});assert.equal(r.status,200,JSON.stringify(r));} + assert.equal((await call(owner,'POST','/v1/dir/cells/chat/release',{fence:2})).status,204); + await h.start(); + const cell=await call(a,'GET','/v1/dir/cells/chat');assert.equal(cell.status,200,JSON.stringify(cell));assert.equal(cell.body.cell.head,'winner');assert.equal(cell.body.cell.lease.expires,0); + assert.equal((await call(a,'POST','/v1/dir/cells/chat/acquire',{force:true})).status,200); + assert.equal((await call(a,'POST','/v1/dir/devices/'+b.device+'/revoke')).status,204); + assert.deepEqual(await call(b,'GET','/v1/dir/list'),{status:401,body:{err:'revoked'}}); + }finally{await h.close();} +}); + +// H1/H2/R3/R8/R24 forbid recreating a published voter from the source backup. +test('H1 H2 R3 R8 R24 public Worker excludes lost stores and refuses a lone voter',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();const made=await h.call(a,'POST','/v1/dir/cells/chat',{head:'saved',class:'work',size:0,keys:{}});assert.equal(made.status,200); + const source=await h.control({op:'source'}),cluster=JSON.parse(source.gate).config.cluster; + await h.control({op:'discard-member',cluster,node:'1'});await h.start(); + const won=await h.call(a,'POST','/v1/dir/cells/chat/acquire',{force:true,expect:{device:a.device,fence:1}});assert.equal(won.status,200,JSON.stringify(won));assert.equal(won.body.cell.lease.fence,2); + assert.equal(await h.control({op:'has-member',cluster,node:'1'}),false); + await h.control({op:'discard-member',cluster,node:'2'});await h.start(); + assert.equal((await h.call(a,'GET','/v1/dir/cells/chat')).status,503); + for(const node of ['1','2'])assert.equal(await h.control({op:'has-member',cluster,node}),false); + assert.equal(JSON.parse((await h.control({op:'source'})).gate).stage,'raft-owned'); + }finally{await h.close();} +}); + +// H1/R16/R21/R25 route both blob authorization and rotation through the same +// committed namespace, including the production /fabric signing domain. +test('H1 H3 R16 R21 R25 rotation guards blob writes and erases every replica after grace',{timeout:20000},async()=>{ + const h=await hostedFixture({fixture:true,base:'/fabric'}),a=h.client(),b=h.client(); + try{ + await h.start();const rid='a'.repeat(64),raw=new TextEncoder().encode('AGEO\x01sealed'); + const frame=encode('0'.repeat(32),[{rid,bytes:raw}]); + assert.equal((await h.call(a,'POST','/v1/store/frames',frame)).status,200); + assert.equal((await h.request(a,'GET','/v1/store/objects/'+rid)).status,200); + const rotate=(c,op)=>h.call(c,'POST','/v1/identity/rotation',{V:1,op,...(op==='retire'?{grace_ms:2000}:{})}); + assert.equal((await rotate(a,'freeze')).status,200); + assert.deepEqual(await h.call(b,'POST','/v1/store/frames',frame),{status:410,body:{err:'rotated'}}); + assert.equal((await h.request(a,'GET','/v1/store/objects/'+rid)).status,200); + assert.equal((await rotate(b,'thaw')).status,200); + assert.equal((await rotate(a,'freeze')).status,200); + const cluster=JSON.parse((await h.control({op:'source'})).gate).config.cluster; + for(const node of ['1','2','3'])await h.control({op:'seed-transport-tail',cluster,node}); + const retired=await rotate(a,'retire');assert.equal(retired.status,200);assert.equal(retired.body.rotation.retire_at,retired.body.now+2000); + const deadline=Date.now()+8000;let gone; + do{await delay(50);gone=await h.call(a,'GET','/v1/dir/list');}while(gone.status!==410&&Date.now(){ + const h=await hostedFixture(),a=h.client(),b=h.client(); + try{ + await h.start(); + const open=async()=>{ + const r=await h.mf.dispatchFetch('http://local/v1/link/requests',{method:'POST',body:JSON.stringify({pubkey:Buffer.from(b.cert.device,'hex').toString('base64url'),x25519:randomBytes(32).toString('base64url'),name_sealed:randomBytes(45).toString('base64url'),platform:'linux'})});assert.equal(r.status,201);return r.json(); + }; + const made=await open(),grant=randomBytes(64).toString('base64url'),record={V:1,name:'c2VhbGVk',platform:'linux'}; + const approval={device:record,cert:Buffer.from(JSON.stringify(b.cert)).toString('base64url'),grant}; + const accepted=await h.call(a,'POST','/v1/dir/requests/'+made.code+'/approve',approval);assert.equal(accepted.status,204,JSON.stringify(accepted)); + const read=await h.mf.dispatchFetch('http://local/v1/link/requests/'+made.code),view=await read.json();assert.equal(view.state,'approved');assert.equal(view.grant,grant); + const list=await h.call(b,'GET','/v1/dir/list');assert.equal(list.status,200);assert.equal(list.body.devices[b.device].name,record.name); + await h.start();assert.equal((await h.call(a,'POST','/v1/dir/requests/'+made.code+'/approve',approval)).status,204); + const denied=await open();assert.equal((await h.call(a,'POST','/v1/dir/requests/'+denied.code+'/deny')).status,204); + assert.equal((await (await h.mf.dispatchFetch('http://local/v1/link/requests/'+denied.code)).json()).state,'denied'); + }finally{await h.close();} +}); + +// H1/H2/R13 retain a healthy member when delayed transport carries a snapshot +// older than the checkpoint it has already durably applied. +test('H1 H2 R13 delayed old snapshot cannot exclude a caught-up hosted member',{timeout:20000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();assert.equal((await h.call(a,'POST','/v1/dir/cells/chat',{head:'current',class:'work',size:0,keys:{}})).status,200); + assert.equal(await h.control({op:'stale-snapshot'}),3); + const row=await h.call(a,'GET','/v1/dir/cells/chat');assert.equal(row.status,200);assert.equal(row.body.cell.head,'current'); + }finally{await h.close();} +}); + +// H2/R14/R24 retain the exact legacy uint64 JSON bytes even for inventory rows +// whose size exceeds JavaScript's safe integer range. Opaque map words remain +// opaque; only the typed Go projector changes protocol number representation. +test('H2 R14 R24 imported wide legacy numbers survive the public cell and list wires',{timeout:20000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();const raw=JSON.stringify({V:1,head:'saved',durable_at:1000,class:'work',size:'WIDE',keys:{fence:{size:'opaque'}},lease:{device:a.device,fence:7,expires:0,pending:0}}).replace('"WIDE"','9223372036854775847'); + await h.control({op:'seed',records:[{kind:'cells',id:'chat',raw}]}); + for(const path of ['/v1/dir/cells/chat','/v1/dir/list']){ + const reply=await h.request(a,'GET',path);assert.equal(reply.status,200);const text=await reply.text();assert.match(text,/"size":9223372036854775847/);assert.match(text,/"fence":\{"size":"opaque"\}/); + } + }finally{await h.close();} +}); + +// H1/R3/R4 never turns an in-memory result from an earlier applied occurrence +// into acknowledgment of a new uncommitted legacy proposal on a minority. +test('H1 R3 R4 replayed v1 proof cannot borrow an older result without quorum',{timeout:20000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();assert.equal((await h.call(a,'POST','/v1/dir/cells/chat',{head:'saved',class:'work',size:0,keys:{}})).status,200); + assert.equal((await h.call(a,'GET','/v1/dir/cells/chat')).status,200); + assert.equal(await h.control({op:'retry-without-quorum'}),'unavailable'); + }finally{await h.close();} +}); + +// H1/R16/R21 make erasure wait for in-flight bytes without owning the queue +// those uploads need for their final committed authorization check. +test('H1 R16 R21 retirement drains an in-flight upload without blocking its authority check',{timeout:20000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client();let uploading; + try{ + await h.start();assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + const cluster=JSON.parse((await h.control({op:'source'})).gate).config.cluster; + await h.control({op:'hold-upload'}); + uploading=h.request(a,'POST','/v1/store/frames',encode('0'.repeat(32),[{rid:'b'.repeat(64),bytes:new TextEncoder().encode('AGEO\x01sealed')}])); + assert.equal((await h.call(a,'POST','/v1/identity/rotation',{op:'freeze'})).status,200); + const retired=await h.call(a,'POST','/v1/identity/rotation',{op:'retire',grace_ms:2000});assert.equal(retired.status,200); + const deadline=Date.now()+8000;let response; + do{await delay(50);response=await h.call(a,'GET','/v1/dir/list');}while(response.body?.err!=='gone'&&Date.now(){});if(uploading)await uploading.catch(()=>{});await h.close();} +}); + +// H1/R25 use the existing base-path normalizer at BOTH ingress and original +// proof capture, so an operator's equivalent prefix spelling cannot break v1. +test('H1 R25 original proof uses the normalized production prefix',{timeout:20000},async()=>{ + const h=await hostedFixture({base:' fabric/ '}),a=h.client(); + try{await h.start();const r=await h.call(a,'POST','/v1/dir/cells/chat',{head:'saved',class:'work',size:0,keys:{}});assert.equal(r.status,200,JSON.stringify(r));assert.equal((await h.call(a,'GET','/v1/dir/cells/chat')).status,200);} + finally{await h.close();} +}); diff --git a/relay/hosted/test/rotation.test.js b/relay/hosted/test/rotation.test.js index 105f94feb8..eb9e68af81 100644 --- a/relay/hosted/test/rotation.test.js +++ b/relay/hosted/test/rotation.test.js @@ -97,3 +97,12 @@ test('a retire answers a deadline of exactly the answer\'s own time plus the gra const view = tenant.rotate('a', { op: 'retire', grace_ms: DEFAULTS.minGraceMs }); assert.equal(view.rotation.retire_at, view.now + DEFAULTS.minGraceMs); }); + +// R28 keeps local platform metadata outside explicit owner erasure while +// removing every actual application table, including interrupted migrations. +test('R28 owner erasure preserves the local platform name table',()=>{ + const sql=memorySql();sql.exec('CREATE TABLE __miniflare_do_name(name TEXT)');sql.exec('INSERT INTO __miniflare_do_name VALUES(?)','identity'); + new Directory(sql,'id_t',()=>NOW,STAGE1).setVault('','sealed');dropTables(sql);dropTables(sql); + assert.deepEqual(sql.exec("SELECT name FROM sqlite_master WHERE type='table'").toArray().map(r=>r.name),['__miniflare_do_name']); + assert.equal(sql.exec('SELECT name FROM __miniflare_do_name').one().name,'identity'); +}); From 16ed24d2f9bdf01490a4ecdbd96ddfc11b5f3e57 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 07:57:52 -0400 Subject: [PATCH 047/103] Document the irreversible hosted directory upgrade. Describe the exact v3-raft-members migration, preserved source backups, quorum refusals, older-client behavior and all-replica retirement in the hosted relay README, the reactor notes and the shipped manual. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../continuing-a-chat-on-another-computer.md | 15 ++++++++ relay/hosted/README.md | 35 ++++++++++++------- relay/hosted/src/raft/README.md | 25 +++++++++++++ 3 files changed, 63 insertions(+), 12 deletions(-) diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index 504fb3ce34..bb5886ed18 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -348,3 +348,18 @@ blob store. No switch selects the former directory writer. Encoded directory commands currently have a 64 KiB limit; a larger older request can be refused before it changes ownership. Larger imported records stay retained. + +## Hosted relay directory upgrade and older clients + +The hosted relay records directory changes on three persistent members. Two must +agree before a take, publish or fresh listing succeeds. Pairing one or two +computers does not require either computer to stay awake. The three hosted copies +still depend on the same hosting provider and relay connection. + +An existing identity upgrades on first authenticated use. An interrupted upgrade +can briefly ask you to retry; it keeps the saved directory and encrypted frames. +The old directory never becomes a second writer. Older clients keep pairing, +listing, publishing, releasing and continuing a chat they observed. A very old +client that blindly forces a live chat receives `another device continued this chat first`. +Quit on the holder first, or update that client before continuing a live chat. +Watch sockets show presence; older clients still send their normal heartbeats. diff --git a/relay/hosted/README.md b/relay/hosted/README.md index 5cf9abb351..a841da2da7 100644 --- a/relay/hosted/README.md +++ b/relay/hosted/README.md @@ -10,10 +10,12 @@ The relay sees only ciphertext and keeps no account table: an identity is the ha ``` request --> Worker (stateless) --+--> /v1/pair* --> PairGate (one object) --> Mailbox (one per nameplate) | - +--> /v1/store/*, /v1/dir/* --> IdentityDO (one per identity) --> R2 + +--> identity ingress --> IdentityDO (router/blob index) --> R2 + | + +--> RaftMemberDO A/B/C ``` -The Worker (`src/worker.js`) decides where a request goes and does nothing else. It reads the +The Worker (`src/raft-worker.js`, exporting `src/worker.js`) decides where a request goes and does nothing else. It reads the identity from the unsigned `Codeaf-Identity` header, refuses an oversized `Content-Length` with 413 before waking anything, and forwards the request, body unread, to that identity's Durable Object. Routing before reading is what keeps one identity's slow put from ever delaying another identity's @@ -26,9 +28,13 @@ it, checks the device is not revoked, spends the caller's rate budget, and serve - `Has` never answers no while a put for the same identity is still being received. It waits for the puts in flight (`src/flight.js`, a port of `internal/blobstore/flight.go`) and fails with 503 after 30 s rather than say no. A caller with no cert of the identity cannot hold a put open. -- The directory lives in the object's SQLite (`src/directory.js`). The object is single-threaded and - every operation is one synchronous turn, which is the compare-and-swap the lease rules need. The - rules (`src/rules.js`) are a JS port of `internal/directory/rules.go`. +- Directory and authority decisions live in three separately addressed `RaftMemberDO` SQLite + stores. `src/raft/hosted.js` freezes the original source on first use, streams P11's same + genesis to A/B/C and publishes ownership only after a fresh quorum read. Commands use the + original v1 proof and the shared Go state machine; JavaScript has no authoritative lease + writer. `src/directory.js` and its old rules remain only for immutable source conversion and + legacy fixture tests. Source backups are retained until explicit owner cleanup or retirement. + An absent quorum returns retryable 503 and never reopens the old writer. - Frames live in R2 at `/f/` (`src/store.js`), each validated (magics, header, `cell_key_id`) before it is stored. The index that finds an object inside its frame is rows in the object's SQLite (`objs`: rid -> frame, offset, length, primary key on rid), so it takes no memory @@ -56,20 +62,20 @@ a device record cannot set or clear its own flag. Directory watch is `GET /v1/dir/watch`, a signed WebSocket upgrade on the same object (`src/watch.js`). It replaces the home screen's polling: the object keeps a durable version of the -directory (`dirver` in its SQLite), bumps it on every change a person could see (a heartbeat that only +directory (the replicated application version), bumps it on every change a person could see (a heartbeat that only moves a lease's expiry is not one), and sends `{"v":N}` to every socket. It accepts with the hibernation API and answers the client's `ping` with the platform's auto-response, so an open, quiet socket wakes nothing. `test/request-scoped.test.js` keeps the object free of timers (but `Has`'s bounded wait) and of outbound connections. `maxWatchers` (default 1000) caps sockets per identity. -Rotation is `POST|GET /v1/identity/rotation` on the same object (`src/rotation.js`, a -port of `internal/directory/rotation.go`). The state is the `rotation` field of the identity record in -the object's SQLite. The first device to freeze owns the rotation, any device may thaw a frozen +Rotation is `POST|GET /v1/identity/rotation` through the same coordinator. +The shared Go state machine owns its `rotation` field in the replicated identity record. The first device to freeze owns the rotation, any device may thaw a frozen identity, nothing thaws a retired one. Routes marked `write` in `src/routes.js` (every directory verb that changes a record, and a frame put) are `410 rotated` once the identity is frozen or retired; reads stay open. A retire arms the object's alarm at its deadline. The alarm (`src/erasure.js`) writes the tombstone first (`gone` in the object's storage), deletes the identity's R2 prefix a page per turn, -then drops the SQLite tables, so a crash half way is finished by the next alarm; afterwards every +then erases all member copies/snapshots and source SQLite tables through a retained journal, +so a crash half way is finished by the next alarm; afterwards every request is `410 gone` and makes no table and no R2 object. The one alarm slot is shared with the stats flush, so `#arm` only ever moves it earlier and the alarm handler re-arms the deadline. @@ -137,7 +143,8 @@ canonical names, so the stricter side is the safe one. | Binding | Kind | Purpose | |---|---|---| | `FRAMES` | R2 bucket | frames and the object index, under `/` | -| `IDENTITY` | Durable Object `IdentityDO` (SQLite) | directory, puts in flight, caps, one per identity | +| `IDENTITY` | Durable Object `IdentityDO` (SQLite) | source gate/backup, coordinator, blob index, puts in flight, caps | +| `RAFT_MEMBERS` | Durable Object `RaftMemberDO` (SQLite) | three independently addressed coordination stores per identity | | `NEWCOMERS` | Durable Object `NewcomerGate` (SQLite) | new identities per IP, one object | | `PAIR_GATE` | Durable Object `PairGate` (SQLite) | nameplates, per-IP counters, one object | | `MAILBOX` | Durable Object `Mailbox` | one pairing, one per nameplate | @@ -149,7 +156,11 @@ canonical names, so the stricter side is the safe one. | `TRUST_PROXY` | var, optional | `1` takes the caller's network from the first `X-Forwarded-For` address (the self-hosted relay's `--trust-proxy`); for test rigs only, never production, where `CF-Connecting-IP` is the one truth | `wrangler.toml` declares all of them and the migration `v1` that creates the four classes as -SQLite-backed; migration `v2` adds the link classes. +SQLite-backed; migration `v2` adds the link classes and `v3-raft-members` adds the three-anchor +namespace. A deployment's own configuration must carry the `RAFT_MEMBERS` binding, the +`v3-raft-members` migration, the `src/raft-worker.js` entry and the build command that +`wrangler.toml` declares; the first authenticated use of an existing identity then upgrades it in place, and that upgrade cannot +be rolled back to the single-object writer. ## Base path diff --git a/relay/hosted/src/raft/README.md b/relay/hosted/src/raft/README.md index 0860e48616..e67e77204e 100644 --- a/relay/hosted/src/raft/README.md +++ b/relay/hosted/src/raft/README.md @@ -142,3 +142,28 @@ never append or application success. The sole P17 router must consume those original peer bytes through the authenticated exchange bridge, propagate wake to current voters, and supply queued assignment/outbox activity conservatively. No host listener, provider service or alternate directory writer is added here. + +## Hosted public assembly + +`HostedIdentity` now runs at every authenticated public identity request. The +source object keeps blob/index/quota/socket admission and its irreversible P11 +gate; all directory, authority, rotation and blob authorization goes through +three separately addressed `RaftMemberDO` actors. Imports advance bounded pages, +reuse source UUID/keys/ceiling and require an actual quorum ReadIndex before +ownership publication. After publication, only stores with durable matching +provenance can participate. A missing quorum refuses; it never reimports a lost +voter under its former ID or reopens the source writer. + +ABI 30 prepares restricted effects using the source's protocol seed (never a +command field), 31 projects typed canonical rows to legacy numeric JSON, and 32 +checks committed personal certificate/revocation/rotation. Step 16 requests an +explicit real campaign instead of fabricating ticks. Original v1 mutations still +use ABI 29 and the same deterministic evaluation. Empty legacy frame plans are +normalized before original-proof comparison, preserving replacement semantics. + +Public source serialization spans coordination, while external blob I/O leaves +that queue and passes a new authority barrier afterward. Public watch sockets +carry presence without the legacy socket-vouch promise; v1 clients keep ordinary +beats. P17C retains personal/worker transport, leadership maintenance and the full +committed-hold/quiescence assembly. Explicit retirement has its own retained +all-replica erasure journal and never serves as pressure cleanup. From cc26bc5052897ad18e938c71ba216af1ce447e52 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 10:38:08 -0400 Subject: [PATCH 048/103] Bind scoped cluster ingress to original credentials and committed authority. Authenticate selected requests before activation and repeat scope/time checks at the quorum-backed view. Retain original personal certificates without overriding committed roles or revocation, and initialize imported anchor authority through the original manifest instead of rewriting frozen rows. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterstate/apply.go | 35 ++++++- internal/clusterstate/ingress.go | 134 ++++++++++++++++++++++++++ internal/clusterstate/ingress_test.go | 121 +++++++++++++++++++++++ internal/clusterstate/membership.go | 29 +++++- internal/clusterstate/types.go | 1 + internal/clusterstate/v1.go | 12 +++ internal/raftabi/abi.go | 6 ++ internal/raftabi/hosted.go | 34 ++++++- internal/raftabi/membership.go | 15 +++ internal/raftabi/scheduler.go | 17 ++-- internal/raftabi/worker.go | 20 ++++ 11 files changed, 413 insertions(+), 11 deletions(-) create mode 100644 internal/clusterstate/ingress.go create mode 100644 internal/clusterstate/ingress_test.go diff --git a/internal/clusterstate/apply.go b/internal/clusterstate/apply.go index 5d87e85371..1251497e01 100644 --- a/internal/clusterstate/apply.go +++ b/internal/clusterstate/apply.go @@ -140,6 +140,9 @@ func Sign(c Command, key ed25519.PrivateKey) (Command, error) { // these immutable records, but must distinguish absent rows from unread rows. func Dependencies(c Command) []RecordKey { keys := []RecordKey{QuiescenceKey(), IdentityKey(), DeviceKey(c.Actor), AuthorityKey(c.Actor), SessionKey(c.Actor, c.Session), SessionsKey()} + if c.Change.InitMembership != nil { + keys = append(keys, LegacyImportKey()) + } m := c.Change if m.RenewHolds != nil { keys = append(keys, DeviceKey(m.RenewHolds.Device), AuthorityKey(m.RenewHolds.Device)) @@ -279,6 +282,8 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er target = new(WorkerRecord) case k == QuiescenceKey(): target = new(Quiescence) + case k == LegacyImportKey(): + target = new(LegacyManifest) case k == MembershipKey(): var h Membership if Decode(b, &h) != nil || h.Validate() != nil { @@ -325,10 +330,38 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er if decode(AuthorityKey(c.Actor), &auth) != nil || decode(DeviceKey(c.Actor), &actor) != nil || decode(IdentityKey(), &identity) != nil { return out, ErrDependency } + // Old imported groups retain their exact manifest and rows. Their FIRST + // registry command may use only a manifest-provisioned anchor key; it logs + // the authority installation instead of rewriting a frozen import snapshot. + if rows[AuthorityKey(c.Actor)] == nil && c.Change.InitMembership != nil { + var manifest LegacyManifest + if raw := rows[LegacyImportKey()]; raw != nil && Decode(raw, &manifest) == nil && manifest.Validate() == nil && manifest.Config.Cluster == c.Cluster && bytes.Equal(manifest.Config.SigningKey, identity.SigningKey) { + for i, node := range manifest.Config.Anchors { + if c.Actor == AnchorActor(node) && i < len(manifest.Config.AnchorKeys) { + auth = Authority{Cluster: c.Cluster, Key: manifest.Config.AnchorKeys[i], Anchor: true} + } + } + } + } digest, _ := Digest(c) signed, e := SigningBytes(c) authenticated := e == nil && len(auth.Key) == ed25519.PublicKeySize && ed25519.Verify(ed25519.PublicKey(auth.Key), signed, c.Signature) + if c.Certificate != "" { + certificateAuth, err := PersonalCommandAuthority(c, identity) + if err != nil { + return out, nil + } + // A retained role takes precedence over a certificate. A scoped or retired + // protocol key cannot turn itself into a personal device at this seam. + if rows[AuthorityKey(c.Actor)] == nil { + auth = certificateAuth + } + authenticated = e == nil && bytes.Equal(auth.Key, certificateAuth.Key) && auth.Personal && ed25519.Verify(auth.Key, signed, c.Signature) + } if c.V1 != nil { + if c.Certificate != "" { + return out, nil + } auth, authenticated = V1AuthorityAt(c, identity, meta.Now) } if !authenticated || c.Cluster != meta.Cluster || auth.Cluster != c.Cluster || !auth.ValidRole() || len(auth.Key) != ed25519.PublicKeySize || digest != c.RequestDigest { @@ -350,7 +383,7 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er } } service := c.Change.RenewHolds != nil || c.Change.RecordUpload != nil || c.Change.AdmitJoin != nil || c.Change.CompleteJoin != nil || auth.Effects && (c.Change.WorkerBeatBatch != nil || c.Change.DeviceSeen != nil) - if service != auth.Effects || auth.Personal && auth.Effects || auth.Anchor && (auth.Personal || auth.Effects || c.Change.PrepareQuiescence == nil && c.Change.SettleMembership == nil && c.Change.SettleLeadership == nil && c.Change.CompleteLeadership == nil && c.Change.RestoreAnchors == nil && c.Change.CloseSession == nil) { + if service != auth.Effects || auth.Personal && auth.Effects || auth.Anchor && (auth.Personal || auth.Effects || c.Change.InitMembership == nil && c.Change.PrepareQuiescence == nil && c.Change.SettleMembership == nil && c.Change.SettleLeadership == nil && c.Change.CompleteLeadership == nil && c.Change.RestoreAnchors == nil && c.Change.CloseSession == nil) { return out, nil } if (c.Change.PrepareQuiescence != nil || c.Change.RestoreAnchors != nil || c.Change.CompleteLeadership != nil) && !auth.Anchor { diff --git a/internal/clusterstate/ingress.go b/internal/clusterstate/ingress.go new file mode 100644 index 0000000000..5cb91ea5be --- /dev/null +++ b/internal/clusterstate/ingress.go @@ -0,0 +1,134 @@ +package clusterstate + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/hex" +) + +const ClusterAPIPath = "/v2/cluster/api" + +// ClusterRequest retains the caller's original signature across routing. The +// domain covers the selected read as well as commands; a scoped delegation is +// never accepted as a personal-device certificate. +type ClusterRequest struct { + IdentityKey []byte `json:"identity_key"` + Cluster string `json:"cluster"` + Certificate string `json:"certificate,omitempty"` + Worker *WorkerCertificate `json:"worker,omitempty"` + At int64 `json:"at"` + Action string `json:"action"` + Incarnation uint64 `json:"incarnation"` + Cell string `json:"cell,omitempty"` + Assignment *AssignmentRef `json:"assignment,omitempty"` + Command *Command `json:"command,omitempty"` + Signature []byte `json:"signature"` +} + +type ClusterResponse struct { + Result *Result `json:"result,omitempty"` + View *CellView `json:"view,omitempty"` + Grant []byte `json:"grant,omitempty"` + Assignment *Assignment `json:"assignment,omitempty"` +} + +func ClusterRequestBytes(request ClusterRequest) ([]byte, error) { + request.Signature = nil + raw, err := Encode(request) + return append([]byte("codeaf/cluster-api/1\n"), raw...), err +} + +// VerifyClusterRequest authenticates before opening a namespace. Hosts repeat +// it at committed time under the actor queue and check the retained root, +// revocation, scope and incarnation at that same quorum-backed view. +func VerifyClusterRequest(r ClusterRequest, now int64) (string, error) { + if len(r.IdentityKey) != ed25519.PublicKeySize || r.Cluster == "" || r.At < 0 || r.At > int64(MaxSafeInteger) || now < 0 || r.At-now > V1SkewMillis || now-r.At > V1SkewMillis { + return "", ErrCodec + } + var public []byte + var actor string + if r.Worker != nil { + if r.Certificate != "" || VerifyWorkerCertificate(*r.Worker, r.IdentityKey, r.Cluster, now) != nil { + return "", ErrCodec + } + public = r.Worker.Key + actor = r.Worker.DeviceID() + } else { + var cert v1Certificate + raw, err := base64.RawURLEncoding.DecodeString(r.Certificate) + if err != nil || len(raw) > 1024 || legacyBody(raw, &cert, false) != nil { + return "", ErrCodec + } + public, err = hex.DecodeString(cert.Device) + if err != nil || len(public) != ed25519.PublicKeySize { + return "", ErrCodec + } + actor = DeviceIDOfKey(public) + if PersonalCertificate(IdentityRec{Identity: IdentityID(r.IdentityKey), SigningKey: r.IdentityKey}, actor, r.Certificate) != nil || r.Incarnation != 0 { + return "", ErrCodec + } + } + switch r.Action { + case "submit": + if r.Command == nil || r.Cell != "" || r.Assignment != nil || r.Command.Actor != actor || r.Command.Cluster != r.Cluster || r.Command.WorkerIncarnation != r.Incarnation || r.Command.Now > r.At || ClientCommand(*r.Command) != nil { + return "", ErrCodec + } + // Reject unrelated or malformed certificate bytes BEFORE they can enter + // the replicated log. Apply still checks the committed root and role. + if r.Command.Certificate != "" && (r.Worker != nil || PersonalCertificate(IdentityRec{Identity: IdentityID(r.IdentityKey), SigningKey: r.IdentityKey}, actor, r.Command.Certificate) != nil) { + return "", ErrCodec + } + // Membership belongs to the enrollment/control protocol. A generic + // selected-work submission cannot leave a half-enrolled personal member. + if membershipMutation(r.Command.Change) { + return "", ErrCodec + } + raw, err := SigningBytes(*r.Command) + if err != nil || !ed25519.Verify(public, raw, r.Command.Signature) { + return "", ErrCodec + } + case "cell": + if r.Cell == "" || r.Command != nil || r.Assignment != nil || r.Worker == nil || r.Incarnation == 0 { + return "", ErrCodec + } + case "assignment": + if r.Assignment == nil || r.Command != nil || r.Cell != "" || r.Worker == nil || r.Incarnation == 0 || r.Assignment.Target != actor || r.Assignment.Incarnation != r.Incarnation { + return "", ErrCodec + } + default: + return "", ErrCodec + } + raw, err := ClusterRequestBytes(r) + if err != nil || !ed25519.Verify(public, raw, r.Signature) { + return "", ErrCodec + } + return actor, nil +} + +// NextSession resumes an anchor's one bounded control session from durable +// application state. Each idle generation advances the cursor instead of +// allocating another retained session ID forever. +func NextSession(raw []byte) (sequence, ack uint64, err error) { + if len(raw) == 0 { + return 1, 0, nil + } + var saved session + if Decode(raw, &saved) != nil || saved.Closed || saved.Sequence == ^uint64(0) { + return 0, 0, ErrCodec + } + return saved.Sequence + 1, saved.Sequence, nil +} + +// PersonalCommandAuthority accepts the existing root-signed personal certificate +// on an original signed v2 command. It grants no worker or service role and +// never overrides a retained role or committed device revocation. +func PersonalCommandAuthority(c Command, identity IdentityRec) (Authority, error) { + if c.V1 != nil || c.WorkerIncarnation != 0 || c.Certificate == "" { + return Authority{}, ErrCodec + } + key, err := legacyCertificate(LegacyConfig{Identity: identity.Identity, SigningKey: identity.SigningKey}, c.Certificate) + if err != nil || DeviceIDOfKey(key) != c.Actor { + return Authority{}, ErrCodec + } + return Authority{Cluster: c.Cluster, Key: key, Personal: true}, nil +} diff --git a/internal/clusterstate/ingress_test.go b/internal/clusterstate/ingress_test.go new file mode 100644 index 0000000000..798588801e --- /dev/null +++ b/internal/clusterstate/ingress_test.go @@ -0,0 +1,121 @@ +package clusterstate_test + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/hex" + "encoding/json" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +func personalProof(t *testing.T, key ed25519.PrivateKey) string { + t.Helper() + c := struct { + V int `json:"V"` + Device string `json:"device"` + Created int64 `json:"created"` + Signature string `json:"sig"` + }{V: 1, Device: hex.EncodeToString(key.Public().(ed25519.PublicKey)), Created: 1} + raw, _ := json.Marshal(c) + c.Signature = hex.EncodeToString(ed25519.Sign(actorKey("root"), append([]byte("codeaf/device-cert/v1\n"), raw...))) + raw, _ = json.Marshal(c) + return base64.RawURLEncoding.EncodeToString(raw) +} + +// R16/R18 bind selected requests, incarnations, role, root and time BEFORE +// namespace activation. A valid delegation can never become a personal proof. +func TestR16R18ClusterIngressOriginalProof(t *testing.T) { + key := actorKey("scoped-http") + root := actorKey("root").Public().(ed25519.PublicKey) + cert := cs.WorkerCertificate{V: 1, Identity: cs.IdentityID(root), Cluster: "test", Key: key.Public().(ed25519.PublicKey), Made: 1, Expires: 1000000, Cells: []string{"selected"}} + raw, err := cs.WorkerCertificateBytes(cert) + if err != nil { + t.Fatal(err) + } + cert.Signature = ed25519.Sign(actorKey("root"), raw) + request := cs.ClusterRequest{IdentityKey: root, Cluster: "test", Worker: &cert, At: 1000, Incarnation: 1, Action: "cell", Cell: "selected"} + raw, err = cs.ClusterRequestBytes(request) + if err != nil { + t.Fatal(err) + } + request.Signature = ed25519.Sign(key, raw) + actor, err := cs.VerifyClusterRequest(request, 1000) + if err != nil || actor != cert.DeviceID() { + t.Fatal(actor, err) + } + for _, test := range []struct { + name string + change func(*cs.ClusterRequest) + }{ + {"selected chat", func(q *cs.ClusterRequest) { q.Cell = "outside" }}, + {"incarnation", func(q *cs.ClusterRequest) { q.Incarnation = 2 }}, + {"cluster", func(q *cs.ClusterRequest) { q.Cluster = "other" }}, + {"root", func(q *cs.ClusterRequest) { q.IdentityKey = actorKey("other-root").Public().(ed25519.PublicKey) }}, + {"action", func(q *cs.ClusterRequest) { q.Action = "submit" }}, + {"personal confusion", func(q *cs.ClusterRequest) { q.Certificate = personalProof(t, key) }}, + {"time", func(q *cs.ClusterRequest) { q.At = 0 }}, + } { + t.Run(test.name, func(t *testing.T) { + q := request + test.change(&q) + if _, err := cs.VerifyClusterRequest(q, 1000); err == nil { + t.Fatal("R16 rewritten request accepted") + } + }) + } + // Fresh signatures on both sides of the exclusive boundary isolate + // certificate expiry from the independent request freshness check. + for _, at := range []int64{cert.Expires - 1, cert.Expires} { + q := request + q.At = at + raw, err = cs.ClusterRequestBytes(q) + if err != nil { + t.Fatal(err) + } + q.Signature = ed25519.Sign(key, raw) + _, err = cs.VerifyClusterRequest(q, at) + if (err == nil) != (at < cert.Expires) { + t.Fatalf("R16 exclusive expiry at %d: %v", at, err) + } + } + if _, err = cs.VerifyClusterRequest(request, 1000+cs.V1SkewMillis+1); err == nil { + t.Fatal("R16 delayed original request accepted") + } +} + +// R16/R24 let an old personal home authenticate its original v2 request while +// retaining committed role and revocation precedence. Metadata alone grants no +// signing authority, and a foreign root cannot rewrite the committed root. +func TestR16R24PersonalCommandCertificate(t *testing.T) { + key := actorKey("old-personal") + actor := cs.DeviceIDOfKey(key.Public().(ed25519.PublicKey)) + certificate := personalProof(t, key) + for _, test := range []struct { + name string + setup func(*application) + want cs.Code + }{ + {"original root", func(a *application) {}, cs.OK}, + {"revoked", func(a *application) { a.put(cs.DeviceKey(actor), cs.Device{V: 1, Revoked: true}) }, cs.Revoked}, + {"retained service role", func(a *application) { + a.put(cs.AuthorityKey(actor), cs.Authority{Cluster: "test", Key: key.Public().(ed25519.PublicKey), Effects: true}) + }, cs.Unauthorized}, + {"foreign root", func(a *application) { + a.put(cs.IdentityKey(), cs.IdentityRec{Identity: cs.IdentityID(actorKey("foreign").Public().(ed25519.PublicKey)), SigningKey: actorKey("foreign").Public().(ed25519.PublicKey)}) + }, cs.Unauthorized}, + } { + t.Run(test.name, func(t *testing.T) { + a := newApplication(t) + test.setup(a) + c, err := cs.Sign(cs.Command{Schema: 1, Cluster: "test", Actor: actor, Certificate: certificate, Session: "personal", Sequence: 1, Now: 1000, Change: cs.Mutation{AdvanceTime: &cs.AdvanceTime{}}}, key) + if err != nil { + t.Fatal(err) + } + if out := a.evaluate(c); out.Result.Code != test.want { + t.Fatal(out.Result.Code, test.want) + } + }) + } +} diff --git a/internal/clusterstate/membership.go b/internal/clusterstate/membership.go index 1e1a89779c..e0a6fbbe40 100644 --- a/internal/clusterstate/membership.go +++ b/internal/clusterstate/membership.go @@ -146,6 +146,17 @@ func applyMembership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, if len(v.Anchors) != 3 || meta.Configuration.Joint() || len(meta.Configuration.Voters) != 3 || len(meta.Configuration.Learners) != 0 || v.Limit < 3 || v.Limit > MaxMemberRecords || meta.Index == 0 { return BadRequest } + if raw := rows[LegacyImportKey()]; raw != nil { + var manifest LegacyManifest + if Decode(raw, &manifest) != nil || manifest.Validate() != nil || manifest.Config.Cluster != c.Cluster || len(manifest.Config.AnchorKeys) != len(v.Anchors) { + return BadRequest + } + for i, anchor := range v.Anchors { + if anchor.Node != manifest.Config.Anchors[i] || !bytes.Equal(anchor.Key, manifest.Config.AnchorKeys[i]) { + return Unauthorized + } + } + } seen := map[string]bool{} h = Membership{Revision: 1, Index: meta.Index, Limit: v.Limit} for i, a := range v.Anchors { @@ -192,7 +203,20 @@ func applyMembership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, var authority Authority var device Device var prior DeviceMember - if Decode(rows[AuthorityKey(v.Device)], &authority) != nil || authority.Cluster != c.Cluster || !authority.ValidRole() || !authority.Personal && authority.Worker == nil || len(authority.Key) != ed25519.PublicKeySize { + if rows[AuthorityKey(v.Device)] == nil && v.Device == c.Actor && c.Certificate != "" { + var identity IdentityRec + if Decode(rows[IdentityKey()], &identity) != nil { + return BadRequest + } + var err error + authority, err = PersonalCommandAuthority(c, identity) + if err != nil { + return Unauthorized + } + } else if Decode(rows[AuthorityKey(v.Device)], &authority) != nil { + return Unauthorized + } + if authority.Cluster != c.Cluster || !authority.ValidRole() || !authority.Personal && authority.Worker == nil || len(authority.Key) != ed25519.PublicKeySize { return Unauthorized } if b := rows[DeviceKey(v.Device)]; b != nil { @@ -246,6 +270,9 @@ func applyMembership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, h.Live++ h.Nodes = append(h.Nodes, node) h.Pending = &MembershipIntent{Node: node, Revision: h.Revision} + if rows[AuthorityKey(v.Device)] == nil { + put(AuthorityKey(v.Device), authority) + } put(MemberKey(node), MemberRecord{Node: node, Device: v.Device, Key: bytes.Clone(authority.Key), Incarnation: incarnation, Protocol: ClusterProtocol, Worker: authority.Worker != nil, WorkerExpires: workerExpires}) put(MemberDeviceKey(v.Device), DeviceMember{Node: node}) put(MembershipKey(), h) diff --git a/internal/clusterstate/types.go b/internal/clusterstate/types.go index 19da648e82..0b0bed7431 100644 --- a/internal/clusterstate/types.go +++ b/internal/clusterstate/types.go @@ -28,6 +28,7 @@ var ( // client sequence. Signature authentication and committed authorization belong // to ingress and apply respectively; decoding alone grants no authority. type Command struct { + Certificate string `json:"certificate,omitempty"` V1 *V1Request `json:"v1_request,omitempty"` WorkerIncarnation uint64 `json:"worker_incarnation,omitempty"` Schema uint16 `json:"schema"` diff --git a/internal/clusterstate/v1.go b/internal/clusterstate/v1.go index 4ce872df0b..0a4adb8b14 100644 --- a/internal/clusterstate/v1.go +++ b/internal/clusterstate/v1.go @@ -107,6 +107,18 @@ func V1SigningMessage(method, uri, at string, body []byte) []byte { // ClientCommand refuses protocol provenance on the public v2 door. Only an // original v1 HTTP proof may select the compatibility rules at that ingress. func ClientCommand(c Command) error { + if c.Certificate != "" { + var cert v1Certificate + raw, err := base64.RawURLEncoding.DecodeString(c.Certificate) + if err != nil || len(raw) > 1024 || legacyBody(raw, &cert, false) != nil { + return ErrCodec + } + key, err := hex.DecodeString(cert.Device) + signature, sigErr := hex.DecodeString(cert.Signature) + if err != nil || sigErr != nil || len(key) != ed25519.PublicKeySize || len(signature) != ed25519.SignatureSize || cert.V != 1 || cert.Created < 0 || DeviceIDOfKey(key) != c.Actor || c.WorkerIncarnation != 0 { + return ErrCodec + } + } if c.V1 != nil || c.Change.Create != nil && c.Change.Create.V1 || c.Change.Publish != nil && c.Change.Publish.V1 || c.Change.Acquire != nil && (c.Change.Acquire.V1 || c.Change.Acquire.Legacy) { return ErrCodec } diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index 66092e9ec3..9628be32dc 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -164,6 +164,12 @@ func (r *Runtime) Call(b []byte) (out []byte) { func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { e := encoder{} switch op { + case 35: + return configurationView(d) + case 34: + return hostedAnchor(d) + case 33: + return clusterIngress(d) case 30: return hostedEffect(d) case 31: diff --git a/internal/raftabi/hosted.go b/internal/raftabi/hosted.go index 59b27db031..932527f328 100644 --- a/internal/raftabi/hosted.go +++ b/internal/raftabi/hosted.go @@ -19,7 +19,7 @@ func hostedEffect(d *decoder) ([]byte, error) { var change cs.Mutation j := json.NewDecoder(bytes.NewReader(raw)) j.DisallowUnknownFields() - if j.Decode(&change) != nil || j.Decode(new(any)) != io.EOF { + if cs.Decode(raw, &change) != nil && (j.Decode(&change) != nil || j.Decode(new(any)) != io.EOF) { return nil, errFrame } if change.DeviceSeen == nil && change.RenewHolds == nil && change.AdmitJoin == nil && change.CompleteJoin == nil && change.RecordUpload == nil { @@ -102,3 +102,35 @@ func hostedAuthority(d *decoder) ([]byte, error) { e.str(string(code)) return e.b, nil } + +// Only a provisioned anchor key can produce these restricted host commands. +// The evaluator checks authority and the actual applied configuration again. +func hostedAnchor(d *decoder) ([]byte, error) { + cluster, node, now, seed, saved, raw := d.str(), d.u64(), d.u64(), d.blob(), d.blob(), d.blob() + if d.end() != nil || len(seed) != ed25519.SeedSize || now > cs.MaxSafeInteger { + return nil, errFrame + } + var change cs.Mutation + if cs.Decode(raw, &change) != nil { + return nil, errFrame + } + if change.InitMembership == nil && change.PrepareQuiescence == nil && change.SettleMembership == nil && change.SettleLeadership == nil && change.CompleteLeadership == nil && change.RestoreAnchors == nil { + return nil, errFrame + } + sequence, ack, err := cs.NextSession(saved) + if err != nil { + return nil, err + } + key := ed25519.NewKeyFromSeed(seed) + command, err := cs.Sign(cs.Command{Schema: cs.SchemaVersion, Cluster: cluster, Actor: cs.AnchorActor(node), Session: "maintenance", Sequence: sequence, Ack: ack, Now: int64(now), Change: change}, key) + if err != nil { + return nil, err + } + encoded, err := cs.EncodeCommand(command) + if err != nil { + return nil, err + } + e := encoder{} + e.blob(encoded) + return e.b, nil +} diff --git a/internal/raftabi/membership.go b/internal/raftabi/membership.go index 4307be1177..8a56e153d8 100644 --- a/internal/raftabi/membership.go +++ b/internal/raftabi/membership.go @@ -87,3 +87,18 @@ func configurationAtEntry(d *decoder) ([]byte, error) { e.message(after) return e.b, nil } + +func configurationView(d *decoder) ([]byte, error) { + conf := new(pb.ConfState) + d.message(conf) + if d.end() != nil { + return nil, errFrame + } + raw, err := clusterstate.Encode(raftcore.Configuration(conf)) + if err != nil { + return nil, err + } + e := encoder{} + e.blob(raw) + return e.b, nil +} diff --git a/internal/raftabi/scheduler.go b/internal/raftabi/scheduler.go index 1264c19e7a..29a4e35777 100644 --- a/internal/raftabi/scheduler.go +++ b/internal/raftabi/scheduler.go @@ -50,14 +50,15 @@ func coordinationPolicy(d *decoder) ([]byte, error) { return nil, err } b, err := clusterstate.Encode(struct { - TickEvery int64 `json:"tick_every"` - RenewEvery int64 `json:"renew_every"` - PresenceEvery int64 `json:"presence_every"` - EvidenceWindow int64 `json:"evidence_window"` - IdleAfter int64 `json:"idle_after"` - MaxRenewHolds int `json:"max_renew_holds"` - MaxMessageBytes int `json:"max_message_bytes"` - }{clusterstate.TickEveryMillis, clusterstate.RenewEveryMillis, clusterstate.PresenceEveryMillis, clusterstate.EvidenceWindowMillis, clusterstate.IdleAfterMillis, clusterstate.MaxRenewHolds, clusterstate.MaxCommandBytes}) + TickEvery int64 `json:"tick_every"` + RenewEvery int64 `json:"renew_every"` + PresenceEvery int64 `json:"presence_every"` + EvidenceWindow int64 `json:"evidence_window"` + IdleAfter int64 `json:"idle_after"` + MaxRenewHolds int `json:"max_renew_holds"` + MaxMessageBytes int `json:"max_message_bytes"` + MaxMemberRecords uint32 `json:"max_member_records"` + }{clusterstate.TickEveryMillis, clusterstate.RenewEveryMillis, clusterstate.PresenceEveryMillis, clusterstate.EvidenceWindowMillis, clusterstate.IdleAfterMillis, clusterstate.MaxRenewHolds, clusterstate.MaxCommandBytes, clusterstate.MaxMemberRecords}) if err != nil { return nil, err } diff --git a/internal/raftabi/worker.go b/internal/raftabi/worker.go index 95318251a3..18af59e4c3 100644 --- a/internal/raftabi/worker.go +++ b/internal/raftabi/worker.go @@ -57,3 +57,23 @@ func assignmentRead(d *decoder) ([]byte, error) { e.blob(raw) return e.b, nil } + +// clusterIngress uses the same closed Go verifier on both runtimes. The caller +// repeats this verification with committed logical time after its read barrier. +func clusterIngress(d *decoder) ([]byte, error) { + raw, now := d.blob(), d.u64() + if d.end() != nil || now > clusterstate.MaxSafeInteger || len(raw) > clusterstate.MaxCommandBytes { + return nil, errFrame + } + var request clusterstate.ClusterRequest + if err := clusterstate.Decode(raw, &request); err != nil { + return nil, err + } + actor, err := clusterstate.VerifyClusterRequest(request, int64(now)) + if err != nil { + return nil, err + } + e := encoder{} + e.str(actor) + return e.b, nil +} From 67a056af0871592364514d4a98287859b13aabf4 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 10:38:09 -0400 Subject: [PATCH 049/103] Connect native scoped requests and watch receipts to durable coordination. Serialize current-time commit, fresh ReadIndex and selected reads. Carry only committed hold receipts, replay bounded outboxes and stop production timers through the shared all-voter sleep protocol. Exercise actual HTTP workers, delayed renewals and ten minutes of virtual idle. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterapi/client.go | 133 +++++++++++ internal/directory/directorytest/lease.go | 10 +- internal/directory/directorytest/watch.go | 3 + internal/directory/feed.go | 53 ++++- internal/directory/watch.go | 45 +++- internal/raftdirectory/ingress.go | 59 +++++ internal/raftdirectory/maintenance.go | 214 ++++++++++++++++++ .../maintenance_internal_test.go | 96 ++++++++ internal/relayconf/watch_test.go | 3 + internal/relayserve/committed_watch_test.go | 142 ++++++++++++ internal/relayserve/ingress.go | 51 +++++ internal/relayserve/ingress_test.go | 153 +++++++++++++ internal/relayserve/maintenance.go | 71 ++++++ internal/relayserve/native.go | 37 ++- internal/relayserve/service.go | 2 + internal/relayserve/watch_test.go | 1 + internal/syncsetup/clusterapi.go | 53 +++++ internal/syncsetup/worker_quiet_test.go | 65 ++++++ 18 files changed, 1167 insertions(+), 24 deletions(-) create mode 100644 internal/clusterapi/client.go create mode 100644 internal/raftdirectory/ingress.go create mode 100644 internal/raftdirectory/maintenance.go create mode 100644 internal/raftdirectory/maintenance_internal_test.go create mode 100644 internal/relayserve/committed_watch_test.go create mode 100644 internal/relayserve/ingress.go create mode 100644 internal/relayserve/ingress_test.go create mode 100644 internal/relayserve/maintenance.go create mode 100644 internal/syncsetup/clusterapi.go create mode 100644 internal/syncsetup/worker_quiet_test.go diff --git a/internal/clusterapi/client.go b/internal/clusterapi/client.go new file mode 100644 index 0000000000..06dce94827 --- /dev/null +++ b/internal/clusterapi/client.go @@ -0,0 +1,133 @@ +// Package clusterapi carries signed scoped requests to the one coordination +// path. It owns no retry loop, authority cache, member store or tool execution. +package clusterapi + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "io" + "net/http" + "strings" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +type Client struct { + Now func() time.Time + Base string + HTTP *http.Client + Request func() cs.ClusterRequest + Sign func([]byte) []byte + Allowed func() bool +} + +func (c Client) call(ctx context.Context, action string, command *cs.Command, cell string, ref *cs.AssignmentRef) (cs.ClusterResponse, error) { + var result cs.ClusterResponse + if c.Allowed == nil || !c.Allowed() { + return result, errors.New("sync is waiting until this computer is allowed to connect") + } + if c.Request == nil || c.Sign == nil { + return result, errors.New("cluster credentials are unavailable") + } + request := c.Request() + request.Action, request.Command, request.Cell, request.Assignment = action, command, cell, ref + now := c.Now + if now == nil { + now = time.Now + } + request.At = now().UnixMilli() + raw, err := cs.ClusterRequestBytes(request) + if err != nil { + return result, err + } + request.Signature = c.Sign(raw) + raw, err = cs.Encode(request) + if err != nil || len(raw) > cs.MaxCommandBytes { + return result, cs.ErrCodec + } + if _, err = cs.VerifyClusterRequest(request, request.At); err != nil { + return result, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.Base, "/")+cs.ClusterAPIPath, bytes.NewReader(raw)) + if err != nil { + return result, err + } + // The root header is a routing hint only; the signed body binds the same key. + req.Header.Set("codeaf-identity", base64.RawURLEncoding.EncodeToString(request.IdentityKey)) + req.Header.Set("Content-Type", "application/json") + client := http.Client{Timeout: 30 * time.Second} + if c.HTTP != nil { + client = *c.HTTP + } + if client.Timeout <= 0 || client.Timeout > 30*time.Second { + client.Timeout = 30 * time.Second + } + client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + // Recheck at the actual send as retained handles may outlive an off decision. + client.Transport = guard{next: client.Transport, allowed: c.Allowed} + res, err := client.Do(req) + if err != nil { + return result, err + } + defer res.Body.Close() + raw, err = io.ReadAll(io.LimitReader(res.Body, cs.MaxEvaluationBytes+1)) + if err != nil { + return result, err + } + if res.StatusCode != http.StatusOK || len(raw) > cs.MaxEvaluationBytes { + return result, errors.New("the selected work is unavailable; refresh before trying again") + } + if err = cs.Decode(raw, &result); err != nil { + return result, err + } + return result, nil +} +func (c Client) Submit(ctx context.Context, command cs.Command) (cs.Result, error) { + out, err := c.call(ctx, "submit", &command, "", nil) + if err == nil && out.Result == nil { + err = cs.ErrCodec + } + if err != nil { + return cs.Result{}, err + } + return *out.Result, nil +} +func (c Client) Cell(ctx context.Context, cell string) (cs.CellView, []byte, error) { + out, err := c.call(ctx, "cell", nil, cell, nil) + if err == nil && out.View == nil { + err = cs.ErrCodec + } + if err != nil { + return cs.CellView{}, nil, err + } + return *out.View, out.Grant, nil +} +func (c Client) Current(ctx context.Context, ref cs.AssignmentRef) (cs.Assignment, cs.CellView, error) { + out, err := c.call(ctx, "assignment", nil, "", &ref) + if err == nil && (out.Assignment == nil || out.View == nil) { + err = cs.ErrCodec + } + if err != nil { + return cs.Assignment{}, cs.CellView{}, err + } + return *out.Assignment, *out.View, nil +} + +type guard struct { + next http.RoundTripper + allowed func() bool +} + +func (g guard) RoundTrip(r *http.Request) (*http.Response, error) { + if !g.allowed() { + return nil, errors.New("sync is off on this computer") + } + next := g.next + if next == nil { + next = http.DefaultTransport + } + return next.RoundTrip(r) +} diff --git a/internal/directory/directorytest/lease.go b/internal/directory/directorytest/lease.go index 4f24340135..9b9c6b7df5 100644 --- a/internal/directory/directorytest/lease.go +++ b/internal/directory/directorytest/lease.go @@ -349,11 +349,17 @@ func (w watchEnv) wantHoldQuery(query string, status int) { wantRefusal(w.t, conn, resp, err, status, "bad_request") } +// R22 supersedes the old socket-authority advertisement on consensus routes. +// The same suite still exercises the original legacy directory protocol. func upgradeAnnouncesVouching(w watchEnv) { + want := "1" + if w.rig.Committed { + want = "" + } for _, holds := range [][]directory.Hold{nil, {hold(cell, 1)}} { s := w.dialHolding(devA, holds) - if got := s.resp.Header.Get(directory.VouchHeader); got != "1" { - w.t.Fatalf("101 answer has %s=%q with holds %v, want 1", directory.VouchHeader, got, holds) + if got := s.resp.Header.Get(directory.VouchHeader); got != want { + w.t.Fatalf("101 answer has %s=%q with holds %v, want %q", directory.VouchHeader, got, holds, want) } } } diff --git a/internal/directory/directorytest/watch.go b/internal/directory/directorytest/watch.go index 67b38adf9c..b03eed1d91 100644 --- a/internal/directory/directorytest/watch.go +++ b/internal/directory/directorytest/watch.go @@ -16,6 +16,9 @@ import ( // act as a stranger, and to know how many sockets the relay lets one identity // hold. type WatchRig struct { + // Committed selects R22's receipt protocol for a consensus backend. Legacy + // in-process directory fixtures retain their original socket-vouch contract. + Committed bool Rig // Base is the relay's base URL, without a trailing slash. Base string diff --git a/internal/directory/feed.go b/internal/directory/feed.go index 3edf052c58..c6ad22def6 100644 --- a/internal/directory/feed.go +++ b/internal/directory/feed.go @@ -169,6 +169,23 @@ func (f *Feed) SubscribeBeatWithEvidence(device string, holds []Hold, beat time. return s, nil } +// SubscribeCommitted carries only durable exact-fence receipts. It deliberately +// leaves socket vouching disabled: a pong cannot authorize a lease extension. +func (f *Feed) SubscribeCommitted(device string, holds []Hold, beat time.Duration, renew func(string, []Hold, int64) ([]dirwatch.HoldReceipt, error)) (*Sub, error) { + s, err := f.subscribe(device, holds, beat) + if err != nil { + return nil, err + } + s.committed = true + s.renew = renew + if err = s.PingCommitted(); err != nil { + s.leave() + return nil, err + } + f.pres.opened(s) + return s, nil +} + func (f *Feed) SubscribeBeat(device string, holds []Hold, beat time.Duration) (*Sub, error) { return f.SubscribeBeatWithEvidence(device, holds, beat, nil) } @@ -249,11 +266,15 @@ func (f *Feed) differs(id string, old, next any, now int64) bool { // Sub is one watcher. It keeps no queue: it always reads the newest Status, so // a slow watcher skips intermediate versions and never sees one go down. type Sub struct { - evidence func(string, []Hold, int64) error - f *Feed - poke chan struct{} - sent uint64 - started bool + evidence func(string, []Hold, int64) error + committed bool + renew func(string, []Hold, int64) ([]dirwatch.HoldReceipt, error) + receiptMu sync.Mutex + receipts []dirwatch.HoldReceipt + f *Feed + poke chan struct{} + sent uint64 + started bool device string shut atomic.Bool @@ -291,12 +312,20 @@ func (s *Sub) Ping() { _ = s.PingCommitted() } // PingCommitted withholds a pong when a consensus renewal cannot commit. func (s *Sub) PingCommitted() error { at := s.f.clock().UnixMilli() - if s.evidence != nil { + if s.evidence != nil || s.renew != nil { holds := make([]Hold, len(s.keys)) for i, k := range s.keys { holds[i] = Hold{Cell: k.cell, Fence: k.fence} } - if err := s.evidence(s.device, holds, at); err != nil { + if s.renew != nil { + receipts, err := s.renew(s.device, holds, at) + if err != nil { + return err + } + s.receiptMu.Lock() + s.receipts = receipts + s.receiptMu.Unlock() + } else if err := s.evidence(s.device, holds, at); err != nil { return err } } @@ -304,6 +333,16 @@ func (s *Sub) PingCommitted() error { return nil } +// takeReceipts transfers the last committed batch to the socket writer. Only +// the bounded set returned by the renewal callback is retained per socket. +func (s *Sub) takeReceipts() []dirwatch.HoldReceipt { + s.receiptMu.Lock() + defer s.receiptMu.Unlock() + receipts := s.receipts + s.receipts = nil + return receipts +} + func (s *Sub) wake() { select { case s.poke <- struct{}{}: diff --git a/internal/directory/watch.go b/internal/directory/watch.go index 73a5905578..884056583e 100644 --- a/internal/directory/watch.go +++ b/internal/directory/watch.go @@ -47,14 +47,16 @@ func (h *handler) watch(w http.ResponseWriter, r *http.Request) { } defer sub.Close() // Accept keeps the headers set on w, so this one rides the 101 answer. - w.Header().Set(VouchHeader, "1") + if !sub.committed { + w.Header().Set(VouchHeader, "1") + } w.Header().Set(PresenceHeader, "1") conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{CompressionMode: websocket.CompressionDisabled}) if err != nil { return } conn.SetReadLimit(watchRead) - serveWatch(r.Context(), conn, sub, c.device, wantsEvents(r)) + serveWatchReceipts(r.Context(), conn, sub, c.device, wantsEvents(r), r.URL.Query().Get("receipts") == "1") } // wantsEvents is whether the socket opted in to event frames (EventsQuery). @@ -111,16 +113,31 @@ func isUpgrade(r *http.Request) bool { // "ping" with "pong" and ignores every other frame, so its error is what ends // the socket when the peer goes. func serveWatch(ctx context.Context, conn *websocket.Conn, sub *Sub, device string, events bool) { + serveWatchReceipts(ctx, conn, sub, device, events, false) +} + +func serveWatchReceipts(ctx context.Context, conn *websocket.Conn, sub *Sub, device string, events, receipts bool) { ctx, cancel := context.WithCancel(ctx) defer cancel() + // The first version precedes receipt frames just as it precedes events. + ready := make(chan struct{}) go func() { defer cancel() - answerPings(ctx, conn, sub) + select { + case <-ready: + case <-ctx.Done(): + return + } + if sendReceipts(ctx, conn, sub, receipts) != nil { + return + } + answerPingsReceipts(ctx, conn, sub, receipts) }() // The version is the first frame; an event socket then hears who is online. - first := func() {} + first := func() { close(ready) } if events { first = func() { + close(ready) events := sub.Events() go sendEvents(ctx, cancel, conn, events) } @@ -202,6 +219,24 @@ func sendVersion(ctx context.Context, conn *websocket.Conn, v uint64) error { // answerPings answers each ping, stamping the socket's sign of life first so a // peer that has heard the pong can rely on it. func answerPings(ctx context.Context, conn *websocket.Conn, sub *Sub) { + answerPingsReceipts(ctx, conn, sub, false) +} + +func sendReceipts(ctx context.Context, conn *websocket.Conn, sub *Sub, enabled bool) error { + holds := sub.takeReceipts() + if !enabled || len(holds) == 0 { + return nil + } + raw, err := json.Marshal(struct { + Holds []dirwatch.HoldReceipt `json:"holds"` + }{holds}) + if err != nil { + return err + } + return sendText(ctx, conn, string(raw)) +} + +func answerPingsReceipts(ctx context.Context, conn *websocket.Conn, sub *Sub, receipts bool) { for { kind, msg, err := conn.Read(ctx) if err != nil { @@ -211,7 +246,7 @@ func answerPings(ctx context.Context, conn *websocket.Conn, sub *Sub) { if sub.PingCommitted() != nil { return } - if sendText(ctx, conn, "pong") != nil { + if sendReceipts(ctx, conn, sub, receipts) != nil || sendText(ctx, conn, "pong") != nil { return } } diff --git a/internal/raftdirectory/ingress.go b/internal/raftdirectory/ingress.go new file mode 100644 index 0000000000..7dd1b1965d --- /dev/null +++ b/internal/raftdirectory/ingress.go @@ -0,0 +1,59 @@ +package raftdirectory + +import ( + "bytes" + "context" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" +) + +// ClusterAPI serializes current-time commit, fresh ReadIndex and every selected +// read. Caller cancellation cannot interrupt an admitted durability transaction. +// The restricted effects witness advances time; it cannot authorize the caller. +func (g *Native) ClusterAPI(ctx context.Context, request cs.ClusterRequest, clock cs.Command) (cs.ClusterResponse, error) { + ctx = context.WithoutCancel(ctx) + g.mu.Lock() + defer g.mu.Unlock() + var out cs.ClusterResponse + actor, err := cs.VerifyClusterRequest(request, clock.Now) + if err != nil || request.Cluster != g.manifest.Config.Cluster || !bytes.Equal(request.IdentityKey, g.manifest.Config.SigningKey) || clock.Cluster != request.Cluster || clock.Change.DeviceSeen == nil || clock.Change.DeviceSeen.Device != actor { + return out, directory.ErrBadRequest + } + result, err := g.submit(ctx, clock) + if err != nil { + return out, err + } + if err = directory.ResultError(result.Code); err != nil { + return out, err + } + i, app, err := g.barrier(ctx) + if err != nil { + return out, err + } + if _, err = cs.VerifyClusterRequest(request, app.LogicalTime); err != nil { + return out, directory.ErrBadRequest + } + if request.Action == "submit" { + result, err = g.submit(ctx, *request.Command) + if err != nil { + return out, err + } + out.Result = &result + return out, nil + } + if request.Action == "cell" { + view, grant, err := AuthorizedWorkerRead(ctx, g.stores[i], request.Cluster, actor, request.Cell, request.Incarnation) + if err != nil { + return out, err + } + out.View, out.Grant = &view, grant + return out, nil + } + assignment, view, err := AuthorizedWorkerAssignment(ctx, g.stores[i], request.Cluster, actor, request.Incarnation, *request.Assignment) + if err != nil { + return out, err + } + out.Assignment, out.View = &assignment, &view + return out, nil +} diff --git a/internal/raftdirectory/maintenance.go b/internal/raftdirectory/maintenance.go new file mode 100644 index 0000000000..ceacc3e96c --- /dev/null +++ b/internal/raftdirectory/maintenance.go @@ -0,0 +1,214 @@ +package raftdirectory + +import ( + "context" + "crypto/ed25519" + "fmt" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftstore" +) + +// InitializeMembership records the provisioned manifest's anchors through the +// ordinary log. It never edits a voter list or invents missing authoritative +// stores. Existing committed registry rows are left intact on every activation. +func (g *Native) InitializeMembership(ctx context.Context, key ed25519.PrivateKey) error { + ctx = context.WithoutCancel(ctx) + g.mu.Lock() + defer g.mu.Unlock() + i, app, err := g.barrier(ctx) + if err != nil { + return err + } + k := raftstore.Key{Namespace: raftstore.Membership, Key: cs.MembershipKey().Key} + rows, _, err := g.stores[i].Read(ctx, app.Applied, []raftstore.Key{k}) + if err != nil { + return err + } + if rows[k] != nil { + return nil + } + if len(g.manifest.Config.AnchorKeys) != len(g.manifest.Config.Anchors) { + return cs.ErrMigration + } + anchors := make([]cs.MemberRecord, len(g.manifest.Config.Anchors)) + for j, id := range g.manifest.Config.Anchors { + anchors[j] = cs.MemberRecord{Node: id, Key: g.manifest.Config.AnchorKeys[j], Incarnation: 1, Protocol: raftcore.ProtocolVersion, Anchor: true} + } + c, err := cs.Sign(cs.Command{Schema: cs.SchemaVersion, Cluster: g.manifest.Config.Cluster, Actor: cs.AnchorActor(g.manifest.Config.Anchors[0]), Session: "initialize-membership", Sequence: 1, Now: app.LogicalTime, Change: cs.Mutation{InitMembership: &cs.InitMembership{Anchors: anchors, Limit: cs.MaxMemberRecords}}}, key) + if err != nil { + return err + } + r, err := g.submit(ctx, c) + if err != nil { + return err + } + return directory.ResultError(r.Code) +} + +// Maintain connects the production host to the shared scheduler. Every final +// sleep acknowledgment is a local durable marker, and an incomplete delivery +// keeps ordinary ticks. Reads still need their own fresh quorum on wake. +func (g *Native) Maintain(ctx context.Context, elapsed, now int64, key ed25519.PrivateKey) (int64, error) { + ctx = context.WithoutCancel(ctx) + g.mu.Lock() + defer g.mu.Unlock() + after := cs.TickEveryMillis + // Election ticks must run before requiring an authority read. A missing + // leader is exactly when the surviving followers need this clock. + plans := make([]raftcore.SchedulePlan, len(g.nodes)) + for i, node := range g.nodes { + if node == nil { + continue + } + p, err := Schedule(ctx, g.stores[i], node, elapsed, now, false, nil) + if err != nil { + return after, err + } + plans[i] = p + if p.Tick { + if err = g.drive(ctx, i, raftcore.Tick{}); err != nil { + return after, err + } + } + } + if err := g.drain(ctx); err != nil { + return after, err + } + leader := g.leader() + if leader < 0 { + return after, directory.ErrUnreachable + } + plan := plans[leader] + if plan.Barrier { + id := fmt.Sprintf("idle-%d", g.serial) + g.serial++ + if err := g.drive(ctx, leader, raftcore.IdleRead{RequestID: id}); err != nil { + return after, err + } + if err := g.drain(ctx); err != nil { + return after, err + } + ok := g.reads[id] + delete(g.reads, id) + if !ok { + return after, directory.ErrUnreachable + } + var err error + plan, err = Schedule(ctx, g.stores[leader], g.nodes[leader], elapsed, now, false, nil) + if err != nil { + return after, err + } + } + if plan.Prepare { + app, _, err := g.stores[leader].ApplicationView(ctx) + if err != nil { + return after, err + } + c, err := cs.Sign(cs.Command{Schema: cs.SchemaVersion, Cluster: g.manifest.Config.Cluster, Actor: cs.AnchorActor(g.manifest.Config.Anchors[0]), Session: "sleep", Sequence: 1, Now: now, Change: cs.Mutation{PrepareQuiescence: &cs.PrepareQuiescence{Barrier: app.Applied, Term: g.status[leader].Term, Voters: raftcore.Configuration(app.ConfState).Voters}}}, key) + if err != nil { + return after, err + } + sk := cs.SessionKey(c.Actor, c.Session) + keys := []raftstore.Key{{Namespace: raftstore.Namespace(sk.Namespace), Key: sk.Key}} + records, _, err := g.stores[leader].Read(ctx, app.Applied, keys) + if err != nil { + return after, err + } + c.Sequence, c.Ack, err = cs.NextSession(records[keys[0]]) + if err != nil { + return after, err + } + c, err = cs.Sign(c, key) + if err != nil { + return after, err + } + r, err := g.submit(ctx, c) + if err != nil { + return after, err + } + if err = directory.ResultError(r.Code); err != nil { + return after, err + } + // Deliver the committed marker to every reachable follower before collecting + // durable acknowledgments. Missing voters remain active and prevent sleep. + if err = g.drive(ctx, leader, raftcore.Tick{}); err != nil { + return after, err + } + if err = g.drain(ctx); err != nil { + return after, err + } + } + acks := []raftcore.SleepAck{} + for i, store := range g.stores { + if store == nil { + continue + } + app, _, err := store.ApplicationView(ctx) + if err != nil { + return after, err + } + k := raftstore.Key{Namespace: raftstore.Membership, Key: cs.QuiescenceKey().Key} + rows, _, err := store.Read(ctx, app.Applied, []raftstore.Key{k}) + if err != nil { + return after, err + } + var m cs.Quiescence + if rows[k] != nil && cs.Decode(rows[k], &m) == nil && app.Applied == m.Index && g.status[i].Term == m.Term { + acks = append(acks, raftcore.SleepAck{Node: g.manifest.Config.Anchors[i], Index: m.Index, Term: m.Term}) + } + } + allSleep := true + for i, node := range g.nodes { + if node == nil { + allSleep = false + continue + } + p, err := Schedule(ctx, g.stores[i], node, elapsed, now, false, acks) + if err != nil { + return after, err + } + allSleep = allSleep && p.Sleeping + } + if allSleep { + return 0, nil + } + return after, nil +} + +// PendingEffects is only a timer hint. Actual delivery still uses Effects.Replay +// and its fresh authorization barrier; an idle scan grants no capability. +func (g *Native) PendingEffects(ctx context.Context) (bool, error) { + g.mu.Lock() + defer g.mu.Unlock() + i := g.leader() + if i < 0 { + return false, directory.ErrUnreachable + } + app, _, err := g.stores[i].ApplicationView(ctx) + if err != nil { + return false, err + } + after := "" + for { + rows, _, err := g.stores[i].ScanPage(ctx, app.Applied, raftstore.Outbox, after) + if err != nil { + return false, err + } + if len(rows) == 0 { + return false, nil + } + for _, row := range rows { + active, err := cs.RecordActive(cs.RecordKey{Namespace: string(row.Namespace), Key: row.Key.Key}, row.Value, app.LogicalTime) + if err != nil { + return false, err + } + if active { + return true, nil + } + after = row.Key.Key + } + } +} diff --git a/internal/raftdirectory/maintenance_internal_test.go b/internal/raftdirectory/maintenance_internal_test.go new file mode 100644 index 0000000000..aab9b3f7be --- /dev/null +++ b/internal/raftdirectory/maintenance_internal_test.go @@ -0,0 +1,96 @@ +package raftdirectory + +import ( + "context" + "crypto/ed25519" + "path/filepath" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftcore" +) + +// R5/R8/R26 require the actual native maintenance clock to elect a surviving +// anchor after leader loss. A failed authority read must not suppress ticks. +func TestR5R8R26NativeMaintenanceElectsWithoutLeader(t *testing.T) { + ctx := context.Background() + keys := make([]ed25519.PrivateKey, 3) + public := make([][]byte, 3) + for i := range keys { + seed := make([]byte, ed25519.SeedSize) + seed[0] = byte(i + 1) + keys[i] = ed25519.NewKeyFromSeed(seed) + public[i] = keys[i].Public().(ed25519.PublicKey) + } + root := ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize)).Public().(ed25519.PublicKey) + effectsSeed := make([]byte, ed25519.SeedSize) + effectsSeed[0] = 4 + effects := ed25519.NewKeyFromSeed(effectsSeed).Public().(ed25519.PublicKey) + now := int64(1700000000000) + source, err := directory.OpenSQLite(filepath.Join(t.TempDir(), "source.db"), func() time.Time { return time.UnixMilli(now) }) + if err != nil { + t.Fatal(err) + } + defer source.Close() + cfg := cs.LegacyConfig{Cluster: "maintenance-loss", Generation: 1, Identity: cs.IdentityID(root), SigningKey: root, EffectsKey: effects, Anchors: []uint64{1, 2, 3}, AnchorKeys: public, Now: now} + if err = source.FreezeLegacy(ctx, cfg); err != nil { + t.Fatal(err) + } + manifest, err := source.InventoryLegacy(ctx) + if err != nil { + t.Fatal(err) + } + host, err := OpenNative(ctx, t.TempDir(), source, manifest) + if err != nil { + t.Fatal(err) + } + defer host.Close() + if err = host.InitializeMembership(ctx, keys[0]); err != nil { + t.Fatal(err) + } + lost := host.leader() + if lost < 0 { + t.Fatal("no initial leader") + } + if err = host.stores[lost].Close(); err != nil { + t.Fatal(err) + } + host.stores[lost] = nil + host.nodes[lost] = nil + host.status[lost] = raftcore.Status{} + // Recover surviving actors from their own stores with fixed timing seeds. + // The production clock, not startup's explicit campaigns, must elect them. + for i, store := range host.stores { + if store == nil { + continue + } + recovered, e := store.Recover(ctx) + if e != nil { + t.Fatal(e) + } + host.nodes[i], e = raftcore.New(raftcore.Config{ClusterUUID: cfg.Cluster, NodeID: uint64(i + 1), Protocol: raftcore.ProtocolVersion, Seed: uint64(i + 8), Anchors: cfg.Anchors}, recovered.Core) + if e != nil { + t.Fatal(e) + } + if e = host.drive(ctx, i, raftcore.Poll{}); e != nil { + t.Fatal(e) + } + } + if err = host.drain(ctx); err != nil { + t.Fatal(err) + } + for elapsed := int64(0); elapsed <= 60000; elapsed += cs.TickEveryMillis { + _, _ = host.Maintain(ctx, elapsed, now+elapsed, keys[0]) + if host.leader() >= 0 { + _, _, err = host.barrier(ctx) + if err != nil { + t.Fatal("surviving majority cannot read", err) + } + t.Logf("surviving majority elected after %d virtual ms", elapsed) + return + } + } + t.Fatal("R5/R8/R26 maintenance stopped election ticks without a leader") +} diff --git a/internal/relayconf/watch_test.go b/internal/relayconf/watch_test.go index a01e7ee368..3f2c5ca273 100644 --- a/internal/relayconf/watch_test.go +++ b/internal/relayconf/watch_test.go @@ -27,6 +27,7 @@ func TestWatchConformance(t *testing.T) { } acct := newAccount(t) return directorytest.WatchRig{ + Committed: true, Rig: directorytest.Rig{ Clock: relayClock{t, service}, Devices: func(name string) directory.Client { @@ -48,6 +49,7 @@ func TestLeaseConformance(t *testing.T) { directorytest.RunLease(t, func(t *testing.T) directorytest.WatchRig { acct := newAccount(t) return directorytest.WatchRig{ + Committed: true, Rig: directorytest.Rig{ Clock: relayClock{t, base}, Devices: func(name string) directory.Client { @@ -69,6 +71,7 @@ func TestPresenceConformance(t *testing.T) { directorytest.RunPresence(t, func(t *testing.T) directorytest.WatchRig { acct := newAccount(t) return directorytest.WatchRig{ + Committed: true, Rig: directorytest.Rig{ Clock: relayClock{t, base}, Devices: func(name string) directory.Client { diff --git a/internal/relayserve/committed_watch_test.go b/internal/relayserve/committed_watch_test.go new file mode 100644 index 0000000000..4b31d84dfd --- /dev/null +++ b/internal/relayserve/committed_watch_test.go @@ -0,0 +1,142 @@ +package relayserve_test + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftstore" +) + +// R22 proves the production watch emits an actual durable receipt instead of +// advertising socket vouching. Release and a delayed ping cannot revive a hold. +func TestR22ProductionWatchReceipts(t *testing.T) { + r := newRig(t) + _, a := newIdentity(t) + created, err := r.dir(a).Create(bg, cell, directory.CellInit{Head: head1}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(bg, 10*time.Second) + defer cancel() + stream, err := r.dir(a).WatchHolding(ctx, []directory.Hold{{Cell: cell, Fence: created.Cell.Lease.Fence}}) + if err != nil { + t.Fatal(err) + } + defer stream.Close() + if stream.Vouching() { + t.Fatal("R22 advertised socket vouching") + } + if _, err = stream.Next(ctx); err != nil { + t.Fatal(err) + } + r.clock.Advance(30 * time.Second) + if err = stream.Ping(ctx); err != nil { + t.Fatal(err) + } + frame, err := stream.Next(ctx) + if err != nil { + t.Fatal(err) + } + if len(frame.Holds) != 1 { + t.Fatalf("R22 no committed receipt: %+v", frame) + } + receipt := frame.Holds[0] + if receipt.Cell != cell || receipt.Fence != 1 || receipt.Index == 0 || receipt.Until != r.clock.Now().Add(directory.LeaseTTL).UnixMilli() { + t.Fatal(receipt) + } + got, err := r.dir(a).Cell(bg, cell) + if err != nil { + t.Fatal(err) + } + if got.Cell.Lease.Expires != receipt.Until { + t.Fatal("receipt differs from committed lease", got, receipt) + } + if err = r.dir(a).Release(bg, cell, 1); err != nil { + t.Fatal(err) + } + r.clock.Advance(10 * time.Second) + if err = stream.Ping(ctx); err != nil { + t.Fatal(err) + } + got, err = r.dir(a).Cell(bg, cell) + if err != nil || got.Cell.Lease.Expires != 0 { + t.Fatal("R22 delayed ping revived released lease", got, err) + } +} + +// R23/R26/R27 exercise ten minutes of virtual idle through the REAL production +// scheduler. No client request or application write recurs after all voters +// applied the sleep marker. A first new read wakes and passes quorum again. +func TestR23R26R27ProductionTenMinuteIdle(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + if err := r.dir(a).PutDevice(bg, a.dev.ID(), directory.Device{V: 1}); err != nil { + t.Fatal(err) + } + var p struct{ Cluster string } + raw, err := os.ReadFile(filepath.Join(r.store, "raft", id.ID(), "provision.json")) + if err != nil { + t.Fatal(err) + } + if json.Unmarshal(raw, &p) != nil { + t.Fatal("provision") + } + inspect := func() (raftstore.Application, cs.Quiescence) { + t.Helper() + s, err := raftstore.Open(filepath.Join(r.store, "raft", id.ID(), "1.db"), raftstore.Identity{ClusterUUID: p.Cluster, NodeID: 1, Protocol: 1}) + if err != nil { + t.Fatal(err) + } + defer s.Close() + app, _, err := s.ApplicationView(bg) + if err != nil { + t.Fatal(err) + } + k := raftstore.Key{Namespace: raftstore.Membership, Key: cs.QuiescenceKey().Key} + rows, _, err := s.Read(bg, app.Applied, []raftstore.Key{k}) + if err != nil { + t.Fatal(err) + } + var marker cs.Quiescence + if len(rows[k]) > 0 && cs.Decode(rows[k], &marker) != nil { + t.Fatal("sleep marker") + } + return app, marker + } + for n := 0; n < 20; n++ { + r.clock.Advance(5 * time.Second) + } + before, marker := inspect() + if marker.Index == 0 || marker.Index != before.Applied { + t.Fatalf("R26 production did not quiesce: %+v %+v", before, marker) + } + r.mu.Lock() + requests := len(r.logs) + r.mu.Unlock() + for n := 0; n < 120; n++ { + r.clock.Advance(5 * time.Second) + } + after, _ := inspect() + r.mu.Lock() + idleRequests := len(r.logs) - requests + r.mu.Unlock() + t.Logf("R23/R27 simulated idle=10m requests=%d applied-before=%d applied-after=%d", idleRequests, before.Applied, after.Applied) + if idleRequests != 0 || after.Applied != before.Applied { + t.Fatal("idle caused recurring requests or application writes") + } + if _, err = r.dir(a).List(bg); err != nil { + t.Fatal(err) + } + awake, _ := inspect() + if awake.Applied <= after.Applied { + t.Fatal("R26 first read did not commit current ingress time") + } + // An idle Home socket pings without making directory HTTP requests. Its + // presence hints are separate from the sleeping consensus group. +} diff --git a/internal/relayserve/ingress.go b/internal/relayserve/ingress.go new file mode 100644 index 0000000000..88af19e443 --- /dev/null +++ b/internal/relayserve/ingress.go @@ -0,0 +1,51 @@ +package relayserve + +import ( + "io" + "net/http" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +// clusterAPI authenticates the entire selected request before activation. A +// worker delegation is accepted only here, never at personal pairing doors. +func (n *namespaces) clusterAPI(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.RawQuery != "" { + http.NotFound(w, r) + return + } + raw, err := io.ReadAll(http.MaxBytesReader(w, r.Body, cs.MaxCommandBytes)) + var request cs.ClusterRequest + if err != nil || cs.Decode(raw, &request) != nil { + http.Error(w, "invalid cluster request", 400) + return + } + actor, err := cs.VerifyClusterRequest(request, n.now().UnixMilli()) + if err != nil { + http.Error(w, "this computer is not allowed to use this cluster", 403) + return + } + d, err := n.activate(r.Context(), cs.IdentityID(request.IdentityKey), request.IdentityKey) + if err != nil { + http.Error(w, cs.LeadershipUnavailable, 503) + return + } + clock, err := cs.Sign(cs.Command{Schema: cs.SchemaVersion, Cluster: d.Cluster, Actor: d.effects.Actor, Session: "ingress", Sequence: 1, Now: n.now().UnixMilli(), Change: cs.Mutation{DeviceSeen: &cs.DeviceSeen{Device: actor}}}, d.effects.Key) + if err != nil { + http.Error(w, cs.LeadershipUnavailable, 503) + return + } + result, err := d.host.ClusterAPI(r.Context(), request, clock) + if err != nil { + http.Error(w, "the selected work is unavailable; refresh before trying again", 503) + return + } + raw, err = cs.Encode(result) + if err != nil || len(raw) > cs.MaxEvaluationBytes { + http.Error(w, cs.LeadershipUnavailable, 503) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + _, _ = w.Write(raw) +} diff --git a/internal/relayserve/ingress_test.go b/internal/relayserve/ingress_test.go new file mode 100644 index 0000000000..873009c5b7 --- /dev/null +++ b/internal/relayserve/ingress_test.go @@ -0,0 +1,153 @@ +package relayserve_test + +import ( + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterapi" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/identity" +) + +// R16/R18/R21 drive original personal and scoped worker signatures through the +// production HTTP door. Selected reads cannot exceed the grant, use stale +// incarnations, survive revocation or turn an old claimed result into authority. +func TestR16R18ProductionScopedWorkerIngress(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + if _, err := r.dir(a).Create(bg, cell, directory.CellInit{Head: head1}); err != nil { + t.Fatal(err) + } + if err := r.dir(a).Release(bg, cell, 1); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(filepath.Join(r.store, "raft", id.ID(), "provision.json")) + if err != nil { + t.Fatal(err) + } + var p struct{ Cluster string } + if json.Unmarshal(raw, &p) != nil { + t.Fatal("provision") + } + certRaw, _ := json.Marshal(a.dev.Cert) + personal := base64.RawURLEncoding.EncodeToString(certRaw) + owner := clusterapi.Client{Base: r.srv.URL, HTTP: r.srv.Client(), Now: r.clock.Now, Allowed: func() bool { return true }, Sign: a.dev.Sign, Request: func() cs.ClusterRequest { + return cs.ClusterRequest{IdentityKey: id.PublicKey(), Cluster: p.Cluster, Certificate: personal} + }} + _, key, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + cert, err := identity.IssueWorkerCert(id, key.Public().(ed25519.PublicKey), p.Cluster, []string{cell}, r.clock.Now(), r.clock.Now().Add(time.Hour), false) + if err != nil { + t.Fatal(err) + } + worker := cert.DeviceID() + incarnation := uint64(0) + wire := clusterapi.Client{Base: r.srv.URL, HTTP: r.srv.Client(), Now: r.clock.Now, Allowed: func() bool { return true }, Sign: func(b []byte) []byte { return ed25519.Sign(key, b) }, Request: func() cs.ClusterRequest { + return cs.ClusterRequest{IdentityKey: id.PublicKey(), Cluster: p.Cluster, Worker: &cert, Incarnation: incarnation} + }} + sequences := map[string]uint64{} + sign := func(actor string, mutation cs.Mutation) cs.Command { + t.Helper() + sequences[actor]++ + n := sequences[actor] + c := cs.Command{Schema: 1, Cluster: p.Cluster, Actor: actor, Session: "integration", Sequence: n, Ack: n - 1, Now: r.clock.Now().UnixMilli(), Change: mutation} + signer := a.dev.Sign + if actor == worker { + c.WorkerIncarnation = incarnation + signer = func(b []byte) []byte { return ed25519.Sign(key, b) } + } else { + c.Certificate = personal + } + c.RequestDigest, err = cs.Digest(c) + if err != nil { + t.Fatal(err) + } + b, err := cs.SigningBytes(c) + if err != nil { + t.Fatal(err) + } + c.Signature = signer(b) + return c + } + submit := func(client clusterapi.Client, c cs.Command, want cs.Code) cs.Result { + t.Helper() + result, err := client.Submit(bg, c) + if err != nil || result.Code != want { + t.Fatalf("want %s: %+v %v", want, result, err) + } + return result + } + grant := []byte{0x81, 0x82, 0x83} + submit(owner, sign(a.dev.ID(), cs.Mutation{EnrollWorker: &cs.EnrollWorker{Certificate: cert, Grants: map[string][]byte{cell: grant}}}), cs.OK) + registration := sign(worker, cs.Mutation{RegisterWorker: &cs.RegisterWorker{Incarnation: 1}}) + submit(wire, registration, cs.OK) + incarnation = 1 + view, got, err := wire.Cell(bg, cell) + if err != nil || view.Cell.Head != head1 || string(got) != string(grant) { + t.Fatal(view, got, err) + } + if _, _, err = wire.Cell(bg, "outside"); err == nil { + t.Fatal("R16 read outside grant") + } + heartbeat := cs.WorkerHeartbeat{Worker: worker, Incarnation: 1, Sequence: 1, SeenAt: r.clock.Now().UnixMilli()} + b, err := cs.WorkerHeartbeatBytes(p.Cluster, heartbeat) + if err != nil { + t.Fatal(err) + } + heartbeat.Signature = ed25519.Sign(key, b) + submit(wire, sign(worker, cs.Mutation{WorkerBeatBatch: &cs.WorkerBeatBatch{Items: []cs.WorkerHeartbeat{heartbeat}}}), cs.OK) + assignment := cs.Assign{ID: "opaque-run", Revision: 1, Cell: cell, Expect: cs.LeaseExpectation{Device: a.dev.ID(), Fence: 1}, Target: worker, Incarnation: 1, Expires: r.clock.Now().Add(time.Minute).UnixMilli(), SealedPayload: []byte{0x91}} + submit(owner, sign(a.dev.ID(), cs.Mutation{Assign: &assignment}), cs.OK) + ref := (cs.Assignment{Spec: assignment}).Ref() + current, _, err := wire.Current(bg, ref) + if err != nil || current.Status != "pending" { + t.Fatal(current, err) + } + claim := sign(worker, cs.Mutation{ClaimAssignment: &cs.ClaimAssignment{Ref: ref, Expect: assignment.Expect}}) + claimed := submit(wire, claim, cs.OK) + if claimed.View == nil || claimed.View.Cell.Lease.Fence != 2 { + t.Fatal(claimed) + } + submit(wire, claim, cs.OK) + // Retrying coordination never invokes a tool. Current state independently + // rejects the saved claim once an ordinary conditional take has displaced it. + expect := directory.LeaseExpectation{Device: worker, Fence: 2} + if _, err = r.dir(a).Acquire(bg, cell, directory.AcquireOpts{Force: true, Expect: &expect}); err != nil { + t.Fatal(err) + } + current, currentView, err := wire.Current(bg, ref) + if err != nil { + t.Fatal(err) + } + if current.Fence != 2 || currentView.Cell.Lease.Fence != 3 || currentView.Cell.Lease.Device != a.dev.ID() { + t.Fatal("R18 current read returned the saved claim as current ownership", current, currentView) + } + incarnation = 2 + if _, _, err = wire.Cell(bg, cell); err == nil { + t.Fatal("R16 stale incarnation read") + } + incarnation = 1 + // The worker cannot turn its delegation into a personal control request. + submit(wire, sign(worker, cs.Mutation{Rotate: &cs.RotateRequest{Op: "freeze"}}), cs.Unauthorized) + if err = r.dir(a).Revoke(bg, worker); err != nil { + t.Fatal(err) + } + if _, _, err = wire.Cell(bg, cell); err == nil { + t.Fatal("R16 revoked worker read") + } + submit(wire, claim, cs.Revoked) + // A grant expires at COMMITTED ingress time even with no other active writer. + r.clock.Advance(2 * time.Hour) + if _, _, err = wire.Cell(bg, cell); err == nil { + t.Fatal("R16 expired worker read") + } +} diff --git a/internal/relayserve/maintenance.go b/internal/relayserve/maintenance.go new file mode 100644 index 0000000000..3db4d73634 --- /dev/null +++ b/internal/relayserve/maintenance.go @@ -0,0 +1,71 @@ +package relayserve + +import ( + "context" + "crypto/ed25519" + "sync" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +// memberTimer has one deadline per identity. A new ingress wakes a quiescent +// group; repeated ingress cannot postpone its already armed maintenance turn. +// Timers stop after the shared all-voter sleep protocol, without idle polling. +type memberTimer struct { + mu sync.Mutex + turn sync.Mutex + stop func() bool + generation uint64 + closed bool + started time.Time + after func(time.Duration, func()) func() bool + anchor ed25519.PrivateKey + cursor string +} + +func (d *nativeNamespace) armMembers() { + m := &d.timer + m.mu.Lock() + defer m.mu.Unlock() + if m.closed || m.stop != nil { + return + } + m.generation++ + generation := m.generation + m.stop = m.after(time.Duration(cs.TickEveryMillis)*time.Millisecond, func() { d.maintainMembers(generation) }) +} +func (d *nativeNamespace) maintainMembers(generation uint64) { + m := &d.timer + m.turn.Lock() + defer m.turn.Unlock() + m.mu.Lock() + if m.closed || generation != m.generation { + m.mu.Unlock() + return + } + m.stop = nil + m.mu.Unlock() + ctx := context.Background() + pending, err := d.host.PendingEffects(ctx) + if err == nil && pending { + m.cursor, err = d.effects.Replay(ctx, d.now().UnixMilli(), d.links, m.cursor) + } + // An unavailable outbox read cannot suppress the election clock. Its own + // authority checks still precede delivery when a leader becomes available. + after, maintenanceErr := d.host.Maintain(ctx, max(int64(0), d.now().Sub(m.started).Milliseconds()), d.now().UnixMilli(), m.anchor) + if err != nil || maintenanceErr != nil || after != 0 { + d.armMembers() + } +} +func (d *nativeNamespace) stopMembers() { + m := &d.timer + m.mu.Lock() + defer m.mu.Unlock() + m.closed = true + m.generation++ + if m.stop != nil { + m.stop() + m.stop = nil + } +} diff --git a/internal/relayserve/native.go b/internal/relayserve/native.go index 2f66b29825..11ae6c48ac 100644 --- a/internal/relayserve/native.go +++ b/internal/relayserve/native.go @@ -16,6 +16,7 @@ import ( "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/dirwatch" "github.com/Agent-Field/codeaf/internal/raftdirectory" "github.com/Agent-Field/codeaf/internal/raftstore" "github.com/Agent-Field/codeaf/internal/reqsign" @@ -31,6 +32,7 @@ type nativeProvision struct { Anchors [3][]byte `json:"anchors"` } type nativeNamespace struct { + timer memberTimer *raftdirectory.V1Directory host *raftdirectory.Native feed *directory.Feed @@ -42,7 +44,7 @@ type nativeNamespace struct { } func (d *nativeNamespace) Feed() *directory.Feed { return d.feed } -func (d *nativeNamespace) Close() error { d.feed.Close(); return d.host.Close() } +func (d *nativeNamespace) Close() error { d.stopMembers(); d.feed.Close(); return d.host.Close() } // activate runs outside every legacy admission lock. The frozen source drains // admitted writes itself and can never become a fallback directory afterward. @@ -189,7 +191,18 @@ func (n *namespaces) activate(ctx context.Context, id string, root []byte) (*nat } d = &nativeNamespace{V1Directory: &raftdirectory.V1Directory{Host: host, Cluster: cfg.Cluster, Now: n.now, Grace: grace}, identity: clusterstate.IdentityRec{V: 1, Identity: id, SigningKey: cfg.SigningKey}, host: host, feed: feed, links: n.links.(*directory.Links), now: n.now, grace: grace} d.effects = raftdirectory.Effects{Host: host, Cluster: cfg.Cluster, Actor: clusterstate.DeviceIDOfKey(cfg.EffectsKey), Key: ed25519.PrivateKey(provision.Effects)} - host.Changed = d.project + if err = host.InitializeMembership(ctx, ed25519.PrivateKey(provision.Anchors[0])); err != nil { + d.Close() + return nil, err + } + d.timer = memberTimer{started: n.now(), anchor: ed25519.PrivateKey(provision.Anchors[0]), after: func(delay time.Duration, fn func()) func() bool { return time.AfterFunc(delay, fn).Stop }} + if n.after != nil { + d.timer.after = n.after + } + host.Changed = func(store *raftstore.SQLite, app raftstore.Application) error { + d.armMembers() + return d.project(store, app) + } feed.OnSeen(func(device string, at int64) { _ = d.seen(device, at) }) if err = host.Inspect(ctx, func(ctx context.Context, s *raftstore.SQLite, a raftstore.Application) error { return d.project(s, a) }); err != nil { d.Close() @@ -202,6 +215,7 @@ func (n *namespaces) activate(ctx context.Context, id string, root []byte) (*nat return nil, err } n.mu.Lock() + d.armMembers() n.groups[id] = d n.mu.Unlock() return d, nil @@ -290,14 +304,18 @@ func (d *nativeNamespace) SubscribeWatch(device string, r *http.Request, holds [ if clusterstate.PersonalCertificate(d.identity, device, certificate) != nil { return nil, directory.ErrBadRequest } - return d.feed.SubscribeBeatWithEvidence(device, holds, beat, func(device string, holds []directory.Hold, at int64) error { + return d.feed.SubscribeCommitted(device, holds, beat, func(device string, holds []directory.Hold, at int64) ([]dirwatch.HoldReceipt, error) { return d.renew(device, holds, at, certificate) }) } -func (d *nativeNamespace) renew(device string, holds []directory.Hold, at int64, certificate string) error { +func (d *nativeNamespace) renew(device string, holds []directory.Hold, at int64, certificate string) ([]dirwatch.HoldReceipt, error) { + // Idle presence never wakes consensus just because a hibernating socket pings. + if len(holds) == 0 { + return nil, nil + } ctx := context.Background() if err := d.authority(ctx, device, false); err != nil { - return err + return nil, err } // Old sockets may name zero, missing or several historical fences. Only // their one current held tuple is evidence; none of the others can renew it. @@ -309,14 +327,14 @@ func (d *nativeNamespace) renew(device string, holds []directory.Hold, at int64, } raw, err := d.host.ReadEffect(ctx, clusterstate.CellKey(h.Cell)) if err != nil { - return err + return nil, err } if raw == nil { continue } var cell clusterstate.Cell if err = clusterstate.Decode(raw, &cell); err != nil { - return err + return nil, err } if cell.Lease.Device != device || cell.Lease.Fence != h.Fence || cell.Lease.Expires == 0 { continue @@ -325,10 +343,9 @@ func (d *nativeNamespace) renew(device string, holds []directory.Hold, at int64, items = append(items, clusterstate.HoldFence{Cell: h.Cell, Fence: h.Fence}) } if len(items) == 0 { - return nil + return nil, nil } - _, err := d.effects.Renew(ctx, at, clusterstate.RenewHolds{Certificate: certificate, Device: device, SeenAt: at, Items: items}) - return err + return d.effects.Renew(ctx, at, clusterstate.RenewHolds{Certificate: certificate, Device: device, SeenAt: at, Items: items}) } func (d *nativeNamespace) seen(device string, at int64) error { diff --git a/internal/relayserve/service.go b/internal/relayserve/service.go index 94c1b84904..df8313aefb 100644 --- a/internal/relayserve/service.go +++ b/internal/relayserve/service.go @@ -11,6 +11,7 @@ import ( "time" "github.com/Agent-Field/codeaf/internal/blobstore" + "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/directory" "github.com/Agent-Field/codeaf/internal/pairbox" "github.com/Agent-Field/codeaf/internal/relay" @@ -117,6 +118,7 @@ func New(cfg Config) *Service { svc.ns = newNamespaces(cfg.Store, now, cfg.Grace, cfg.MaxWatchers) svc.ns.after = cfg.After svc.ns.links = links + mux.HandleFunc(clusterstate.ClusterAPIPath, svc.ns.clusterAPI) auth := noting(svc.ns.authenticate) dirs := logged(cfg.Logf, directory.Handler(auth, svc.ns.directory)) mux.Handle("/v1/dir/", dirs) diff --git a/internal/relayserve/watch_test.go b/internal/relayserve/watch_test.go index 9c3e0625d7..b3de6453fe 100644 --- a/internal/relayserve/watch_test.go +++ b/internal/relayserve/watch_test.go @@ -49,6 +49,7 @@ func (r *rig) signer(d device) wireauth.Sign { return reqsign.SignFor(d, r.clock func (r *rig) watchRig(t *testing.T) directorytest.WatchRig { home := newHousehold(t) return directorytest.WatchRig{ + Committed: true, Rig: directorytest.Rig{ Clock: r.clock, Devices: func(name string) directory.Client { return r.dir(home.device(name)) }, diff --git a/internal/syncsetup/clusterapi.go b/internal/syncsetup/clusterapi.go new file mode 100644 index 0000000000..134dfaf20c --- /dev/null +++ b/internal/syncsetup/clusterapi.go @@ -0,0 +1,53 @@ +package syncsetup + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "net/http" + + "github.com/Agent-Field/codeaf/internal/clusterapi" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/identity" +) + +// ClusterAPI uses the same use-time network gate as directory and blob clients. +// Constructing it does not enroll a member, discover a relay or open a socket. +func (s *Sync) ClusterAPI(cluster string) clusterapi.Client { + return clusterapi.Client{Base: s.Relay, HTTP: s.ClusterHTTP(nil), Allowed: func() bool { return MayTalk(s.Home) }, Sign: s.Device.Sign, Request: func() cs.ClusterRequest { + cert, _ := json.Marshal(s.Device.Cert) + return cs.ClusterRequest{IdentityKey: s.Identity.PublicKey(), Cluster: cluster, Certificate: base64.RawURLEncoding.EncodeToString(cert)} + }} +} + +// WorkerClusterAPI accepts only delegated credentials and an explicit use-time +// policy. It never loads a personal identity or discovers a relay on its behalf. +// The caller supplies the SAME policy for peer frames and its content transport. +// The worker home and process sync-off decision still take precedence at use time. +func WorkerClusterAPI(home, base string, root ed25519.PublicKey, worker identity.Worker, incarnation uint64, client *http.Client, allowed func() bool) clusterapi.Client { + return clusterapi.Client{Base: base, HTTP: client, Allowed: func() bool { return Resolve(home).On() && allowed != nil && allowed() }, Sign: worker.Sign, Request: func() cs.ClusterRequest { + cert := worker.Certificate + return cs.ClusterRequest{IdentityKey: append([]byte(nil), root...), Cluster: cert.Cluster, Worker: &cert, Incarnation: incarnation} + }} +} + +// SignClusterCommand retains the original personal signature and certificate +// for hosts whose legacy directory never saved a personal authority row. +func (s *Sync) SignClusterCommand(c cs.Command) (cs.Command, error) { + cert, err := json.Marshal(s.Device.Cert) + if err != nil { + return c, err + } + c.Certificate = base64.RawURLEncoding.EncodeToString(cert) + c.Actor = s.Device.ID() + c.RequestDigest, err = cs.Digest(c) + if err != nil { + return c, err + } + raw, err := cs.SigningBytes(c) + if err != nil { + return c, err + } + c.Signature = s.Device.Sign(raw) + return c, nil +} diff --git a/internal/syncsetup/worker_quiet_test.go b/internal/syncsetup/worker_quiet_test.go new file mode 100644 index 0000000000..0c66dbd3cd --- /dev/null +++ b/internal/syncsetup/worker_quiet_test.go @@ -0,0 +1,65 @@ +package syncsetup + +import ( + "bytes" + "context" + "crypto/ed25519" + "encoding/hex" + "io" + "net/http" + "strings" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/identity" +) + +type workerGateTransport struct{ calls int } + +func (w *workerGateTransport) RoundTrip(r *http.Request) (*http.Response, error) { + w.calls++ + return &http.Response{StatusCode: http.StatusServiceUnavailable, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(""))}, nil +} + +// R16/R23 keep a standalone delegated worker independent of a personal identity +// while making a later process/home off decision override its retained handle. +func TestR16R23WorkerClusterHandleStopsAfterOff(t *testing.T) { + home := t.TempDir() + id, err := identity.Mint() + if err != nil { + t.Fatal(err) + } + seed := bytes.Repeat([]byte{91}, ed25519.SeedSize) + key := ed25519.NewKeyFromSeed(seed) + now := time.Now() + cert, err := identity.IssueWorkerCert(id, key.Public().(ed25519.PublicKey), "scoped-cluster", []string{"selected"}, now.Add(-time.Minute), now.Add(time.Hour), false) + if err != nil { + t.Fatal(err) + } + raw, err := cs.Encode(struct { + V uint16 `json:"v"` + Seed string `json:"worker_seed"` + Certificate cs.WorkerCertificate `json:"certificate"` + }{1, hex.EncodeToString(seed), cert}) + if err != nil { + t.Fatal(err) + } + worker, err := identity.UnmarshalWorker(raw, id.PublicKey(), "scoped-cluster", now) + if err != nil { + t.Fatal(err) + } + transport := &workerGateTransport{} + t.Setenv(URLVar, "http://worker.invalid") + client := WorkerClusterAPI(home, "http://worker.invalid", id.PublicKey(), worker, 1, &http.Client{Transport: transport}, func() bool { return true }) + if _, _, err = client.Cell(context.Background(), "selected"); err == nil || transport.calls != 1 { + t.Fatal("R23 explicit worker did not use its controlled wire", transport.calls, err) + } + t.Setenv(URLVar, "off") + if _, _, err = client.Cell(context.Background(), "selected"); err == nil || transport.calls != 1 { + t.Fatal("R23 retained worker sent after off", transport.calls, err) + } + if _, err = identity.Load(home); err == nil { + t.Fatal("R16 worker client created a personal identity") + } +} From e944e7a64cd0d7c6b45f77bc78f65719f1e1c9b4 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 10:38:09 -0400 Subject: [PATCH 050/103] Connect hosted scoped ingress and idle maintenance to the shared Go policy. Use additive ABI doors rather than JavaScript authorization or lease rules. Preserve hibernating certificate evidence, original request bytes and durable results, and prove production worker claims and idle scheduling through the local Worker. Co-Authored-By: Claude Opus 5.5 (1M context) --- relay/hosted/src/front.js | 6 +- relay/hosted/src/identity.js | 30 ++++- relay/hosted/src/raft/hosted.js | 117 ++++++++++++++++++- relay/hosted/src/raft/ingress.js | 16 +++ relay/hosted/src/raft/liveness.js | 2 +- relay/hosted/src/raft/member.js | 17 ++- relay/hosted/src/watch.js | 6 +- relay/hosted/test/raft/client-wiring.test.js | 70 +++++++++++ relay/hosted/test/raft/hosted-fixture.js | 14 ++- relay/hosted/test/raft/hosted-go.test.js | 14 +++ relay/hosted/test/raft/hosted-worker.js | 14 +++ 11 files changed, 291 insertions(+), 15 deletions(-) create mode 100644 relay/hosted/src/raft/ingress.js create mode 100644 relay/hosted/test/raft/client-wiring.test.js diff --git a/relay/hosted/src/front.js b/relay/hosted/src/front.js index 50377b3420..de0ca5eed3 100644 --- a/relay/hosted/src/front.js +++ b/relay/hosted/src/front.js @@ -1,6 +1,8 @@ // The front door's routing, kept apart from the Worker entry so a test can drive it without the // Durable Object classes (which only load inside the Workers runtime). import { stripBase } from './base.js'; +import { clusterRequestLimit } from './raft/ingress.js'; +import { sharedRuntime } from './raft/actor.js'; import { checkDeclared } from './body.js'; import { serveLink } from './link/front.js'; import { isPagePath, servePage } from './page.js'; @@ -12,7 +14,7 @@ import { notFound, respond } from './wire.js'; /** forward hands an identity's request to its Durable Object, after refusing what needs no object to refuse. */ async function forward(request, env) { - const route = matchRoute(request.method, new URL(request.url).pathname); + const route = new URL(request.url).pathname==='/v2/cluster/api' ? {limit:clusterRequestLimit(sharedRuntime()),over:"too_large"} : matchRoute(request.method, new URL(request.url).pathname); checkDeclared(request, route.limit, route.over); const identity = await identityOf(request.headers.get('codeaf-identity')); return env.IDENTITY.get(env.IDENTITY.idFromName(identity)).fetch(request); @@ -26,6 +28,6 @@ export function dispatch(outer, env) { const { pathname } = new URL(request.url); if (pathname.startsWith('/v1/link/')) return serveLink(request, env); if (pathname.startsWith('/v1/pair')) return servePair(request, env); - if (pathname.startsWith('/v1/')) return forward(request, env); + if (pathname.startsWith('/v1/') || pathname==='/v2/cluster/api') return forward(request, env); throw notFound(); } diff --git a/relay/hosted/src/identity.js b/relay/hosted/src/identity.js index 7ac99526d9..3ed8d50f0d 100644 --- a/relay/hosted/src/identity.js +++ b/relay/hosted/src/identity.js @@ -4,6 +4,11 @@ import { DurableObject } from 'cloudflare:workers'; import { LegacyAdmission } from './raft/legacy.js'; import { HostedIdentity } from './raft/hosted.js'; +import { clusterIngress,clusterRequestLimit } from './raft/ingress.js'; +import { sharedRuntime } from './raft/actor.js'; +import { identityOf } from './verify.js'; +import { b64,unb64 } from './raft/storage.js'; +import { fromBase64Url } from './codec.js'; import { signedUri } from './base.js'; import { readBody } from './body.js'; import { dropTables, erasePage } from './erasure.js'; @@ -15,7 +20,7 @@ import { policyOf } from './rules.js'; import { Tenant, HAS_WAIT_MS } from './tenant.js'; import { finishClose, Watchers } from './watch.js'; import { checkCert, checkRequest } from './verify.js'; -import { gone, ipOf, rateLimited, respond, unwrap } from './wire.js'; +import { gone, ipOf, rateLimited, respond, unwrap, Wire } from './wire.js'; const SIGNED = { identity: 'codeaf-identity', cert: 'codeaf-cert', time: 'codeaf-time', sig: 'codeaf-sig' }; const signedHeaders = (request) => Object.fromEntries(Object.entries(SIGNED).map(([k, h]) => [k, request.headers.get(h)])); @@ -73,6 +78,21 @@ export class IdentityDO extends DurableObject { async #serve(request) { const url = new URL(request.url); + if(url.pathname==='/v2/cluster/api'){ + if(request.method!=='POST'||url.search)throw new Wire('bad_request',400); + const raw=await readBody(request,clusterRequestLimit(sharedRuntime()),'too_large'); + let verified;try{verified=clusterIngress(sharedRuntime(),raw,Date.now());}catch{throw new Wire("unauthorized",401);} + const {actor,request:proof}=verified; + const root=fromBase64Url(request.headers.get('codeaf-identity')); + if(!root||b64(root)!==proof.identity_key)throw new Wire('unauthorized',401); + await this.#refuseGone(); + await this.#admitted.get(ipOf(request,this.env)); + const identity=await identityOf(request.headers.get('codeaf-identity')); + const tenant=this.#tenantOf(identity);this.#admit(tenant,actor); + this.host??=new HostedIdentity(this.ctx,this.env,tenant,this.legacyAdmission,at=>this.#arm(at)); + const reply=await this.host.serial(()=>this.host.api(raw,proof,actor)); + this.watchers=this.host.watchers;return reply; + } const route = matchRoute(request.method, url.pathname); const caller = await checkCert(signedHeaders(request)); const leave = route.frames ? this.#enterPut() : () => {}; @@ -83,7 +103,7 @@ export class IdentityDO extends DurableObject { await this.#admitted.get(ipOf(request, this.env)); const tenant = this.#tenantOf(who.identity); this.#admit(tenant, who.device); - this.host ??= new HostedIdentity(this.ctx,this.env,tenant,this.legacyAdmission); + this.host ??= new HostedIdentity(this.ctx,this.env,tenant,this.legacyAdmission,at=>this.#arm(at)); const reply=await this.host.dispatch({tenant,device:who.device,body,request,env:this.env},route); this.watchers=this.host.watchers;return reply; } finally { @@ -132,12 +152,12 @@ export class IdentityDO extends DurableObject { // request supplies the public root; an alarm cannot invent that key. this.watchers.closeAll(1012,'reconnecting');return; } - const tenant=this.#tenantOf(identity);this.host ??= new HostedIdentity(this.ctx,this.env,tenant,this.legacyAdmission); + const tenant=this.#tenantOf(identity);this.host ??= new HostedIdentity(this.ctx,this.env,tenant,this.legacyAdmission,at=>this.#arm(at)); const erase=await this.host.serial(async()=>{ await this.host.ensure(Uint8Array.from(atob(provision.config.signing_key),c=>c.charCodeAt(0))); this.watchers=this.host.watchers;this.watchers.expire();tenant.stats.flush(); - const raw=await this.host.barrierRecord({namespace:'state',key:'identity'}); - const record=this.host.project('identity',raw),at=record.rotation?.retire_at; + await this.host.maintenance(Date.now()); + const at=this.host.retireAt; if(at && at<=Date.now())return true; if(at)await this.#arm(at);return false; }); diff --git a/relay/hosted/src/raft/hosted.js b/relay/hosted/src/raft/hosted.js index a68b302d1e..1d35ee6299 100644 --- a/relay/hosted/src/raft/hosted.js +++ b/relay/hosted/src/raft/hosted.js @@ -1,3 +1,5 @@ +import { coordinationPolicy,CommittedHolds } from './liveness.js'; +import { clusterIngress } from './ingress.js'; import { frame, response, HostCodec } from './abi.js'; import { sharedRuntime } from './actor.js'; import { encodeCanonical, decodeCanonical } from './codec.js'; @@ -30,7 +32,9 @@ async function keypair(){ // gate prevents it ever becoming a second directory writer. All coordination // replies below depend on the members' actual durable application and quorum. export class HostedIdentity { - constructor(ctx,env,tenant,admission){ + constructor(ctx,env,tenant,admission,arm){ + this.armAlarm=arm??(async at=>{const old=await ctx.storage.getAlarm();if(old===null||atthis.ctx.waitUntil(this.ctx.storage.setAlarm(at))); this.tenant.watchers=this.watchers;this.ready=true; this.effects=new RaftEffectsFacade(this,this.manifest.config.cluster); + this.holds=new CommittedHolds(this,this.policy); + this.holds.opened(Date.now(),this.watchers.holdEvidence(Date.now()).some(socket=>socket.holds.length)); + const b=await this.barrier(),header=await this.members.get(b.node).hostView(b.index,'membership','registry'); + if(!header.raw){ + const anchors=this.config(this.manifest.config.anchors[0]).anchors; + requireOK((await this.anchorCommand({init_membership:{anchors,limit:this.policy.max_member_records}},Date.now())).value.code); + } + const root=await this.barrierRecord({namespace:'state',key:'identity'}); + this.retireAt=this.project('identity',root).rotation?.retire_at; + await this.wakeMaintenance(); } accept(id,e){if(!e)return;if(e.status)this.states.set(String(id),e.status);this.queue.push(...(e.send??[]).map(m=>({...m,from:String(id),to:String(m.to)})));this.reads.push(...(e.reads??[]).map(r=>({...r,node:String(id)})));} async input(id,tag,value){const result=await this.members.get(String(id)).hostInput(tag,value);this.accept(id,result);return result;} @@ -184,6 +198,100 @@ export class HostedIdentity { } jsonBytes(raw){return new Response(raw,{headers:{'content-type':'application/json'}});} proof(request,body){return {method:request.method,uri:signedUri(this.env,new URL(request.url)),identity_key:b64(fromBase64Url(request.headers.get('codeaf-identity'))),certificate:request.headers.get('codeaf-cert'),at:request.headers.get('codeaf-time'),signature:b64(fromBase64Url(request.headers.get('codeaf-sig'))),body:b64(body)};} + configuration(raw){const r=response(this.runtime.call(frame(35).blob(unb64(raw)).finish())),conf=decodeCanonical(dec.decode(r.blob()));r.end();return conf;} + async anchorCommand(change,now){ + const id=this.manifest.config.anchors[0],actor='anchor/'+id; + const saved=await this.barrierRecord({namespace:'sessions',key:segment(actor)+'/'+segment('maintenance')}); + const key=await crypto.subtle.importKey('pkcs8',unb64(this.provisioning.anchors[0]),'Ed25519',true,['sign']); + const pkcs8=new Uint8Array(await crypto.subtle.exportKey('pkcs8',key)); + const r=response(this.runtime.call(frame(34).str(this.manifest.config.cluster).u64(id).u64(now).blob(pkcs8.slice(-32)).blob(saved??new Uint8Array()).blob(data(change)).finish())); + const command=r.blob();r.end();return this.commit(command); + } + async wakeMaintenance(){this.asleep=false;await this.armAlarm(Date.now()+this.policy.tick_every);} + async maintenance(now=Date.now(),elapsed=Math.floor(performance.now())){ + if(this.asleep)return; + this.elapsed=Math.max(this.elapsed,elapsed-this.started); + // Alarm/recovery effects are retained until their exact batch is consumed. + for(const [id,member]of this.members){ + this.accept(id,await member.drainRecovery()); + const batch=await member.scheduled();if(!batch)continue; + this.accept(id,{send:batch.send.map(m=>({to:m.to,bytes:unb64(m.bytes)}))});await this.pump();await member.acknowledgeScheduled(batch.id); + } + if(this.holds.next!==null)await this.holds.renew(now,this.watchers.holdEvidence(now)); + // Scanning is an internal activity hint only. Delivery obtains its own + // fresh barrier and authority check before touching a public link request. + const leader=this.leader(); + if(!leader){ + // A missing leader still needs ordinary follower election ticks. An + // unavailable authority read cannot be a reason to cancel that clock. + for(const [id,member]of this.members){const effects=await member.schedule(this.elapsed,now,false);this.accept(id,effects);} + await this.pump();await this.armAlarm(now+this.policy.tick_every);return; + } + let scan=this.outboxScan??''; + const app=await this.members.get(leader).hostView(this.states.get(leader).applied,'outbox','',scan); + this.outboxScan=app.rows.at(-1)?.key??''; + if(app.rows.some(row=>{const value=decodeCanonical(dec.decode(row.data));return value.decision&&!value.outcome})){ + await this.effects.replay(code=>this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(code)),scan,now); + } + let plan; + for(const [id,member]of this.members){const effects=await member.schedule(this.elapsed,now,false);this.accept(id,effects);if(id===this.leader())plan=effects.plan;} + await this.pump(); + if(plan?.barrier){ + const id=crypto.randomUUID();this.reads=[];await this.input(this.leader(),15,id);await this.pump(); + if(!this.reads.some(r=>r.id===id))throw unavailable();this.reads=[]; + const effects=await this.members.get(this.leader()).schedule(this.elapsed,now,false);this.accept(this.leader(),effects);await this.pump();plan=effects.plan; + } + if(plan?.prepare){ + const id=this.leader(),s=this.states.get(id),view=await this.members.get(id).hostView(s.applied,'state','identity'); + // The IdleRead freshness is consumed at the SAME applied position. Read + // the anchor session locally so command construction does not wake it. + const saved=await this.members.get(id).hostView(s.applied,'sessions',segment('anchor/'+this.manifest.config.anchors[0])+'/'+segment('maintenance')); + const pkcs8=unb64(this.provisioning.anchors[0]); + const r=response(this.runtime.call(frame(34).str(this.manifest.config.cluster).u64(this.manifest.config.anchors[0]).u64(now).blob(pkcs8.slice(-32)).blob(saved.raw??new Uint8Array()).blob(data({prepare_quiescence:{barrier:String(s.applied),term:String(s.term),voters:this.configuration(view.app.conf).voters}})).finish())); + const raw=r.blob();r.end();requireOK((await this.commit(raw)).value.code); + await this.input(this.leader(),1);await this.pump(); + } + const id=this.leader(),s=this.states.get(id),row=await this.members.get(id).hostView(s.applied,'membership','quiescence'); + if(row.raw){ + const marker=decodeCanonical(dec.decode(row.raw)); + try{ + const acks=[];for(const voter of marker.voters)acks.push(await this.members.get(voter).sleepAck(marker.index,marker.term)); + let sleeping=true; + for(const [node,member]of this.members){const effects=await member.schedule(this.elapsed,now,false,acks);this.accept(node,effects);sleeping&&=effects.plan.sleeping;} + await this.pump();this.asleep=sleeping; + }catch{this.asleep=false;} + } + if(!this.asleep)await this.armAlarm(now+this.policy.tick_every); + if(this.holds.next!==null)await this.armAlarm(this.holds.next); + } + async api(raw,q,actor) { + await this.ensure(unb64(q.identity_key)); + await this.wakeMaintenance(); + if(q.cluster!==this.manifest.config.cluster||q.identity_key!==this.manifest.config.signing_key)throw new Wire('unauthorized',401); + requireOK((await this.commitEffect({device_seen:{device:actor}},Date.now())).code); + const b=await this.barrier(),member=this.members.get(b.node); + const root=await member.hostView(b.index,'state','identity'); + clusterIngress(this.runtime,raw,root.app.now); + if(q.action==='submit'){ + const result=await this.commit(data(q.command)); + if(result.value.code==='ok'){ + const b=await this.barrier(),row=await this.members.get(b.node).hostView(b.index,'state','identity'); + const identity=this.project('identity',row.raw);this.retireAt=identity.rotation?.retire_at; + this.watchers.broadcast(row.app.version); + if(identity.rotation)this.watchers.closeAll(CLOSE_ROTATED,'rotated'); + if(this.retireAt)await this.armAlarm(this.retireAt); + if(q.command.change.revoke)this.watchers.closeDevice(q.command.change.revoke.device,CLOSE_REVOKED,'revoked'); + } + return this.jsonBytes(encodeCanonical({result:result.value})); + } + const selected=decodeCanonical(dec.decode(await member.hostSelected(raw,b.index))); + requireOK(selected.code); + const out={};for(const key of ['view','grant','assignment'])if(selected[key]!==undefined)out[key]=selected[key]; + // Current-assignment matches the native door, which exposes the selected + // grant through the cell door alone. + if(q.action==='assignment')delete out.grant; + return this.jsonBytes(encodeCanonical(out)); + } async dispatch(c,route){ if(!new URL(c.request.url).pathname.startsWith('/v1/store/'))return this.serial(()=>this.serve(c,route)); const cert=c.request.headers.get('codeaf-cert'); @@ -199,17 +307,22 @@ export class HostedIdentity { async serve(c,route){ const {request,device,body}=c,url=new URL(request.url),path=url.pathname,cert=request.headers.get('codeaf-cert'); await this.ensure(fromBase64Url(request.headers.get('codeaf-identity'))); + await this.wakeMaintenance(); if(path==='/v1/dir/watch'){ if(request.headers.get('upgrade')?.toLowerCase()!=='websocket')throw new Wire('upgrade_required',426); const holds=parseHolds(url.searchParams.getAll('hold'),this.tenant.limits.maxHolds),beat=parseBeat(url.searchParams.getAll('beat')); const a=await this.authorize(device,cert,true); - return this.watchers.accept(device,Number(a.app.version),Date.now(),{holds,beat,events:url.searchParams.get('events')==='1',receipts:url.searchParams.get('receipts')==='1',certificate:cert}); + const reply=this.watchers.accept(device,Number(a.app.version),Date.now(),{holds,beat,events:url.searchParams.get('events')==='1',receipts:url.searchParams.get('receipts')==='1',certificate:cert}); + this.holds.opened(Date.now(),holds.length>0); + if(holds.length)await this.holds.renew(Date.now(),this.watchers.holdEvidence(Date.now())); + if(this.holds.next!==null)await this.armAlarm(this.holds.next);return reply; } const join=/^\/v1\/dir\/requests\/([^/]+)\/(approve|deny)$/.exec(path); if(join){await this.authorize(device,cert,true);const code=decodeURIComponent(join[1]).trim().toLowerCase();const box=this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(code));await this.effects.admit(box,code,Date.now());} let raw;try{raw=v1Command(this.runtime,this.manifest.config.cluster,Date.now(),this.proof(request,body));}catch{throw new Wire('bad_request',400);} const result=await this.commit(raw);requireOK(result.value.code); const b=await this.barrier(),snapshot=await this.members.get(b.node).hostView(b.index,'state','identity'),identity=this.project('identity',snapshot.raw),version=snapshot.app.version; + this.retireAt=identity.rotation?.retire_at; this.watchers.broadcast(version); if(join){const code=decodeURIComponent(join[1]).trim().toLowerCase();await this.effects.deliver(this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(code)),code,Date.now());unwrap(await this.env.LINK_GATE.get(this.env.LINK_GATE.idFromName('gate')).decided(code));} if(request.method==='GET'){ diff --git a/relay/hosted/src/raft/ingress.js b/relay/hosted/src/raft/ingress.js new file mode 100644 index 0000000000..624ad37c01 --- /dev/null +++ b/relay/hosted/src/raft/ingress.js @@ -0,0 +1,16 @@ +import { frame, response } from './abi.js'; +import { decodeCanonical } from './codec.js'; +import { coordinationPolicy } from './liveness.js'; + +let requestLimit; +// The front and identity doors use Go's actual protocol bound, so a schema +// limit cannot drift between pre-body admission and the committed verifier. +export function clusterRequestLimit(runtime){return requestLimit??=coordinationPolicy(runtime).max_message_bytes;} + +// The same closed verifier authenticates native and hosted ingress. Repeating +// it at committed time prevents a delayed request reviving expired authority. +export function clusterIngress(runtime, raw, now) { + const r=response(runtime.call(frame(33).blob(raw).u64(now).finish())); + const actor=r.str();r.end(); + return {actor,request:decodeCanonical(new TextDecoder().decode(raw))}; +} diff --git a/relay/hosted/src/raft/liveness.js b/relay/hosted/src/raft/liveness.js index 71ff710ecc..ba4f8c31bd 100644 --- a/relay/hosted/src/raft/liveness.js +++ b/relay/hosted/src/raft/liveness.js @@ -44,7 +44,7 @@ export class CommittedHolds { } } catch {socket.failed?.();continue;} if(!items.length)continue; let result; - try { result=await this.host.commitEffect({renew_holds:{device:socket.device,seen_at:socket.seenAt,items}},now); } + try { result=await this.host.commitEffect({renew_holds:{device:socket.device,seen_at:socket.seenAt,items,...(socket.certificate?{certificate:socket.certificate}:{})}},now); } catch {socket.failed?.();continue;} if(result.code!=='ok'||!result.index||result.index==='0'){socket.failed?.();continue;} const holds=(result.items??[]).filter(item=>item.code==='ok'&&item.view).map(item=>({cell:item.cell,fence:item.view.cell.lease.fence,index:result.index,at:result.now,until:item.view.cell.lease.expires})); diff --git a/relay/hosted/src/raft/member.js b/relay/hosted/src/raft/member.js index 2c5692c684..056d2c239c 100644 --- a/relay/hosted/src/raft/member.js +++ b/relay/hosted/src/raft/member.js @@ -1,3 +1,5 @@ +import { clusterIngress } from './ingress.js'; +import { workerCell,workerAssignment } from './worker.js'; import { DurableObject } from 'cloudflare:workers'; import { MemberActor, sharedRuntime } from './actor.js'; import { MemberInbox, INBOX_BYTES } from './inbox.js'; @@ -65,7 +67,7 @@ export class RaftMemberDO extends DurableObject { await this.ctx.storage.delete('legacy-pending');await this.ctx.storage.sync(); } async hostInput(tag,value) { - if(![0,1,3,4,10,12,16].includes(tag))throw new Error('raft member: unsupported coordinator input'); + if(![0,1,3,4,10,12,15,16].includes(tag))throw new Error('raft member: unsupported coordinator input'); await this.load(); if(tag===3){ const raw=value.slice();return this.actor.serial(async()=>{ @@ -88,6 +90,19 @@ export class RaftMemberDO extends DurableObject { return {app,rows}; }); } + // The coordinator's fresh ReadIndex must have applied before a scoped + // read runs. Authority/time and selected rows share this member queue. + async hostSelected(raw,index) { + await this.load();return this.actor.serial(()=>{ + if(this.actor.stopped||this.actor.store.stopped||this.actor.status.applied{ if(this.actor.stopped||this.actor.store.stopped)throw new Error('raft member: stopped'); diff --git a/relay/hosted/src/watch.js b/relay/hosted/src/watch.js index 6803655c77..0eb087bf6d 100644 --- a/relay/hosted/src/watch.js +++ b/relay/hosted/src/watch.js @@ -82,11 +82,11 @@ export class Watchers { * it is also the viewer that makes the presence sweep worth arming (contract 21.12.4). `beat` is the ping period the * client declared, from which the socket's window follows. */ - accept(device, version, now, { holds, beat, events = false, receipts = false }) { + accept(device, version, now, { holds, beat, events = false, receipts = false, certificate = undefined }) { if (this.ctx.getWebSockets().length >= this.cap) throw new Wire('too_many_watchers', 429); const { 0: client, 1: server } = new WebSocketPair(); this.ctx.acceptWebSocket(server, [device]); - server.serializeAttachment({ ...FRESH, at: now, holds, events, beat, receipts }); + server.serializeAttachment({ ...FRESH, at: now, holds, events, beat, receipts, ...(certificate?{certificate}:{}) }); server.send(frame(version)); if (events) for (const text of this.liveness.snapshot(device)) server.send(text); this.liveness.opened(device); @@ -114,7 +114,7 @@ export class Watchers { let seenAt=0; try {seenAt=Math.max(att.at,this.ctx.getWebSocketAutoResponseTimestamp(ws)?.getTime()??0);} catch {} return {device:this.ctx.getTags(ws)[0],holds:att.lapsed ? [] : att.holds.map(([cell,fence])=>({cell,fence:String(fence)})), - seenAt, + seenAt,certificate:att.certificate, send:text=>{if(att.receipts)attempt(()=>ws.send(text));},failed:()=>attempt(()=>ws.close(1013,'retry'))}; }); } diff --git a/relay/hosted/test/raft/client-wiring.test.js b/relay/hosted/test/raft/client-wiring.test.js new file mode 100644 index 0000000000..134d822c8c --- /dev/null +++ b/relay/hosted/test/raft/client-wiring.test.js @@ -0,0 +1,70 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { generateKeyPairSync,createHash,sign } from 'node:crypto'; +import { encodeCanonical } from '../../src/raft/codec.js'; +import { hostedFixture } from './hosted-fixture.js'; + +const hash=b=>createHash('sha256').update(b).digest('hex'); +const encoded=v=>Buffer.from(encodeCanonical(v)); +// R16/R18 use public ingress and original signatures, never direct actor apply. +// The same scoped reads recheck expiry/incarnation after a new quorum barrier. +test('R16 R18 production hosted scoped ingress preserves grants and conditional claims',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();assert.equal((await h.call(a,'POST','/v1/dir/cells/chat',{head:'sealed',class:'work',size:0,keys:{}})).status,200); + assert.equal((await h.call(a,'POST','/v1/dir/cells/chat/release',{fence:1})).status,204); + const cluster=JSON.parse((await h.control({op:'source'})).gate).config.cluster; + const key=generateKeyPairSync('ed25519'),pub=key.publicKey.export({format:'der',type:'spki'}).subarray(-32),worker='dev_'+hash(pub).slice(0,32); + const cert=h.delegate({cluster,key:pub.toString('base64'),made:Date.now()-1000,expires:Date.now()+600000,cells:['chat'],learner:false}); + const w={key,device:worker};let incarnation='0';const sequence={}; + const command=(c,change)=>{ + const n=sequence[c.device]=(sequence[c.device]??0)+1; + const cmd={schema:1,cluster,actor:c.device,session:'integration',sequence:String(n),...(n>1?{ack:String(n-1)}:{}),now:Date.now(),change,request_digest:'',signature:null}; + if(c===w&&incarnation!=='0')cmd.worker_incarnation=incarnation; + if(c===a)cmd.certificate=Buffer.from(JSON.stringify(a.cert)).toString('base64url'); + cmd.request_digest=hash(encoded({...(cmd.worker_incarnation?{worker_incarnation:cmd.worker_incarnation}:{}),cluster,actor:cmd.actor,session:cmd.session,sequence:cmd.sequence,change})); + cmd.signature=sign(null,Buffer.concat([Buffer.from('codeaf.directory.command.v1\0'),encoded(cmd)]),c.key.privateKey).toString('base64');return cmd; + }; + const api=(c,q)=>h.api(c,{cluster,incarnation:c===w?incarnation:'0',...(c===w?{worker:cert}:{}),...q}); + const submit=async(c,change,want='ok')=>{const r=await api(c,{action:'submit',command:command(c,change)});assert.equal(r.status,200,JSON.stringify(r));assert.equal(r.body.result.code,want);return r.body.result;}; + await submit(a,{enroll_worker:{certificate:cert,grants:{chat:'gYKD'}}}); + await submit(w,{register_worker:{previous:'0',incarnation:'1',caps:{os:'',arch:'',sandbox:null,container:null,gpu:null,cow:''},sealed_labels:null}});incarnation='1'; + const selected=await api(w,{action:'cell',cell:'chat'});assert.equal(selected.status,200,JSON.stringify(selected));assert.equal(selected.body.view.cell.head,'sealed');assert.equal(selected.body.grant,'gYKD'); + assert.notEqual((await api(w,{action:'cell',cell:'outside'})).status,200); + const beat={worker,incarnation:'1',sequence:'1',seen_at:Date.now(),work:null,signature:null};beat.signature=sign(null,Buffer.from('codeaf/worker-beat/v1\0'+encodeCanonical({cluster,beat})),key.privateKey).toString('base64'); + await submit(w,{worker_beat_batch:{items:[beat]}}); + const spec={id:'opaque',revision:'1',cell:'chat',parent_cell:'',expect:{device:a.device,fence:'1'},target:worker,incarnation:'1',expires:Date.now()+60000,sealed_payload:'kQ=='}; + await submit(a,{assign:spec}); + const ref={id:'opaque',revision:'1',cell:'chat',target:worker,incarnation:'1'}; + const pending=await api(w,{action:'assignment',assignment:ref});assert.equal(pending.status,200,JSON.stringify(pending));assert.equal(pending.body.assignment.status,'pending'); + const claim=command(w,{claim_assignment:{ref,expect:spec.expect}}); + const injected={...claim,certificate:'private-field',signature:null}; + injected.signature=sign(null,Buffer.concat([Buffer.from('codeaf.directory.command.v1\0'),encoded(injected)]),key.privateKey).toString('base64'); + assert.notEqual((await api(w,{action:'submit',command:injected})).status,200,'R15 invalid certificate reached the log'); + for(let n=0;n<2;n++){const r=await api(w,{action:'submit',command:claim});assert.equal(r.status,200);assert.equal(r.body.result.view.cell.lease.fence,'2');} + assert.equal((await h.call(a,'POST','/v1/dir/cells/chat/acquire',{force:true,expect:{device:worker,fence:2}})).status,200); + const current=await api(w,{action:'assignment',assignment:ref});assert.equal(current.status,200);assert.equal(current.body.assignment.fence,'2');assert.equal(current.body.view.cell.lease.fence,'3'); + await submit(w,{rotate:{op:'freeze'}},'unauthorized'); + incarnation='2';assert.notEqual((await api(w,{action:'cell',cell:'chat'})).status,200);incarnation='1'; + await submit(a,{drain_worker:{worker,incarnation:'1'}}); + await h.start();assert.equal((await api(w,{action:'cell',cell:'chat'})).status,200,'restart lost selected grant'); + assert.equal((await h.call(a,'POST','/v1/dir/devices/'+worker+'/revoke')).status,204); + assert.notEqual((await api(w,{action:'cell',cell:'chat'})).status,200); + const delayed=await api(w,{action:'submit',command:claim});assert.equal(delayed.status,200);assert.equal(delayed.body.result.code,'revoked'); + }finally{await h.close();} +}); + +// R23/R26/R27 advance ten idle minutes without sleeping the laptop. The fixture +// clock driver is internal; every counted member call is production work. +test('R23 R26 R27 production hosted ten-minute idle has no recurring member work',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + // R26 recovers the same durable identity before starting the idle clock. + await h.start(); + const measured=await h.control({op:'idle-schedule'}); + assert.equal(measured.sleeping,true,JSON.stringify(measured)); + assert.equal(measured.idleMilliseconds,600000);assert.equal(measured.recurringMemberCalls,0);assert.equal(measured.appliedBefore,measured.appliedAfter); + console.log('R23 R27 ten-minute idle',JSON.stringify(measured)); + }finally{await h.close();} +}); diff --git a/relay/hosted/test/raft/hosted-fixture.js b/relay/hosted/test/raft/hosted-fixture.js index 8c775c7f36..2e1137283e 100644 --- a/relay/hosted/test/raft/hosted-fixture.js +++ b/relay/hosted/test/raft/hosted-fixture.js @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { prefixOf } from '../../src/base.js'; +import { encodeCanonical } from '../../src/raft/codec.js'; import { Miniflare, convertV4MiniflareOptions } from 'miniflare'; import { build } from 'esbuild'; import { mkdtemp, copyFile, rm, readFile } from 'node:fs/promises'; @@ -43,5 +44,16 @@ export async function hostedFixture({fixture=false,base='',go=false}={}) { }; const call=async(...args)=>{const response=await request(...args),text=await response.text();return {status:response.status,body:text?JSON.parse(text):null};}; const control=async value=>{assert(fixture);const r=await mf.dispatchFetch('http://local/fixture',{method:'POST',body:JSON.stringify({identity,...value})});assert.equal(r.status,200,await r.clone().text());return r.json();}; - return {identity,client,dir,start,request,call,control,get mf(){return mf;},async close(){if(mf)await mf.dispose();await rm(dir,{recursive:true,force:true});}}; + const api=async(c,q)=>{ + q={identity_key:rootKey.toString('base64'),...q,at:q.at??Date.now(),signature:null}; + if(!q.worker)q.certificate=Buffer.from(JSON.stringify(c.cert)).toString('base64url'); + q.signature=sign(null,Buffer.from('codeaf/cluster-api/1\n'+encodeCanonical(q)),c.key.privateKey).toString('base64'); + const r=await mf.dispatchFetch('http://local'+prefix+'/v2/cluster/api',{method:'POST',headers:{'Codeaf-Identity':rootKey.toString('base64url')},body:encodeCanonical(q)}); + const text=await r.text();return {status:r.status,body:text?JSON.parse(text):null}; + }; + const delegate=certificate=>{ + certificate={v:1,identity,...certificate,signature:null}; + certificate.signature=sign(null,Buffer.from('codeaf/worker-cert/v1\0'+encodeCanonical(certificate)),root.privateKey).toString('base64');return certificate; + }; + return {identity,client,dir,start,request,call,control,api,delegate,get mf(){return mf;},async close(){if(mf)await mf.dispose();await rm(dir,{recursive:true,force:true});}}; } diff --git a/relay/hosted/test/raft/hosted-go.test.js b/relay/hosted/test/raft/hosted-go.test.js index 14cf2c244c..297ebd8602 100644 --- a/relay/hosted/test/raft/hosted-go.test.js +++ b/relay/hosted/test/raft/hosted-go.test.js @@ -3,6 +3,20 @@ import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; import { hostedFixture } from './hosted-fixture.js'; +const root=new URL('../../../../',import.meta.url); +const footprint=new URL('../../../../scripts/raft-proof-footprint.sh',import.meta.url).pathname; +async function prepare(args){ + const child=spawn('sh',[footprint,...args],{cwd:root,stdio:['ignore','pipe','pipe']}); + let output='';child.stdout.on('data',b=>{output+=b});child.stderr.on('data',b=>{output+=b}); + const code=await new Promise((resolve,reject)=>{child.once('error',reject);child.once('exit',resolve)}); + console.log(output);assert.equal(code,0,output); +} +// R27: compilation can leave a disk backlog after initial admission. Compile +// before starting workerd and admit the actual runtime separately, so package +// linking cannot consume the journey's ordinary network request deadlines. +await prepare(['env','GOMEMLIMIT=192MiB','go','test','-p=1','./internal/raftdirectory','-run','^$','-count=1']); +await prepare(['true']); + // H4/R1/R2/R23/R25/R29 run ordinary Go Sync clients and the unchanged real // handoff assertions through the public Worker, with two disposable homes. test('H4 R1 R2 R23 R25 R29 two Go homes continue, take back, quit, switch off and revoke through workerd',{timeout:180000},async()=>{ diff --git a/relay/hosted/test/raft/hosted-worker.js b/relay/hosted/test/raft/hosted-worker.js index 247edd21ac..de80434936 100644 --- a/relay/hosted/test/raft/hosted-worker.js +++ b/relay/hosted/test/raft/hosted-worker.js @@ -50,6 +50,19 @@ export class IdentityDO extends Source { this.host.queue.push({from,to,bytes:raw});await this.host.pump();return this.host.members.size; }); } + async idleSchedule(){ + // R26 allows eviction between the HTTP request and this fixture RPC. + // Restore through the production alarm before driving its virtual clock. + if(!this.host)await super.alarm(); + return this.host.serial(async()=>{ + const base=Date.now(); + for(let n=1;n<=20;n++)await this.host.maintenance(base+n*this.host.policy.tick_every,this.host.started+n*this.host.policy.tick_every); + const id=this.host.leader(),before=this.host.states.get(id).applied; + const sleeping=this.host.asleep; + const turns=measurements.turns; + for(let n=21;n<=140;n++)await this.host.maintenance(base+n*this.host.policy.tick_every,this.host.started+n*this.host.policy.tick_every); + return {sleeping,appliedBefore:String(before),appliedAfter:String(this.host.states.get(id).applied),recurringMemberCalls:measurements.turns-turns,idleMilliseconds:120*this.host.policy.tick_every}; + });} inspect() { return {gate:this.ctx.storage.sql.exec('SELECT doc FROM legacy_gate WHERE one=1').toArray()[0]?.doc, originals:this.ctx.storage.sql.exec('SELECT kind,id,doc FROM dir ORDER BY kind,id').toArray()}; @@ -88,6 +101,7 @@ export default { if(op==='release-upload'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).releaseHeldUpload();return Response.json(true);} if(op==='retry-without-quorum')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).retryWithoutQuorum()); if(op==='stale-snapshot')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).replayStale()); + if(op==='idle-schedule')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).idleSchedule()); if(op==='source')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).inspect()); if(op==='seed-transport-tail'){await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).seedTransportTail();return Response.json(true);} if(op==='retained-control')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).retainedControl()); From f7b20aecfabf66bfcb012fad54aab8fb2f84cb36 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 10:38:10 -0400 Subject: [PATCH 051/103] Keep the missing personal-member journey visible before release. The production personal learner transport and leadership door remain incomplete. Retain an end-to-end failing acceptance test and describe the actual limit in the manual so adapter proofs cannot be mistaken for a finished journey. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../continuing-a-chat-on-another-computer.md | 42 +++++++------------ internal/syncsetup/member_production_test.go | 38 +++++++++++++++++ 2 files changed, 52 insertions(+), 28 deletions(-) create mode 100644 internal/syncsetup/member_production_test.go diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index bb5886ed18..f605b74b92 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -256,38 +256,17 @@ A relay directory upgrade can briefly pause pairing and chat moves. Retry after ## Joined computers, learner capacity and an outdated build -Joining the relay's coordination group does not make a computer a voter. Its -learner role does not require other laptops to stay awake, and a slow learner -cannot hold back relay writes or keep old coordination logs forever. A learner -is ready only after it has applied the committed join barrier. Removing one -retires its node ID permanently; joining again allocates a fresh ID. - -The group admits at most 32 live learners. Retired enrollment records are kept, -with a separate operator capacity limit; reaching either limit refuses new -enrollment instead of deleting saved chats or old records. The capacity sentence -is `this cluster has reached its enrollment limit; remove a joined computer or ask the relay operator for capacity`. -An unsupported protocol says `this codeaf build must be updated before it can join this cluster; your saved chats are still here`. -Joining alone does not request leadership. Use `codeaf cluster lead` after this computer has joined with persistent storage. A worker setup command is not available yet. +Paired computers currently use the relay as authenticated directory clients. The production client does not yet activate its own persistent coordination member, so pairing alone does not make this computer a learner or a voter. The relay keeps three persistent voters and does not need either of your paired laptops to stay awake. -## Which computer coordinates: cluster status and cluster lead - -`codeaf cluster status` asks the relay for the current coordinator and voting computers. `codeaf cluster lead` requests leadership for this computer after it has joined with persistent storage. The target must catch up and exchange recent traffic; a pending request is not success. A successful line names the computer, term, revision and read barrier. Leadership moves coordination metadata; it does not move chat ownership or download chat content. Continue a chat through the usual continue door. +The membership protocol admits at most 32 live learners and retains retired records instead of reusing their authority. A protocol mismatch says `this codeaf build must be updated before it can join this cluster; your saved chats are still here`. Its capacity refusal is `this cluster has reached its enrollment limit; remove a joined computer or ask the relay operator for capacity`. Personal-member activation and a worker setup command are not available through the production client yet. -The default has three persistent relay voters. Paired computers join without votes. To lead, one caught-up personal computer replaces one anchor through a majority-approved change, retaining three voters; that anchor stays caught up without voting. When the personal leader disappears, the two remaining anchor voters can elect a replacement and restore the third anchor. The returning computer does not automatically lead again. Request leadership explicitly. - -Three voters tolerate one unavailable member. Losing the personal leader and one of its two voting anchors pauses coordination. Saved chats remain. The refusal is `this cluster cannot coordinate right now; reconnect enough of its voting computers and try again; your saved chats are still here`. Network partitions and scheduling stalls can delay elections beyond the healthy fallback target. A relay without this door says `this relay does not offer cluster leadership yet; your saved chats are still here`. Persistent member/client assembly is required; a paired API client alone cannot lead. It says `this computer must join the cluster with persistent storage before it can lead`. If a reply is lost, running `codeaf cluster lead` again resumes the saved request rather than silently choosing a new configuration. +## Which computer coordinates: cluster status and cluster lead -## How long does a replacement leader take after my laptop sleeps? +`codeaf cluster status` and `codeaf cluster lead` exist, but the production personal-member door is not connected yet. The relay currently says `this relay does not offer cluster leadership yet; your saved chats are still here`. A paired API client cannot lead. A relay that provides status without a joined member says `this computer must join the cluster with persistent storage before it can lead`. -With the two remaining relay voters reachable, codeaf aims to accept a new -conditional operation within about 65 seconds. Relay anchors ask to lead in -ordered randomized windows: the first at 30–35 seconds, the second at 40–45, -and the third at 50–55. A promoted personal computer uses 60–65 seconds. -Network partitions or scheduling stalls can take longer. If the personal leader -and either remaining relay voter are both unavailable before the third anchor -is restored, writes wait for a majority to return. Returning computers stay -learners; leadership returns only when you request it again. +Do not treat pairing or a saved leadership request as a successful transfer. Success requires the selected computer to be elected, commit its current term and pass a fresh majority read; its receipt names the computer, term, revision and read barrier. Retrying `codeaf cluster lead` preserves an already saved signed request. Continue chats through the ordinary continue door. +Three voters tolerate one unavailable member. Losing a majority pauses coordination; saved chats remain. The refusal is `this cluster cannot coordinate right now; reconnect enough of its voting computers and try again; your saved chats are still here`. ## Does an open connection keep my chat running on this computer? @@ -307,7 +286,7 @@ Worker setup and assignment execution are not available as a terminal command ye A worker restart registers a newer process number. Delayed requests and heartbeats from its previous process cannot refresh availability or save work. A drained worker cannot receive new claims; already held work can still save and let go. Restarting does not clear draining. Expired and removed enrollment records remain retained; reaching their capacity limit refuses new enrollment instead of deleting records or saved chats. -A scoped API worker does not join the coordination log. Joining as a learner requires explicit consent in its certificate: the full log reveals identity-wide routing and ownership metadata, while chat content, names and task instructions remain encrypted. Use scoped API access for an untrusted third-party worker. A delegated learner gets no vote and cannot become leader. Existing paired personal computers keep their shared-root trust model. +A scoped API worker does not join the coordination log. Joining as a learner requires explicit consent in its certificate: the full log reveals identity-wide routing and ownership metadata, while chat content, names and task instructions remain encrypted. Use scoped API access for an untrusted third-party worker. A delegated learner gets no vote and cannot become leader. The production client does not yet activate delegated learners; selected API access is available without joining the log. Existing paired personal computers keep their shared-root trust model. ## How does a worker claim an assignment or run parallel tasks? @@ -363,3 +342,10 @@ listing, publishing, releasing and continuing a chat they observed. A very old client that blindly forces a live chat receives `another device continued this chat first`. Quit on the holder first, or update that client before continuing a live chat. Watch sockets show presence; older clients still send their normal heartbeats. + + +## Scoped worker requests and committed watch receipts + +The relay's selected-worker API accepts an original signed command or a signed request for one chat or assignment. A delegated worker receives only its selected sealed grant. Each read commits current time, waits for a fresh majority response and checks its current scope and process number. A retained claim alone cannot authorize execution after another computer takes the chat. A worker cannot use this API to pair computers, rotate the identity or promote itself. + +Native and hosted watch connections carry committed receipts to clients that ask for them. Each receipt names the exact chat and ownership number, durable position and hold-until time. Older clients receive their original version and presence frames and keep ordinary heartbeats. A ping without a receipt cannot suppress those heartbeats. A released chat cannot be revived by a delayed ping. Idle presence pings do not start a recurring directory request loop. diff --git a/internal/syncsetup/member_production_test.go b/internal/syncsetup/member_production_test.go new file mode 100644 index 0000000000..1764eceee6 --- /dev/null +++ b/internal/syncsetup/member_production_test.go @@ -0,0 +1,38 @@ +package syncsetup + +import ( + "context" + "net/http/httptest" + "testing" + + "github.com/Agent-Field/codeaf/internal/relayserve" +) + +// R6/R9/R23 require the REAL codeaf client and production relay to activate a +// durable personal learner before status/lead can report it. An internal core +// fixture or a hosted actor standing in for this computer cannot prove this. +func TestR6R9R23PersonalMemberProductionLifecycle(t *testing.T) { + service := relayserve.New(relayserve.Config{Store: t.TempDir()}) + defer service.Close() + server := httptest.NewServer(service.Handler) + defer server.Close() + home := machine(t, server.URL) + client, ok, err := Open(home) + if err != nil || !ok { + t.Fatal(err) + } + status, err := client.ClusterStatus(context.Background()) + if err != nil { + t.Fatalf("R6/R9 personal activation/status door: %v", err) + } + if status.Self == 0 || len(status.Configuration.Voters) != 3 || len(status.Configuration.Learners) == 0 { + t.Fatalf("R6/R9 personal home did not join as a learner: %+v", status) + } + receipt, err := client.LeadCluster(context.Background(), 0) + if err != nil { + t.Fatalf("R6 actual personal leadership: %v", err) + } + if receipt.Node != status.Self || receipt.Term == 0 || receipt.Barrier == 0 { + t.Fatalf("R6 pending intent presented as leadership: %+v", receipt) + } +} From ffbb16cd6189de1be9070ae177c861568f3b0cf1 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 10:38:10 -0400 Subject: [PATCH 052/103] Wait for owned journey engines to exit before removing their homes. The keyless journey can finish its assertions while an engine daemon still writes the temporary store. Wait for the same owned processes after killing them so cleanup records real process termination before TempDir removal. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/e2e/pairjourney_e2e_test.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/internal/e2e/pairjourney_e2e_test.go b/internal/e2e/pairjourney_e2e_test.go index caeb5c5f38..b7a1101024 100644 --- a/internal/e2e/pairjourney_e2e_test.go +++ b/internal/e2e/pairjourney_e2e_test.go @@ -229,9 +229,22 @@ func runPairJourney(t *testing.T, key, mode string, online bool) { // it also ends the fresh-install probe's. testRoot := filepath.Dir(filepath.Dir(homeA)) t.Cleanup(func() { - for _, pid := range engineDaemonsUnder(testRoot) { + // C28: signaling an owned engine does not wait for its last filesystem + // operation to finish. Wait for exit before TempDir cleanup, + // or a successful journey races a daemon recreating its shared store. + pids := engineDaemonsUnder(testRoot) + for _, pid := range pids { _ = syscall.Kill(pid, syscall.SIGKILL) } + deadline := time.Now().Add(10 * time.Second) + for _, pid := range pids { + for !terminalProcessExited(pid) && time.Now().Before(deadline) { + time.Sleep(20 * time.Millisecond) + } + if !terminalProcessExited(pid) { + t.Errorf("the journey engine process %d did not exit before home cleanup", pid) + } + } }) if err := os.WriteFile(filepath.Join(wsB, "Makefile"), []byte("test:\n\t@echo 3 passed\n"), 0o644); err != nil { t.Fatal(err) From 8e83dd7336f34a19242f1ce4eff1cf8e38e855f4 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 12:23:19 -0400 Subject: [PATCH 053/103] Expose committed native cluster status while refusing unfinished leadership. A signed personal status request must cross a fresh majority barrier and current revocation checks. Report only committed membership and refuse leadership before saving or proposing an intent that the production host cannot complete. Keep the missing learner lifecycle visible in its unchanged acceptance test. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterctl/control.go | 21 +++- .../continuing-a-chat-on-another-computer.md | 4 +- internal/raftdirectory/control.go | 85 ++++++++++++++ internal/relayserve/control.go | 46 ++++++++ internal/relayserve/control_test.go | 111 ++++++++++++++++++ internal/relayserve/service.go | 2 + internal/syncsetup/cluster.go | 3 + internal/syncsetup/member_control_test.go | 80 +++++++++++++ 8 files changed, 345 insertions(+), 7 deletions(-) create mode 100644 internal/raftdirectory/control.go create mode 100644 internal/relayserve/control.go create mode 100644 internal/relayserve/control_test.go create mode 100644 internal/syncsetup/member_control_test.go diff --git a/internal/clusterctl/control.go b/internal/clusterctl/control.go index eba51f127f..3491d1c476 100644 --- a/internal/clusterctl/control.go +++ b/internal/clusterctl/control.go @@ -18,6 +18,10 @@ const Path = "/v2/cluster/control" var ErrUnavailable = errors.New(cs.LeadershipUnavailable) +// ErrUnsupported distinguishes an absent leadership capability from a lost +// quorum. Clients can refuse before saving an intent the relay cannot finish. +var ErrUnsupported = errors.New("this relay does not offer cluster leadership yet; your saved chats are still here") + type CommittedRefusal struct { Code cs.Code `json:"code"` Sequence uint64 `json:"sequence"` @@ -28,11 +32,12 @@ func (e CommittedRefusal) Error() string { } type Status struct { - Cluster string `json:"cluster"` - Self uint64 `json:"self"` - Leader uint64 `json:"leader"` - Membership cs.Membership `json:"membership"` - Configuration cs.Configuration `json:"configuration"` + LeadershipUnavailable bool `json:"leadership_unavailable,omitempty"` + Cluster string `json:"cluster"` + Self uint64 `json:"self"` + Leader uint64 `json:"leader"` + Membership cs.Membership `json:"membership"` + Configuration cs.Configuration `json:"configuration"` } // Backend must authorize against committed state after a fresh quorum read. @@ -83,6 +88,10 @@ func Handler(auth wireauth.Authenticate, open func(string) (Backend, error)) htt result, err = backend.Lead(r.Context(), c) } if err != nil { + if errors.Is(err, ErrUnsupported) { + http.Error(w, ErrUnsupported.Error(), http.StatusNotFound) + return + } var refused CommittedRefusal if errors.As(err, &refused) { b, _ := cs.Encode(refused) @@ -147,7 +156,7 @@ func (c Client) request(ctx context.Context, method string, body []byte, out any } if res.StatusCode != 200 { if res.StatusCode == 404 { - return errors.New("this relay does not offer cluster leadership yet; your saved chats are still here") + return ErrUnsupported } return ErrUnavailable } diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index f605b74b92..b6ff9f1791 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -262,7 +262,9 @@ The membership protocol admits at most 32 live learners and retains retired reco ## Which computer coordinates: cluster status and cluster lead -`codeaf cluster status` and `codeaf cluster lead` exist, but the production personal-member door is not connected yet. The relay currently says `this relay does not offer cluster leadership yet; your saved chats are still here`. A paired API client cannot lead. A relay that provides status without a joined member says `this computer must join the cluster with persistent storage before it can lead`. +`codeaf cluster status` on a self-hosted `codeaf relay` with storage reports the committed voters and current leader after a fresh majority read. Personal-member activation is not connected yet: pairing alone does not give this computer a node in that list. `codeaf cluster lead` on an unjoined computer says `this computer must join the cluster with persistent storage before it can lead`. + +Leadership is not offered yet on any relay. A direct leadership request to the self-hosted relay is refused without changing membership. The hosted relay has no cluster control door and says `this relay does not offer cluster leadership yet; your saved chats are still here`. Ordinary chat sync remains available. Do not treat pairing or a saved leadership request as a successful transfer. Success requires the selected computer to be elected, commit its current term and pass a fresh majority read; its receipt names the computer, term, revision and read barrier. Retrying `codeaf cluster lead` preserves an already saved signed request. Continue chats through the ordinary continue door. diff --git a/internal/raftdirectory/control.go b/internal/raftdirectory/control.go new file mode 100644 index 0000000000..097d25b2aa --- /dev/null +++ b/internal/raftdirectory/control.go @@ -0,0 +1,85 @@ +package raftdirectory + +import ( + "context" + + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftstore" +) + +// PersonalStatus binds the original certificate to CURRENT committed identity, +// revocation and membership under the same actor barrier. Pairing alone creates +// no member: an absent device mapping remains Self=0 rather than a guessed ID. +func (g *Native) PersonalStatus(ctx context.Context, actor, certificate string) (out clusterctl.Status, err error) { + err = g.Inspect(ctx, func(ctx context.Context, store *raftstore.SQLite, app raftstore.Application) error { + read := func(k cs.RecordKey) ([]byte, error) { + key := raftstore.Key{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key} + rows, _, err := store.Read(ctx, app.Applied, []raftstore.Key{key}) + return rows[key], err + } + var identity cs.IdentityRec + raw, err := read(cs.IdentityKey()) + if err != nil { + return err + } + if cs.Decode(raw, &identity) != nil || cs.PersonalCertificate(identity, actor, certificate) != nil { + return directory.ErrUnauthorized + } + raw, err = read(cs.DeviceKey(actor)) + if err != nil { + return err + } + if raw != nil { + var device cs.Device + if cs.Decode(raw, &device) != nil || device.Revoked { + return directory.ErrUnauthorized + } + } + raw, err = read(cs.AuthorityKey(actor)) + if err != nil { + return err + } + if raw != nil { + var authority cs.Authority + if cs.Decode(raw, &authority) != nil || !authority.ValidRole() || !authority.Personal || authority.Cluster != g.manifest.Config.Cluster { + return directory.ErrUnauthorized + } + } + conf := raftcore.Configuration(app.ConfState) + membership, _, err := cs.MembershipView(read, conf, app.Applied) + if err != nil || membership.Revision == 0 { + return cs.ErrDependency + } + out = clusterctl.Status{Cluster: g.manifest.Config.Cluster, Leader: g.status[g.leader()].NodeID, Membership: membership, Configuration: conf, LeadershipUnavailable: true} + raw, err = read(cs.MemberDeviceKey(actor)) + if err != nil || raw == nil { + return err + } + var mapping cs.DeviceMember + if cs.Decode(raw, &mapping) != nil { + return cs.ErrDependency + } + if mapping.Node == 0 { + return nil + } + raw, err = read(cs.MemberKey(mapping.Node)) + if err != nil { + return err + } + var member cs.MemberRecord + if cs.Decode(raw, &member) != nil || member.Node != mapping.Node || member.Device != actor || member.Anchor || member.Worker { + return cs.ErrDependency + } + if !member.Retired { + out.Self = member.Node + } + return nil + }) + if err != nil { + return clusterctl.Status{}, err + } + return out, nil +} diff --git a/internal/relayserve/control.go b/internal/relayserve/control.go new file mode 100644 index 0000000000..973a99d0db --- /dev/null +++ b/internal/relayserve/control.go @@ -0,0 +1,46 @@ +package relayserve + +import ( + "context" + "net/http" + + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/reqsign" +) + +type controlCertificateKey struct{} + +// controlAuthenticate retains this request's original certificate for the +// backend's second authorization under its own fresh quorum barrier. +func (n *namespaces) controlAuthenticate(r *http.Request, body []byte) (string, string, error) { + identity, actor, err := n.authenticate(r, body) + if err == nil { + *r = *r.WithContext(context.WithValue(r.Context(), controlCertificateKey{}, r.Header.Get(reqsign.HeaderCert))) + } + return identity, actor, err +} + +func (n *namespaces) controlBackend(identity string) (clusterctl.Backend, error) { + n.mu.Lock() + d := n.groups[identity] + n.mu.Unlock() + if d == nil { + return nil, clusterctl.ErrUnavailable + } + return nativeControl{d}, nil +} + +type nativeControl struct{ owner *nativeNamespace } + +func (b nativeControl) Status(ctx context.Context, actor string) (clusterctl.Status, error) { + b.owner.armMembers() + certificate, _ := ctx.Value(controlCertificateKey{}).(string) + return b.owner.host.PersonalStatus(ctx, actor, certificate) +} + +// Lead cannot create an intent until the production personal-member lifecycle +// can finish it. In particular, refusal never proposes a configuration change. +func (b nativeControl) Lead(context.Context, cs.Command) (cs.LeadershipReceipt, error) { + return cs.LeadershipReceipt{}, clusterctl.ErrUnsupported +} diff --git a/internal/relayserve/control_test.go b/internal/relayserve/control_test.go new file mode 100644 index 0000000000..5e7020df96 --- /dev/null +++ b/internal/relayserve/control_test.go @@ -0,0 +1,111 @@ +package relayserve_test + +import ( + "errors" + "net/http" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/reqsign" +) + +// R9/R30 expose the actual committed anchor topology without manufacturing a +// personal node from a pairing certificate. R6 refuses before proposing an +// unsupported transfer, leaving both registry intent and voters unchanged. +func TestR6R9R30NativeControlReportsCommittedAnchorsAndRefusesLead(t *testing.T) { + r := newRig(t) + _, a := newIdentity(t) + client := clusterctl.Client{Base: r.srv.URL, Sign: reqsign.SignFor(a, r.clock.Now), HTTP: r.srv.Client()} + status, err := client.Status(bg) + if err != nil { + t.Fatal(err) + } + if status.Cluster == "" || status.Leader == 0 || status.Self != 0 || status.Membership.Pending != nil || !status.LeadershipUnavailable || !reflect.DeepEqual(status.Configuration.Voters, []uint64{1, 2, 3}) || len(status.Configuration.Learners) != 0 { + t.Fatalf("R9 status invented a personal member or changed the default: %+v", status) + } + c := cs.Command{Schema: cs.SchemaVersion, Cluster: status.Cluster, Actor: a.dev.ID(), Session: "leadership", Sequence: 1, Now: r.clock.Now().UnixMilli(), Change: cs.Mutation{RequestLeadership: &cs.RequestLeadership{Revision: status.Membership.Revision, Node: 4, Device: a.dev.ID()}}} + c.RequestDigest, err = cs.Digest(c) + if err != nil { + t.Fatal(err) + } + b, err := cs.SigningBytes(c) + if err != nil { + t.Fatal(err) + } + c.Signature = a.dev.Sign(b) + if _, err = client.Lead(bg, c); !errors.Is(err, clusterctl.ErrUnsupported) || err.Error() != "this relay does not offer cluster leadership yet; your saved chats are still here" { + t.Fatalf("R6 unsupported lead: %v", err) + } + after, err := client.Status(bg) + if err != nil || !reflect.DeepEqual(after, status) { + t.Fatalf("R6 refusal changed committed membership: before=%+v after=%+v err=%v", status, after, err) + } +} + +// R16 checks the production control door with its real certificate and then +// the same key after committed removal. A cached status cannot vouch it back in. +func TestR16R30NativeControlRequiresCurrentPersonalAuthorization(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + b := newDevice(t, id, t.TempDir()) + response, err := http.Get(r.srv.URL + clusterctl.Path) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != http.StatusForbidden { + t.Fatalf("unsigned status: %d", response.StatusCode) + } + client := clusterctl.Client{Base: r.srv.URL, Sign: reqsign.SignFor(b, r.clock.Now), HTTP: r.srv.Client()} + if _, err := client.Status(bg); err != nil { + t.Fatal(err) + } + if err := r.dir(b).PutDevice(bg, b.dev.ID(), directory.Device{V: 1, Name: "c2VhbGVk"}); err != nil { + t.Fatal(err) + } + if err := r.dir(a).Revoke(bg, b.dev.ID()); err != nil { + t.Fatal(err) + } + if _, err := client.Status(bg); err == nil { + t.Fatal("R16 revoked certificate read cached cluster status") + } +} + +// R4/R30 require the status door to lose service with quorum, even when its +// previous answer and the remaining anchor's committed registry are intact. +func TestR4R30NativeControlCannotServeCachedStatusWithoutQuorum(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + client := func() clusterctl.Client { + return clusterctl.Client{Base: r.srv.URL, Sign: reqsign.SignFor(a, r.clock.Now), HTTP: r.srv.Client()} + } + before, err := client().Status(bg) + if err != nil { + t.Fatal(err) + } + r.stop() + path := filepath.Join(r.store, "raft", id.ID()) + for _, node := range []string{"2", "3"} { + if err := os.Rename(filepath.Join(path, node+".db"), filepath.Join(path, node+".asleep")); err != nil { + t.Fatal(err) + } + } + r.start() + if _, err := client().Status(bg); err == nil { + t.Fatal("R4 cached status answered without a majority") + } + r.stop() + if err := os.Rename(filepath.Join(path, "2.asleep"), filepath.Join(path, "2.db")); err != nil { + t.Fatal(err) + } + r.start() + after, err := client().Status(bg) + if err != nil || before.Cluster != after.Cluster || len(after.Configuration.Voters) != 3 { + t.Fatalf("R4 surviving majority did not recover the original topology: %+v %v", after, err) + } +} diff --git a/internal/relayserve/service.go b/internal/relayserve/service.go index df8313aefb..461dc268b6 100644 --- a/internal/relayserve/service.go +++ b/internal/relayserve/service.go @@ -11,6 +11,7 @@ import ( "time" "github.com/Agent-Field/codeaf/internal/blobstore" + "github.com/Agent-Field/codeaf/internal/clusterctl" "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/directory" "github.com/Agent-Field/codeaf/internal/pairbox" @@ -119,6 +120,7 @@ func New(cfg Config) *Service { svc.ns.after = cfg.After svc.ns.links = links mux.HandleFunc(clusterstate.ClusterAPIPath, svc.ns.clusterAPI) + mux.Handle(clusterctl.Path, clusterctl.Handler(svc.ns.controlAuthenticate, svc.ns.controlBackend)) auth := noting(svc.ns.authenticate) dirs := logged(cfg.Logf, directory.Handler(auth, svc.ns.directory)) mux.Handle("/v1/dir/", dirs) diff --git a/internal/syncsetup/cluster.go b/internal/syncsetup/cluster.go index eca4322888..3cd8f61aef 100644 --- a/internal/syncsetup/cluster.go +++ b/internal/syncsetup/cluster.go @@ -68,6 +68,9 @@ func (s *Sync) LeadCluster(ctx context.Context, node uint64) (cs.LeadershipRecei if node == 0 { return receipt, errors.New("this computer must join the cluster with persistent storage before it can lead") } + if status.LeadershipUnavailable { + return receipt, clusterctl.ErrUnsupported + } if saved.Sequence == ^uint64(0) { return receipt, errors.New("this computer's leadership request sequence is full") } diff --git a/internal/syncsetup/member_control_test.go b/internal/syncsetup/member_control_test.go new file mode 100644 index 0000000000..bdb27253da --- /dev/null +++ b/internal/syncsetup/member_control_test.go @@ -0,0 +1,80 @@ +package syncsetup + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/reqsign" +) + +type unavailableControl struct { + self uint64 + t *testing.T +} + +func (b unavailableControl) Status(context.Context, string) (clusterctl.Status, error) { + return clusterctl.Status{Cluster: "native", Self: b.self, LeadershipUnavailable: true}, nil +} +func (b unavailableControl) Lead(context.Context, cs.Command) (cs.LeadershipReceipt, error) { + b.t.Fatal("R6 an unavailable leadership door received a proposal") + return cs.LeadershipReceipt{}, clusterctl.ErrUnsupported +} + +// R6/R23 refuse before saving a new request when the authenticated relay says +// this home has no durable member or leadership is unavailable. Neither case +// may leave an intent for a later activation to accidentally complete. +func TestR6R23UnavailableLeadershipCannotLeaveNewIntent(t *testing.T) { + for _, tc := range []struct { + name string + self uint64 + want string + }{ + {"unjoined", 0, "this computer must join the cluster with persistent storage before it can lead"}, + {"joined", 4, "this relay does not offer cluster leadership yet; your saved chats are still here"}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(clusterctl.Handler(func(r *http.Request, b []byte) (string, string, error) { return reqsign.Verify(r, b, time.Now()) }, func(string) (clusterctl.Backend, error) { return unavailableControl{tc.self, t}, nil })) + defer server.Close() + home := machine(t, server.URL) + client, ok, err := Open(home) + if err != nil || !ok { + t.Fatal(err) + } + _, err = client.LeadCluster(context.Background(), 0) + if err == nil || err.Error() != tc.want { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(home, "cluster-leadership-"+client.Device.ID()+".json")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("R6 refused lead saved an unfinishable request: %v", err) + } + t.Setenv(URLVar, "off") + if _, err := client.ClusterStatus(context.Background()); !errors.Is(err, ErrQuiet) { + t.Fatalf("R23 retained status handle bypassed quiet: %v", err) + } + }) + } +} + +// R6/R30 keep the hosted refusal's exact words at the real client boundary. +func TestR6R30AbsentHostedControlKeepsLeadershipSentence(t *testing.T) { + server := httptest.NewServer(http.NotFoundHandler()) + defer server.Close() + client, ok, err := Open(machine(t, server.URL)) + if err != nil || !ok { + t.Fatal(err) + } + if _, err := client.ClusterStatus(context.Background()); !errors.Is(err, clusterctl.ErrUnsupported) || err.Error() != "this relay does not offer cluster leadership yet; your saved chats are still here" { + t.Fatal(err) + } + if _, err := client.LeadCluster(context.Background(), 0); !errors.Is(err, clusterctl.ErrUnsupported) { + t.Fatal(err) + } +} From 8e3e231165933524820e83aff2b33620d6b7cb7a Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 12:23:19 -0400 Subject: [PATCH 054/103] Reuse hosted fixture bundles so runtime restarts preserve their program. Cache each normal or injected-crash bundle once per fixture. Restart Miniflare over the same persisted state without rebundling, retaining every scoped-flow assertion and its original timeout. Pin the absent hosted control route across restart. Co-Authored-By: Claude Opus 5.5 (1M context) --- relay/hosted/test/raft/hosted-control.test.js | 24 +++++++++++++++++++ relay/hosted/test/raft/hosted-fixture.js | 13 +++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 relay/hosted/test/raft/hosted-control.test.js diff --git a/relay/hosted/test/raft/hosted-control.test.js b/relay/hosted/test/raft/hosted-control.test.js new file mode 100644 index 0000000000..c821316c45 --- /dev/null +++ b/relay/hosted/test/raft/hosted-control.test.js @@ -0,0 +1,24 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { stat } from 'node:fs/promises'; +import { join } from 'node:path'; +import { hostedFixture } from './hosted-fixture.js'; + +// R6/R30 keep hosted leadership absent rather than accepting an intent that +// cannot finish. Restart runs the same bundle over the same persistent state. +test('R6 R30 hosted control stays absent across a runtime restart',{timeout:30000},async()=>{ + const h=await hostedFixture({base:'/fabric'}),a=h.client(); + try{ + await h.start(); + for(const method of ['GET','POST']){ + const r=await h.request(a,method,'/v2/cluster/control'); + assert.equal(r.status,404);await r.text(); + } + const bundle=await stat(join(h.dir,'worker.js')); + await h.start(); + const restarted=await stat(join(h.dir,'worker.js')); + assert.equal(restarted.mtimeNs??restarted.mtimeMs,bundle.mtimeNs??bundle.mtimeMs,'restart rebuilt the program'); + const r=await h.request(a,'GET','/v2/cluster/control'); + assert.equal(r.status,404);await r.text(); + }finally{await h.close();} +}); diff --git a/relay/hosted/test/raft/hosted-fixture.js b/relay/hosted/test/raft/hosted-fixture.js index 2e1137283e..2e3b506e09 100644 --- a/relay/hosted/test/raft/hosted-fixture.js +++ b/relay/hosted/test/raft/hosted-fixture.js @@ -22,14 +22,25 @@ export async function hostedFixture({fixture=false,base='',go=false}={}) { return {key,cert,device:'dev_'+sha(pub).slice(0,32)}; }; await copyFile(join(out,'core.wasm'),join(dir,'core.wasm')); - let mf; + let mf, compiledCut; + const bundles=new Map(); const compile=async(cut)=>{ + // A restart must retain the same program and storage. Cache each injected + // crash variant separately so recovery restores the ordinary bundle too. + const variant=JSON.stringify(cut??null); + if(compiledCut===variant)return; + if(bundles.has(variant)){ + await copyFile(bundles.get(variant),join(dir,'worker.js')); + compiledCut=variant;return; + } const plugins=[{name:'local-wasm',setup(b){b.onResolve({filter:/\.wasm$/},()=>({path:'core.wasm',external:true}));}}]; if(cut)plugins.push({name:'process-cut',setup(b){b.onLoad({filter:/\/src\/raft\/hosted.js$/},async args=>{ let text=await readFile(args.path,'utf8');const needle=cut.needle;assert.equal(text.split(needle).length,2,'cut must identify exactly one boundary'); text=text.replace(needle,needle+(cut.after??"throw new Error('H2 injected crash after durable boundary');"));return {contents:text,loader:'js'}; });}}); await build({entryPoints:[fixture?'test/raft/hosted-worker.js':'src/raft-worker.js'],bundle:true,format:'esm',target:'es2022',external:['cloudflare:*'],plugins,outfile:join(dir,'worker.js')}); + const saved=join(dir,'bundle-'+bundles.size+'.js'); + await copyFile(join(dir,'worker.js'),saved);bundles.set(variant,saved);compiledCut=variant; }; const start=async(cut)=>{ if(mf)await mf.dispose();await compile(cut); From 986ae39e6947450c7b148cd1c492edb3a15cb398 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 13:50:58 -0400 Subject: [PATCH 055/103] Serve durable personal learners through the native relay's committed membership. Preserve each paired device's original signed admission, bind node authority permanently to its store, and reconcile separate learner configuration after a restart. Route authenticated exchange and socket traffic through the native member queue without waiting on learners, stream bounded snapshots independently of compaction, and require signed durable application evidence for readiness. Keep the provision immutable while fencing restart cursors, and preserve legal maximum command writes through snapshot chunks. Keep three anchor voters and all-voter sleep with fresh quorum wake, and pin the hosted member doors absent. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterctl/control.go | 1 + internal/clusterstate/apply.go | 2 +- internal/clusterstate/membership.go | 80 +- internal/clusterstate/personal.go | 46 + internal/clusterstate/personal_test.go | 73 ++ internal/raftdirectory/control.go | 16 + internal/raftdirectory/maintenance.go | 11 + internal/raftdirectory/native.go | 29 +- internal/raftdirectory/personal.go | 234 +++++ internal/raftdirectory/transport.go | 334 +++++++ internal/rafttransport/README.md | 68 +- internal/rafttransport/exchange.go | 11 +- internal/rafttransport/router.go | 44 + internal/relayserve/native.go | 74 ++ internal/relayserve/personal.go | 103 ++ internal/relayserve/personal_test.go | 880 ++++++++++++++++++ internal/relayserve/service.go | 6 + relay/hosted/test/raft/hosted-control.test.js | 19 +- 18 files changed, 2004 insertions(+), 27 deletions(-) create mode 100644 internal/clusterstate/personal.go create mode 100644 internal/clusterstate/personal_test.go create mode 100644 internal/raftdirectory/personal.go create mode 100644 internal/raftdirectory/transport.go create mode 100644 internal/relayserve/personal.go create mode 100644 internal/relayserve/personal_test.go diff --git a/internal/clusterctl/control.go b/internal/clusterctl/control.go index 3491d1c476..0c09c2f63b 100644 --- a/internal/clusterctl/control.go +++ b/internal/clusterctl/control.go @@ -32,6 +32,7 @@ func (e CommittedRefusal) Error() string { } type Status struct { + Ready []uint64 `json:"ready,omitempty"` LeadershipUnavailable bool `json:"leadership_unavailable,omitempty"` Cluster string `json:"cluster"` Self uint64 `json:"self"` diff --git a/internal/clusterstate/apply.go b/internal/clusterstate/apply.go index 1251497e01..ab6aadb7ce 100644 --- a/internal/clusterstate/apply.go +++ b/internal/clusterstate/apply.go @@ -383,7 +383,7 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er } } service := c.Change.RenewHolds != nil || c.Change.RecordUpload != nil || c.Change.AdmitJoin != nil || c.Change.CompleteJoin != nil || auth.Effects && (c.Change.WorkerBeatBatch != nil || c.Change.DeviceSeen != nil) - if service != auth.Effects || auth.Personal && auth.Effects || auth.Anchor && (auth.Personal || auth.Effects || c.Change.InitMembership == nil && c.Change.PrepareQuiescence == nil && c.Change.SettleMembership == nil && c.Change.SettleLeadership == nil && c.Change.CompleteLeadership == nil && c.Change.RestoreAnchors == nil && c.Change.CloseSession == nil) { + if service != auth.Effects || auth.Personal && auth.Effects || auth.Anchor && (auth.Personal || auth.Effects || (c.Change.EnrollMember == nil || c.Change.EnrollMember.Admission == nil) && (c.Change.RemoveMember == nil || c.Change.RemoveMember.Admission == nil) && c.Change.InitMembership == nil && c.Change.PrepareQuiescence == nil && c.Change.SettleMembership == nil && c.Change.SettleLeadership == nil && c.Change.CompleteLeadership == nil && c.Change.RestoreAnchors == nil && c.Change.CloseSession == nil) { return out, nil } if (c.Change.PrepareQuiescence != nil || c.Change.RestoreAnchors != nil || c.Change.CompleteLeadership != nil) && !auth.Anchor { diff --git a/internal/clusterstate/membership.go b/internal/clusterstate/membership.go index e0a6fbbe40..f9aa29edac 100644 --- a/internal/clusterstate/membership.go +++ b/internal/clusterstate/membership.go @@ -37,6 +37,7 @@ func hasID(ids []uint64, id uint64) bool { // MemberRecord contains protocol authority only. Retired IDs are never deleted; // a device that rejoins receives a fresh ID and a fresh transport incarnation. type MemberRecord struct { + Store string `json:"store,omitempty"` Worker bool `json:"worker,omitempty"` WorkerExpires int64 `json:"worker_expires,omitempty"` Node uint64 `json:"node"` @@ -69,15 +70,18 @@ type InitMembership struct { Limit uint32 `json:"limit"` } type EnrollMember struct { - Node uint64 `json:"node"` - Revision uint64 `json:"revision"` - Device string `json:"device"` - Protocol uint16 `json:"protocol"` + Store string `json:"store,omitempty"` + Admission *V1Request `json:"admission,omitempty"` + Node uint64 `json:"node"` + Revision uint64 `json:"revision"` + Device string `json:"device"` + Protocol uint16 `json:"protocol"` } type RemoveMember struct { - Revision uint64 `json:"revision"` - Node uint64 `json:"node"` - Device string `json:"device"` + Admission *V1Request `json:"admission,omitempty"` + Revision uint64 `json:"revision"` + Node uint64 `json:"node"` + Device string `json:"device"` } type SettleMembership struct { Revision uint64 `json:"revision"` @@ -117,10 +121,16 @@ func membershipDependencies(m Mutation) []RecordKey { keys = append(keys, MemberKey(a.Node), AuthorityKey(AnchorActor(a.Node))) } case m.EnrollMember != nil: - keys = append(keys, MembershipKey(), DeviceKey(m.EnrollMember.Device), AuthorityKey(m.EnrollMember.Device), MemberDeviceKey(m.EnrollMember.Device), MemberKey(m.EnrollMember.Node), MemberTombstoneKey(m.EnrollMember.Node)) + keys = append(keys, MemberStoreKey(m.EnrollMember.Device, m.EnrollMember.Store), MembershipKey(), DeviceKey(m.EnrollMember.Device), AuthorityKey(m.EnrollMember.Device), MemberDeviceKey(m.EnrollMember.Device), MemberKey(m.EnrollMember.Node), MemberTombstoneKey(m.EnrollMember.Node)) case m.RemoveMember != nil: v := m.RemoveMember - keys = append(keys, MembershipKey(), MemberKey(v.Node), MemberTombstoneKey(v.Node), MemberDeviceKey(v.Device)) + keys = append(keys, MembershipKey(), MemberKey(v.Node), MemberTombstoneKey(v.Node), MemberDeviceKey(v.Device), DeviceKey(v.Device), AuthorityKey(v.Device)) + if v.Admission != nil { + var request PersonalAdmission + if legacyBody(v.Admission.Body, &request, false) == nil { + keys = append(keys, MemberStoreKey(v.Device, request.Store)) + } + } case m.SettleMembership != nil: v := m.SettleMembership keys = append(keys, MembershipKey(), MemberKey(v.Node)) @@ -201,9 +211,30 @@ func applyMembership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, return UpdateRequired } var authority Authority + var admissionKey []byte + if v.Admission != nil { + var identity IdentityRec + if Decode(rows[IdentityKey()], &identity) != nil { + return BadRequest + } + request, granted, actor, err := VerifyPersonalAdmission(*v.Admission, identity, result.Now) + if err != nil || actor != v.Device || request.Store != v.Store || request.Protocol != v.Protocol { + return Unauthorized + } + granted.Cluster = c.Cluster + authority = granted + admissionKey = granted.Key + if rows[MemberStoreKey(v.Device, v.Store)] != nil { + return Conflict + } + } else if v.Store != "" { + return Unauthorized + } var device Device var prior DeviceMember - if rows[AuthorityKey(v.Device)] == nil && v.Device == c.Actor && c.Certificate != "" { + if rows[AuthorityKey(v.Device)] == nil && v.Admission != nil { + // The original device proof supplied authority above. + } else if rows[AuthorityKey(v.Device)] == nil && v.Device == c.Actor && c.Certificate != "" { var identity IdentityRec if Decode(rows[IdentityKey()], &identity) != nil { return BadRequest @@ -216,6 +247,9 @@ func applyMembership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, } else if Decode(rows[AuthorityKey(v.Device)], &authority) != nil { return Unauthorized } + if v.Admission != nil && (!authority.Personal || !bytes.Equal(authority.Key, admissionKey)) { + return Unauthorized + } if authority.Cluster != c.Cluster || !authority.ValidRole() || !authority.Personal && authority.Worker == nil || len(authority.Key) != ed25519.PublicKeySize { return Unauthorized } @@ -273,8 +307,11 @@ func applyMembership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, if rows[AuthorityKey(v.Device)] == nil { put(AuthorityKey(v.Device), authority) } - put(MemberKey(node), MemberRecord{Node: node, Device: v.Device, Key: bytes.Clone(authority.Key), Incarnation: incarnation, Protocol: ClusterProtocol, Worker: authority.Worker != nil, WorkerExpires: workerExpires}) + put(MemberKey(node), MemberRecord{Store: v.Store, Node: node, Device: v.Device, Key: bytes.Clone(authority.Key), Incarnation: incarnation, Protocol: ClusterProtocol, Worker: authority.Worker != nil, WorkerExpires: workerExpires}) put(MemberDeviceKey(v.Device), DeviceMember{Node: node}) + if v.Store != "" { + put(MemberStoreKey(v.Device, v.Store), DeviceMember{Node: node}) + } put(MembershipKey(), h) result.Membership = &MembershipResult{Node: node, Revision: h.Revision} return OK @@ -287,6 +324,27 @@ func applyMembership(c Command, meta ApplicationMeta, rows map[RecordKey][]byte, if Decode(rows[MemberKey(v.Node)], &member) != nil { return NotFound } + if v.Admission != nil { + var identity IdentityRec + if Decode(rows[IdentityKey()], &identity) != nil { + return BadRequest + } + request, granted, actor, err := VerifyPersonalAdmission(*v.Admission, identity, result.Now) + var device Device + if err != nil || actor != v.Device || request.Protocol != ClusterProtocol || request.Store == member.Store { + return Unauthorized + } + var authority Authority + if Decode(rows[AuthorityKey(v.Device)], &authority) != nil || !authority.ValidRole() || !authority.Personal || authority.Cluster != c.Cluster || !bytes.Equal(authority.Key, granted.Key) { + return Unauthorized + } + if rows[MemberStoreKey(v.Device, request.Store)] != nil { + return Conflict + } + if raw := rows[DeviceKey(v.Device)]; raw != nil && (Decode(raw, &device) != nil || device.Revoked) { + return Revoked + } + } if member.Anchor || member.Retired || member.Node != v.Node || member.Device != v.Device || hasID(meta.Configuration.Voters, v.Node) { return Conflict } diff --git a/internal/clusterstate/personal.go b/internal/clusterstate/personal.go new file mode 100644 index 0000000000..e2aa338853 --- /dev/null +++ b/internal/clusterstate/personal.go @@ -0,0 +1,46 @@ +package clusterstate + +import ( + "bytes" + "crypto/ed25519" + "encoding/hex" + "strconv" +) + +const PersonalAdmissionPath = "/v2/cluster/enroll" + +// PersonalAdmission binds a new durable store to an existing paired device. +// The store token is created once WITH the store, not once per connection. +type PersonalAdmission struct { + Store string `json:"store"` + Protocol uint16 `json:"protocol"` +} + +// VerifyPersonalAdmission retains the original HTTP proof in the replicated +// command. Neither a host signature nor a presented transport key substitutes +// for this certificate under the currently committed identity root. +func VerifyPersonalAdmission(p V1Request, identity IdentityRec, now int64) (PersonalAdmission, Authority, string, error) { + var request PersonalAdmission + if p.Method != "POST" || p.URI != PersonalAdmissionPath || !bytes.Equal(p.IdentityKey, identity.SigningKey) || legacyBody(p.Body, &request, false) != nil { + return request, Authority{}, "", ErrCodec + } + token, err := hex.DecodeString(request.Store) + if err != nil || len(token) != 32 || hex.EncodeToString(token) != request.Store { + return request, Authority{}, "", ErrCodec + } + at, err := strconv.ParseInt(p.At, 10, 64) + if err != nil || at < 0 || now < 0 || at > int64(MaxSafeInteger) || now > int64(MaxSafeInteger) || at < now-V1SkewMillis || at > now+V1SkewMillis { + return request, Authority{}, "", ErrCodec + } + key, err := legacyCertificate(LegacyConfig{Identity: identity.Identity, SigningKey: identity.SigningKey}, p.Certificate) + if err != nil || !ed25519.Verify(key, V1SigningMessage(p.Method, p.URI, p.At, p.Body), p.Signature) { + return request, Authority{}, "", ErrCodec + } + return request, Authority{Key: key, Personal: true}, DeviceIDOfKey(key), nil +} + +// MemberStoreKey is permanent even after retirement. Retrying an old store +// cannot retire its replacement or allocate another durable node identity. +func MemberStoreKey(device, store string) RecordKey { + return RecordKey{"membership", "store/" + segment(device) + "/" + store} +} diff --git a/internal/clusterstate/personal_test.go b/internal/clusterstate/personal_test.go new file mode 100644 index 0000000000..7761e3a06a --- /dev/null +++ b/internal/clusterstate/personal_test.go @@ -0,0 +1,73 @@ +package clusterstate_test + +import ( + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "encoding/json" + "net/http" + "strings" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/identity" +) + +// R9/R12 retain the original personal proof through committed application. +// Changing the root, store, route or certificate cannot authorize allocation. +func TestR9R12PersonalAdmissionProofBindsStoreAndCommittedRoot(t *testing.T) { + id, err := identity.Ensure(t.TempDir()) + if err != nil { + t.Fatal(err) + } + pub, key, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + now := time.UnixMilli(1700000000000) + cert := identity.IssueCert(id, pub, now) + raw, _ := json.Marshal(cert) + request := cs.PersonalAdmission{Store: strings.Repeat("01", 32), Protocol: cs.ClusterProtocol} + body, _ := json.Marshal(request) + proof := cs.V1Request{Method: http.MethodPost, URI: cs.PersonalAdmissionPath, IdentityKey: id.PublicKey(), Certificate: base64.RawURLEncoding.EncodeToString(raw), At: "1700000000000", Body: body} + proof.Signature = ed25519.Sign(key, cs.V1SigningMessage(proof.Method, proof.URI, proof.At, proof.Body)) + root := cs.IdentityRec{V: 1, Identity: id.ID(), SigningKey: id.PublicKey()} + got, authority, actor, err := cs.VerifyPersonalAdmission(proof, root, now.UnixMilli()) + if err != nil || got != request || !authority.Personal || !ed25519.PublicKey(authority.Key).Equal(pub) || actor != cert.DeviceID() { + t.Fatalf("original paired store refused: %+v %v", got, err) + } + for _, change := range []string{"store", "route", "certificate", "root", "stale", "malformed-store"} { + t.Run(change, func(t *testing.T) { + candidate := proof + committed := root + at := now.UnixMilli() + switch change { + case "store": + altered := request + altered.Store = strings.Repeat("02", 32) + candidate.Body, _ = json.Marshal(altered) + case "route": + candidate.URI = "/v2/cluster/control" + case "certificate": + candidate.Certificate = "" + case "root": + _, other, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + committed.SigningKey = other.Public().(ed25519.PublicKey) + case "stale": + at += cs.V1SkewMillis + 1 + case "malformed-store": + altered := request + altered.Store = "new-store" + candidate.Body, _ = json.Marshal(altered) + candidate.Signature = ed25519.Sign(key, cs.V1SigningMessage(candidate.Method, candidate.URI, candidate.At, candidate.Body)) + } + if _, _, _, err := cs.VerifyPersonalAdmission(candidate, committed, at); err == nil { + t.Fatal("altered or stale enrollment gained authority") + } + }) + } +} diff --git a/internal/raftdirectory/control.go b/internal/raftdirectory/control.go index 097d25b2aa..10376d54eb 100644 --- a/internal/raftdirectory/control.go +++ b/internal/raftdirectory/control.go @@ -54,6 +54,21 @@ func (g *Native) PersonalStatus(ctx context.Context, actor, certificate string) return cs.ErrDependency } out = clusterctl.Status{Cluster: g.manifest.Config.Cluster, Leader: g.status[g.leader()].NodeID, Membership: membership, Configuration: conf, LeadershipUnavailable: true} + if g.peers != nil { + for _, id := range conf.Learners { + raw, err := read(cs.MemberKey(id)) + if err != nil { + return err + } + var record cs.MemberRecord + if cs.Decode(raw, &record) != nil { + return cs.ErrDependency + } + if peer, ok := g.peer(id); ok && !peer.Revoked && cs.LearnerReady(record, conf, g.peers.applied[id], record.ReadyIndex) { + out.Ready = append(out.Ready, id) + } + } + } raw, err = read(cs.MemberDeviceKey(actor)) if err != nil || raw == nil { return err @@ -74,6 +89,7 @@ func (g *Native) PersonalStatus(ctx context.Context, actor, certificate string) return cs.ErrDependency } if !member.Retired { + out.Self = member.Node } return nil diff --git a/internal/raftdirectory/maintenance.go b/internal/raftdirectory/maintenance.go index ceacc3e96c..b622dab13c 100644 --- a/internal/raftdirectory/maintenance.go +++ b/internal/raftdirectory/maintenance.go @@ -81,6 +81,12 @@ func (g *Native) Maintain(ctx context.Context, elapsed, now int64, key ed25519.P if leader < 0 { return after, directory.ErrUnreachable } + if g.peers != nil { + g.peers.now = max(g.peers.now, now) + if err := g.reconcileMembership(ctx); err != nil { + return after, err + } + } plan := plans[leader] if plan.Barrier { id := fmt.Sprintf("idle-%d", g.serial) @@ -173,6 +179,11 @@ func (g *Native) Maintain(ctx context.Context, elapsed, now int64, key ed25519.P allSleep = allSleep && p.Sleeping } if allSleep { + if g.peers != nil { + if err := g.quiescePeers(ctx); err != nil { + return after, err + } + } return 0, nil } return after, nil diff --git a/internal/raftdirectory/native.go b/internal/raftdirectory/native.go index edc94f11f5..fc95b52690 100644 --- a/internal/raftdirectory/native.go +++ b/internal/raftdirectory/native.go @@ -23,9 +23,11 @@ import ( // Native owns three independent durable actors on one host. The queue covers // each persistence completion, authenticated local delivery and read callback; // no caller can inspect a replica between its barrier and its application read. -// All peer bytes originate inside this queue, never at a public HTTP door. +// Peer bytes reach a core only inside this queue, after committed enrollment +// authenticates external input. Public operations never wait on a learner. type Native struct { mu sync.Mutex + peers *nativePeers manifest clusterstate.LegacyManifest stores [3]*raftstore.SQLite nodes [3]*raftcore.Core @@ -269,7 +271,14 @@ func (g *Native) drain(ctx context.Context) error { } } if !found { - return directory.ErrUnauthorized + if g.peers == nil { + // Activation can recover voters before their committed router + // opens. An unreachable learner cannot gate that recovery. + continue + } + if err := g.sendExternal(ctx, m); err != nil { + return err + } } } return nil @@ -325,6 +334,10 @@ func (g *Native) stageSnapshot(ctx context.Context, target int, message *pb.Mess } func (g *Native) submit(ctx context.Context, c clusterstate.Command) (clusterstate.Result, error) { + if g.peers != nil { + g.peers.sleeping = false + g.peers.now = max(g.peers.now, c.Now) + } i := g.leader() if i < 0 { return clusterstate.Result{}, directory.ErrUnreachable @@ -374,6 +387,10 @@ func (g *Native) submit(ctx context.Context, c clusterstate.Command) (clustersta } } } + if err := g.checkpointExternal(ctx); err != nil { + g.failed = err + return r, err + } if g.Changed != nil { app, _, e := g.stores[i].ApplicationView(ctx) if e != nil { @@ -383,9 +400,17 @@ func (g *Native) submit(ctx context.Context, c clusterstate.Command) (clustersta return r, e } } + if g.peers != nil { + if err := g.installPeers(ctx); err != nil { + return r, err + } + } return r, nil } func (g *Native) barrier(ctx context.Context) (int, raftstore.Application, error) { + if g.peers != nil { + g.peers.sleeping = false + } i := g.leader() if i < 0 { return i, raftstore.Application{}, directory.ErrUnreachable diff --git a/internal/raftdirectory/personal.go b/internal/raftdirectory/personal.go new file mode 100644 index 0000000000..cae252093a --- /dev/null +++ b/internal/raftdirectory/personal.go @@ -0,0 +1,234 @@ +package raftdirectory + +import ( + "context" + "crypto/ed25519" + "errors" + "strconv" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/rafttransport" +) + +// PersonalEnrollment advertises a committed barrier, never a readiness claim. +// A returning store receives the same node, incarnation and protocol authority. +type PersonalEnrollment struct { + Scope rafttransport.Scope `json:"scope"` + Generation uint64 `json:"generation,string"` + Node uint64 `json:"node,string"` + Incarnation uint64 `json:"incarnation,string"` + Barrier uint64 `json:"barrier,string"` + Members []rafttransport.Member `json:"members"` +} + +// EnrollmentRefusal carries the shared manual sentence without inventing a +// different capacity or protocol limit at this HTTP boundary. +type EnrollmentRefusal struct{ Code clusterstate.Code } + +func (e EnrollmentRefusal) Error() string { return clusterstate.EnrollmentMessage(e.Code) } + +func recordReader(ctx context.Context, s *raftstore.SQLite, app raftstore.Application) clusterstate.ReadRecord { + return func(k clusterstate.RecordKey) ([]byte, error) { + key := raftstore.Key{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key} + rows, _, err := s.Read(ctx, app.Applied, []raftstore.Key{key}) + return rows[key], err + } +} +func (g *Native) membership(ctx context.Context) (int, raftstore.Application, clusterstate.Membership, clusterstate.ReadRecord, error) { + i := g.leader() + if i < 0 { + return i, raftstore.Application{}, clusterstate.Membership{}, nil, directory.ErrUnreachable + } + app, _, err := g.stores[i].ApplicationView(ctx) + if err != nil { + return i, app, clusterstate.Membership{}, nil, err + } + read := recordReader(ctx, g.stores[i], app) + h, _, err := clusterstate.MembershipView(read, raftcore.Configuration(app.ConfState), app.Applied) + return i, app, h, read, err +} +func (g *Native) controlChange(ctx context.Context, change clusterstate.Mutation, now int64) (clusterstate.Result, error) { + _, app, _, read, err := g.membership(ctx) + if err != nil { + return clusterstate.Result{}, err + } + actor := clusterstate.AnchorActor(g.manifest.Config.Anchors[0]) + raw, err := read(clusterstate.SessionKey(actor, "member-maintenance")) + if err != nil { + return clusterstate.Result{}, err + } + seq, ack, err := clusterstate.NextSession(raw) + if err != nil { + return clusterstate.Result{}, err + } + c, err := clusterstate.Sign(clusterstate.Command{Schema: clusterstate.SchemaVersion, Cluster: g.manifest.Config.Cluster, Actor: actor, Session: "member-maintenance", Sequence: seq, Ack: ack, Now: max(now, app.LogicalTime), Change: change}, g.peers.keys[0]) + if err != nil { + return clusterstate.Result{}, err + } + r, err := g.submit(ctx, c) + if err == nil && r.Code != clusterstate.OK { + err = EnrollmentRefusal{r.Code} + if err.Error() == "" { + err = directory.ResultError(r.Code) + } + } + return r, err +} + +// reconcileMembership runs at activation and maintenance, independently of a +// retrying client. Registry intent alone is enough to finish either stage. +func (g *Native) reconcileMembership(ctx context.Context) error { + for stage := 0; stage < 3; stage++ { + i, app, h, _, err := g.membership(ctx) + if err != nil { + return err + } + if h.Pending == nil { + return g.installPeers(ctx) + } + plan, err := g.nodes[i].PlanMembership(h) + if errors.Is(err, raftcore.ErrBusy) { + return nil + } + if err != nil { + return err + } + if len(plan.Change) > 0 { + if err = g.drive(ctx, i, raftcore.Configure{Change: plan.Change}); err != nil { + return err + } + if err = g.drain(ctx); err != nil { + return err + } + continue + } + if !plan.Settle { + return nil + } + if _, err = g.controlChange(ctx, clusterstate.Mutation{SettleMembership: &clusterstate.SettleMembership{Revision: h.Revision, Node: h.Pending.Node}}, app.LogicalTime); err != nil { + return err + } + if plan.Checkpoint { + p, err := g.stores[i].CreateSnapshot(ctx) + if err != nil { + return err + } + if err = g.stores[i].Compact(ctx, p.GetMetadata().GetIndex()); err != nil { + return err + } + if err = g.drive(ctx, i, raftcore.CompactSnapshot{Snapshot: p}); err != nil { + return err + } + } + } + return nil +} + +// EnrollPersonal serializes allocation, replacement and the original signed +// proof with all other member input. No public call waits on the learner. +func (g *Native) EnrollPersonal(ctx context.Context, proof clusterstate.V1Request) (PersonalEnrollment, error) { + ctx = context.WithoutCancel(ctx) + g.mu.Lock() + defer g.mu.Unlock() + var out PersonalEnrollment + if g.peers == nil { + return out, directory.ErrUnreachable + } + if _, _, err := g.barrier(ctx); err != nil { + return out, err + } + if err := g.reconcileMembership(ctx); err != nil { + return out, err + } + _, app, h, read, err := g.membership(ctx) + if err != nil { + return out, err + } + var identity clusterstate.IdentityRec + raw, err := read(clusterstate.IdentityKey()) + if err != nil || clusterstate.Decode(raw, &identity) != nil { + return out, clusterstate.ErrDependency + } + at, _ := strconv.ParseInt(proof.At, 10, 64) + now := max(at, app.LogicalTime) + request, granted, actor, err := clusterstate.VerifyPersonalAdmission(proof, identity, now) + if err != nil { + return out, directory.ErrUnauthorized + } + var device clusterstate.Device + raw, err = read(clusterstate.DeviceKey(actor)) + if err != nil { + return out, err + } + if raw != nil && (clusterstate.Decode(raw, &device) != nil || device.Revoked) { + return out, directory.ErrUnauthorized + } + var authority clusterstate.Authority + raw, err = read(clusterstate.AuthorityKey(actor)) + if err != nil { + return out, err + } + if raw != nil && (clusterstate.Decode(raw, &authority) != nil || !authority.ValidRole() || !authority.Personal || authority.Cluster != g.manifest.Config.Cluster || !ed25519.PublicKey(authority.Key).Equal(ed25519.PublicKey(granted.Key))) { + return out, directory.ErrUnauthorized + } + if request.Protocol != raftcore.ProtocolVersion { + return out, EnrollmentRefusal{clusterstate.UpdateRequired} + } + var mapping clusterstate.DeviceMember + raw, err = read(clusterstate.MemberStoreKey(actor, request.Store)) + if err != nil { + return out, err + } + if raw != nil { + if clusterstate.Decode(raw, &mapping) != nil { + return out, clusterstate.ErrDependency + } + } else { + if len(h.Nodes) >= int(h.Limit) { + return out, EnrollmentRefusal{clusterstate.Capacity} + } + raw, err = read(clusterstate.MemberDeviceKey(actor)) + if err != nil { + return out, err + } + if raw != nil && clusterstate.Decode(raw, &mapping) != nil { + return out, clusterstate.ErrDependency + } + if mapping.Node != 0 { + if _, err = g.controlChange(ctx, clusterstate.Mutation{RemoveMember: &clusterstate.RemoveMember{Node: mapping.Node, Device: actor, Revision: h.Revision, Admission: &proof}}, now); err != nil { + return out, err + } + if err = g.reconcileMembership(ctx); err != nil { + return out, err + } + _, app, h, read, err = g.membership(ctx) + if err != nil { + return out, err + } + } + r, err := g.controlChange(ctx, clusterstate.Mutation{EnrollMember: &clusterstate.EnrollMember{Node: h.NextNode, Revision: h.Revision, Device: actor, Protocol: request.Protocol, Store: request.Store, Admission: &proof}}, now) + if err != nil { + return out, err + } + mapping.Node = r.Membership.Node + } + // Configuration is a separate maintenance stage. The enrollment reply can + // survive a crash with Barrier=0; activation repairs that committed intent. + _, app, _, read, err = g.membership(ctx) + if err != nil { + return out, err + } + raw, err = read(clusterstate.MemberKey(mapping.Node)) + if err != nil { + return out, err + } + var member clusterstate.MemberRecord + if clusterstate.Decode(raw, &member) != nil || member.Device != actor || member.Store != request.Store || member.Retired || member.Worker || member.Anchor { + return out, directory.ErrUnauthorized + } + out = PersonalEnrollment{Scope: g.peers.view.Scope, Generation: g.peers.view.Generation, Node: member.Node, Incarnation: member.Incarnation, Barrier: member.ReadyIndex, Members: g.peers.view.Members} + return out, nil +} diff --git a/internal/raftdirectory/transport.go b/internal/raftdirectory/transport.go new file mode 100644 index 0000000000..289399d43e --- /dev/null +++ b/internal/raftdirectory/transport.go @@ -0,0 +1,334 @@ +package raftdirectory + +import ( + "context" + "crypto/ed25519" + "errors" + "reflect" + + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + "github.com/Agent-Field/codeaf/internal/rafttransport" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +type snapshotTransfer struct { + from int + message []byte + manifest raftsnapshot.Manifest + generation string + offset uint64 + started int64 +} +type nativePeers struct { + router *rafttransport.Router + view rafttransport.View + keys [3]ed25519.PrivateKey + sequence [3]uint64 + ack [3][3]uint64 + applied map[uint64]uint64 + snapshots map[uint64]*snapshotTransfer + now int64 + sleeping bool + checkpoint bool +} + +// AttachTransport restores authority from applied membership before exposing +// the listener. Generation was durably advanced by the host before this call. +func (g *Native) AttachTransport(ctx context.Context, generation uint64, keys [3]ed25519.PrivateKey) (*rafttransport.Router, error) { + g.mu.Lock() + defer g.mu.Unlock() + router, err := rafttransport.NewRouter(rafttransport.DefaultLimits()) + if err != nil { + return nil, err + } + for i, key := range keys { + if len(key) != ed25519.PrivateKeySize || !key.Public().(ed25519.PublicKey).Equal(ed25519.PublicKey(g.manifest.Config.AnchorKeys[i])) { + return nil, rafttransport.ErrAuth + } + } + g.peers = &nativePeers{router: router, keys: keys, view: rafttransport.View{Scope: rafttransport.Scope{Identity: g.manifest.Config.Identity, Cluster: g.manifest.Config.Cluster, Epoch: g.manifest.Config.Generation, Protocol: raftcore.ProtocolVersion}, Generation: generation}, applied: map[uint64]uint64{}, snapshots: map[uint64]*snapshotTransfer{}} + if err = g.installPeers(ctx); err != nil { + return nil, err + } + if err = g.reconcileMembership(ctx); err != nil { + return nil, err + } + router.Pump = g.pumpPeers + return router, nil +} +func (g *Native) installPeers(ctx context.Context) error { + p := g.peers + if p == nil { + return nil + } + _, app, _, read, err := g.membership(ctx) + if err != nil { + return err + } + v, err := rafttransport.CommittedView(p.view.Scope, p.view.Generation, app.Applied, raftcore.Configuration(app.ConfState), read) + if err != nil { + return err + } + if reflect.DeepEqual(v.Members, p.view.Members) { + return nil + } + if err = p.router.Install(v); err != nil { + return err + } + p.view = v + // Registry changes discard expendable inboxes. Report an unfinished snapshot + // as failed so Raft can restart it under the new committed transport view. + for to, transfer := range p.snapshots { + delete(p.snapshots, to) + if err = g.drive(ctx, transfer.from, raftcore.TransportResult{Peer: to, Snapshot: true}); err != nil { + return err + } + } + return nil +} +func (g *Native) peer(id uint64) (rafttransport.Member, bool) { + for _, m := range g.peers.view.Members { + if m.Node == id && !m.Revoked { + return m, true + } + } + return rafttransport.Member{}, false +} +func (g *Native) envelope(from int, to uint64, payload []byte, chunk *rafttransport.Chunk) (rafttransport.Envelope, error) { + p := g.peers + target, ok := g.peer(to) + if !ok { + return rafttransport.Envelope{}, rafttransport.ErrAuth + } + if p.sequence[from] == ^uint64(0) { + return rafttransport.Envelope{}, rafttransport.ErrGeneration + } + p.sequence[from]++ + e := rafttransport.Envelope{Scope: p.view.Scope, From: g.manifest.Config.Anchors[from], To: to, Incarnation: 1, Destination: target.Incarnation, Sequence: p.sequence[from], Payload: payload, Chunk: chunk} + err := e.Sign(p.keys[from]) + return e, err +} +func (g *Native) localExchange(from int, messages []rafttransport.Envelope) (rafttransport.Response, error) { + p := g.peers + request := rafttransport.Request{Scope: p.view.Scope, Generation: p.view.Generation, Node: g.manifest.Config.Anchors[from], Incarnation: 1, Ack: p.ack[from], Messages: messages} + if err := request.Sign(p.keys[from]); err != nil { + return rafttransport.Response{}, err + } + return p.router.LocalExchange(request, p.now) +} +func (g *Native) queueExternal(from int, to uint64, payload []byte, chunk *rafttransport.Chunk) error { + e, err := g.envelope(from, to, payload, chunk) + if err != nil { + return err + } + _, err = g.localExchange(from, []rafttransport.Envelope{e}) + return err +} +func (g *Native) sendExternal(ctx context.Context, m raftcore.Outbound) error { + p := g.peers + target, ok := g.peer(m.To) + if !ok { + return nil + } + if target.Voter { + return directory.ErrUnreachable + } + if p.sleeping { + return nil + } + message := new(pb.Message) + if proto.Unmarshal(m.Bytes, message) != nil { + return raftcore.ErrPeer + } + from := -1 + for i, id := range g.manifest.Config.Anchors { + if id == message.GetFrom() { + from = i + } + } + if from < 0 { + return raftcore.ErrPeer + } + // A legal maximum command can gain protobuf framing beyond the normal + // transport envelope. Keep BOTH caps: checkpoint its applied prefix and + // let that learner receive chunks, rather than failing a voter write. + if len(m.Bytes) > rafttransport.MaxBatchBytes { + p.checkpoint = true + return g.drive(ctx, from, raftcore.TransportResult{Peer: m.To}) + } + if message.GetType() == pb.MsgSnap { + manifest, err := raftsnapshot.Decode(message.GetSnapshot().GetData()) + if err != nil { + return err + } + p.snapshots[m.To] = &snapshotTransfer{from: from, message: m.Bytes, manifest: manifest, generation: raftsnapshot.ID(message.GetSnapshot().GetData()), started: p.now} + // Start with the manifest; chunks occupy their independently bounded lane. + if err = g.queueExternal(from, m.To, m.Bytes, nil); err != nil { + delete(p.snapshots, m.To) + if errors.Is(err, rafttransport.ErrCapacity) { + return g.drive(ctx, from, raftcore.TransportResult{Peer: m.To, Snapshot: true}) + } + return err + } + return nil + } + if err := g.queueExternal(from, m.To, m.Bytes, nil); err != nil { + if errors.Is(err, rafttransport.ErrCapacity) { + return g.drive(ctx, from, raftcore.TransportResult{Peer: m.To}) + } + return err + } + return nil +} + +// pumpPeers steps authenticated external input through the same persistence +// queue as local messages. The router lock is released before acquiring mu; +// public writes never wait for a learner, its socket or a network callback. +func (g *Native) pumpPeers(request rafttransport.Request, now int64) error { + ctx := context.Background() + g.mu.Lock() + defer g.mu.Unlock() + if g.closed || g.failed != nil { + return directory.ErrUnreachable + } + p := g.peers + p.now = max(p.now, now) + if err := g.installPeers(ctx); err != nil { + return err + } + if _, ok := g.peer(request.Node); !ok || request.Scope != p.view.Scope || request.Generation != p.view.Generation { + return rafttransport.ErrAuth + } + if request.Applied != 0 { + _, app, _, _, err := g.membership(ctx) + if err != nil { + return err + } + if request.Applied > app.Applied { + return rafttransport.ErrWire + } + p.applied[request.Node] = max(p.applied[request.Node], request.Applied) + } + // Empty polls of a sleeping learner must not manufacture heartbeats or wake + // work. A peer frame is real ingress and wakes on the ordinary Step path. + if len(request.Messages) > 0 { + p.sleeping = false + } + for round := 0; round < rafttransport.MaxInflight; round++ { + progressed := false + for i, node := range g.nodes { + if node == nil { + continue + } + response, err := g.localExchange(i, nil) + if err != nil { + return err + } + for _, delivery := range response.Messages { + e := delivery.Envelope + member, ok := g.peer(e.From) + if !ok || e.Scope != p.view.Scope || e.Destination != 1 || e.Verify(member.Key) != nil || e.Chunk != nil { + return rafttransport.ErrAuth + } + if err = g.drive(ctx, i, raftcore.PeerMessage{Bytes: e.Payload}); err != nil { + return err + } + p.ack[i][delivery.Lane] = max(p.ack[i][delivery.Lane], delivery.Cursor) + progressed = true + } + } + if err := g.drain(ctx); err != nil { + return err + } + if !progressed { + break + } + } + if err := g.checkpointExternal(ctx); err != nil { + return err + } + if p.sleeping { + return nil + } + // Read a single chunk at a time. Backpressure discards no authoritative + // state and never pins compaction; an expired transfer is retried by Raft. + for to, transfer := range p.snapshots { + if p.applied[to] >= transfer.manifest.Index { + delete(p.snapshots, to) + if err := g.drive(ctx, transfer.from, raftcore.TransportResult{Peer: to, Snapshot: true, Success: true}); err != nil { + return err + } + continue + } + if p.now-transfer.started >= rafttransport.RetentionMillis { + delete(p.snapshots, to) + if err := g.drive(ctx, transfer.from, raftcore.TransportResult{Peer: to, Snapshot: true}); err != nil { + return err + } + continue + } + if transfer.offset >= transfer.manifest.Chunks { + continue + } + chunk, err := g.stores[transfer.from].ReadSnapshotChunk(ctx, transfer.generation, transfer.offset) + if err != nil { + delete(p.snapshots, to) + if err = g.drive(ctx, transfer.from, raftcore.TransportResult{Peer: to, Snapshot: true}); err != nil { + return err + } + continue + } + err = g.queueExternal(transfer.from, to, chunk, &rafttransport.Chunk{Generation: transfer.generation, Offset: transfer.offset, Hash: raftsnapshot.ID(chunk)}) + if errors.Is(err, rafttransport.ErrCapacity) { + continue + } + if err != nil { + return err + } + transfer.offset++ + } + return g.drain(ctx) +} + +// quiescePeers abandons expendable partial transfers as well as inboxes. The +// next real ingress can immediately retry from the current compacted prefix. +func (g *Native) quiescePeers(ctx context.Context) error { + p := g.peers + p.sleeping = true + p.router.Quiesce(p.view.Scope) + for to, transfer := range p.snapshots { + delete(p.snapshots, to) + if err := g.drive(ctx, transfer.from, raftcore.TransportResult{Peer: to, Snapshot: true}); err != nil { + return err + } + } + return g.drain(ctx) +} + +// checkpointExternal handles normal framing overflow with the existing streamed +// snapshot lane. Only the durably applied prefix is compacted; learner progress +// never delays publication or changes an application or transport byte limit. +func (g *Native) checkpointExternal(ctx context.Context) error { + if g.peers == nil || !g.peers.checkpoint { + return nil + } + i := g.leader() + if i < 0 { + return directory.ErrUnreachable + } + snapshot, err := g.stores[i].CreateSnapshot(ctx) + if err != nil { + return err + } + if err = g.stores[i].Compact(ctx, snapshot.GetMetadata().GetIndex()); err != nil { + return err + } + if err = g.drive(ctx, i, raftcore.CompactSnapshot{Snapshot: snapshot}); err != nil { + return err + } + g.peers.checkpoint = false + return g.drain(ctx) +} diff --git a/internal/rafttransport/README.md b/internal/rafttransport/README.md index 18aba5c962..dce952f312 100644 --- a/internal/rafttransport/README.md +++ b/internal/rafttransport/README.md @@ -1,10 +1,11 @@ # Native peer transport -This is P07's internal relay bridge, not an enrollment service or a directory -writer. `relayserve.Config.Cluster` receives `rafttransport.Handler(router, now)` from -the member host after it loads committed enrollment. An ordinary relay has no peer routes until that assembly -exists. P08 supplies hosted actors, P12 supplies durable enrollment and P17 wires -production client routing. No environment switch selects another writer. +This is the authenticated peer bridge, separate from application ingress. The +persistent native relay assembles its router from committed membership and serves +personal admission, HTTP exchange and the outbound push socket on its ordinary +listener. `relayserve.Config.Cluster` is the internal custom-host handler seam. +The hosted relay's member/control doors remain absent, and the codeaf client does +not yet activate its own learner. No environment switch selects another writer. ## Host ordering and authorization @@ -160,3 +161,60 @@ consent, exclusive certificate expiry, revocation, rotation and current process incarnation. The old metadata-only CommittedView marks workers revoked, so a host that has not supplied these dependencies fails closed. Worker restarts cannot resurrect a retired peer: remove/re-enroll with a fresh node ID. + +## Native personal learners + +The persistent native relay now owns its committed router and mounts enrollment, +exchange and socket routes on its ordinary listener. `Config.Cluster` remains the +internal custom-host assembly seam; the default native service installs its own +router. Hosted member/control routes remain absent. + +`POST /v2/cluster/enroll` uses the existing `reqsign` headers, original paired +personal certificate and body `{"store":"<64 lowercase hex characters>","protocol":1}`. +The client generates that random store token once when creating its SQLite store +and persists it alongside the store. Reopening it reuses the token. Wiping it +requires a new token: the old node is permanently retired, including its store +mapping and tombstone. Admission allocates from committed `NextNode`; caller IDs +and keys are never allocation authority. The original signed request is retained +and checked by the replicated evaluator against the committed root and revocation. +Worker certificates and retained worker/service roles cannot use personal admission. + +The reply is `raftdirectory.PersonalEnrollment`: scope, durable router generation, +node, incarnation, advertised barrier and committed member public keys. The four +uint64 fields cross JSON as decimal strings. A first reply may have barrier `"0"`: +registry admission committed, and activation/maintenance completes the separate +learner configuration. Repeating the same request refreshes the grant without +allocating another node. A nonzero barrier is the committed settlement index; +readiness additionally requires actual learner-local application through it. + +An exchange Request may include `applied` as a decimal string, signed with all its +other fields. The faithful host sets it only after its SQLite application commit. +The control Status `ready` list contains current learners whose signed durable +progress covers their advertised barrier. Leader replication Match and transport +acknowledgments alone never set ready. Router restart forgets this advisory evidence +and requires a new signed report. Registered keys and generations are checked on +HTTP and every socket frame/push. Ordinary public ingress or a returning peer may +activate an existing durable namespace; a peer cannot bootstrap an unknown one. + +The router's Pump callback runs outside its lock. The native actor drains its +anchor inboxes through LocalExchange and completes persistence/application under +its queue, then makes bounded messages available to the outward-dialing learner. +It never dials or waits on that learner. Snapshot manifests precede ≤256 KiB chunks +on the separate chunk lane; the recipient stages all chunks before stepping the +snapshot into its core and reporting application. Retention or a lost transfer +reports failure to Raft for retry, while normal snapshots compact independently +of learner lag. All-voter quiescence discards expendable learner deliveries and +stops scheduling without learner acknowledgments. Fresh public read/write input +wakes the ordinary quorum path. + +The codeaf client does not start its own learner yet. This server assembly adds +no automatic client discovery, network attempts, worker setup or leadership. + +The native host advances its checksummed `transport-generation` file with atomic +rename and directory fsync before opening its router; it never mutates the source +provision. Counter corruption refuses activation and retains evidence. During cold +voter activation external deliveries are unreachable until the committed router +is installed. A maximum-sized legal command can acquire protobuf framing larger +than the normal peer budget. In that case the host checkpoints only the applied +prefix and uses snapshot chunks for catch-up, keeping both existing byte limits +and successful voter operations intact. diff --git a/internal/rafttransport/exchange.go b/internal/rafttransport/exchange.go index 549be92059..ad7d56673a 100644 --- a/internal/rafttransport/exchange.go +++ b/internal/rafttransport/exchange.go @@ -24,6 +24,9 @@ const ( // Request signs polls and acknowledgments as well as outbound digests. A // stolen unsigned cursor must not allow a third party to drain a peer's inbox. type Request struct { + // Applied is signed evidence from the member after its application transaction. + // A transport acknowledgment or leader Match alone never makes it ready. + Applied uint64 `json:"applied,string,omitempty"` Scope Scope `json:"scope"` Node uint64 `json:"node,string"` Incarnation uint64 `json:"incarnation,string"` @@ -296,7 +299,13 @@ func serveSocket(ctx context.Context, conn *websocket.Conn, router *Router, init return } response, err = router.exchange(req, now().UnixMilli(), receiveOnly) - if err != nil || !writeSocket(ctx, conn, response) { + if err != nil { + return + } + if receiveOnly && len(response.Messages) == 0 { + continue + } + if !writeSocket(ctx, conn, response) { return } } diff --git a/internal/rafttransport/router.go b/internal/rafttransport/router.go index 123f2c8a55..3622fcb236 100644 --- a/internal/rafttransport/router.go +++ b/internal/rafttransport/router.go @@ -70,6 +70,10 @@ type group struct { now int64 } type Router struct { + // Pump services locally hosted members after routing, outside the router lock. + // It must never wait for an external member or perform network I/O. + Pump func(Request, int64) error + mu sync.Mutex limits Limits groups map[Scope]*group @@ -232,6 +236,23 @@ func (r *Router) Exchange(req Request, now int64) (Response, error) { return r.exchange(req, now, false) } func (r *Router) exchange(req Request, now int64, receiveOnly bool) (Response, error) { + response, err := r.route(req, now, receiveOnly) + if err != nil || r.Pump == nil { + return response, err + } + if err = r.Pump(req, now); err != nil { + return Response{}, err + } + return r.route(req, now, true) +} + +// LocalExchange is the actor's internal side of the SAME authenticated router. +// It avoids recursively entering Pump while that actor holds its own queue. +func (r *Router) LocalExchange(req Request, now int64) (Response, error) { + return r.route(req, now, false) +} + +func (r *Router) route(req Request, now int64, receiveOnly bool) (Response, error) { if now < 0 || now > math.MaxInt64-RetentionMillis { return Response{}, ErrWire } @@ -396,3 +417,26 @@ func (r *Router) Stats(scope Scope, now int64) Stats { } return s } + +// Quiesce discards only expendable deliveries to nonvoters at the all-voter +// sleep boundary. Learner queues cannot keep a group active or push old work +// after sleep. Raft sends fresh append or snapshot traffic after public wake. +func (r *Router) Quiesce(scope Scope) { + r.mu.Lock() + defer r.mu.Unlock() + g := r.groups[scope] + if g == nil { + return + } + for id, box := range g.inboxes { + if g.members[id].Voter { + continue + } + for lane, queue := range box.queues { + for _, q := range queue { + g.bytes[lane] -= q.size + } + box.queues[lane] = nil + } + } +} diff --git a/internal/relayserve/native.go b/internal/relayserve/native.go index 11ae6c48ac..eeef9eb25c 100644 --- a/internal/relayserve/native.go +++ b/internal/relayserve/native.go @@ -5,6 +5,8 @@ import ( "context" "crypto/ed25519" "crypto/rand" + "crypto/sha256" + "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -19,6 +21,7 @@ import ( "github.com/Agent-Field/codeaf/internal/dirwatch" "github.com/Agent-Field/codeaf/internal/raftdirectory" "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/rafttransport" "github.com/Agent-Field/codeaf/internal/reqsign" "github.com/Agent-Field/codeaf/internal/wireauth" ) @@ -33,6 +36,7 @@ type nativeProvision struct { } type nativeNamespace struct { timer memberTimer + peers http.Handler *raftdirectory.V1Directory host *raftdirectory.Native feed *directory.Feed @@ -195,6 +199,31 @@ func (n *namespaces) activate(ctx context.Context, id string, root []byte) (*nat d.Close() return nil, err } + // Inbox loss fences old cursors, independently of the IMMUTABLE source + // provision. A restart must never rewrite that manifest or its keys. + generation, e := advanceNativeTransport(filepath.Join(path, "transport-generation")) + if e != nil { + d.Close() + return nil, e + } + var keys [3]ed25519.PrivateKey + for i, key := range provision.Anchors { + keys[i] = ed25519.PrivateKey(key) + } + router, e := host.AttachTransport(ctx, generation, keys) + if e != nil { + d.Close() + return nil, e + } + pump := router.Pump + router.Pump = func(request rafttransport.Request, now int64) error { + err := pump(request, now) + if err == nil && len(request.Messages) > 0 { + d.armMembers() + } + return err + } + d.peers = rafttransport.Handler(router, n.now) d.timer = memberTimer{started: n.now(), anchor: ed25519.PrivateKey(provision.Anchors[0]), after: func(delay time.Duration, fn func()) func() bool { return time.AfterFunc(delay, fn).Stop }} if n.after != nil { d.timer.after = n.after @@ -457,3 +486,48 @@ func (c *nativeClient) DenyRequest(ctx context.Context, code string) error { } return c.owner.effects.Deliver(ctx, c.owner.now().UnixMilli(), c.owner.links, code) } + +// advanceNativeTransport publishes a checksummed counter before exchange can +// open. Its own file is mutable host state; the source provision never changes. +func advanceNativeTransport(path string) (uint64, error) { + raw, err := os.ReadFile(path) + generation := uint64(0) + if err == nil { + if len(raw) != 8+sha256.Size { + return 0, rafttransport.ErrGeneration + } + sum := sha256.Sum256(raw[:8]) + if !bytes.Equal(raw[8:], sum[:]) { + return 0, rafttransport.ErrGeneration + } + generation = binary.BigEndian.Uint64(raw[:8]) + if generation == 0 || generation == ^uint64(0) { + return 0, rafttransport.ErrGeneration + } + } else if !errors.Is(err, os.ErrNotExist) { + return 0, err + } + generation++ + raw = make([]byte, 8, 8+sha256.Size) + binary.BigEndian.PutUint64(raw, generation) + sum := sha256.Sum256(raw) + raw = append(raw, sum[:]...) + f, err := os.CreateTemp(filepath.Dir(path), ".transport-") + if err != nil { + return 0, err + } + defer os.Remove(f.Name()) + _, err = f.Write(raw) + err = errors.Join(err, f.Sync(), f.Close()) + if err != nil { + return 0, err + } + if err = os.Rename(f.Name(), path); err != nil { + return 0, err + } + dir, err := os.Open(filepath.Dir(path)) + if err != nil { + return 0, err + } + return generation, errors.Join(dir.Sync(), dir.Close()) +} diff --git a/internal/relayserve/personal.go b/internal/relayserve/personal.go new file mode 100644 index 0000000000..7f13251276 --- /dev/null +++ b/internal/relayserve/personal.go @@ -0,0 +1,103 @@ +package relayserve + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "io" + "net/http" + "os" + "path/filepath" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/rafttransport" + "github.com/Agent-Field/codeaf/internal/reqsign" +) + +func (n *namespaces) enrollPersonal(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + if r.Method != http.MethodPost || r.URL.RawQuery != "" { + http.NotFound(w, r) + return + } + raw, err := io.ReadAll(http.MaxBytesReader(w, r.Body, cs.MaxCommandBytes)) + if err != nil { + http.Error(w, "invalid enrollment", 400) + return + } + id, _, err := n.authenticate(r, raw) + if err != nil { + http.Error(w, "this computer is not allowed to join this cluster", 403) + return + } + proof, err := reqsign.Capture(r, raw) + if err != nil { + http.Error(w, "invalid enrollment", 403) + return + } + n.mu.Lock() + d := n.groups[id] + n.mu.Unlock() + if d == nil { + http.Error(w, "cluster is unavailable", 503) + return + } + out, err := d.host.EnrollPersonal(r.Context(), proof) + if err != nil { + var refusal raftdirectory.EnrollmentRefusal + if errors.As(err, &refusal) { + status := http.StatusConflict + if refusal.Code == cs.Capacity { + status = http.StatusTooManyRequests + } + http.Error(w, refusal.Error(), status) + return + } + http.Error(w, "this computer is not allowed to join this cluster", 403) + return + } + d.armMembers() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(out) +} + +// exchangePersonal never discovers or creates a namespace from a peer key. +// Only admission or ordinary signed public ingress can activate its committed +// stores. Established sockets use the router's per-frame current view checks. +func (n *namespaces) exchangePersonal(w http.ResponseWriter, r *http.Request) { + var raw []byte + var err error + if r.URL.Path == rafttransport.SocketPath { + raw, err = base64.RawURLEncoding.DecodeString(r.Header.Get(rafttransport.SocketAuthHeader)) + } else { + raw, err = io.ReadAll(http.MaxBytesReader(w, r.Body, rafttransport.MaxExchangeBytes)) + r.Body = io.NopCloser(bytes.NewReader(raw)) + } + var request rafttransport.Request + if err != nil || rafttransport.Decode(raw, &request) != nil { + http.Error(w, "invalid peer request", 400) + return + } + n.mu.Lock() + d := n.groups[request.Scope.Identity] + n.mu.Unlock() + if d == nil && checkedID(request.Scope.Identity) == nil { + // A returning peer may activate an EXISTING durable namespace, but an + // unregistered key cannot create storage or invent a cluster root. + if _, err := os.Stat(filepath.Join(n.root, "raft", request.Scope.Identity, "provision.json")); err == nil { + d, _ = n.activate(r.Context(), request.Scope.Identity, nil) + } + } + if d == nil || d.peers == nil { + http.Error(w, "member authorization refused", 403) + return + } + d.peers.ServeHTTP(w, r) + // Merely polling an idle learner does not arm recurring work. Actual peer + // ingress wakes the same scheduler as every other consensus input. + if len(request.Messages) > 0 { + d.armMembers() + } +} diff --git a/internal/relayserve/personal_test.go b/internal/relayserve/personal_test.go new file mode 100644 index 0000000000..ff5346ff40 --- /dev/null +++ b/internal/relayserve/personal_test.go @@ -0,0 +1,880 @@ +package relayserve_test + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterapi" + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/rafttransport" + "github.com/Agent-Field/codeaf/internal/reqsign" + "github.com/coder/websocket" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +type personalSigner struct { + root ed25519.PublicKey + key ed25519.PrivateKey + cert identity.Cert +} + +func (s personalSigner) IdentityKey() ed25519.PublicKey { return s.root } +func (s personalSigner) Cert() identity.Cert { return s.cert } +func (s personalSigner) Sign(b []byte) []byte { return ed25519.Sign(s.key, b) } +func personalDevice(t *testing.T, id identity.Identity) personalSigner { + t.Helper() + pub, key, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + return personalSigner{id.PublicKey(), key, identity.IssueCert(id, pub, time.UnixMilli(1000))} +} +func personalToken(t *testing.T) string { + t.Helper() + var b [32]byte + if _, err := rand.Read(b[:]); err != nil { + t.Fatal(err) + } + return hex.EncodeToString(b[:]) +} +func admission(t *testing.T, r *rig, s reqsign.Signer, store string, protocol uint16) (raftdirectory.PersonalEnrollment, int, string) { + t.Helper() + raw, _ := json.Marshal(cs.PersonalAdmission{Store: store, Protocol: protocol}) + req, err := http.NewRequest(http.MethodPost, r.srv.URL+cs.PersonalAdmissionPath, bytes.NewReader(raw)) + if err != nil { + t.Fatal(err) + } + reqsign.Sign(req, raw, s, r.clock.Now()) + response, err := r.srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + var out raftdirectory.PersonalEnrollment + if response.StatusCode == 200 && json.Unmarshal(body, &out) != nil { + t.Fatal("invalid enrollment reply") + } + return out, response.StatusCode, string(body) +} +func personalStatus(t *testing.T, r *rig, s reqsign.Signer) clusterctl.Status { + t.Helper() + out, err := (clusterctl.Client{Base: r.srv.URL, Sign: reqsign.SignFor(s, r.clock.Now), HTTP: r.srv.Client()}).Status(bg) + if err != nil { + t.Fatal(err) + } + return out +} +func joined(t *testing.T, r *rig, s reqsign.Signer, store string) raftdirectory.PersonalEnrollment { + t.Helper() + out, status, body := admission(t, r, s, store, 1) + if status != 200 { + t.Fatalf("enrollment %d: %s", status, body) + } + if out.Barrier == 0 { + r.clock.Advance(time.Duration(cs.TickEveryMillis) * time.Millisecond) + out, status, body = admission(t, r, s, store, 1) + if status != 200 { + t.Fatalf("retry %d: %s", status, body) + } + } + if out.Node <= 3 || out.Barrier == 0 { + t.Fatalf("no durable learner grant: %+v", out) + } + return out +} + +// The test learner has its own real core and FULL SQLite store. Only its +// outbound Client reaches the relay. It completes Persist/Apply before signing +// durable progress, and retains an incoming snapshot until every chunk is saved. +type personalLearner struct { + t *testing.T + signer personalSigner + grant raftdirectory.PersonalEnrollment + store *raftstore.SQLite + core *raftcore.Core + path string + status raftcore.Status + pending []rafttransport.Envelope + sequence uint64 + ack [3]uint64 + snapshot *pb.Message + chunks map[uint64]bool + receivedChunks int + maxChunk int + snapshots int +} + +func learner(t *testing.T, s personalSigner, grant raftdirectory.PersonalEnrollment) *personalLearner { + t.Helper() + l := &personalLearner{t: t, signer: s, grant: grant, path: filepath.Join(t.TempDir(), "learner.db")} + l.open() + t.Cleanup(func() { l.store.Close() }) + return l +} +func (l *personalLearner) open() { + l.t.Helper() + var err error + l.store, err = raftstore.Open(l.path, raftstore.Identity{ClusterUUID: l.grant.Scope.Cluster, NodeID: l.grant.Node, Protocol: l.grant.Scope.Protocol}) + if err != nil { + l.t.Fatal(err) + } + recovered, err := l.store.Recover(bg) + if err != nil { + l.t.Fatal(err) + } + l.core, err = raftcore.New(raftcore.Config{ClusterUUID: l.grant.Scope.Cluster, NodeID: l.grant.Node, Protocol: l.grant.Scope.Protocol, Seed: l.grant.Node}, recovered.Core) + if err != nil { + l.t.Fatal(err) + } + l.step(raftcore.Poll{}) +} +func (l *personalLearner) step(input raftcore.Input) { + l.t.Helper() + for n := 0; n < 10000; n++ { + effects, err := l.core.Step(input) + if err != nil { + l.t.Fatal(err) + } + l.status = effects.Status + for _, m := range effects.Send { + var destination uint64 + for _, member := range l.grant.Members { + if member.Node == m.To { + destination = member.Incarnation + } + } + l.sequence++ + e := rafttransport.Envelope{Scope: l.grant.Scope, From: l.grant.Node, To: m.To, Incarnation: l.grant.Incarnation, Destination: destination, Sequence: l.sequence, Payload: m.Bytes} + if err = e.Sign(l.signer.key); err != nil { + l.t.Fatal(err) + } + l.pending = append(l.pending, e) + } + if effects.Persist != nil { + if err = l.store.Persist(bg, *effects.Persist); err != nil { + l.t.Fatal(err) + } + input = raftcore.Persisted{BatchID: effects.Persist.ID} + continue + } + if effects.Apply != nil { + if _, err = raftdirectory.Apply(bg, l.store, *effects.Apply, l.grant.Scope.Cluster, 0); err != nil { + l.t.Fatal(err) + } + input = raftcore.Applied{BatchID: effects.Apply.ID} + continue + } + return + } + l.t.Fatal("learner exceeded bounded completion budget") +} +func (l *personalLearner) request() rafttransport.Request { + l.t.Helper() + app, _, err := l.store.ApplicationView(bg) + if err != nil { + l.t.Fatal(err) + } + req := rafttransport.Request{Scope: l.grant.Scope, Node: l.grant.Node, Incarnation: l.grant.Incarnation, Generation: l.grant.Generation, Ack: l.ack, Applied: app.Applied} + if len(l.pending) > 0 { + req.Messages = l.pending[:1] + } + if err = req.Sign(l.signer.key); err != nil { + l.t.Fatal(err) + } + return req +} +func (l *personalLearner) exchange(r *rig) (rafttransport.Response, error) { + l.t.Helper() + req := l.request() + response, err := (rafttransport.Client{URL: r.srv.URL, HTTP: r.srv.Client()}).Exchange(bg, req) + if err != nil { + return response, err + } + if len(req.Messages) > 0 { + l.pending = l.pending[1:] + } + l.receive(response) + return response, nil +} +func (l *personalLearner) receive(response rafttransport.Response) { + l.t.Helper() + for _, delivery := range response.Messages { + if delivery.Cursor <= l.ack[delivery.Lane] { + continue + } + e := delivery.Envelope + var source ed25519.PublicKey + for _, m := range l.grant.Members { + if m.Node == e.From && m.Incarnation == e.Incarnation && !m.Revoked { + source = m.Key + } + } + if e.Verify(source) != nil || e.Scope != l.grant.Scope || e.To != l.grant.Node || e.Destination != l.grant.Incarnation { + l.t.Fatal("unauthorized peer delivery") + } + if e.Chunk != nil { + if l.snapshot == nil || e.Chunk.Generation != raftsnapshot.ID(l.snapshot.GetSnapshot().GetData()) { + l.t.Fatal("chunk without its signed manifest") + } + if err := l.store.PutSnapshotChunk(bg, e.Chunk.Generation, e.Chunk.Offset, e.Payload, e.Chunk.Hash); err != nil { + l.t.Fatal(err) + } + l.chunks[e.Chunk.Offset] = true + l.receivedChunks++ + l.maxChunk = max(l.maxChunk, len(e.Payload)) + manifest, err := raftsnapshot.Decode(l.snapshot.GetSnapshot().GetData()) + if err != nil { + l.t.Fatal(err) + } + if uint64(len(l.chunks)) == manifest.Chunks { + pointer, err := l.store.CompleteSnapshot(bg, e.Chunk.Generation) + if err != nil || !proto.Equal(pointer, l.snapshot.GetSnapshot()) { + l.t.Fatalf("invalid completed snapshot: %v", err) + } + raw, err := proto.Marshal(l.snapshot) + if err != nil { + l.t.Fatal(err) + } + l.step(raftcore.PeerMessage{Bytes: raw}) + l.snapshot = nil + l.snapshots++ + } + } else { + m := new(pb.Message) + if proto.Unmarshal(e.Payload, m) != nil { + l.t.Fatal("invalid raft message") + } + if m.GetType() == pb.MsgSnap && m.GetSnapshot().GetMetadata().GetIndex() > l.status.Applied { + if l.snapshot == nil || !proto.Equal(l.snapshot.GetSnapshot(), m.GetSnapshot()) { + if _, err := l.store.BeginSnapshot(bg, m.GetSnapshot().GetData()); err != nil { + l.t.Fatal(err) + } + l.snapshot = m + l.chunks = map[uint64]bool{} + } + } else { + l.step(raftcore.PeerMessage{Bytes: e.Payload}) + } + } + l.ack[delivery.Lane] = delivery.Cursor + } +} +func (l *personalLearner) catchUp(r *rig, index uint64) { + l.t.Helper() + for round := 0; round < 120; round++ { + if _, err := l.exchange(r); err != nil { + l.t.Fatalf("catch-up round %d: %v", round, err) + } + if l.status.Applied >= index && len(l.pending) == 0 { + if _, err := l.exchange(r); err != nil { + l.t.Fatal(err) + } + return + } + if round%10 == 9 { + r.clock.Advance(time.Duration(cs.TickEveryMillis) * time.Millisecond) + } + } + l.t.Fatalf("R9 learner did not apply advertised barrier %d: %+v", index, l.status) +} + +// E1/R9/R12 prove allocation at the authenticated production door, including +// exact-store retries, wipe retirement, protocol and personal-only authority. +func TestR9R12PersonalEnrollmentDoor(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + s := personalDevice(t, id) + token := personalToken(t) + first := joined(t, r, s, token) + again := joined(t, r, s, token) + if first.Node != again.Node || first.Barrier != again.Barrier { + t.Fatal("same store allocated twice") + } + replacement := joined(t, r, s, personalToken(t)) + if replacement.Node <= first.Node { + t.Fatal("wiped store reused a node ID") + } + if _, status, _ := admission(t, r, s, token, 1); status == 200 { + t.Fatal("retired store retired its replacement") + } + status := personalStatus(t, r, s) + if status.Self != replacement.Node || !reflect.DeepEqual(status.Configuration.Voters, []uint64{1, 2, 3}) || !reflect.DeepEqual(status.Configuration.Learners, []uint64{replacement.Node}) { + t.Fatalf("wrong topology: %+v", status) + } + if _, code, body := admission(t, r, s, personalToken(t), 2); code == 200 || !strings.Contains(body, cs.EnrollmentMessage(cs.UpdateRequired)) { + t.Fatalf("protocol refusal %d %s", code, body) + } + // The old transport generation and node remain fenced after replacement. + old := learner(t, s, first) + if _, err := old.exchange(r); !errors.Is(err, rafttransport.ErrAuth) { + t.Fatalf("retired peer accepted: %v", err) + } + if err := directory.NewHTTP(r.srv.URL, reqsign.SignFor(s, r.clock.Now), r.srv.Client()).PutDevice(bg, s.cert.DeviceID(), directory.Device{V: 1}); err != nil { + t.Fatal(err) + } + if err := r.dir(a).Revoke(bg, s.cert.DeviceID()); err != nil { + t.Fatal(err) + } + if _, code, _ := admission(t, r, s, personalToken(t), 1); code == 200 { + t.Fatal("revoked device enrolled") + } + unsigned, err := http.Post(r.srv.URL+cs.PersonalAdmissionPath, "application/json", strings.NewReader(`{}`)) + if err != nil { + t.Fatal(err) + } + unsigned.Body.Close() + if unsigned.StatusCode != 403 { + t.Fatal("unpaired admission accepted") + } +} + +// E1/R9/R27 count live learners at the real admission boundary; the 33rd +// request uses the existing manual sentence and leaves the voters unchanged. +func TestR9R27PersonalEnrollmentCapacity(t *testing.T) { + r := newRig(t) + id, _ := newIdentity(t) + for n := 0; n < cs.MaxLearners; n++ { + joined(t, r, personalDevice(t, id), personalToken(t)) + } + s := personalDevice(t, id) + if _, status, body := admission(t, r, s, personalToken(t), 1); status != 429 || !strings.Contains(body, cs.EnrollmentMessage(cs.Capacity)) { + t.Fatalf("capacity: %d %s", status, body) + } + state := personalStatus(t, r, s) + if len(state.Configuration.Learners) != cs.MaxLearners || !reflect.DeepEqual(state.Configuration.Voters, []uint64{1, 2, 3}) { + t.Fatalf("capacity changed quorum: %+v", state) + } +} + +// E2/R12 cuts the relay after its enrollment reply, while the committed intent +// is still outside the configuration. Activation, with NO second admission, +// must complete that same node's learner configuration and advertised barrier. +func TestR12PersonalEnrollmentCrashBetweenStages(t *testing.T) { + r := newRig(t) + id, _ := newIdentity(t) + s := personalDevice(t, id) + token := personalToken(t) + admitted, code, body := admission(t, r, s, token, 1) + if code != 200 || admitted.Barrier != 0 { + t.Fatalf("expected separate enrollment stage: %+v %d %s", admitted, code, body) + } + before := personalStatus(t, r, s) + if before.Membership.Pending == nil || len(before.Configuration.Learners) != 0 { + t.Fatal("fixture did not cut between stages") + } + r.restart() + after := personalStatus(t, r, s) + if after.Self != admitted.Node || after.Membership.Pending != nil || !reflect.DeepEqual(after.Configuration.Learners, []uint64{admitted.Node}) { + t.Fatalf("activation did not finish intent: %+v", after) + } + resumed := joined(t, r, s, token) + if resumed.Node != admitted.Node || resumed.Barrier == 0 || resumed.Generation <= admitted.Generation { + t.Fatal("restart reused transport generation or allocated again") + } + stale := learner(t, s, admitted) + if _, err := stale.exchange(r); !errors.Is(err, rafttransport.ErrGeneration) { + t.Fatalf("old generation accepted: %v", err) + } + l := learner(t, s, resumed) + l.catchUp(r, resumed.Barrier) +} + +// E3/R23 verifies HTTP and the outbound push socket on the ordinary native +// listener. Registered keys alone may send consensus, and revocation fences an +// already-open socket as well as future polls and cross-identity addressing. +func TestR16R23PersonalExchangeAndSocket(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + s := personalDevice(t, id) + grant := joined(t, r, s, personalToken(t)) + l := learner(t, s, grant) + bad := l.request() + _, wrong, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + bad.Sign(wrong) + if _, err = (rafttransport.Client{URL: r.srv.URL}).Exchange(bg, bad); !errors.Is(err, rafttransport.ErrAuth) { + t.Fatalf("unregistered key accepted: %v", err) + } + foreign := l.request() + foreign.Scope.Identity = "id_00000000000000000000000000000000" + foreign.Sign(s.key) + if _, err = (rafttransport.Client{URL: r.srv.URL}).Exchange(bg, foreign); !errors.Is(err, rafttransport.ErrAuth) { + t.Fatalf("cross-identity peer accepted: %v", err) + } + for _, kind := range []pb.MessageType{pb.MsgProp, pb.MsgHup, pb.MsgStorageAppend} { + raw, _ := proto.Marshal(&pb.Message{Type: kind.Enum(), From: proto.Uint64(grant.Node), To: proto.Uint64(1), Term: proto.Uint64(1)}) + e := rafttransport.Envelope{Scope: grant.Scope, From: grant.Node, To: 1, Incarnation: 1, Destination: 1, Sequence: 1, Payload: raw} + if e.Sign(s.key) == nil { + t.Fatalf("peer %s was offered", kind) + } + // A hostile sender can bypass its local Sign validator. The actual + // relay still refuses the signed envelope before any core input. + digest := sha256.Sum256(raw) + e.Digest = hex.EncodeToString(digest[:]) + unsigned, _ := json.Marshal(e) + e.Signature = ed25519.Sign(s.key, append([]byte("codeaf/raft/envelope/1\n"), unsigned...)) + request := l.request() + request.Messages = []rafttransport.Envelope{e} + if err = request.Sign(s.key); err != nil { + t.Fatal(err) + } + if _, err = (rafttransport.Client{URL: r.srv.URL}).Exchange(bg, request); err == nil { + t.Fatalf("relay accepted peer %s", kind) + } + } + req := l.request() + raw, _ := rafttransport.Encode(req) + headers := http.Header{rafttransport.SocketAuthHeader: []string{base64.RawURLEncoding.EncodeToString(raw)}} + ctx, cancel := context.WithTimeout(bg, 10*time.Second) + defer cancel() + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(r.srv.URL, "http")+rafttransport.SocketPath, &websocket.DialOptions{HTTPHeader: headers}) + if err != nil { + t.Fatal(err) + } + defer conn.CloseNow() + conn.SetReadLimit(rafttransport.MaxExchangeBytes) + kind, raw, err := conn.Read(ctx) + if err != nil || kind != websocket.MessageBinary { + t.Fatalf("socket initial delivery: %v", err) + } + var response rafttransport.Response + if rafttransport.Decode(raw, &response) != nil { + t.Fatal("invalid pushed response") + } + l.receive(response) + l.catchUp(r, grant.Barrier) + if err = directory.NewHTTP(r.srv.URL, reqsign.SignFor(s, r.clock.Now), r.srv.Client()).PutDevice(bg, s.cert.DeviceID(), directory.Device{V: 1}); err != nil { + t.Fatal(err) + } + if err = r.dir(a).Revoke(bg, s.cert.DeviceID()); err != nil { + t.Fatal(err) + } + raw, _ = rafttransport.Encode(l.request()) + _ = conn.Write(ctx, websocket.MessageBinary, raw) + for n := 0; n < 8; n++ { + if _, _, err = conn.Read(ctx); err != nil { + break + } + } + if err == nil { + t.Fatal("revoked socket kept serving") + } + if _, err = l.exchange(r); !errors.Is(err, rafttransport.ErrAuth) { + t.Fatalf("revoked poll accepted: %v", err) + } +} + +// E4/R9 requires evidence from each learner's durable application, not just a +// configuration row or uploaded snapshot. Both learners remain nonvoting. +func TestR9PersonalLearnersSnapshotLogAndReady(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + // Multiple bounded sealed rows produce full-size snapshot chunks while + // keeping every public command below the existing 64 KiB command cap. + for n := 0; n < 18; n++ { + d := personalDevice(t, id) + if err := directory.NewHTTP(r.srv.URL, reqsign.SignFor(d, r.clock.Now), r.srv.Client()).PutDevice(bg, d.cert.DeviceID(), directory.Device{V: 1, Name: strings.Repeat("e", 20000)}); err != nil { + t.Fatal(err) + } + } + s := personalDevice(t, id) + grant := joined(t, r, s, personalToken(t)) + l := learner(t, s, grant) + initial := personalStatus(t, r, s) + if len(initial.Ready) != 0 { + t.Fatal("ready before learner applied") + } + l.catchUp(r, grant.Barrier) + if l.snapshots == 0 || l.receivedChunks < 2 || l.maxChunk != raftsnapshot.ChunkBytes { + t.Fatalf("no streamed snapshot: snapshots=%d chunks=%d max=%d", l.snapshots, l.receivedChunks, l.maxChunk) + } + if err := r.dir(a).PutDevice(bg, a.dev.ID(), directory.Device{V: 1, Name: "after snapshot"}); err != nil { + t.Fatal(err) + } + target := personalStatus(t, r, s).Membership.Index + l.catchUp(r, max(target, l.status.Applied+1)) + s2 := personalDevice(t, id) + g2 := joined(t, r, s2, personalToken(t)) + l2 := learner(t, s2, g2) + l2.catchUp(r, g2.Barrier) + // Refresh routing metadata from committed admission before accepting the + // second learner's node as a possible peer source. + l.grant.Members = g2.Members + l.catchUp(r, g2.Barrier) + state := personalStatus(t, r, s) + if !reflect.DeepEqual(state.Ready, []uint64{grant.Node, g2.Node}) || !reflect.DeepEqual(state.Configuration.Voters, []uint64{1, 2, 3}) || !reflect.DeepEqual(state.Configuration.Learners, []uint64{grant.Node, g2.Node}) { + t.Fatalf("wrong ready learners: %+v", state) + } + if err := l.store.Close(); err != nil { + t.Fatal(err) + } + l.open() + l.catchUp(r, g2.Barrier) + if l.status.Applied < g2.Barrier { + t.Fatal("same learner store lost durable catch-up") + } +} + +// E5/R9/R27 proves a blocked or killed exchange cannot gate public operations +// or compaction. The resumed learner catches up from a later snapshot. +func TestR9R27BlockedLearnerCannotDelayVotersOrCompaction(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + s := personalDevice(t, id) + grant := joined(t, r, s, personalToken(t)) + l := learner(t, s, grant) + l.catchUp(r, grant.Barrier) + // Hold an authenticated exchange at its response boundary. Public work + // must finish while this learner is blocked, then survive its cancellation. + entered := make(chan struct{}) + release := make(chan struct{}) + blocked := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + r.svc.Handler.ServeHTTP(&blockedPeerWriter{ResponseWriter: w, entered: entered, release: release}, request) + })) + ctx, cancel := context.WithCancel(bg) + finished := make(chan error, 1) + go func() { + _, err := (rafttransport.Client{URL: blocked.URL, HTTP: blocked.Client()}).Exchange(ctx, l.request()) + finished <- err + }() + select { + case <-entered: + case <-time.After(10 * time.Second): + cancel() + close(release) + blocked.Close() + t.Fatal("learner exchange did not reach response boundary") + } + done := make(chan error, 1) + go func() { + done <- r.dir(a).PutDevice(bg, a.dev.ID(), directory.Device{V: 1, Name: "while learner blocked"}) + }() + select { + case err := <-done: + if err != nil { + t.Error(err) + } + case <-time.After(10 * time.Second): + t.Error("public write waited on blocked learner") + } + if _, err := r.dir(a).List(bg); err != nil { + t.Error(err) + } + cancel() + close(release) + <-finished + blocked.Close() + var err error + // Bounded real signed writes cross the existing 4 MiB log threshold + // while the learner is absent. No compaction threshold is changed. + for n := 0; n < 120; n++ { + if err = r.dir(a).PutDevice(bg, a.dev.ID(), directory.Device{V: 1, Name: strings.Repeat("s", 20000) + fmt.Sprint(n)}); err != nil { + t.Fatal(err) + } + } + before := l.snapshots + r.clock.Advance(time.Duration(cs.TickEveryMillis) * time.Millisecond) + l.catchUp(r, l.status.Applied+1) + if l.snapshots <= before { + t.Fatal("dead learner pinned compaction instead of getting a new snapshot") + } +} + +// E5/E6/R26/R27 use the production fake-clock scheduler with a real learner. +// All voters sleep without learner acks; no peer frames recur for ten minutes, +// and first read then write wake before the learner receives the new log. +func TestR9R26R27PersonalLearnerSleepAndWake(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + s := personalDevice(t, id) + grant := joined(t, r, s, personalToken(t)) + l := learner(t, s, grant) + // This learner stalls before catch-up. Neither its progress nor its + // final acknowledgment may be required for the voters to sleep. + for n := 0; n < 20; n++ { + r.clock.Advance(5 * time.Second) + } + inspect := func() (raftstore.Application, cs.Quiescence) { + t.Helper() + store, err := raftstore.Open(filepath.Join(r.store, "raft", id.ID(), "1.db"), raftstore.Identity{ClusterUUID: grant.Scope.Cluster, NodeID: 1, Protocol: 1}) + if err != nil { + t.Fatal(err) + } + defer store.Close() + app, _, err := store.ApplicationView(bg) + if err != nil { + t.Fatal(err) + } + key := raftstore.Key{Namespace: raftstore.Membership, Key: cs.QuiescenceKey().Key} + rows, _, err := store.Read(bg, app.Applied, []raftstore.Key{key}) + if err != nil { + t.Fatal(err) + } + var marker cs.Quiescence + if cs.Decode(rows[key], &marker) != nil { + t.Fatal("no durable sleep marker") + } + return app, marker + } + asleep, marker := inspect() + if marker.Index != asleep.Applied { + t.Fatal("learner prevented all-voter sleep") + } + for n := 0; n < 120; n++ { + r.clock.Advance(5 * time.Second) + } + after, _ := inspect() + if after.Applied != asleep.Applied { + t.Fatal("idle learner caused recurring application work") + } + response, err := l.exchange(r) + if err != nil || len(response.Messages) != 0 { + t.Fatalf("sleeping relay sent learner traffic: %+v %v", response, err) + } + if _, err = r.dir(a).List(bg); err != nil { + t.Fatal("first quorum read after sleep:", err) + } + read, _ := inspect() + if read.Applied <= after.Applied { + t.Fatal("first read reused sleep authority") + } + if err = r.dir(a).PutDevice(bg, a.dev.ID(), directory.Device{V: 1, Name: "after wake"}); err != nil { + t.Fatal(err) + } + written, _ := inspect() + if written.Applied <= read.Applied { + t.Fatal("first write did not durably wake") + } + l.catchUp(r, written.Applied) + t.Logf("R9/R26/R27 ten-minute idle applied=%d peer deliveries=0; fresh read=%d write=%d learner=%d", asleep.Applied, read.Applied, written.Applied, l.status.Applied) +} + +// E1/R9/R16 keeps scoped credentials out of personal admission, even if the +// shared root also issued a personal certificate for that retained worker key. +func TestR9R16PersonalAdmissionRefusesWorkerAuthority(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + if _, err := r.dir(a).Create(bg, cell, directory.CellInit{Head: head1}); err != nil { + t.Fatal(err) + } + state := personalStatus(t, r, a) + s := personalDevice(t, id) + workerCert, err := identity.IssueWorkerCert(id, s.key.Public().(ed25519.PublicKey), state.Cluster, []string{cell}, r.clock.Now(), r.clock.Now().Add(time.Hour), true) + if err != nil { + t.Fatal(err) + } + certRaw, _ := json.Marshal(a.Cert()) + cert := base64.RawURLEncoding.EncodeToString(certRaw) + command := cs.Command{Schema: 1, Cluster: state.Cluster, Actor: a.dev.ID(), Certificate: cert, Session: "enroll-worker-test", Sequence: 1, Now: r.clock.Now().UnixMilli(), Change: cs.Mutation{EnrollWorker: &cs.EnrollWorker{Certificate: workerCert, Grants: map[string][]byte{cell: {1}}}}} + command.RequestDigest, err = cs.Digest(command) + if err != nil { + t.Fatal(err) + } + raw, err := cs.SigningBytes(command) + if err != nil { + t.Fatal(err) + } + command.Signature = a.Sign(raw) + client := clusterapi.Client{Base: r.srv.URL, HTTP: r.srv.Client(), Now: r.clock.Now, Allowed: func() bool { return true }, Sign: a.Sign, Request: func() cs.ClusterRequest { + return cs.ClusterRequest{IdentityKey: id.PublicKey(), Cluster: state.Cluster, Certificate: cert} + }} + result, err := client.Submit(bg, command) + if err != nil || result.Code != cs.OK { + t.Fatalf("worker setup: %+v %v", result, err) + } + if _, status, _ := admission(t, r, s, personalToken(t), 1); status != 403 { + t.Fatalf("retained worker role enrolled as personal: %d", status) + } + body, _ := json.Marshal(cs.PersonalAdmission{Store: personalToken(t), Protocol: 1}) + req, err := http.NewRequest(http.MethodPost, r.srv.URL+cs.PersonalAdmissionPath, bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + reqsign.Sign(req, body, s, r.clock.Now()) + raw, _ = json.Marshal(workerCert) + req.Header.Set(reqsign.HeaderCert, base64.RawURLEncoding.EncodeToString(raw)) + response, err := r.srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != 403 { + t.Fatal("worker certificate enrolled as personal") + } + if after := personalStatus(t, r, a); len(after.Configuration.Learners) != 0 || after.Membership.Live != 0 { + t.Fatal("refused worker changed membership") + } +} + +// blockedPeerWriter holds the response only after the production peer handler +// has authorized and serviced its input. Voter work must not share this wait. +type blockedPeerWriter struct { + http.ResponseWriter + entered chan struct{} + release <-chan struct{} +} + +func (w *blockedPeerWriter) Write(raw []byte) (int, error) { + close(w.entered) + <-w.release + return w.ResponseWriter.Write(raw) +} + +// R12/R23 keep generation fencing durable without mutating the original +// provision. Damaged counter evidence is retained and cannot reset authority. +func TestR12R23PersonalTransportRetainsCounterAndImmutableProvision(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + s := personalDevice(t, id) + token := personalToken(t) + grant := joined(t, r, s, token) + live := learner(t, s, grant) + live.catchUp(r, grant.Barrier) + base := filepath.Join(r.store, "raft", id.ID()) + provision, err := os.ReadFile(filepath.Join(base, "provision.json")) + if err != nil { + t.Fatal(err) + } + counterPath := filepath.Join(base, "transport-generation") + counter, err := os.ReadFile(counterPath) + if err != nil { + t.Fatal(err) + } + r.stop() + damaged := append([]byte(nil), counter...) + damaged[len(damaged)-1] ^= 1 + if err = os.WriteFile(counterPath, damaged, 0600); err != nil { + t.Fatal(err) + } + r.start() + _, err = (clusterctl.Client{Base: r.srv.URL, Sign: reqsign.SignFor(a, r.clock.Now), HTTP: r.srv.Client()}).Status(bg) + if err == nil { + t.Fatal("damaged generation reset its authority") + } + retained, err := os.ReadFile(counterPath) + if err != nil || !bytes.Equal(retained, damaged) { + t.Fatal("damaged counter evidence was overwritten") + } + r.stop() + if err = os.WriteFile(counterPath, counter, 0600); err != nil { + t.Fatal(err) + } + r.start() + _ = personalStatus(t, r, a) + after, err := os.ReadFile(filepath.Join(base, "provision.json")) + if err != nil || !bytes.Equal(after, provision) { + t.Fatal("restart rewrote immutable provision") + } + stale := learner(t, s, grant) + if _, err = stale.exchange(r); !errors.Is(err, rafttransport.ErrGeneration) { + t.Fatalf("restored counter reused an advertised generation: %v", err) + } + current := joined(t, r, s, token) + if current.Node != grant.Node || current.Generation <= grant.Generation { + t.Fatal("cold restart changed durable member identity") + } + live.grant = current + live.ack = [3]uint64{} + if err := r.dir(a).PutDevice(bg, a.dev.ID(), directory.Device{V: 1, Name: "after router restart"}); err != nil { + t.Fatal(err) + } + live.catchUp(r, live.status.Applied+1) + +} + +// R9/R27 require every admitted public command to remain usable with a +// learner. Protobuf framing does not fit inside a maximum-sized command's +// own byte budget; transport must catch up without changing either cap. +func TestR9R27PersonalLearnerCannotBreakMaximumCommand(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + s := personalDevice(t, id) + grant := joined(t, r, s, personalToken(t)) + l := learner(t, s, grant) + l.catchUp(r, grant.Barrier) + encodedSize := func(n int) int { + t.Helper() + body, _ := json.Marshal(directory.Device{V: 1, Name: strings.Repeat("x", n)}) + request, err := http.NewRequest(http.MethodPut, r.srv.URL+"/v1/dir/devices/"+a.dev.ID(), bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + reqsign.Sign(request, body, a, r.clock.Now()) + proof, err := reqsign.Capture(request, body) + if err != nil { + t.Fatal(err) + } + command, err := cs.V1Command(grant.Scope.Cluster, r.clock.Now().UnixMilli(), proof) + if err != nil { + t.Fatal(err) + } + raw, err := cs.EncodeCommand(command) + if err != nil { + t.Fatal(err) + } + return len(raw) + } + low, high := 0, cs.MaxCommandBytes + for low < high { + middle := (low + high + 1) / 2 + if encodedSize(middle) <= cs.MaxCommandBytes { + low = middle + } else { + high = middle - 1 + } + } + if encodedSize(low) > cs.MaxCommandBytes || encodedSize(low) < cs.MaxCommandBytes-4 { + t.Fatal("fixture missed the legal command boundary") + } + if err := r.dir(a).PutDevice(bg, a.dev.ID(), directory.Device{V: 1, Name: strings.Repeat("x", low)}); err != nil { + t.Fatal("learner broke a legal public write:", err) + } + if _, err := r.dir(a).List(bg); err != nil { + t.Fatal("learner stopped quorum reads:", err) + } + l.catchUp(r, l.status.Applied+1) + app, _, err := l.store.ApplicationView(bg) + if err != nil { + t.Fatal(err) + } + record := raftstore.Key{Namespace: raftstore.State, Key: cs.DeviceKey(a.dev.ID()).Key} + rows, _, err := l.store.Read(bg, app.Applied, []raftstore.Key{record}) + if err != nil { + t.Fatal(err) + } + var saved cs.Device + if cs.Decode(rows[record], &saved) != nil || saved.Name != strings.Repeat("x", low) { + t.Fatal("learner did not retain the maximum legal command") + } +} diff --git a/internal/relayserve/service.go b/internal/relayserve/service.go index 461dc268b6..d43e1ed302 100644 --- a/internal/relayserve/service.go +++ b/internal/relayserve/service.go @@ -15,6 +15,7 @@ import ( "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/directory" "github.com/Agent-Field/codeaf/internal/pairbox" + "github.com/Agent-Field/codeaf/internal/rafttransport" "github.com/Agent-Field/codeaf/internal/relay" ) @@ -120,6 +121,11 @@ func New(cfg Config) *Service { svc.ns.after = cfg.After svc.ns.links = links mux.HandleFunc(clusterstate.ClusterAPIPath, svc.ns.clusterAPI) + mux.HandleFunc(clusterstate.PersonalAdmissionPath, svc.ns.enrollPersonal) + if cfg.Cluster == nil { + mux.HandleFunc(rafttransport.ExchangePath, svc.ns.exchangePersonal) + mux.HandleFunc(rafttransport.SocketPath, svc.ns.exchangePersonal) + } mux.Handle(clusterctl.Path, clusterctl.Handler(svc.ns.controlAuthenticate, svc.ns.controlBackend)) auth := noting(svc.ns.authenticate) dirs := logged(cfg.Logf, directory.Handler(auth, svc.ns.directory)) diff --git a/relay/hosted/test/raft/hosted-control.test.js b/relay/hosted/test/raft/hosted-control.test.js index c821316c45..8fa85736e4 100644 --- a/relay/hosted/test/raft/hosted-control.test.js +++ b/relay/hosted/test/raft/hosted-control.test.js @@ -4,21 +4,26 @@ import { stat } from 'node:fs/promises'; import { join } from 'node:path'; import { hostedFixture } from './hosted-fixture.js'; -// R6/R30 keep hosted leadership absent rather than accepting an intent that +// R6/R9/R23/R30 keep hosted member and leadership doors absent until their +// runtime can serve an external member. Refusing avoids an intent that // cannot finish. Restart runs the same bundle over the same persistent state. -test('R6 R30 hosted control stays absent across a runtime restart',{timeout:30000},async()=>{ +test('R6 R9 R23 R30 hosted member doors stay absent across a runtime restart',{timeout:30000},async()=>{ const h=await hostedFixture({base:'/fabric'}),a=h.client(); try{ await h.start(); - for(const method of ['GET','POST']){ - const r=await h.request(a,method,'/v2/cluster/control'); - assert.equal(r.status,404);await r.text(); + for(const path of ['/v2/cluster/control','/v2/cluster/enroll','/v2/cluster/exchange','/v2/cluster/socket']){ + for(const method of ['GET','POST']){ + const r=await h.request(a,method,path); + assert.equal(r.status,404,path);await r.text(); + } } const bundle=await stat(join(h.dir,'worker.js')); await h.start(); const restarted=await stat(join(h.dir,'worker.js')); assert.equal(restarted.mtimeNs??restarted.mtimeMs,bundle.mtimeNs??bundle.mtimeMs,'restart rebuilt the program'); - const r=await h.request(a,'GET','/v2/cluster/control'); - assert.equal(r.status,404);await r.text(); + for(const path of ['/v2/cluster/control','/v2/cluster/enroll','/v2/cluster/exchange','/v2/cluster/socket']){ + const r=await h.request(a,'GET',path); + assert.equal(r.status,404,path);await r.text(); + } }finally{await h.close();} }); From e04121960e0a3ed4333173d8376915babd661834 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 13:50:58 -0400 Subject: [PATCH 056/103] Keep a completed device removal visible when a later list read fails. Retain the action result independently of refresh errors so a delayed devices-list failure cannot erase removal confirmation. Preserve initial-load error behavior and test both callback orders without altering the retained assertion or its predicate. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../manual/chat/add-or-remove-a-computer.md | 4 ++ internal/tui3/devices.go | 16 ++++++- internal/tui3/devices_refresh_test.go | 43 +++++++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 internal/tui3/devices_refresh_test.go diff --git a/internal/manual/chat/add-or-remove-a-computer.md b/internal/manual/chat/add-or-remove-a-computer.md index 239aafcff0..5f8ca329cb 100644 --- a/internal/manual/chat/add-or-remove-a-computer.md +++ b/internal/manual/chat/add-or-remove-a-computer.md @@ -164,6 +164,10 @@ desk was removed — it can no longer reach your chats. The list is in a fixed order: this device first, then devices that are online, then the rest, each by name. Two devices with the same name show a short tail, such as `tower #a1b2`, so you can tell them apart. Your own computer has no `r`: it cannot remove itself, and if you press `r` there the list says `this device cannot remove itself — choose another row.` A computer that is already removed does nothing when you press `r`. A removed computer can only come back as a new request that you approve. If you remove a computer from another of your devices while this list is open, its row changes to `removed` here too, without closing the list. Removing a computer does not undo what it already holds: for a lost computer read the page *Locking out a lost computer for good*. +## A device was removed but refreshing the list failed + +After a successful removal, its confirmation stays visible even if refreshing the device list fails. The card shows the refresh error too, then clears that error when a later refresh succeeds. A failed refresh does not restore the removed computer's access. If the removal itself fails, the card shows that error and does not confirm a removal. + ## I was told this device was removed — how to bring it back Moving chats and pairing are available by default on Linux, macOS and native Windows on every build with a compatible engine. Pair both computers first. diff --git a/internal/tui3/devices.go b/internal/tui3/devices.go index efb18c9995..bb6f813fc2 100644 --- a/internal/tui3/devices.go +++ b/internal/tui3/devices.go @@ -57,6 +57,9 @@ type deviceCard struct { loaded bool cursor int line string + // A list read cannot undo the outcome of a completed rename or removal. + // Keep its failure separately so both facts survive delayed feed callbacks. + readErr error // naming is the name being typed, and nil when no name is being typed. naming *string // read is the newest directory version the rows were read at, so a change another computer @@ -87,9 +90,14 @@ func (m devicesMsg) land(a *app) tea.Cmd { return nil } if m.err != nil { - m.card.line = m.err.Error() + if !m.card.loaded { + m.card.line = m.err.Error() + } else { + m.card.readErr = m.err + } return nil } + m.card.readErr = nil online, _ := m.card.livePresence() for i := range m.rows { m.rows[i].Online = online[m.rows[i].ID] && !m.rows[i].Revoked @@ -321,6 +329,9 @@ func (c *deviceCard) rows(width int, now time.Time, pal palette) []string { if c.line != "" { out = append(out, dressed(wrap(c.line, max(width, 4)), pal.accent)...) } + if c.readErr != nil { + out = append(out, dressed(wrap(c.readErr.Error(), max(width, 4)), pal.accent)...) + } return out } @@ -328,6 +339,9 @@ func (c *deviceCard) rows(width int, now time.Time, pal palette) []string { // once the look has failed, why. A failure that landed on a card still saying // "looking up" would never be read. func (c *deviceCard) waiting() string { + if c.readErr != nil { + return c.readErr.Error() + } if c.line != "" { return c.line } diff --git a/internal/tui3/devices_refresh_test.go b/internal/tui3/devices_refresh_test.go new file mode 100644 index 0000000000..210d953668 --- /dev/null +++ b/internal/tui3/devices_refresh_test.go @@ -0,0 +1,43 @@ +package tui3 + +import ( + "errors" + "strings" + "testing" +) + +// R16/R30 keep a completed removal visible when a feed-triggered list read +// fails before or after its response. A read timeout does not restore access +// or erase the successful operation; both outcomes must remain visible. +func TestR16R30DeviceRemovalSurvivesDelayedListFailure(t *testing.T) { + for _, before := range []bool{false, true} { + t.Run(map[bool]string{false: "after", true: "before"}[before], func(t *testing.T) { + a := newTestApp(&fakeAgent{model: "test/model"}) + rows := []DeviceRow{{ID: "self", Name: "here", Self: true}, {ID: "other", Name: "spark", Online: true}} + card := &deviceCard{rowsOf: rows, loaded: true, cursor: 1} + a.showCard(card) + failed := devicesMsg{card: card, err: errors.New("the device list did not finish loading")} + if before { + failed.land(a) + } + revokedMsg{card: card, row: rows[1]}.land(a) + if !before { + failed.land(a) + } + shown := plain(frame(a)) + for _, want := range []string{"spark was removed", "the device list did not finish loading"} { + if !strings.Contains(shown, want) { + t.Fatalf("R16/R30 lost %q after committed removal:\n%s", want, shown) + } + } + if !card.rowsOf[1].Revoked || card.rowsOf[1].Online { + t.Fatal("R16 failed list read restored the removed device") + } + devicesMsg{card: card, rows: rows}.land(a) + shown = plain(frame(a)) + if !strings.Contains(shown, "spark was removed") || strings.Contains(shown, "did not finish loading") { + t.Fatalf("R30 successful refresh did not clear only its own error:\n%s", shown) + } + }) + } +} From f1a0e9b7a7f0750db20296857b3192113506e494 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 16:20:22 -0400 Subject: [PATCH 057/103] Keep passive learner transport authenticated without waking an idle quorum. Learner replies that arrive after the all-voter sleep barrier must not restart periodic coordination. Keep the existing response deadline while deliveries await acknowledgment, and let empty subscriptions wake passively. Device signing callbacks retain the canonical wire without exporting private seeds. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/raftdirectory/transport.go | 38 +++++++++++++--- internal/rafttransport/envelope.go | 13 +++++- internal/rafttransport/exchange.go | 45 +++++++++++++++++-- .../rafttransport/signing_callback_test.go | 35 +++++++++++++++ 4 files changed, 120 insertions(+), 11 deletions(-) create mode 100644 internal/rafttransport/signing_callback_test.go diff --git a/internal/raftdirectory/transport.go b/internal/raftdirectory/transport.go index 289399d43e..e4f368b6dd 100644 --- a/internal/raftdirectory/transport.go +++ b/internal/raftdirectory/transport.go @@ -212,10 +212,13 @@ func (g *Native) pumpPeers(request rafttransport.Request, now int64) error { } p.applied[request.Node] = max(p.applied[request.Node], request.Applied) } - // Empty polls of a sleeping learner must not manufacture heartbeats or wake - // work. A peer frame is real ingress and wakes on the ordinary Step path. - if len(request.Messages) > 0 { - p.sleeping = false + // A final learner response can arrive after all voters agreed to sleep. + // It has no quorum weight and must not restart periodic work. New requests + // and higher-term evidence still wake through the ordinary peer path. + for _, envelope := range request.Messages { + if !g.idleLearnerResponse(envelope) { + p.sleeping = false + } } for round := 0; round < rafttransport.MaxInflight; round++ { progressed := false @@ -233,8 +236,10 @@ func (g *Native) pumpPeers(request rafttransport.Request, now int64) error { if !ok || e.Scope != p.view.Scope || e.Destination != 1 || e.Verify(member.Key) != nil || e.Chunk != nil { return rafttransport.ErrAuth } - if err = g.drive(ctx, i, raftcore.PeerMessage{Bytes: e.Payload}); err != nil { - return err + if !g.idleLearnerResponse(e) { + if err = g.drive(ctx, i, raftcore.PeerMessage{Bytes: e.Payload}); err != nil { + return err + } } p.ack[i][delivery.Lane] = max(p.ack[i][delivery.Lane], delivery.Cursor) progressed = true @@ -332,3 +337,24 @@ func (g *Native) checkpointExternal(ctx context.Context) error { g.peers.checkpoint = false return g.drain(ctx) } + +// idleLearnerResponse discards only response traffic rendered unnecessary by +// the completed all-voter sleep barrier. It cannot hide a newer term or a vote. +func (g *Native) idleLearnerResponse(e rafttransport.Envelope) bool { + if !g.peers.sleeping { + return false + } + member, ok := g.peer(e.From) + if !ok || member.Voter { + return false + } + leader := g.leader() + if leader < 0 { + return false + } + message := new(pb.Message) + if proto.Unmarshal(e.Payload, message) != nil || message.GetTerm() > g.status[leader].Term { + return false + } + return message.GetType() == pb.MsgAppResp || message.GetType() == pb.MsgHeartbeatResp +} diff --git a/internal/rafttransport/envelope.go b/internal/rafttransport/envelope.go index e1c6ec7690..66028354ec 100644 --- a/internal/rafttransport/envelope.go +++ b/internal/rafttransport/envelope.go @@ -103,11 +103,22 @@ func (e *Envelope) Sign(key ed25519.PrivateKey) error { if len(key) != ed25519.PrivateKeySize { return ErrAuth } + return e.SignWith(func(b []byte) []byte { return ed25519.Sign(key, b) }) +} + +// SignWith keeps device private keys behind their existing signing boundary. +func (e *Envelope) SignWith(sign func([]byte) []byte) error { + if sign == nil { + return ErrAuth + } e.Digest = digest(e.Payload) if err := e.validate(); err != nil { return err } - e.Signature = ed25519.Sign(key, e.signingBytes()) + e.Signature = sign(e.signingBytes()) + if len(e.Signature) != ed25519.SignatureSize { + return ErrAuth + } return nil } func (e Envelope) Verify(key ed25519.PublicKey) error { diff --git a/internal/rafttransport/exchange.go b/internal/rafttransport/exchange.go index ad7d56673a..cf7de24a6f 100644 --- a/internal/rafttransport/exchange.go +++ b/internal/rafttransport/exchange.go @@ -83,10 +83,21 @@ func (r *Request) Sign(key ed25519.PrivateKey) error { if len(key) != ed25519.PrivateKeySize { return ErrAuth } + return r.SignWith(func(b []byte) []byte { return ed25519.Sign(key, b) }) +} + +// SignWith uses the same canonical bytes without exporting a device seed. +func (r *Request) SignWith(sign func([]byte) []byte) error { + if sign == nil { + return ErrAuth + } if err := r.validate(); err != nil { return err } - r.Signature = ed25519.Sign(key, r.signingBytes()) + r.Signature = sign(r.signingBytes()) + if len(r.Signature) != ed25519.SignatureSize { + return ErrAuth + } return nil } func (r Request) Verify(key ed25519.PublicKey) error { @@ -251,9 +262,7 @@ func serveSocket(ctx context.Context, conn *websocket.Conn, router *Router, init frames := make(chan frame, 1) go func() { for { - readCtx, stop := context.WithTimeout(ctx, socketTimeout) - kind, data, err := conn.Read(readCtx) - stop() + kind, data, err := conn.Read(ctx) if kind != websocket.MessageBinary && err == nil { err = ErrWire } @@ -267,6 +276,30 @@ func serveSocket(ctx context.Context, conn *websocket.Conn, router *Router, init } } }() + // An idle member owes no keepalive. The existing response deadline applies + // only while a bounded delivery awaits acknowledgment, so a sleeping group + // can wake this socket without a recurring request loop. + var deadline <-chan time.Time + var timer *time.Timer + defer func() { + if timer != nil { + timer.Stop() + } + }() + arm := func(resp Response, received bool) { + if len(resp.Messages) != 0 && timer != nil && !received { + return + } + if timer != nil { + timer.Stop() + timer = nil + } + deadline = nil + if len(resp.Messages) != 0 { + timer = time.NewTimer(socketTimeout) + deadline = timer.C + } + } req := initial changed, err := router.changed(req) if err != nil { @@ -276,11 +309,14 @@ func serveSocket(ctx context.Context, conn *websocket.Conn, router *Router, init if err != nil || !writeSocket(ctx, conn, response) { return } + arm(response, true) for { receiveOnly := false select { case <-ctx.Done(): return + case <-deadline: + return case <-changed: receiveOnly = true case f := <-frames: @@ -308,6 +344,7 @@ func serveSocket(ctx context.Context, conn *websocket.Conn, router *Router, init if !writeSocket(ctx, conn, response) { return } + arm(response, !receiveOnly) } } diff --git a/internal/rafttransport/signing_callback_test.go b/internal/rafttransport/signing_callback_test.go new file mode 100644 index 0000000000..43421ffed5 --- /dev/null +++ b/internal/rafttransport/signing_callback_test.go @@ -0,0 +1,35 @@ +package rafttransport + +import ( + "crypto/ed25519" + "reflect" + "testing" + + pb "go.etcd.io/raft/v3/raftpb" +) + +// R23 keeps the canonical authenticated bytes unchanged when a personal device +// signs through its existing callback instead of exporting its private seed. +func TestR23DeviceSigningCallbackKeepsPeerWire(t *testing.T) { + key := key(1) + callback := func(b []byte) []byte { return ed25519.Sign(key, b) } + original := message(t, 1, 2, 1, pb.MsgApp, 0) + actual := original + actual.Signature = nil + must(t, actual.SignWith(callback)) + if !reflect.DeepEqual(actual, original) { + t.Fatal("R23 callback changed canonical envelope") + } + originalRequest := request(t, 1, original) + actualRequest := originalRequest + actualRequest.Signature = nil + must(t, actualRequest.SignWith(callback)) + if !reflect.DeepEqual(actualRequest, originalRequest) { + t.Fatal("R23 callback changed canonical request") + } + for _, sign := range []func([]byte) []byte{nil, func([]byte) []byte { return nil }} { + if actual.SignWith(sign) != ErrAuth || actualRequest.SignWith(sign) != ErrAuth { + t.Fatal("R23 empty signer admitted") + } + } +} From 51282243a0110b6714db2e7386ced8d2b5254d7b Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 16:20:22 -0400 Subject: [PATCH 058/103] Run durable personal learners inside the existing home sync lifetime. Persist store identity before authenticated admission and finish real snapshot and log application before reporting readiness. Reuse authority on reopen, replace it after store loss, and check current quiet policy at every HTTP, socket and delayed callback. Share local applied display and follower hints across Home handles so idle native homes stop recurring requests while ownership actions retain fresh majority reads and hosted sync keeps its existing polling. Expose committed learner identity and readiness in cluster status. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/codeaf/cluster.go | 18 +- cmd/codeaf/cluster_learner_test.go | 40 ++ internal/raftdirectory/display.go | 58 ++ internal/raftdirectory/v1.go | 47 +- internal/rafttransport/README.md | 14 +- internal/syncsetup/cluster.go | 24 +- internal/syncsetup/fleet.go | 14 + internal/syncsetup/member.go | 660 +++++++++++++++++ internal/syncsetup/member_client_test.go | 868 +++++++++++++++++++++++ internal/syncsetup/member_display.go | 267 +++++++ internal/syncsetup/quiet.go | 20 + internal/syncsetup/state.go | 36 +- internal/syncsetup/syncsetup.go | 75 +- internal/vaultsync/syncer.go | 24 +- 14 files changed, 2084 insertions(+), 81 deletions(-) create mode 100644 cmd/codeaf/cluster_learner_test.go create mode 100644 internal/raftdirectory/display.go create mode 100644 internal/syncsetup/member.go create mode 100644 internal/syncsetup/member_client_test.go create mode 100644 internal/syncsetup/member_display.go diff --git a/cmd/codeaf/cluster.go b/cmd/codeaf/cluster.go index 1935e3063d..590ca0437a 100644 --- a/cmd/codeaf/cluster.go +++ b/cmd/codeaf/cluster.go @@ -14,12 +14,11 @@ import ( ) const clusterHelp = `codeaf cluster status - Show the voting computers and the current coordinator. + Show this joined computer, learners, voters and the current coordinator. codeaf cluster lead - Ask this joined computer to coordinate. Keep its persistent store available. - Success includes the elected computer, term and quorum read barrier. - Three voters tolerate one unavailable member. A sleeping personal leader is - replaced by the relay anchors; it leads again only after another request. + Leadership is not offered yet; your saved chats are still here. + A paired home on a self-hosted relay with storage joins as a learner. + Three voters tolerate one unavailable member. ` type clusterDoor interface { @@ -68,6 +67,15 @@ func clusterCommand(ctx context.Context, c clusterDoor, verb string, out io.Writ if len(s.Configuration.Voters) > 0 { fmt.Fprintf(out, "voting computers %v\n", s.Configuration.Voters) } + if s.Self != 0 { + fmt.Fprintf(out, "this computer %d\n", s.Self) + } + if len(s.Configuration.Learners) > 0 { + fmt.Fprintf(out, "learning computers %v\n", s.Configuration.Learners) + } + if len(s.Ready) > 0 { + fmt.Fprintf(out, "ready computers %v\n", s.Ready) + } if len(s.Configuration.Outgoing) > 0 { fmt.Fprintf(out, "finishing the change from %v\n", s.Configuration.Outgoing) } diff --git a/cmd/codeaf/cluster_learner_test.go b/cmd/codeaf/cluster_learner_test.go new file mode 100644 index 0000000000..9e3ab10a02 --- /dev/null +++ b/cmd/codeaf/cluster_learner_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +type personalStatusFixture struct { + clusterFixture + status clusterctl.Status +} + +func (f personalStatusFixture) ClusterStatus(context.Context) (clusterctl.Status, error) { + return f.status, nil +} + +// H1/R9 makes the real status command expose this home's committed learner +// and readiness, with the three anchors still the only voting computers. +func TestR9ClusterStatusShowsPersonalLearner(t *testing.T) { + fixture := personalStatusFixture{status: clusterctl.Status{Self: 4, Leader: 1, Configuration: cs.Configuration{Voters: []uint64{1, 2, 3}, Learners: []uint64{4, 5}}, Ready: []uint64{4}}} + var out bytes.Buffer + if err := clusterCommand(t.Context(), fixture, "status", &out); err != nil { + t.Fatal(err) + } + for _, line := range []string{"coordinator 1\n", "this computer 4\n", "voting computers [1 2 3]\n", "learning computers [4 5]\n", "ready computers [4]\n"} { + if !strings.Contains(out.String(), line) { + t.Errorf("R9 status omitted %q: %s", line, out.String()) + } + } + out.Reset() + fixture.status = clusterctl.Status{} + if err := clusterCommand(t.Context(), fixture, "status", &out); err != nil || out.Len() != 0 { + t.Fatal("R9 unknown status did not remain absent", err, out.String()) + } +} diff --git a/internal/raftdirectory/display.go b/internal/raftdirectory/display.go new file mode 100644 index 0000000000..be084a1932 --- /dev/null +++ b/internal/raftdirectory/display.go @@ -0,0 +1,58 @@ +package raftdirectory + +import ( + "context" + "encoding/hex" + "strings" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftstore" +) + +// DisplayListing projects one applied view. Public reads call it only inside +// their fresh quorum barrier; a personal learner uses it solely for Home's +// display hints. Taking, publishing and authorization never use this shortcut. +func DisplayListing(ctx context.Context, s *raftstore.SQLite, app raftstore.Application) (directory.Listing, error) { + l := directory.Listing{Devices: map[string]directory.Device{}, Cells: map[string]directory.Cell{}} + l.Now, l.Version = app.LogicalTime, app.VisibleVersion + after := "" + for { + rows, _, err := s.ScanPage(ctx, app.Applied, raftstore.State, after) + if err != nil { + return l, err + } + if len(rows) == 0 { + return l, nil + } + for _, r := range rows { + after = r.Key.Key + switch { + case after == "identity": + if err := clusterstate.Decode(r.Value, &l.Identity); err != nil { + return l, err + } + case strings.HasPrefix(after, "device/"): + id, e := hex.DecodeString(strings.TrimPrefix(after, "device/")) + if e != nil { + return l, clusterstate.ErrDependency + } + var v directory.Device + if e = clusterstate.Decode(r.Value, &v); e != nil { + return l, e + } + l.Devices[string(id)] = v + case strings.HasPrefix(after, "cell/"): + id, e := hex.DecodeString(strings.TrimPrefix(after, "cell/")) + if e != nil { + return l, clusterstate.ErrDependency + } + var v directory.Cell + if e = clusterstate.Decode(r.Value, &v); e != nil { + return l, e + } + l.Cells[string(id)] = v + } + } + } +} diff --git a/internal/raftdirectory/v1.go b/internal/raftdirectory/v1.go index cb4e1e0765..9f94f844c4 100644 --- a/internal/raftdirectory/v1.go +++ b/internal/raftdirectory/v1.go @@ -2,10 +2,8 @@ package raftdirectory import ( "context" - "encoding/hex" "net/http" "reflect" - "strings" "time" "github.com/Agent-Field/codeaf/internal/clusterstate" @@ -133,48 +131,11 @@ func authorizeV1Read(ctx context.Context, s *raftstore.SQLite, app raftstore.App } func (b *v1Backend) Listing(ctx context.Context, actor string) (directory.Listing, error) { - l := directory.Listing{Devices: map[string]directory.Device{}, Cells: map[string]directory.Cell{}} + var l directory.Listing err := b.read(ctx, actor, func(ctx context.Context, s *raftstore.SQLite, app raftstore.Application) error { - l.Now, l.Version = app.LogicalTime, app.VisibleVersion - after := "" - for { - rows, _, err := s.ScanPage(ctx, app.Applied, raftstore.State, after) - if err != nil { - return err - } - if len(rows) == 0 { - return nil - } - for _, r := range rows { - after = r.Key.Key - switch { - case after == "identity": - if err := clusterstate.Decode(r.Value, &l.Identity); err != nil { - return err - } - case strings.HasPrefix(after, "device/"): - id, e := hex.DecodeString(strings.TrimPrefix(after, "device/")) - if e != nil { - return clusterstate.ErrDependency - } - var v directory.Device - if e = clusterstate.Decode(r.Value, &v); e != nil { - return e - } - l.Devices[string(id)] = v - case strings.HasPrefix(after, "cell/"): - id, e := hex.DecodeString(strings.TrimPrefix(after, "cell/")) - if e != nil { - return clusterstate.ErrDependency - } - var v directory.Cell - if e = clusterstate.Decode(r.Value, &v); e != nil { - return e - } - l.Cells[string(id)] = v - } - } - } + var err error + l, err = DisplayListing(ctx, s, app) + return err }) return l, err } diff --git a/internal/rafttransport/README.md b/internal/rafttransport/README.md index dce952f312..653768994b 100644 --- a/internal/rafttransport/README.md +++ b/internal/rafttransport/README.md @@ -4,8 +4,7 @@ This is the authenticated peer bridge, separate from application ingress. The persistent native relay assembles its router from committed membership and serves personal admission, HTTP exchange and the outbound push socket on its ordinary listener. `relayserve.Config.Cluster` is the internal custom-host handler seam. -The hosted relay's member/control doors remain absent, and the codeaf client does -not yet activate its own learner. No environment switch selects another writer. +The hosted relay's member/control doors remain absent, and a paired codeaf home runs its durable learner inside the terminal state-sync lifetime. No environment switch selects another writer. ## Host ordering and authorization @@ -54,8 +53,9 @@ use the same node/scope/incarnation/router generation and receive responses; queued traffic also pushes a response without requiring a poll. Each frame and push rechecks enrollment. The channel carries no lease or append acknowledgment. Compression is disabled. Initial/body/frame/write/read bounds apply, and at most -eight sockets per member can be open. Send an authenticated frame within 30 seconds -or reconnect using HTTP fallback/backoff. No socket is required on a member. +eight sockets per member can be open. A delivery awaiting acknowledgment has a 30-second response deadline. An empty +subscription needs no keepalive: the socket stays passive until committed work +wakes it. Reconnect failures use HTTP fallback/backoff. No socket is required on a member. Hibernation is a hosted-runtime responsibility; the Go server has ordinary sockets. Responses are hints to transport delivery, not proof of persistence. Before core @@ -207,8 +207,10 @@ of learner lag. All-voter quiescence discards expendable learner deliveries and stops scheduling without learner acknowledgments. Fresh public read/write input wakes the ordinary quorum path. -The codeaf client does not start its own learner yet. This server assembly adds -no automatic client discovery, network attempts, worker setup or leadership. +The codeaf client starts its personal learner only inside the existing terminal +state-sync lifetime, or a scoped status activation, after current quiet guards. +Open remains local. Hosted relays offer neither enrollment nor control. Worker +setup and leadership remain unavailable. The native host advances its checksummed `transport-generation` file with atomic rename and directory fsync before opening its router; it never mutates the source diff --git a/internal/syncsetup/cluster.go b/internal/syncsetup/cluster.go index 3cd8f61aef..0aa24487b8 100644 --- a/internal/syncsetup/cluster.go +++ b/internal/syncsetup/cluster.go @@ -16,9 +16,29 @@ import ( ) func (s *Sync) clusterClient() clusterctl.Client { - return clusterctl.Client{Base: s.Relay, Sign: reqsign.SignFor(deviceSigner{s.Identity, s.Device}, time.Now), HTTP: s.ClusterHTTP(&http.Client{Timeout: 30 * time.Second})} + return clusterctl.Client{Base: s.Relay, Sign: reqsign.SignFor(deviceSigner{s.Identity, s.Device}, s.now), HTTP: s.ClusterHTTP(&http.Client{Timeout: 30 * time.Second})} } func (s *Sync) ClusterStatus(ctx context.Context) (clusterctl.Status, error) { + if !s.memberAllowed() { + return clusterctl.Status{}, ErrQuiet + } + status, err := s.clusterClient().Status(ctx) + if err != nil { + return status, err + } + s.memberMu.Lock() + defer s.memberMu.Unlock() + if s.member != nil { + return status, nil + } + m, e := s.openMember(ctx) + if errors.Is(e, clusterctl.ErrUnsupported) || filelock.IsBusy(e) { + return status, nil + } + if e != nil { + return status, e + } + m.close() return s.clusterClient().Status(ctx) } @@ -115,7 +135,7 @@ func (s *Sync) LeadCluster(ctx context.Context, node uint64) (cs.LeadershipRecei } return receipt, nil } -func saveLeadership(path string, v leadershipSession) error { +func saveLeadership(path string, v any) error { b, err := json.Marshal(v) if err != nil { return err diff --git a/internal/syncsetup/fleet.go b/internal/syncsetup/fleet.go index 4648349195..9ae08e94ce 100644 --- a/internal/syncsetup/fleet.go +++ b/internal/syncsetup/fleet.go @@ -11,6 +11,13 @@ import ( // from the directory. It is the fact the home card that offers another machine // waits on. func (s *Sync) FleetSize(ctx context.Context) (int, error) { + current, err := s.currentDisplaySync() + if err != nil { + return 0, err + } + if current != s { + return current.FleetSize(ctx) + } l, err := s.list(ctx) if err != nil { return 0, err @@ -27,6 +34,13 @@ func (s *Sync) FleetSize(ctx context.Context) (int, error) { // Roster lists the devices that are not revoked, this one first, with the names // opened, for the row that shows who is online. func (s *Sync) Roster(ctx context.Context) ([]chatlist.Device, error) { + current, err := s.currentDisplaySync() + if err != nil { + return nil, err + } + if current != s { + return current.Roster(ctx) + } l, err := s.list(ctx) if err != nil { return nil, err diff --git a/internal/syncsetup/member.go b/internal/syncsetup/member.go new file mode 100644 index 0000000000..5056cac207 --- /dev/null +++ b/internal/syncsetup/member.go @@ -0,0 +1,660 @@ +package syncsetup + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "io" + "net/http" + "os" + "path/filepath" + "sync" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/filelock" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/rafttransport" + "github.com/Agent-Field/codeaf/internal/reqsign" + "github.com/coder/websocket" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +var errMemberWaiting = errors.New("this computer is still joining the cluster; retry shortly; your saved chats are still here") + +// memberRecord belongs to the store, rather than the device's pairing. Losing +// an initialized store must retire its old authority instead of bootstrapping +// another log under the same node. Pending admission keeps its token on retry. +type memberRecord struct { + Store string + Grant raftdirectory.PersonalEnrollment + Initialized bool + Seed uint64 +} + +type personalMember struct { + closed bool + changed chan struct{} + idle bool + mu sync.Mutex + owner *Sync + record memberRecord + path string + lock *os.File + store *raftstore.SQLite + core *raftcore.Core + status raftcore.Status + pending []rafttransport.Envelope + sequence uint64 + ack [3]uint64 + snapshot *pb.Message + chunks map[uint64]bool + conn *websocket.Conn + cancel context.CancelFunc + done chan struct{} + failure error +} + +func (s *Sync) memberPath() string { + sum := sha256.Sum256([]byte(s.Ledger + "/" + s.Device.ID())) + return filepath.Join(s.Home, "cluster-member", hex.EncodeToString(sum[:])) +} + +// memberAllowed fences delayed work as well as new dials. A re-pair can leave +// the old handle paired but bound to a different identity or relay. +func (s *Sync) memberAllowed() bool { + if !MayTalk(s.Home) { + return false + } + id, err := identity.Load(s.Home) + base, e := relayBase(Resolve(s.Home).URL) + return err == nil && e == nil && id.ID() == s.Identity.ID() && base == s.Relay +} + +func (s *Sync) enrollMember(ctx context.Context, token string) (raftdirectory.PersonalEnrollment, error) { + var grant raftdirectory.PersonalEnrollment + if !s.memberAllowed() { + return grant, ErrQuiet + } + raw, _ := json.Marshal(cs.PersonalAdmission{Store: token, Protocol: rafttransport.Protocol}) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.Relay+cs.PersonalAdmissionPath, bytes.NewReader(raw)) + if err != nil { + return grant, err + } + reqsign.Sign(req, raw, deviceSigner{s.Identity, s.Device}, s.now()) + req.Header.Set("Content-Type", "application/json") + client := s.ClusterHTTP(&http.Client{Timeout: requestTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}) + resp, err := client.Do(req) + if err != nil { + return grant, err + } + defer resp.Body.Close() + b, err := io.ReadAll(io.LimitReader(resp.Body, cs.MaxEvaluationBytes+1)) + if err != nil { + return grant, err + } + if resp.StatusCode == http.StatusNotFound { + return grant, clusterctl.ErrUnsupported + } + if resp.StatusCode != http.StatusOK { + return grant, errors.New(string(bytes.TrimSpace(b))) + } + if len(b) > cs.MaxEvaluationBytes || json.Unmarshal(b, &grant) != nil || grant.Scope.Identity != s.Identity.ID() || grant.Scope.Protocol != rafttransport.Protocol || grant.Node == 0 || grant.Incarnation == 0 || grant.Generation == 0 { + return grant, rafttransport.ErrWire + } + return grant, nil +} + +func (s *Sync) openMember(ctx context.Context) (_ *personalMember, err error) { + if !s.memberAllowed() { + return nil, ErrQuiet + } + path := s.memberPath() + if err = os.MkdirAll(path, 0700); err != nil { + return nil, err + } + lock, err := os.OpenFile(filepath.Join(path, "lock"), os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, err + } + m := &personalMember{owner: s, path: path, lock: lock, changed: make(chan struct{})} + defer func() { + if err != nil { + m.close() + } + }() + if err = filelock.Lock(lock, true, true); err != nil { + return nil, err + } + b, e := os.ReadFile(filepath.Join(path, "identity.json")) + if e == nil { + err = json.Unmarshal(b, &m.record) + } else if !errors.Is(e, os.ErrNotExist) { + err = e + } else if _, exists := os.Stat(filepath.Join(path, "member.db")); !errors.Is(exists, os.ErrNotExist) { + // Missing authority beside an existing database is damage, not permission + // to retire the old member and try that database under newly minted bytes. + err = raftstore.ErrIdentity + } + if err != nil { + return nil, err + } + dbPath := filepath.Join(path, "member.db") + _, e = os.Stat(dbPath) + if m.record.Initialized && errors.Is(e, os.ErrNotExist) { + m.record = memberRecord{} + } + if m.record.Store == "" { + var token [32]byte + if _, err = rand.Read(token[:]); err != nil { + return nil, err + } + m.record.Store = hex.EncodeToString(token[:]) + if err = saveLeadership(filepath.Join(path, "identity.json"), m.record); err != nil { + return nil, err + } + } + old := m.record.Grant + pendingStage := old.Node != 0 && old.Barrier == 0 + // Admission is resumable from committed intent. The same store token is + // saved before the first request, including a lost response before any DB. + for { + if err = ctx.Err(); err != nil { + return nil, err + } + m.record.Grant, err = s.enrollMember(ctx, m.record.Store) + if err != nil { + return nil, err + } + g := m.record.Grant + if old.Node != 0 && (old.Node != g.Node || old.Scope != g.Scope || old.Incarnation != g.Incarnation) { + return nil, raftstore.ErrIdentity + } + if err = saveLeadership(filepath.Join(path, "identity.json"), m.record); err != nil { + return nil, err + } + if g.Barrier != 0 { + break + } + if pendingStage { + return nil, errMemberWaiting + } + pendingStage = true + old = g + } + if !s.memberAllowed() { + return nil, ErrQuiet + } + g := m.record.Grant + m.store, err = raftstore.Open(dbPath, raftstore.Identity{ClusterUUID: g.Scope.Cluster, NodeID: g.Node, Protocol: g.Scope.Protocol}) + if err != nil { + return nil, err + } + m.record.Initialized = true + recovered, err := m.store.Recover(ctx) + if err != nil { + return nil, err + } + var entropy [8]byte + if _, err = rand.Read(entropy[:]); err != nil { + return nil, err + } + m.record.Seed = binary.LittleEndian.Uint64(entropy[:]) + if err = saveLeadership(filepath.Join(path, "identity.json"), m.record); err != nil { + return nil, err + } + m.core, err = raftcore.New(raftcore.Config{ClusterUUID: g.Scope.Cluster, NodeID: g.Node, Protocol: g.Scope.Protocol, Seed: m.record.Seed}, recovered.Core) + if err != nil { + return nil, err + } + if err = m.step(ctx, raftcore.Poll{}); err != nil { + return nil, err + } + for { + beforeApplied, beforeAck := m.status.Applied, m.ack + sent := len(m.pending) != 0 + if err = m.exchange(ctx); err != nil { + return nil, err + } + if m.status.Applied >= g.Barrier && len(m.pending) == 0 { + // The extra round reports the durable application, not transport receipt. + if err = m.exchange(ctx); err != nil { + return nil, err + } + break + } + // An unavailable leader must not turn activation into an empty hot poll. + // Retain the same durable store and resume at the owner's next retry round. + if !sent && beforeApplied == m.status.Applied && beforeAck == m.ack && len(m.pending) == 0 { + return nil, errMemberWaiting + } + } + return m, nil +} + +func (m *personalMember) step(ctx context.Context, input raftcore.Input) error { + for { + if !m.owner.memberAllowed() { + return ErrQuiet + } + effects, err := m.core.Step(input) + if err != nil { + return err + } + m.status = effects.Status + for _, out := range effects.Send { + var destination uint64 + for _, peer := range m.record.Grant.Members { + if peer.Node == out.To && !peer.Revoked { + destination = peer.Incarnation + } + } + m.sequence++ + e := rafttransport.Envelope{Scope: m.record.Grant.Scope, From: m.record.Grant.Node, To: out.To, Incarnation: m.record.Grant.Incarnation, Destination: destination, Sequence: m.sequence, Payload: out.Bytes} + if err = e.SignWith(m.owner.Device.Sign); err != nil { + return err + } + m.pending = append(m.pending, e) + } + if effects.Persist != nil { + if err = m.store.Persist(ctx, *effects.Persist); err != nil { + return err + } + input = raftcore.Persisted{BatchID: effects.Persist.ID} + continue + } + if effects.Apply != nil { + if _, err = raftdirectory.Apply(ctx, m.store, *effects.Apply, m.record.Grant.Scope.Cluster, 0); err != nil { + return err + } + input = raftcore.Applied{BatchID: effects.Apply.ID} + continue + } + if effects.SnapshotNeeded != nil { + p, err := m.store.CreateSnapshot(ctx) + if err != nil { + return err + } + if err = m.store.Compact(ctx, p.GetMetadata().GetIndex()); err != nil { + return err + } + input = raftcore.CompactSnapshot{Snapshot: p} + continue + } + return nil + } +} + +func (m *personalMember) request() (rafttransport.Request, error) { + if !m.owner.memberAllowed() { + return rafttransport.Request{}, ErrQuiet + } + g := m.record.Grant + r := rafttransport.Request{Scope: g.Scope, Node: g.Node, Incarnation: g.Incarnation, Generation: g.Generation, Ack: m.ack, Applied: m.status.Applied} + if len(m.pending) > 0 { + r.Messages = m.pending[:1] + } + err := r.SignWith(m.owner.Device.Sign) + return r, err +} +func (m *personalMember) exchange(ctx context.Context) error { + req, err := m.request() + if err != nil { + return err + } + response, err := (rafttransport.Client{URL: m.owner.Relay, HTTP: m.owner.ClusterHTTP(nil)}).Exchange(ctx, req) + if err != nil { + return err + } + if len(req.Messages) > 0 { + m.pending = m.pending[1:] + } + return m.receive(ctx, response) +} + +func (m *personalMember) receive(ctx context.Context, response rafttransport.Response) error { + if !m.owner.memberAllowed() { + return ErrQuiet + } + if response.Generation != m.record.Grant.Generation { + return rafttransport.ErrGeneration + } + if len(response.Messages) > rafttransport.MaxMessages { + return rafttransport.ErrWire + } + for _, delivery := range response.Messages { + if delivery.Cursor == 0 { + return rafttransport.ErrCursor + } + if int(delivery.Lane) >= len(m.ack) { + return rafttransport.ErrWire + } + if delivery.Cursor <= m.ack[delivery.Lane] { + continue + } + e := delivery.Envelope + peers := m.record.Grant.Members + // Once the registry is applied, it replaces the initial authenticated grant. + // A retained grant cannot authorize a subsequently revoked member. + app, _, err := m.store.ApplicationView(ctx) + if err != nil { + return err + } + if app.Applied >= m.record.Grant.Barrier { + view, err := rafttransport.CommittedViewAt(m.record.Grant.Scope, m.record.Grant.Generation, app.Applied, raftcore.Configuration(app.ConfState), app.LogicalTime, func(k cs.RecordKey) ([]byte, error) { + key := raftstore.Key{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key} + rows, _, e := m.store.Read(ctx, app.Applied, []raftstore.Key{key}) + return rows[key], e + }) + if err != nil { + return err + } + peers = view.Members + m.record.Grant.Members = peers + } + var source ed25519.PublicKey + for _, peer := range peers { + if peer.Node == e.From && peer.Incarnation == e.Incarnation && !peer.Revoked { + source = peer.Key + } + } + if e.Verify(source) != nil || e.Scope != m.record.Grant.Scope || e.To != m.record.Grant.Node || e.Destination != m.record.Grant.Incarnation { + return rafttransport.ErrAuth + } + if e.Chunk != nil { + if m.snapshot == nil || e.Chunk.Generation != raftsnapshot.ID(m.snapshot.GetSnapshot().GetData()) { + return rafttransport.ErrWire + } + if err = m.store.PutSnapshotChunk(ctx, e.Chunk.Generation, e.Chunk.Offset, e.Payload, e.Chunk.Hash); err != nil { + return err + } + m.chunks[e.Chunk.Offset] = true + manifest, err := raftsnapshot.Decode(m.snapshot.GetSnapshot().GetData()) + if err != nil { + return err + } + if uint64(len(m.chunks)) == manifest.Chunks { + pointer, err := m.store.CompleteSnapshot(ctx, e.Chunk.Generation) + if err != nil { + return err + } + if !proto.Equal(pointer, m.snapshot.GetSnapshot()) { + return raftstore.ErrSnapshot + } + raw, err := proto.Marshal(m.snapshot) + if err != nil { + return err + } + if err = m.step(ctx, raftcore.PeerMessage{Bytes: raw}); err != nil { + return err + } + m.snapshot = nil + } + } else { + message := new(pb.Message) + if proto.Unmarshal(e.Payload, message) != nil { + return rafttransport.ErrWire + } + if message.GetType() == pb.MsgSnap && message.GetSnapshot().GetMetadata().GetIndex() > m.status.Applied { + if m.snapshot == nil || !proto.Equal(m.snapshot.GetSnapshot(), message.GetSnapshot()) { + if _, err = m.store.BeginSnapshot(ctx, message.GetSnapshot().GetData()); err != nil { + return err + } + m.snapshot = message + m.chunks = map[uint64]bool{} + } + } else if err = m.step(ctx, raftcore.PeerMessage{Bytes: e.Payload}); err != nil { + return err + } + } + m.ack[delivery.Lane] = delivery.Cursor + } + return nil +} + +func (m *personalMember) start(ctx context.Context) error { + if !m.owner.memberAllowed() { + return ErrQuiet + } + req, err := m.request() + if err != nil { + return err + } + // Upgrade authentication carries no outbound messages. Catch-up flushed them. + req.Messages = nil + if err = req.SignWith(m.owner.Device.Sign); err != nil { + return err + } + raw, _ := rafttransport.Encode(req) + header := http.Header{} + header.Set(rafttransport.SocketAuthHeader, base64.RawURLEncoding.EncodeToString(raw)) + ctx, m.cancel = context.WithCancel(ctx) + conn, resp, err := websocket.Dial(ctx, m.owner.Relay+rafttransport.SocketPath, &websocket.DialOptions{HTTPClient: m.owner.ClusterHTTP(&http.Client{Timeout: requestTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}), HTTPHeader: header, CompressionMode: websocket.CompressionDisabled}) + if resp != nil && err != nil && resp.Body != nil { + resp.Body.Close() + } + if err != nil { + m.cancel() + return err + } + conn.SetReadLimit(rafttransport.MaxExchangeBytes) + m.conn = conn + m.done = make(chan struct{}) + go func() { + defer close(m.done) + for { + kind, raw, err := conn.Read(ctx) + m.mu.Lock() + if err == nil && (!m.owner.memberAllowed() || ctx.Err() != nil) { + err = ErrQuiet + } + var response rafttransport.Response + if err == nil && (kind != websocket.MessageBinary || rafttransport.Decode(raw, &response) != nil) { + err = rafttransport.ErrWire + } + if err == nil { + err = m.receive(ctx, response) + } + // An empty push is only a subscription acknowledgment. Answering it would + // create an endless empty frame loop and wake an otherwise idle group. + if err == nil && (len(response.Messages) > 0 || len(m.pending) > 0) { + for { + req, e := m.request() + if e != nil { + err = e + break + } + data, _ := rafttransport.Encode(req) + if !m.owner.memberAllowed() { + err = ErrQuiet + break + } + writeCtx, cancel := context.WithTimeout(ctx, requestTimeout) + m.owner.clusterFrames.Add(1) + err = conn.Write(writeCtx, websocket.MessageBinary, data) + cancel() + if err != nil { + break + } + if len(req.Messages) == 0 { + break + } + m.pending = m.pending[1:] + if len(m.pending) == 0 { + break + } + } + } + if err != nil { + m.failure = err + } + m.idle = err == nil && len(response.Messages) == 0 && len(m.pending) == 0 + close(m.changed) + m.changed = make(chan struct{}) + m.mu.Unlock() + if err != nil { + conn.CloseNow() + return + } + } + }() + return nil +} + +func (m *personalMember) close() { + m.withdrawDisplay() + m.mu.Lock() + if !m.closed { + m.closed = true + close(m.changed) + m.changed = make(chan struct{}) + } + m.mu.Unlock() + if m.cancel != nil { + m.cancel() + } + if m.conn != nil { + m.conn.CloseNow() + } + if m.done != nil { + <-m.done + } + m.mu.Lock() + if m.store != nil { + m.store.Close() + } + m.mu.Unlock() + if m.lock != nil { + filelock.Unlock(m.lock) + m.lock.Close() + } +} + +func (s *Sync) stopMember() { + s.memberMu.Lock() + defer s.memberMu.Unlock() + if s.member != nil { + s.member.close() + s.member = nil + } +} + +// memberRound is called by the existing terminal lifetime, never by Open. +// Unsupported hosted relays keep their original state-sync path and are not +// repeatedly rediscovered. Failed native activation retries with saved identity. +func (s *Sync) memberRound(ctx context.Context) error { + s.memberMu.Lock() + defer s.memberMu.Unlock() + if !s.memberAllowed() { + if s.member != nil { + s.member.close() + s.member = nil + } + return nil + } + if s.member != nil { + s.member.mu.Lock() + err := s.member.failure + s.member.mu.Unlock() + if err == nil { + return nil + } + s.member.close() + s.member = nil + } + if s.memberUnsupported || s.now().Before(s.memberRetry) { + return nil + } + status, err := s.clusterClient().Status(ctx) + if errors.Is(err, clusterctl.ErrUnsupported) { + s.memberUnsupported = true + return nil + } + if err == nil && status.Cluster != "" { + s.member, err = s.openMember(ctx) + if err == nil { + err = s.member.start(ctx) + } + if err == nil { + s.member.publishDisplay() + } + if err != nil && s.member != nil { + s.member.close() + s.member = nil + } + } + if err != nil { + if errors.Is(err, clusterctl.ErrUnsupported) { + s.memberUnsupported = true + } + if s.memberBackoff == 0 { + s.memberBackoff = s.Interval + } else { + s.memberBackoff *= 2 + } + s.memberBackoff = min(s.memberBackoff, time.Minute) + // Seeded jitter separates homes that all wake after the same outage. + // The exponential ceiling still bounds the complete retry delay. + s.memberRetries++ + seed := sha256.Sum256([]byte(s.Home + s.Device.ID())) + sample := binary.LittleEndian.Uint64(seed[:8]) ^ (s.memberRetries * 0x9e3779b97f4a7c15) + sample ^= sample >> 12 + sample ^= sample << 25 + sample ^= sample >> 27 + half := s.memberBackoff / 2 + delay := half + time.Duration(sample%uint64(s.memberBackoff-half+1)) + s.memberRetry = s.now().Add(delay) + } else { + s.memberBackoff = 0 + } + return err +} + +// memberVault is a local change hint, never authority to perform a public read +// or take. Network operations still use their ordinary fresh quorum barriers. +func (s *Sync) memberVault(ctx context.Context) (string, bool, error) { + s.memberMu.Lock() + defer s.memberMu.Unlock() + if s.member == nil { + return "", false, nil + } + m := s.member + m.mu.Lock() + defer m.mu.Unlock() + if !s.memberAllowed() { + return "", false, ErrQuiet + } + if m.failure != nil { + return "", false, m.failure + } + app, _, err := m.store.ApplicationView(ctx) + if err != nil { + return "", false, err + } + if app.Applied < m.record.Grant.Barrier { + return "", false, nil + } + key := raftstore.Key{Namespace: raftstore.State, Key: cs.IdentityKey().Key} + rows, _, err := m.store.Read(ctx, app.Applied, []raftstore.Key{key}) + if err != nil { + return "", false, err + } + var id cs.IdentityRec + if err = cs.Decode(rows[key], &id); err != nil { + return "", false, err + } + return id.Vault, true, nil +} diff --git a/internal/syncsetup/member_client_test.go b/internal/syncsetup/member_client_test.go new file mode 100644 index 0000000000..0f3d0615a3 --- /dev/null +++ b/internal/syncsetup/member_client_test.go @@ -0,0 +1,868 @@ +package syncsetup + +import ( + "bufio" + "context" + "encoding/binary" + "encoding/json" + "errors" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/directory/directorytest" + "github.com/Agent-Field/codeaf/internal/dirwatch" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/rafttransport" + "github.com/Agent-Field/codeaf/internal/relayserve" + "github.com/Agent-Field/codeaf/internal/reqsign" + "github.com/Agent-Field/codeaf/internal/store" +) + +// peerFrameCounter observes actual socket bytes at the relay, independently of +// client counters. It parses complete masked/unmasked frames across I/O splits. +type peerFrameCounter struct { + mu sync.Mutex + b []byte + count atomic.Uint64 +} + +func (c *peerFrameCounter) add(b []byte) { + c.mu.Lock() + defer c.mu.Unlock() + c.b = append(c.b, b...) + for len(c.b) >= 2 { + n := uint64(c.b[1] & 127) + header := 2 + if n == 126 { + if len(c.b) < 4 { + return + } + n = uint64(binary.BigEndian.Uint16(c.b[2:4])) + header = 4 + } + if n == 127 { + if len(c.b) < 10 { + return + } + n = binary.BigEndian.Uint64(c.b[2:10]) + header = 10 + } + if c.b[1]&128 != 0 { + header += 4 + } + if n > rafttransport.MaxExchangeBytes { + panic("unbounded peer frame") + } + if uint64(len(c.b)) < uint64(header)+n { + return + } + c.count.Add(1) + c.b = c.b[header+int(n):] + } +} + +type peerCountConn struct { + net.Conn + in, out *peerFrameCounter +} + +func (c peerCountConn) Read(b []byte) (int, error) { + n, e := c.Conn.Read(b) + c.in.add(b[:n]) + return n, e +} +func (c peerCountConn) Write(b []byte) (int, error) { + n, e := c.Conn.Write(b) + c.out.add(b[:n]) + return n, e +} + +type peerCountWriter struct { + http.ResponseWriter + in, out *peerFrameCounter +} + +func (w peerCountWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + c, rw, e := w.ResponseWriter.(http.Hijacker).Hijack() + if e != nil { + return nil, nil, e + } + wrapped := peerCountConn{c, w.in, w.out} + // The HTTP handshake already flushed through ResponseWriter before Hijack. + rw.Writer.Reset(wrapped) + return wrapped, rw, nil +} + +type memberRig struct { + parkExchanges atomic.Bool + storage string + t *testing.T + clock *directorytest.FakeClock + svc *relayserve.Service + server *httptest.Server + requests atomic.Uint64 + in, out peerFrameCounter + homes []*Sync + interrupt atomic.Bool +} + +func newMemberRig(t *testing.T) *memberRig { + t.Helper() + t.Setenv(config.ProfileDirEnv, "") + r := &memberRig{t: t, clock: directorytest.NewFakeClock()} + r.clock.Advance(time.Now().Sub(r.clock.Now())) + r.storage = t.TempDir() + r.svc = relayserve.New(relayserve.Config{Store: r.storage, Now: r.clock.Now, After: r.clock.After}) + r.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, q *http.Request) { + r.requests.Add(1) + if q.URL.Path == "/v2/cluster/enroll" && r.interrupt.Swap(false) { + // Commit admission and lose its reply before the client's store is opened. + recorder := httptest.NewRecorder() + r.svc.Handler.ServeHTTP(recorder, q) + http.Error(w, "interrupted", 503) + return + } + if q.URL.Path == rafttransport.ExchangePath && r.parkExchanges.Load() { + recorder := httptest.NewRecorder() + r.svc.Handler.ServeHTTP(recorder, q) + if recorder.Code != http.StatusOK { + w.WriteHeader(recorder.Code) + w.Write(recorder.Body.Bytes()) + return + } + var response rafttransport.Response + if json.Unmarshal(recorder.Body.Bytes(), &response) != nil { + panic("invalid real exchange") + } + response.Messages = nil + raw, _ := rafttransport.Encode(response) + w.Write(raw) + return + } + if strings.EqualFold(q.Header.Get("Upgrade"), "websocket") { + r.svc.Handler.ServeHTTP(peerCountWriter{w, &r.in, &r.out}, q) + } else { + r.svc.Handler.ServeHTTP(w, q) + } + })) + t.Cleanup(func() { + for _, s := range r.homes { + s.stopMember() + } + r.server.Close() + r.svc.Close() + }) + return r +} +func (r *memberRig) home(id *identity.Identity) *Sync { + r.t.Helper() + home := machine(r.t, r.server.URL) + if id != nil { + _, err := identity.Adopt(home, *id, true) + mustNil(r.t, err) + } + own, err := identity.Load(home) + mustNil(r.t, err) + dev, err := identity.Device(home) + mustNil(r.t, err) + s := buildAt(home, r.server.URL, own, dev, DefaultInterval, r.clock.Now) + r.homes = append(r.homes, s) + return s +} +func waitMember(t *testing.T, s *Sync, predicate func(*personalMember) bool) { + t.Helper() + timer := time.NewTimer(requestTimeout) + defer timer.Stop() + for { + s.memberMu.Lock() + m := s.member + if m == nil { + s.memberMu.Unlock() + t.Fatal("R9 no running learner") + } + m.mu.Lock() + s.memberMu.Unlock() + err := m.failure + pass := predicate(m) + changed := m.changed + m.mu.Unlock() + if err != nil { + t.Fatalf("R9 peer failed: %v", err) + } + if pass { + return + } + select { + case <-changed: + case <-timer.C: + t.Fatal("R9 learner did not reach its applied state") + } + } +} +func readyMember(t *testing.T, s *Sync) { + t.Helper() + mustNil(t, s.memberRound(t.Context())) + waitMember(t, s, func(m *personalMember) bool { return m.idle && m.status.Applied >= m.record.Grant.Barrier }) +} +func nodeOf(s *Sync) uint64 { + s.memberMu.Lock() + defer s.memberMu.Unlock() + return s.member.record.Grant.Node +} +func recordOf(t *testing.T, s *Sync) memberRecord { + t.Helper() + b, e := os.ReadFile(filepath.Join(s.memberPath(), "identity.json")) + mustNil(t, e) + var record memberRecord + mustNil(t, json.Unmarshal(b, &record)) + return record +} + +// H1/R9 proves both real homes own persistent learners while all three anchors +// alone vote, and ready requires signed evidence after actual local application. +func TestR9PersonalHomesJoinAsDurableLearners(t *testing.T) { + r := newMemberRig(t) + a := r.home(nil) + id := a.Identity + b := r.home(&id) + before := r.requests.Load() + opened, ok, e := Open(a.Home) + mustNil(t, e) + if !ok || opened == nil || r.requests.Load() != before { + t.Fatal("R23 Open used the network") + } + readyMember(t, a) + readyMember(t, b) + status, e := a.ClusterStatus(t.Context()) + mustNil(t, e) + if len(status.Configuration.Voters) != 3 || !slices.Equal(status.Configuration.Voters, []uint64{1, 2, 3}) || len(status.Configuration.Learners) != 2 { + t.Fatalf("R9 topology: %+v", status) + } + for _, s := range []*Sync{a, b} { + node := nodeOf(s) + if !slices.Contains(status.Configuration.Learners, node) || !slices.Contains(status.Ready, node) { + t.Fatalf("R9 learner not ready: %+v", status) + } + if _, e := os.Stat(filepath.Join(s.memberPath(), "member.db")); e != nil { + t.Fatal(e) + } + } + if _, e = a.LeadCluster(t.Context(), 0); !errors.Is(e, clusterctl.ErrUnsupported) { + t.Fatal("R6 leadership was offered", e) + } +} + +// H2/R9 proves reopening retains authority, a lost reply resumes the saved +// enrollment, and wiping only the authoritative store retires the old node. +func TestR9PersonalStoreReopenWipeAndInterruptedEnrollment(t *testing.T) { + r := newMemberRig(t) + s := r.home(nil) + r.interrupt.Store(true) + s.memberRound(t.Context()) + pending := recordOf(t, s) + if pending.Store == "" || s.member != nil { + t.Fatal("R9 interruption lost pending identity") + } + reopened := buildAt(s.Home, s.Relay, s.Identity, s.Device, s.Interval, r.clock.Now) + r.homes = append(r.homes, reopened) + readyMember(t, reopened) + first := recordOf(t, reopened) + if first.Store != pending.Store { + t.Fatal("R9 enrollment retry changed store identity") + } + oldNode := nodeOf(reopened) + reopened.stopMember() + again := buildAt(s.Home, s.Relay, s.Identity, s.Device, s.Interval, r.clock.Now) + r.homes = append(r.homes, again) + readyMember(t, again) + if nodeOf(again) != oldNode || recordOf(t, again).Store != first.Store { + t.Fatal("R9 reopened with new authority") + } + again.stopMember() + mustNil(t, os.Remove(filepath.Join(s.memberPath(), "member.db"))) + // A wiped database includes its WAL sidecars; a normal close checkpointed them. + fresh := buildAt(s.Home, s.Relay, s.Identity, s.Device, s.Interval, r.clock.Now) + r.homes = append(r.homes, fresh) + readyMember(t, fresh) + if nodeOf(fresh) == oldNode || recordOf(t, fresh).Store == first.Store { + t.Fatal("R9 wiped store reused authority") + } + status, e := fresh.ClusterStatus(t.Context()) + mustNil(t, e) + if slices.Contains(status.Configuration.Learners, oldNode) { + t.Fatal("R9 old node remains configured") + } + m := fresh.member + m.mu.Lock() + app, _, e := m.store.ApplicationView(t.Context()) + mustNil(t, e) + key := raftstore.Key{Namespace: raftstore.Enrollment, Key: cs.MemberKey(oldNode).Key} + rows, _, e := m.store.Read(t.Context(), app.Applied, []raftstore.Key{key}) + mustNil(t, e) + var retired cs.MemberRecord + decodeErr := cs.Decode(rows[key], &retired) + m.mu.Unlock() + mustNil(t, decodeErr) + if !retired.Retired { + t.Fatal("R9 old node resurrected") + } +} + +// H3/R23 checks each retained HTTP/frame/callback path after sync-off or an +// unpair, including startup and discovery. No quiet attempt reaches the relay. +func TestR23PersonalLearnerStopsAtEveryUse(t *testing.T) { + for _, unpair := range []bool{false, true} { + t.Run(map[bool]string{false: "off", true: "unpaired"}[unpair], func(t *testing.T) { + r := newMemberRig(t) + s := r.home(nil) + readyMember(t, s) + m := s.member + if unpair { + mustNil(t, os.Remove(filepath.Join(s.Home, identity.File))) + } else { + t.Setenv(URLVar, "off") + } + requests, frames, attempts := r.requests.Load(), r.in.count.Load(), s.relayAttempts.Load()+s.clusterFrames.Load() + m.mu.Lock() + _, e := m.request() + if !errors.Is(e, ErrQuiet) { + t.Fatal(e) + } + e = m.exchange(t.Context()) + if !errors.Is(e, ErrQuiet) { + t.Fatal(e) + } + e = m.receive(t.Context(), rafttransport.Response{Generation: m.record.Grant.Generation}) + if !errors.Is(e, ErrQuiet) { + t.Fatal(e) + } + e = m.start(t.Context()) + if !errors.Is(e, ErrQuiet) { + t.Fatal(e) + } + m.mu.Unlock() + _, e = s.enrollMember(t.Context(), m.record.Store) + if !errors.Is(e, ErrQuiet) { + t.Fatal(e) + } + _, e = s.ClusterStatus(t.Context()) + if !errors.Is(e, ErrQuiet) { + t.Fatal(e) + } + s.memberRound(t.Context()) + mustNil(t, s.SyncState(t.Context())) + if r.requests.Load() != requests || r.in.count.Load() != frames || s.relayAttempts.Load()+s.clusterFrames.Load() != attempts { + t.Fatal("R23 quiet learner sent traffic") + } + }) + } +} + +// H4/H5/R26/R27/C30 drive ten virtual idle minutes on the REAL native relay +// and real open home state rounds, then require remote and local state changes +// to pass the normal quorum-backed read/write doors and arrive within a round. +func TestR26R27C30OpenLearnerIdleAndStateWake(t *testing.T) { + r := newMemberRig(t) + a := r.home(nil) + id := a.Identity + b := r.home(&id) + readyMember(t, a) + readyMember(t, b) + mustNil(t, a.SyncState(t.Context())) + mustNil(t, b.SyncState(t.Context())) + // Settle useful startup sharing before beginning the idle accounting window. + settle := func(s *Sync) { + waitMember(t, s, func(m *personalMember) bool { return m.idle }) + mustNil(t, s.SyncState(t.Context())) + } + for round := 0; round < 3; round++ { + settle(a) + settle(b) + } + for tick := 0; tick < 20; tick++ { + r.clock.Advance(DefaultInterval) + settle(a) + settle(b) + } + // Exercise the same owner used by the real terminal during the entire + // measured window. Host ticks advance explicitly; no wall sleeps are used. + tickHome := func(s *Sync) func() { + ctx, cancel := context.WithCancel(t.Context()) + ticks := make(chan time.Time) + done := make(chan struct{}) + go func() { s.runState(ctx, ticks); close(done) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(requestTimeout): + t.Error("R23 state owner did not stop") + } + }) + tick := func() { + select { + case ticks <- r.clock.Now(): + case <-time.After(requestTimeout): + t.Fatal("R27 state round stalled") + } + } + tick() + tick() + return func() { tick(); tick() } + } + tickA, tickB := tickHome(a), tickHome(b) + // The surface and standing observer hold separate Sync handles. Their + // follower, roster and backstop must all use the terminal-owned learner. + screenA := buildAt(a.Home, a.Relay, a.Identity, a.Device, a.Interval, r.clock.Now) + screenB := buildAt(b.Home, b.Relay, b.Identity, b.Device, b.Interval, r.clock.Now) + followA, followB := screenA.Follow(), screenB.Follow() + defer followA.Close() + defer followB.Close() + streams := []dirwatch.Stream{} + for _, screen := range []*Sync{screenA, screenB} { + watcher := screen.Dir.(directory.Watcher) + stream, err := screen.followStream(t.Context(), watcher) + mustNil(t, err) + defer stream.Close() + streams = append(streams, stream) + } + countAttempts := func() uint64 { + return a.relayAttempts.Load() + a.clusterFrames.Load() + b.relayAttempts.Load() + b.clusterFrames.Load() + screenA.relayAttempts.Load() + screenA.clusterFrames.Load() + screenB.relayAttempts.Load() + screenB.clusterFrames.Load() + } + requests, frames, attempts := r.requests.Load(), r.in.count.Load()+r.out.count.Load(), countAttempts() + for tick := 0; tick < 120; tick++ { + r.clock.Advance(DefaultInterval) + tickA() + tickB() + for i, screen := range []*Sync{screenA, screenB} { + mustNil(t, streams[i].Ping(t.Context())) + for { + frame, err := streams[i].Next(t.Context()) + mustNil(t, err) + if frame.Pong { + break + } + } + _, _, err := screen.RowsAt(t.Context()) + mustNil(t, err) + _, err = screen.Roster(t.Context()) + mustNil(t, err) + _, err = screen.FleetSize(t.Context()) + mustNil(t, err) + } + followA.Probe() + followB.Probe() + } + gotHTTP, gotFrames := r.requests.Load()-requests, r.in.count.Load()+r.out.count.Load()-frames + gotAttempts := countAttempts() - attempts + t.Logf("R27 native ten-minute idle: relay HTTP=%d socket frames=%d client attempts=%d", gotHTTP, gotFrames, gotAttempts) + if gotHTTP != 0 || gotFrames != 0 || gotAttempts != 0 { + t.Fatal("R27 idle home generated traffic") + } + // The first public read after idle and the next state write retain ordinary + // ReadIndex/proposal authority. Local learner state is only a change signal. + _, e := b.Dir.List(t.Context()) + mustNil(t, e) + mustNil(t, config.WriteChatModel(b.Home, "stub/after-idle")) + brain, e := store.Open(filepath.Join(b.Home, "graph.db")) + mustNil(t, e) + memory, e := brain.AddMemory(store.Memory{Owner: store.OwnerUser, Text: "The idle wake carries memory too.", Origin: store.OriginPerson}) + mustNil(t, e) + mustNil(t, brain.Close()) + mustNil(t, b.SyncState(t.Context())) + want, _ := b.Dir.List(t.Context()) + waitMember(t, a, func(m *personalMember) bool { return appliedVault(m) == want.Identity.Vault && m.idle }) + rid, local, e := a.memberVault(t.Context()) + mustNil(t, e) + if !local || rid != want.Identity.Vault { + t.Fatalf("R26 remote committed vault did not arrive: %q != %q", rid, want.Identity.Vault) + } + mustNil(t, a.SyncState(t.Context())) + raw, e := os.ReadFile(filepath.Join(a.Home, "config.json")) + mustNil(t, e) + if !strings.Contains(string(raw), "stub/after-idle") { + t.Fatal("C30 preference did not arrive within one state round") + } + received, e := store.Open(filepath.Join(a.Home, "graph.db")) + mustNil(t, e) + remembered, found, e := received.MemoryRecord(memory.ID) + mustNil(t, e) + mustNil(t, received.Close()) + if !found || remembered.Text != memory.Text { + t.Fatal("C30 shared memory did not arrive within one state round") + } + mustNil(t, config.WriteChatModel(a.Home, "stub/local-wake")) + mustNil(t, a.SyncState(t.Context())) + l, e := a.Dir.List(t.Context()) + mustNil(t, e) + if l.Identity.Vault == want.Identity.Vault { + t.Fatal("C30 local change was not pushed promptly") + } +} + +// H6/R23/C30 pins the hosted capability boundary and its unchanged polling +// behavior. A relay with no control/enrollment door never opens a member store. +func TestR23C30HostedStateKeepsExistingPolling(t *testing.T) { + server := relay(t) + s, ok, e := Open(machine(t, server.URL)) + mustNil(t, e) + if !ok { + t.Fatal("open") + } + s.memberRound(t.Context()) + s.memberRound(t.Context()) + if s.member != nil || !s.memberUnsupported { + t.Fatal("R23 hosted started a learner") + } + if _, e := os.Stat(s.memberPath()); !errors.Is(e, os.ErrNotExist) { + t.Fatal("R23 hosted created learner storage", e) + } + _, e = s.ClusterStatus(t.Context()) + if !errors.Is(e, clusterctl.ErrUnsupported) { + t.Fatal(e) + } + _, e = s.LeadCluster(t.Context(), 0) + if !errors.Is(e, clusterctl.ErrUnsupported) { + t.Fatal(e) + } + mustNil(t, config.WriteChatModel(s.Home, "stub/hosted-state")) + mustNil(t, s.SyncState(t.Context())) + // Count ordinary directory requests independently at the actual HTTP server. + var requests atomic.Uint64 + counted := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); server.Config.Handler.ServeHTTP(w, r) })) + defer counted.Close() + active := build(s.Home, counted.URL, s.Identity, s.Device, s.Interval) + active.stateSent = s.stateSent + for tick := 0; tick < 120; tick++ { + mustNil(t, active.SyncState(t.Context())) + } + t.Logf("R27 hosted ten-minute idle: relay HTTP=%d (polling retained)", requests.Load()) + if requests.Load() != 2*uint64(10*time.Minute/DefaultInterval) { + t.Fatalf("C30 hosted polling changed: %d", requests.Load()) + } +} + +// H1/H3/C30 prove the existing terminal lifetime notices pairing while Home +// stands open, starts the real learner, and joins all its work on cancellation. +func TestR9R23C30PairingStartsLearnerInsideOpenHomeLifetime(t *testing.T) { + r := newMemberRig(t) + home := machine(t, r.server.URL) + // A solo first launch has local identity state but no network authority. + mustNil(t, os.Remove(filepath.Join(home, identity.File))) + _, e := identity.EnsureSolo(home) + mustNil(t, e) + s, ok, e := Open(home) + mustNil(t, e) + if !ok { + t.Fatal("open solo") + } + ctx, cancel := context.WithCancel(t.Context()) + ticks := make(chan time.Time) + done := make(chan struct{}) + go func() { s.runState(ctx, ticks); close(done) }() + t.Cleanup(func() { cancel(); <-done }) + step := func() { + select { + case ticks <- r.clock.Now(): + case <-time.After(requestTimeout): + t.Fatal("C30 state lifetime did not complete round") + } + } + step() + if r.requests.Load() != 0 { + t.Fatal("R23 solo Home attempted discovery") + } + fleet, e := identity.Ensure(t.TempDir()) + mustNil(t, e) + _, e = identity.Adopt(home, fleet, true) + mustNil(t, e) + step() + step() + active, ok, e := Open(home) + mustNil(t, e) + if !ok { + t.Fatal("open paired") + } + status, e := active.clusterClient().Status(t.Context()) + mustNil(t, e) + if status.Self == 0 || !slices.Contains(status.Configuration.Learners, status.Self) { + t.Fatal("R9 live pairing did not start learner") + } + current, e := s.currentDisplaySync() + mustNil(t, e) + if current.Identity.ID() != fleet.ID() { + t.Fatal("C30 Home retained its old metadata key after pairing") + } + before := r.requests.Load() + _, _, e = s.RowsAt(t.Context()) + mustNil(t, e) + if r.requests.Load() != before { + t.Fatal("R27 paired-open Home did not use its newly owned learner") + } + cancel() + select { + case <-done: + case <-time.After(requestTimeout): + t.Fatal("R23 lifetime did not join learner shutdown") + } + // Shutdown must release the sole member lock and SQLite connection. + reopened, e := active.openMember(t.Context()) + mustNil(t, e) + reopened.close() +} + +// H2/R9/R23 require same-store generation recovery after relay inbox loss, +// rather than trusting stale delivery cursors or enrolling a replacement node. +func TestR9R23PersonalLearnerRefreshesColdRouterGeneration(t *testing.T) { + r := newMemberRig(t) + s := r.home(nil) + readyMember(t, s) + old := recordOf(t, s) + s.stopMember() + mustNil(t, r.svc.Close()) + r.svc = relayserve.New(relayserve.Config{Store: r.storage, Now: r.clock.Now, After: r.clock.After}) + readyMember(t, s) + current := recordOf(t, s) + if current.Store != old.Store || current.Grant.Node != old.Grant.Node || current.Grant.Generation <= old.Grant.Generation { + t.Fatal("R9 cold router replaced store or reused generation") + } + mustNil(t, config.WriteChatModel(s.Home, "stub/cold")) + mustNil(t, s.SyncState(t.Context())) + want, e := s.Dir.List(t.Context()) + mustNil(t, e) + waitMember(t, s, func(m *personalMember) bool { return m.idle && appliedVault(m) == want.Identity.Vault }) +} + +// H3/R23 prohibits discovery, enrollment, SQLite creation or delayed callbacks +// before pairing or when sync was already off at the start of the lifetime. +func TestR23PersonalLearnerDoesNotStartWhileQuiet(t *testing.T) { + for _, solo := range []bool{true, false} { + t.Run(map[bool]string{true: "solo", false: "off"}[solo], func(t *testing.T) { + r := newMemberRig(t) + s := r.home(nil) + if solo { + mustNil(t, os.Remove(filepath.Join(s.Home, identity.File))) + _, e := identity.EnsureSolo(s.Home) + mustNil(t, e) + } else { + t.Setenv(URLVar, "off") + } + s.memberRound(t.Context()) + _, e := s.openMember(t.Context()) + if !errors.Is(e, ErrQuiet) { + t.Fatal(e) + } + mustNil(t, s.SyncState(t.Context())) + if s.member != nil || r.requests.Load() != 0 || s.relayAttempts.Load() != 0 || s.clusterFrames.Load() != 0 { + t.Fatal("R23 quiet startup reached the relay") + } + if _, e := os.Stat(s.memberPath()); !errors.Is(e, os.ErrNotExist) { + t.Fatal("R23 quiet startup created store", e) + } + }) + } +} + +// appliedVault reads only the learner's own durable application under its actor. +func appliedVault(m *personalMember) string { + ctx := context.Background() + app, _, err := m.store.ApplicationView(ctx) + if err != nil { + return "" + } + key := raftstore.Key{Namespace: raftstore.State, Key: cs.IdentityKey().Key} + rows, _, err := m.store.Read(ctx, app.Applied, []raftstore.Key{key}) + if err != nil { + return "" + } + var id cs.IdentityRec + if cs.Decode(rows[key], &id) != nil { + return "" + } + return id.Vault +} + +// H1/R9 proves the production learner installs a multi-chunk snapshot and then +// applies later log entries, rather than reporting router delivery as readiness. +func TestR9PersonalHomeInstallsSnapshotChunksAndLog(t *testing.T) { + r := newMemberRig(t) + s := r.home(nil) + id := s.Identity + var last *Sync + for n := 0; n < 14; n++ { + writer := r.home(&id) + last = writer + mustNil(t, writer.Dir.PutDevice(t.Context(), writer.Device.ID(), directory.Device{V: 1, Name: strings.Repeat("e", 20000)})) + } + readyMember(t, s) + m := s.member + m.mu.Lock() + recovered, e := m.store.Recover(t.Context()) + m.mu.Unlock() + mustNil(t, e) + manifest, e := raftsnapshot.Decode(recovered.Core.Snapshot.GetData()) + mustNil(t, e) + if manifest.Chunks < 2 { + t.Fatal("R9 fixture did not install multiple snapshot chunks") + } + mustNil(t, last.Dir.PutDevice(t.Context(), last.Device.ID(), directory.Device{V: 1, Name: "after installed snapshot"})) + key := cs.DeviceKey(last.Device.ID()) + waitMember(t, s, func(m *personalMember) bool { + app, _, err := m.store.ApplicationView(t.Context()) + if err != nil { + return false + } + local := raftstore.Key{Namespace: raftstore.Namespace(key.Namespace), Key: key.Key} + rows, _, err := m.store.Read(t.Context(), app.Applied, []raftstore.Key{local}) + if err != nil { + return false + } + var device cs.Device + return cs.Decode(rows[local], &device) == nil && device.Name == "after installed snapshot" && m.idle + }) + status, e := s.ClusterStatus(t.Context()) + mustNil(t, e) + if !slices.Contains(status.Ready, nodeOf(s)) { + t.Fatal("R9 durable snapshot/log learner not ready") + } +} + +// H3/R23 exercises a real established socket callback after the person's off +// decision. A different authorized machine writes through the public door; +// this home's delayed receive must discard it without sending another frame. +func TestR23OpenLearnerDiscardsDelayedPushAfterOff(t *testing.T) { + r := newMemberRig(t) + s := r.home(nil) + readyMember(t, s) + m := s.member + remote := directory.NewHTTP(r.server.URL, reqsign.SignFor(deviceSigner{s.Identity, s.Device}, r.clock.Now), r.server.Client()) + t.Setenv(URLVar, "off") + attempts, frames := s.relayAttempts.Load(), s.clusterFrames.Load() + m.mu.Lock() + changed := m.changed + m.mu.Unlock() + mustNil(t, remote.PutDevice(t.Context(), s.Device.ID(), directory.Device{V: 1, Name: "delayed while off"})) + select { + case <-changed: + case <-time.After(requestTimeout): + t.Fatal("R23 real delayed socket callback never arrived") + } + m.mu.Lock() + failure := m.failure + m.mu.Unlock() + if !errors.Is(failure, ErrQuiet) { + t.Fatal("R23 delayed callback did not stop", failure) + } + if attempts != s.relayAttempts.Load() || frames != s.clusterFrames.Load() { + t.Fatal("R23 delayed callback sent after off") + } + s.memberRound(t.Context()) +} + +// H2/R9/R27 requires interrupted catch-up to retain its store and retry next +// round without busy polling or treating a transport response as readiness. +func TestR9R27PersonalCatchupResumesWithoutEmptyPolling(t *testing.T) { + r := newMemberRig(t) + s := r.home(nil) + r.parkExchanges.Store(true) + err := s.memberRound(t.Context()) + if !errors.Is(err, errMemberWaiting) || s.member != nil { + t.Fatal("R9 empty catch-up did not remain pending", err) + } + pending := recordOf(t, s) + attempts := s.relayAttempts.Load() + for round := 0; round < 3; round++ { + s.memberRound(t.Context()) + } + if s.relayAttempts.Load() != attempts { + t.Fatal("R27 retry backoff made empty polls") + } + r.parkExchanges.Store(false) + r.clock.Advance(DefaultInterval) + readyMember(t, s) + resumed := recordOf(t, s) + if resumed.Store != pending.Store || resumed.Grant.Node != pending.Grant.Node { + t.Fatal("R9 partial catch-up replaced authority") + } +} + +// H4/H5/R9/R23 require the actual Home follower and its separate Sync handle +// to receive local version/join/refusal signals without periodic network use. +func TestR9R23LocalHomeFollowerSharesTheOwnedLearner(t *testing.T) { + r := newMemberRig(t) + a := r.home(nil) + mustNil(t, a.Rename(t.Context(), "home-a")) + readyMember(t, a) + screen := buildAt(a.Home, a.Relay, a.Identity, a.Device, a.Interval, r.clock.Now) + stream, e := screen.followStream(t.Context(), screen.Dir.(directory.Watcher)) + mustNil(t, e) + defer stream.Close() + drain := func() { + mustNil(t, stream.Ping(t.Context())) + for { + f, e := stream.Next(t.Context()) + mustNil(t, e) + if f.Pong { + return + } + } + } + drain() + baseline := r.requests.Load() + _, _, e = screen.RowsAt(t.Context()) + mustNil(t, e) + roster, e := screen.Roster(t.Context()) + mustNil(t, e) + if len(roster) != 1 || r.requests.Load() != baseline { + t.Fatal("R27 Home display backstop used the network") + } + id := a.Identity + b := r.home(&id) + mustNil(t, b.Rename(t.Context(), "home-b")) + var joined bool + ctx, cancel := context.WithTimeout(t.Context(), requestTimeout) + defer cancel() + for !joined { + frame, e := stream.Next(ctx) + mustNil(t, e) + joined = frame.Event != nil && frame.Event.T == dirwatch.EventJoined && frame.Event.Device == b.Device.ID() + } + roster, e = screen.Roster(t.Context()) + mustNil(t, e) + if len(roster) != 2 { + t.Fatal("R9 committed joined roster did not reach Home") + } + // Display hints grant no ownership or stale-read shortcut to authority. + before := r.requests.Load() + _, e = screen.Dir.List(t.Context()) + mustNil(t, e) + if r.requests.Load() <= before { + t.Fatal("R26 public authority read used local display state") + } + t.Setenv(URLVar, "off") + before = r.requests.Load() + if e = stream.Ping(t.Context()); !errors.Is(e, ErrQuiet) { + t.Fatal(e) + } + if _, e = stream.Next(t.Context()); !errors.Is(e, ErrQuiet) { + t.Fatal(e) + } + if r.requests.Load() != before { + t.Fatal("R23 local follower sent after off") + } +} diff --git a/internal/syncsetup/member_display.go b/internal/syncsetup/member_display.go new file mode 100644 index 0000000000..ee3bd9eba7 --- /dev/null +++ b/internal/syncsetup/member_display.go @@ -0,0 +1,267 @@ +package syncsetup + +import ( + "context" + "errors" + "github.com/Agent-Field/codeaf/internal/identity" + "sort" + "sync" + + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/dirwatch" + "github.com/Agent-Field/codeaf/internal/raftdirectory" +) + +// liveMembers shares the existing terminal-owned learner with other Sync +// handles for the same home. It owns no goroutine, store or network lifetime. +var liveMembers = struct { + sync.Mutex + members map[string]*personalMember +}{members: map[string]*personalMember{}} + +func (m *personalMember) publishDisplay() { + liveMembers.Lock() + defer liveMembers.Unlock() + liveMembers.members[m.owner.memberPath()] = m +} +func (m *personalMember) withdrawDisplay() { + liveMembers.Lock() + defer liveMembers.Unlock() + key := m.owner.memberPath() + if liveMembers.members[key] == m { + delete(liveMembers.members, key) + } +} +func (s *Sync) displayMember() *personalMember { + liveMembers.Lock() + defer liveMembers.Unlock() + return liveMembers.members[s.memberPath()] +} + +// displayListing is a hint for Home, its roster and its backstop. Decisions +// still call Dir.Cell/List and the ordinary mutation doors with fresh quorum. +func (s *Sync) displayListing(ctx context.Context) (directory.Listing, bool, error) { + if !s.memberAllowed() { + return directory.Listing{}, false, nil + } + m := s.displayMember() + if m == nil { + return directory.Listing{}, false, nil + } + m.mu.Lock() + defer m.mu.Unlock() + if m.closed || m.failure != nil { + return directory.Listing{}, false, nil + } + app, _, err := m.store.ApplicationView(ctx) + if err != nil { + return directory.Listing{}, true, err + } + if app.Applied < m.record.Grant.Barrier { + return directory.Listing{}, false, nil + } + l, err := raftdirectory.DisplayListing(ctx, m.store, app) + // This clock ages only display hints. It cannot expire a committed lease + // or authorize a take; those paths use the relay's current-time barrier. + l.Now = max(l.Now, s.now().UnixMilli()) + return l, true, err +} + +var errMemberDisplayChanged = errors.New("the local directory follower changed") + +// switchingDisplay ends the old advisory socket at its next use once the +// terminal's learner is available. Its replacement is entirely local. +type switchingDisplay struct { + dirwatch.Stream + owner *Sync +} + +func (s switchingDisplay) Ping(ctx context.Context) error { + if !s.owner.memberAllowed() { + s.Stream.Close() + return ErrQuiet + } + if s.owner.memberAllowed() && s.owner.displayMember() != nil { + s.Stream.Close() + return errMemberDisplayChanged + } + s.owner.clusterFrames.Add(1) + return s.Stream.Ping(ctx) +} + +func (s *Sync) followStream(ctx context.Context, w directory.Watcher) (dirwatch.Stream, error) { + if !s.memberAllowed() { + return nil, ErrQuiet + } + if m := s.displayMember(); m != nil { + m.mu.Lock() + valid := !m.closed && m.failure == nil + m.mu.Unlock() + if valid { + return &localDisplayStream{owner: s, member: m, pong: make(chan struct{}, 1), closed: make(chan struct{}), devices: map[string]directory.Device{}, online: map[string]bool{}}, nil + } + } + stream, err := w.Watch(ctx) + if err != nil { + return nil, err + } + return switchingDisplay{Stream: stream, owner: s}, nil +} + +type localDisplayStream struct { + owner *Sync + member *personalMember + mu sync.Mutex + pong chan struct{} + closed chan struct{} + once sync.Once + initialized bool + applied uint64 + cached directory.Listing + version uint64 + devices map[string]directory.Device + online map[string]bool + frames []dirwatch.Frame +} + +func (s *localDisplayStream) Vouching() bool { return false } +func (s *localDisplayStream) Close() { s.once.Do(func() { close(s.closed) }) } +func (s *localDisplayStream) Ping(context.Context) error { + if !s.owner.memberAllowed() { + s.Close() + return ErrQuiet + } + select { + case s.pong <- struct{}{}: + default: + } + return nil +} +func (s *localDisplayStream) Next(ctx context.Context) (dirwatch.Frame, error) { + for { + if !s.owner.memberAllowed() { + s.Close() + return dirwatch.Frame{}, ErrQuiet + } + m := s.member + m.mu.Lock() + changed := m.changed + applied := m.status.Applied + closed, failed := m.closed, m.failure + m.mu.Unlock() + if closed || failed != nil { + return dirwatch.Frame{}, errMemberDisplayChanged + } + listing, local, err := s.cached, true, error(nil) + if s.applied != applied || !s.initialized { + listing, local, err = s.owner.displayListing(ctx) + if err == nil && local { + s.cached = listing + s.applied = applied + } + } + listing.Now = max(listing.Now, s.owner.now().UnixMilli()) + if err != nil { + return dirwatch.Frame{}, err + } + if !local { + return dirwatch.Frame{}, errMemberDisplayChanged + } + if listing.Identity.Rotation != nil { + return dirwatch.Frame{}, dirwatch.ErrRotated + } + if listing.Devices[s.owner.Device.ID()].Revoked { + return dirwatch.Frame{}, dirwatch.ErrRevoked + } + s.mu.Lock() + s.project(listing) + if len(s.frames) > 0 { + frame := s.frames[0] + s.frames = s.frames[1:] + s.mu.Unlock() + return frame, nil + } + s.mu.Unlock() + select { + case <-changed: + case <-s.pong: + return dirwatch.Frame{Pong: true}, nil + case <-s.closed: + return dirwatch.Frame{}, context.Canceled + case <-ctx.Done(): + return dirwatch.Frame{}, ctx.Err() + } + } +} + +// project produces only display hints and committed joined/revoked news. A +// quiet neighbour's presence ages normally; a live unexpired held chat is +// also recent evidence. Neither hint can suppress an ownership heartbeat. +func (s *localDisplayStream) project(l directory.Listing) { + if !s.initialized || l.Version != s.version { + s.frames = append(s.frames, dirwatch.Frame{Version: l.Version}) + } + ids := make([]string, 0, len(l.Devices)) + for id := range l.Devices { + ids = append(ids, id) + } + sort.Strings(ids) + online := map[string]bool{s.owner.Device.ID(): true} + for _, cell := range l.Cells { + if cell.Lease.Expires > l.Now { + online[cell.Lease.Device] = true + } + } + for _, id := range ids { + d := l.Devices[id] + active := !d.Revoked && (online[id] || d.LastSeen != 0 && l.Now-d.LastSeen < dirwatch.DeadAfter.Milliseconds()) + if active != s.online[id] { + e := dirwatch.PresenceEvent(id, active, l.Now) + s.frames = append(s.frames, dirwatch.Frame{Event: &e}) + } + old, existed := s.devices[id] + if s.initialized && !existed && !d.Revoked && id != s.owner.Device.ID() { + e := dirwatch.JoinedEvent(id, d.Name, d.Platform, d.Created) + s.frames = append(s.frames, dirwatch.Frame{Event: &e}) + } + if d.Revoked && !old.Revoked { + e := dirwatch.RevokedEvent(id, l.Now) + s.frames = append(s.frames, dirwatch.Frame{Event: &e}) + } + online[id] = active + } + s.initialized = true + s.version = l.Version + s.devices = l.Devices + s.online = online +} + +// currentDisplaySync follows a pairing or relay adoption made while Home is +// open. Its old signer and metadata key must not keep displaying the old fleet. +// Hand-built local sources have no relay and retain their supplied behavior. +func (s *Sync) currentDisplaySync() (*Sync, error) { + if s.Relay == "" || Quiet(s.Home) { + return s, nil + } + id, err := identity.Load(s.Home) + if err != nil { + return nil, err + } + base, err := relayBase(Resolve(s.Home).URL) + if err != nil { + return nil, err + } + if id.ID() == s.Identity.ID() && base == s.Relay { + return s, nil + } + next, ok, err := Open(s.Home) + if err != nil { + return nil, err + } + if !ok { + return s, nil + } + current := buildAt(next.Home, next.Relay, next.Identity, next.Device, next.Interval, s.now) + current.Hosted = next.Hosted + return current, nil +} diff --git a/internal/syncsetup/quiet.go b/internal/syncsetup/quiet.go index 0ce14e509b..facac01903 100644 --- a/internal/syncsetup/quiet.go +++ b/internal/syncsetup/quiet.go @@ -64,6 +64,9 @@ func (s *Sync) list(ctx context.Context) (directory.Listing, error) { if Quiet(s.Home) { return directory.Listing{}, nil } + if l, local, err := s.displayListing(ctx); local || err != nil { + return l, err + } return s.Dir.List(ctx) } @@ -75,6 +78,11 @@ func (s *Sync) ClusterHTTP(client *http.Client) *http.Client { client = &http.Client{} } copyClient := *client + next := copyClient.Transport + if next == nil { + next = http.DefaultTransport + } + copyClient.Transport = relayCountingWire{s: s, next: next} return gated(s.Home, ©Client) } @@ -100,3 +108,15 @@ func (s quietStream) Ping(ctx context.Context) error { return s.Stream.Ping(ctx) } func (s quietStream) Vouching() bool { return !Quiet(s.home) && s.Stream.Vouching() } + +// relayCountingWire counts actual relay attempts inside the quiet gate. +// Frame attempts are counted at their separate guarded socket write boundary. +type relayCountingWire struct { + s *Sync + next http.RoundTripper +} + +func (w relayCountingWire) RoundTrip(r *http.Request) (*http.Response, error) { + w.s.relayAttempts.Add(1) + return w.next.RoundTrip(r) +} diff --git a/internal/syncsetup/state.go b/internal/syncsetup/state.go index a0ac0ba407..9af240b796 100644 --- a/internal/syncsetup/state.go +++ b/internal/syncsetup/state.go @@ -17,6 +17,10 @@ func (s *Sync) SyncState(ctx context.Context) error { if Quiet(s.Home) { return nil } + rid, local, err := s.memberVault(ctx) + if err != nil { + return err + } return s.withVault(func(v vaultsync.Syncer) error { if err := v.Capture(); err != nil { return err @@ -26,8 +30,17 @@ func (s *Sync) SyncState(ctx context.Context) error { if err := v.Push(ctx); err != nil { return err } - } else if err := v.Pull(ctx); err != nil { - return err + s.stateVaultKnown = false + } else if !local || !s.stateVaultKnown || rid != s.stateVault { + if local { + err = v.PullAt(ctx, rid) + } else { + err = v.Pull(ctx) + } + if err != nil { + return err + } + s.stateVault, s.stateVaultKnown = rid, local } s.stateSent, _ = keys.Fingerprint(s.Home) return nil @@ -39,17 +52,24 @@ func (s *Sync) SyncState(ctx context.Context) error { func (s *Sync) RunState(ctx context.Context) { ticker := time.NewTicker(s.Interval) defer ticker.Stop() + s.runState(ctx, ticker.C) +} + +// runState shares the terminal lifetime while accepting sampled host ticks. +func (s *Sync) runState(ctx context.Context, ticks <-chan time.Time) { + defer func() { s.stopMember() }() for { if ctx.Err() != nil { return } - if next, err := s.stateRound(ctx); err == nil { - s = next - } + next, _ := s.stateRound(ctx) + // A newly adopted identity owns its learner even if its first vault round + // fails. Keeping the old handle here would orphan the new lifetime. + s = next select { case <-ctx.Done(): return - case <-ticker.C: + case <-ticks: } } } @@ -59,10 +79,14 @@ func (s *Sync) RunState(ctx context.Context) { func (s *Sync) stateRound(ctx context.Context) (*Sync, error) { next, ok, err := Open(s.Home) if err != nil || !ok { + s.stopMember() return s, err } if next.Identity.ID() == s.Identity.ID() && next.Device.ID() == s.Device.ID() && next.Relay == s.Relay { next = s + } else { + s.stopMember() } + next.memberRound(ctx) return next, next.SyncState(ctx) } diff --git a/internal/syncsetup/syncsetup.go b/internal/syncsetup/syncsetup.go index 3009eb21aa..e6cead1e50 100644 --- a/internal/syncsetup/syncsetup.go +++ b/internal/syncsetup/syncsetup.go @@ -16,6 +16,8 @@ import ( "net/url" "strconv" "strings" + "sync" + "sync/atomic" "time" "github.com/Agent-Field/codeaf/internal/blobstore" @@ -51,14 +53,25 @@ var ErrNoIdentity = errors.New(chatlist.NoIdentity) // Sync is everything that talks to the relay, bound to this device. type Sync struct { - stateSent string // The last completed home-state round, guarded by stateMu. - Dir directory.Client // bound to this device by its signer - Store blobstore.Store // the bare store wire; every use of it goes through a Scope so it is counted - Device identity.Dev - Identity identity.Identity - Relay string // the normalized relay address every client and the ledger name are built from - Hosted bool // Relay is the hosted default, which the first-run line speaks of - Home string // the codeaf home: where the branch map and the stats files live + memberRetries uint64 + clock func() time.Time + relayAttempts atomic.Uint64 + clusterFrames atomic.Uint64 + memberMu sync.Mutex + member *personalMember + memberUnsupported bool + memberRetry time.Time + memberBackoff time.Duration + stateVault string + stateVaultKnown bool + stateSent string // The last completed home-state round, guarded by stateMu. + Dir directory.Client // bound to this device by its signer + Store blobstore.Store // the bare store wire; every use of it goes through a Scope so it is counted + Device identity.Dev + Identity identity.Identity + Relay string // the normalized relay address every client and the ledger name are built from + Hosted bool // Relay is the hosted default, which the first-run line speaks of + Home string // the codeaf home: where the branch map and the stats files live // ProfileDir is where the product keeps its profile files (config.json, // credentials.json): CODEAF_PROFILE_DIR when set, else empty, which means Home. ProfileDir string @@ -161,11 +174,14 @@ func flushInterval() (time.Duration, error) { // build binds both wires to one signer and one client, and leaves the store to be counted by scope. func build(home, base string, id identity.Identity, dev identity.Dev, interval time.Duration) *Sync { - sign := reqsign.SignFor(deviceSigner{id, dev}, time.Now) - hc := gated(home, &http.Client{Timeout: requestTimeout}) - return &Sync{ - Dir: directory.NewHTTP(base, sign, hc), - Store: blobstore.NewHTTP(base, sign, gated(home, &http.Client{})), + return buildAt(home, base, id, dev, interval, time.Now) +} + +// buildAt supplies the host clock for deterministic lifetime and wake proofs. +func buildAt(home, base string, id identity.Identity, dev identity.Dev, interval time.Duration, now func() time.Time) *Sync { + sign := reqsign.SignFor(deviceSigner{id, dev}, now) + s := &Sync{ + clock: now, Device: dev, Identity: id, Relay: base, @@ -174,6 +190,9 @@ func build(home, base string, id identity.Identity, dev identity.Dev, interval t Ledger: cellstore.LedgerName(base, id.ID()), Interval: interval, } + s.Dir = directory.NewHTTP(base, sign, s.ClusterHTTP(&http.Client{Timeout: requestTimeout})) + s.Store = blobstore.NewHTTP(base, sign, s.ClusterHTTP(&http.Client{})) + return s } // deviceSigner is the reqsign.Signer of this machine: the device key signs, @@ -196,6 +215,13 @@ func (s *Sync) Rows(ctx context.Context) ([]chatlist.Row, error) { // RowsAt is Rows and the directory version the listing was read at. func (s *Sync) RowsAt(ctx context.Context) ([]chatlist.Row, uint64, error) { + current, err := s.currentDisplaySync() + if err != nil { + return nil, 0, err + } + if current != s { + return current.RowsAt(ctx) + } l, err := s.list(ctx) if err != nil { return nil, 0, err @@ -214,11 +240,23 @@ func (s *Sync) Follow() dirwatch.Follower { if !ok { return nil } - return dirwatch.Follow(s.Ledger, func(ctx context.Context) (dirwatch.Stream, error) { + return dirwatch.Follow(s.Ledger+"#home:"+s.Home, func(ctx context.Context) (dirwatch.Stream, error) { if Quiet(s.Home) { return nil, ErrQuiet } - stream, err := w.Watch(ctx) + current, err := s.currentDisplaySync() + if err != nil { + return nil, err + } + watcher := w + if current != s { + var ok bool + watcher, ok = current.Dir.(directory.Watcher) + if !ok { + return nil, dirwatch.ErrNoRoute + } + } + stream, err := current.followStream(ctx, watcher) if err != nil { return nil, err } @@ -266,3 +304,10 @@ var ( // The engine glue is the sync seam by method set: no adapter between them. _ cellsync.Engine = cellstore.SyncEngine{} ) + +func (s *Sync) now() time.Time { + if s.clock != nil { + return s.clock() + } + return time.Now() +} diff --git a/internal/vaultsync/syncer.go b/internal/vaultsync/syncer.go index 662e85da31..d0770fc311 100644 --- a/internal/vaultsync/syncer.go +++ b/internal/vaultsync/syncer.go @@ -126,18 +126,34 @@ func (s Syncer) pull(ctx context.Context) (string, error) { if err != nil || rid == "" { return "", err } + return rid, s.pullAt(ctx, rid) +} + +// PullAt merges the content named by the learner's durable applied state. The +// caller uses this only as a change hint; it grants no coordination authority. +func (s Syncer) PullAt(ctx context.Context, rid string) error { + if err := s.each(Carrier.captureEdits); err != nil { + return err + } + if rid == "" { + return nil + } + return s.pullAt(ctx, rid) +} + +func (s Syncer) pullAt(ctx context.Context, rid string) error { obj, err := s.Store.Get(ctx, rid) if err != nil { - return "", err + return err } enc, err := verified(rid, obj) if err != nil { - return "", err + return err } if err := s.Vault.Merge(enc); err != nil { - return "", err + return err } - return rid, s.restoreCarried() + return s.restoreCarried() } // verified returns the vault.enc inside obj when obj hashes to rid. From 9306756084eade285fa02e8aea0e4d87bc0e2511 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 16:20:23 -0400 Subject: [PATCH 059/103] Keep device guidance readable while documenting native learner sync. Move coordination and worker machinery into its own manual page while preserving device sentences and every retained retrieval probe. Explain native durable learners, the local Home follower, the actual healthy backstop, hosted polling and the continuing leadership refusal. Keep delegate and remote-session answers reachable through headings in the askers' words. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../continuing-a-chat-on-another-computer.md | 93 ++------------- .../manual/chat/coordination-and-workers.md | 106 ++++++++++++++++++ internal/manual/chat/delegates.md | 4 + internal/manual/chat/home.md | 22 ++-- .../manual/chat/running-on-another-machine.md | 4 +- internal/manual/chat/worker-harness.md | 2 +- internal/manual/coordination_test.go | 33 ++++++ 7 files changed, 168 insertions(+), 96 deletions(-) create mode 100644 internal/manual/chat/coordination-and-workers.md create mode 100644 internal/manual/coordination_test.go diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index b6ff9f1791..5f4e60b299 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -249,105 +249,26 @@ other computer keeps seeing the previous saved head. Retry when the connection returns. If the final save fails during quit, the work stays local and that computer does not announce a clean release; another computer does not automatically open that incomplete save. Uploaded objects are kept for reuse if another device continued the chat first; this does not discard the work saved on your computer. -## Relay directory upgrade pauses and saved chats - -A relay directory upgrade can briefly pause pairing and chat moves. Retry after the relay is ready. The upgrade refusal is `codeaf: directory upgrade is incomplete; retry shortly; saved chats are still here`. It does not delete saved chats, files, branches or the encrypted vault. The relay keeps the old directory as a backup and never makes it a second writer after the new directory takes over. A row that cannot be copied safely leaves the upgrade paused for the operator; it is never truncated to make it fit. - - -## Joined computers, learner capacity and an outdated build - -Paired computers currently use the relay as authenticated directory clients. The production client does not yet activate its own persistent coordination member, so pairing alone does not make this computer a learner or a voter. The relay keeps three persistent voters and does not need either of your paired laptops to stay awake. - -The membership protocol admits at most 32 live learners and retains retired records instead of reusing their authority. A protocol mismatch says `this codeaf build must be updated before it can join this cluster; your saved chats are still here`. Its capacity refusal is `this cluster has reached its enrollment limit; remove a joined computer or ask the relay operator for capacity`. Personal-member activation and a worker setup command are not available through the production client yet. - ## Which computer coordinates: cluster status and cluster lead -`codeaf cluster status` on a self-hosted `codeaf relay` with storage reports the committed voters and current leader after a fresh majority read. Personal-member activation is not connected yet: pairing alone does not give this computer a node in that list. `codeaf cluster lead` on an unjoined computer says `this computer must join the cluster with persistent storage before it can lead`. - -Leadership is not offered yet on any relay. A direct leadership request to the self-hosted relay is refused without changing membership. The hosted relay has no cluster control door and says `this relay does not offer cluster leadership yet; your saved chats are still here`. Ordinary chat sync remains available. - -Do not treat pairing or a saved leadership request as a successful transfer. Success requires the selected computer to be elected, commit its current term and pass a fresh majority read; its receipt names the computer, term, revision and read barrier. Retrying `codeaf cluster lead` preserves an already saved signed request. Continue chats through the ordinary continue door. - -Three voters tolerate one unavailable member. Losing a majority pauses coordination; saved chats remain. The refusal is `this cluster cannot coordinate right now; reconnect enough of its voting computers and try again; your saved chats are still here`. - -## Does an open connection keep my chat running on this computer? - -The sync connection carries presence hints and committed lease acknowledgments. An open socket or an answered ping alone cannot keep ownership of a chat. codeaf skips an ordinary heartbeat only while it has a recent acknowledgment for that exact chat and ownership number, with enough time left before expiry. Without that acknowledgment it keeps sending ordinary heartbeats. A delayed acknowledgment for an earlier owner cannot take the chat back, and a delayed renewal cannot undo a clean quit. - -## Does syncing keep sending requests when nothing is happening? - -A home that has never paired sends no sync traffic. `CODEAF_SYNC_URL=off` also blocks new sync requests from clients that were opened before the setting changed. Presence pings on an open view are hints; they do not write chat ownership. On the consensus directory, a group with no work or held chats can stop its periodic coordination timer after an idle barrier has reached every voter. The first operation wakes coordination and must obtain a fresh majority response. A saved leader address cannot answer that operation by itself. Losing the majority leaves syncing unavailable and keeps local work intact. +On a self-hosted service with persistent storage, an open paired home joins as a learner. Three persistent voters keep coordination available when your laptops sleep. `codeaf cluster status` shows this computer after joining. Leadership is not offered yet; continue chats through the ordinary continue door. See the coordination and workers page for command refusals and setup details. ## Can a worker access all my chats or approve pairing? -A delegated worker uses its own key and an expiring certificate naming selected chats. It receives only sealed grants for those chats, rather than the full exported identity. It cannot approve pairing, rotate the identity, read the identity vault, enroll other devices or request leadership. Richer capabilities, worker labels and execution credentials stay encrypted; only the existing platform and sandbox capability shape is readable by the relay. - -Worker setup and assignment execution are not available as a terminal command yet. Registration and a fresh signed heartbeat only record availability; they never start tools or give the worker ownership of a chat. Removing a worker blocks later requests after that removal commits. Expiry and removal cannot take back bytes the worker already downloaded. +A delegated worker can access only its selected chats. It cannot approve pairing, rotate your identity, read your key vault or request leadership. Worker setup is not available as a terminal command yet. Personal computers keep their shared trust model. ## Worker restarts, draining and learner metadata visibility -A worker restart registers a newer process number. Delayed requests and heartbeats from its previous process cannot refresh availability or save work. A drained worker cannot receive new claims; already held work can still save and let go. Restarting does not clear draining. Expired and removed enrollment records remain retained; reaching their capacity limit refuses new enrollment instead of deleting records or saved chats. - -A scoped API worker does not join the coordination log. Joining as a learner requires explicit consent in its certificate: the full log reveals identity-wide routing and ownership metadata, while chat content, names and task instructions remain encrypted. Use scoped API access for an untrusted third-party worker. A delegated learner gets no vote and cannot become leader. The production client does not yet activate delegated learners; selected API access is available without joining the log. Existing paired personal computers keep their shared-root trust model. - +A restarted worker uses a newer process number; delayed requests from its previous process cannot save work. Draining prevents new claims, while already held work can finish saving. Joining the full coordination log requires explicit consent and exposes identity-wide routing metadata, with chat contents and names still encrypted. -## How does a worker claim an assignment or run parallel tasks? +## How does a delegated worker claim an assignment or run parallel tasks? -The coordination service accepts an explicit selected chat and target worker. Instructions and credentials stay encrypted. Registration and availability alone cannot start work: the worker first downloads the saved chat, then commits a conditional claim for the assigned chat's observed holder and fence. Continuing that same observed chat elsewhere competes with the claim; only one can win. A changed assignment, expired delegation, drained worker or newer worker process can refuse the claim. A claim returns the committed head, which the worker finishes downloading before opening it. - -Parallel tasks use distinct child chats, each with its own holder. Finishing a child records its already published result; incorporating that result into the parent still requires the parent's current holder and fence. The worker uploads the complete saved result before publishing its head. Leadership needs neither those files nor their keys. - -Worker setup and assignment execution are not available as a terminal command yet. The internal worker adapter supports a small scripted worker; phone controls and creating remote machines come later. +An assignment names the selected chat and target worker. The worker downloads the saved chat, conditionally claims its observed ownership, then downloads the accepted saved head before opening it. Registration alone never starts work. Parallel tasks use separate child chats. Worker setup and execution are not available as a terminal command yet. ## Will an interrupted worker replay tools after a crash or reassignment? -No. The worker saves its claim request before sending it and records an execution intent before invoking tools. A lost coordination reply can retry that same request, retaining its original fence. Once execution may have started, reopening keeps the records and reports `this work may be incomplete; its saved records have been kept`. It never invokes that attempt again automatically. A recorded finish can retry its coordination acknowledgment without running tools. - -Reassignment changes the run revision and requires a new conditional claim. The replacement can recover saved records, but the scripted adapter refuses automatic execution of a reassigned run. Missing or incomplete calls are not instructions to execute them. Cancellation prevents future claims; stopping a claimed assignment does not cancel an already running shell process. Another take changes the fence and blocks the displaced holder from publishing. Already downloaded bytes and uploaded frames stay retained. +No. The worker saves its claim and an execution intent before invoking tools. A lost reply retries the same coordination request. Once execution may have started, reopening reports `this work may be incomplete; its saved records have been kept` and keeps the records. It never runs that attempt again automatically. Reassignment and cancellation cannot undo an already running command. ## Why can an older computer list chats but not continue a chat that is still running? -Older builds can still list, save, publish and release chats through a compatible relay. A relay using quorum-backed coordination refuses an older request that tries to take a live chat without naming its observed holder and ownership number. It uses the existing refusal, `another device continued this chat first`; an old build does not necessarily show an extra update hint. Quit on the computer holding the chat before continuing it elsewhere, or update codeaf on the computer trying to continue. A released or expired chat can still be taken. A conditional choice never silently changes to a newer holder. - - -## Running a relay on one persistent computer - -`codeaf relay --store ` serves pairing, encrypted files and the chat directory -with one command. Each identity uses three local persistent coordination copies, -with separate databases under `/raft/`. Changes and authority reads require -a fresh majority response. All three copies still depend on this computer and -its disk; this setup does not survive losing the relay host. Without `--store`, -the relay is only a transport and pairing mailbox. - -On first use, an existing directory is frozen, copied into the three stores and -checked through a majority before requests resume. The original directory stays -as a frozen backup under `/directory/`. An interrupted upgrade resumes with -the same identity and stores; an incomplete or damaged import refuses requests -instead of reopening the old writer. Keep the whole store directory when -restarting or backing up the relay. Chat content stays encrypted in the existing -blob store. No switch selects the former directory writer. - -Encoded directory commands currently have a 64 KiB limit; a larger older request -can be refused before it changes ownership. Larger imported records stay retained. - -## Hosted relay directory upgrade and older clients - -The hosted relay records directory changes on three persistent members. Two must -agree before a take, publish or fresh listing succeeds. Pairing one or two -computers does not require either computer to stay awake. The three hosted copies -still depend on the same hosting provider and relay connection. - -An existing identity upgrades on first authenticated use. An interrupted upgrade -can briefly ask you to retry; it keeps the saved directory and encrypted frames. -The old directory never becomes a second writer. Older clients keep pairing, -listing, publishing, releasing and continuing a chat they observed. A very old -client that blindly forces a live chat receives `another device continued this chat first`. -Quit on the holder first, or update that client before continuing a live chat. -Watch sockets show presence; older clients still send their normal heartbeats. - - -## Scoped worker requests and committed watch receipts - -The relay's selected-worker API accepts an original signed command or a signed request for one chat or assignment. A delegated worker receives only its selected sealed grant. Each read commits current time, waits for a fresh majority response and checks its current scope and process number. A retained claim alone cannot authorize execution after another computer takes the chat. A worker cannot use this API to pair computers, rotate the identity or promote itself. - -Native and hosted watch connections carry committed receipts to clients that ask for them. Each receipt names the exact chat and ownership number, durable position and hold-until time. Older clients receive their original version and presence frames and keep ordinary heartbeats. A ping without a receipt cannot suppress those heartbeats. A released chat cannot be revived by a delayed ping. Idle presence pings do not start a recurring directory request loop. +Older builds can still list, save, publish and release chats through a compatible service. A service using quorum-backed coordination refuses an older request that tries to take a live chat without naming its observed holder and ownership number. It uses the existing refusal, `another device continued this chat first`; an old build does not necessarily show an extra update hint. Quit on the computer holding the chat before continuing it elsewhere, or update codeaf on the computer trying to continue. A released or expired chat can still be taken. A conditional choice never silently changes to a newer holder. diff --git a/internal/manual/chat/coordination-and-workers.md b/internal/manual/chat/coordination-and-workers.md new file mode 100644 index 0000000000..6b812a8e62 --- /dev/null +++ b/internal/manual/chat/coordination-and-workers.md @@ -0,0 +1,106 @@ +# Coordination and workers + +## Relay directory upgrade pauses and saved chats + +A relay directory upgrade can briefly pause pairing and chat moves. Retry after the relay is ready. The upgrade refusal is `codeaf: directory upgrade is incomplete; retry shortly; saved chats are still here`. It does not delete saved chats, files, branches or the encrypted vault. The relay keeps the old directory as a backup and never makes it a second writer after the new directory takes over. A row that cannot be copied safely leaves the upgrade paused for the operator; it is never truncated to make it fit. + + +## Joined computers, learner capacity and an outdated build + +An open, paired home with sync on joins a self-hosted `codeaf relay --store ` as a durable learner. Its store lives under the home, and reopening reuses its node. Losing that store enrolls a fresh node and retires the old authority. The three anchors remain the only voters, so neither laptop needs to stay awake. Joining starts while the paired home is open, or when you ask for `codeaf cluster status`. + +If joining has no new data to apply, the client keeps its store for the next retry round instead of polling continuously. A status activation can say `this computer is still joining the cluster; retry shortly; your saved chats are still here`. + +The membership protocol admits at most 32 live learners and retains retired records instead of reusing their authority. A protocol mismatch says `this codeaf build must be updated before it can join this cluster; your saved chats are still here`. Its capacity refusal is `this cluster has reached its enrollment limit; remove a joined computer or ask the relay operator for capacity`. A worker setup command is not available through the production client yet. + +## Which computer coordinates: cluster status and cluster lead + +`codeaf cluster status` on a self-hosted `codeaf relay` with storage reports the committed voters and current leader after a fresh majority read. A paired computer joins as a learner with persistent storage before status shows its node. Readiness requires its own durable application through the advertised join position. Status names `this computer`, `learning computers` and `ready computers` when known, alongside `voting computers` and the coordinator. `codeaf cluster lead` on an unjoined computer says `this computer must join the cluster with persistent storage before it can lead`. + +Leadership is not offered yet on any relay. A direct leadership request to the self-hosted relay is refused without changing membership. The hosted relay has no cluster control door and says `this relay does not offer cluster leadership yet; your saved chats are still here`. Ordinary chat sync remains available. + +Do not treat pairing or a saved leadership request as a successful transfer. Success requires the selected computer to be elected, commit its current term and pass a fresh majority read; its receipt names the computer, term, revision and read barrier. Retrying `codeaf cluster lead` preserves an already saved signed request. Continue chats through the ordinary continue door. + +Three voters tolerate one unavailable member. Losing a majority pauses coordination; saved chats remain. The refusal is `this cluster cannot coordinate right now; reconnect enough of its voting computers and try again; your saved chats are still here`. + +## Does an open connection keep my chat running on this computer? + +The sync connection carries presence hints and committed lease acknowledgments. An open socket or an answered ping alone cannot keep ownership of a chat. codeaf skips an ordinary heartbeat only while it has a recent acknowledgment for that exact chat and ownership number, with enough time left before expiry. Without that acknowledgment it keeps sending ordinary heartbeats. A delayed acknowledgment for an earlier owner cannot take the chat back, and a delayed renewal cannot undo a clean quit. + +## Does syncing keep sending requests when nothing is happening? + +A home that has never paired sends no sync traffic. `CODEAF_SYNC_URL=off` also blocks new sync requests from clients that were opened before the setting changed. Presence pings on an open view are hints; they do not write chat ownership. On the native relay, an open idle home uses its learner’s applied vault pointer to notice shared memory and preference changes. Local edits are detected locally and pushed on the next state round. A caught-up idle home sends no recurring state polls or peer keepalives. Home also follows its local applied directory for display changes; its roster and backstop read that same local hint. Quiet neighbours’ presence can age out without forcing a network ping. Continuing a chat and all ownership decisions still use fresh majority reads. A committed change wakes peer delivery and the next state round pulls the changed vault. The hosted relay has no learner door and keeps its existing five-second state polling. On the consensus directory, a group with no work or held chats can stop its periodic coordination timer after an idle barrier has reached every voter. The first operation wakes coordination and must obtain a fresh majority response. A saved leader address cannot answer that operation by itself. Losing the majority leaves syncing unavailable and keeps local work intact. + +## Can a worker access all my chats or approve pairing? + +A delegated worker uses its own key and an expiring certificate naming selected chats. It receives only sealed grants for those chats, rather than the full exported identity. It cannot approve pairing, rotate the identity, read the identity vault, enroll other devices or request leadership. Richer capabilities, worker labels and execution credentials stay encrypted; only the existing platform and sandbox capability shape is readable by the relay. + +Worker setup and assignment execution are not available as a terminal command yet. Registration and a fresh signed heartbeat only record availability; they never start tools or give the worker ownership of a chat. Removing a worker blocks later requests after that removal commits. Expiry and removal cannot take back bytes the worker already downloaded. + +## Worker restarts, draining and learner metadata visibility + +A worker restart registers a newer process number. Delayed requests and heartbeats from its previous process cannot refresh availability or save work. A drained worker cannot receive new claims; already held work can still save and let go. Restarting does not clear draining. Expired and removed enrollment records remain retained; reaching their capacity limit refuses new enrollment instead of deleting records or saved chats. + +A scoped API worker does not join the coordination log. Joining as a learner requires explicit consent in its certificate: the full log reveals identity-wide routing and ownership metadata, while chat content, names and task instructions remain encrypted. Use scoped API access for an untrusted third-party worker. A delegated learner gets no vote and cannot become leader. The production client does not yet activate delegated learners; selected API access is available without joining the log. Existing paired personal computers keep their shared-root trust model. + + +## How does a worker claim an assignment or run parallel tasks? + +The coordination service accepts an explicit selected chat and target worker. Instructions and credentials stay encrypted. Registration and availability alone cannot start work: the worker first downloads the saved chat, then commits a conditional claim for the assigned chat's observed holder and fence. Continuing that same observed chat elsewhere competes with the claim; only one can win. A changed assignment, expired delegation, drained worker or newer worker process can refuse the claim. A claim returns the committed head, which the worker finishes downloading before opening it. + +Parallel tasks use distinct child chats, each with its own holder. Finishing a child records its already published result; incorporating that result into the parent still requires the parent's current holder and fence. The worker uploads the complete saved result before publishing its head. Leadership needs neither those files nor their keys. + +Worker setup and assignment execution are not available as a terminal command yet. The internal worker adapter supports a small scripted worker; phone controls and creating remote machines come later. + +## Will an interrupted worker replay tools after a crash or reassignment? + +No. The worker saves its claim request before sending it and records an execution intent before invoking tools. A lost coordination reply can retry that same request, retaining its original fence. Once execution may have started, reopening keeps the records and reports `this work may be incomplete; its saved records have been kept`. It never invokes that attempt again automatically. A recorded finish can retry its coordination acknowledgment without running tools. + +Reassignment changes the run revision and requires a new conditional claim. The replacement can recover saved records, but the scripted adapter refuses automatic execution of a reassigned run. Missing or incomplete calls are not instructions to execute them. Cancellation prevents future claims; stopping a claimed assignment does not cancel an already running shell process. Another take changes the fence and blocks the displaced holder from publishing. Already downloaded bytes and uploaded frames stay retained. + +## Why can an older computer list chats but not continue a chat that is still running? + +Older builds can still list, save, publish and release chats through a compatible relay. A relay using quorum-backed coordination refuses an older request that tries to take a live chat without naming its observed holder and ownership number. It uses the existing refusal, `another device continued this chat first`; an old build does not necessarily show an extra update hint. Quit on the computer holding the chat before continuing it elsewhere, or update codeaf on the computer trying to continue. A released or expired chat can still be taken. A conditional choice never silently changes to a newer holder. + + +## Hosting a relay with persistent storage + +`codeaf relay --store ` serves pairing, encrypted files and the chat directory +with one command. Each identity uses three local persistent coordination copies, +with separate databases under `/raft/`. Changes and authority reads require +a fresh majority response. All three copies still depend on this computer and +its disk; this setup does not survive losing the relay host. Without `--store`, +the relay is only a transport and pairing mailbox. + +On first use, an existing directory is frozen, copied into the three stores and +checked through a majority before requests resume. The original directory stays +as a frozen backup under `/directory/`. An interrupted upgrade resumes with +the same identity and stores; an incomplete or damaged import refuses requests +instead of reopening the old writer. Keep the whole store directory when +restarting or backing up the relay. Chat content stays encrypted in the existing +blob store. No switch selects the former directory writer. + +Encoded directory commands currently have a 64 KiB limit; a larger older request +can be refused before it changes ownership. Larger imported records stay retained. + +## Hosted relay directory upgrade and older clients + +The hosted relay records directory changes on three persistent members. Two must +agree before a take, publish or fresh listing succeeds. Pairing one or two +computers does not require either computer to stay awake. The three hosted copies +still depend on the same hosting provider and relay connection. + +An existing identity upgrades on first authenticated use. An interrupted upgrade +can briefly ask you to retry; it keeps the saved directory and encrypted frames. +The old directory never becomes a second writer. Older clients keep pairing, +listing, publishing, releasing and continuing a chat they observed. A very old +client that blindly forces a live chat receives `another device continued this chat first`. +Quit on the holder first, or update that client before continuing a live chat. +Watch sockets show presence; older clients still send their normal heartbeats. + + +## Scoped worker requests and committed watch receipts + +The relay's selected-worker API accepts an original signed command or a signed request for one chat or assignment. A delegated worker receives only its selected sealed grant. Each read commits current time, waits for a fresh majority response and checks its current scope and process number. A retained claim alone cannot authorize execution after another computer takes the chat. A worker cannot use this API to pair computers, rotate the identity or promote itself. + +Native and hosted watch connections carry committed receipts to clients that ask for them. Each receipt names the exact chat and ownership number, durable position and hold-until time. Older clients receive their original version and presence frames and keep ordinary heartbeats. A ping without a receipt cannot suppress those heartbeats. A released chat cannot be revived by a delayed ping. Idle presence pings do not start a recurring directory request loop. diff --git a/internal/manual/chat/delegates.md b/internal/manual/chat/delegates.md index c638496280..82c53a86da 100644 --- a/internal/manual/chat/delegates.md +++ b/internal/manual/chat/delegates.md @@ -1,5 +1,9 @@ # Programs codeaf carries +## What is a delegate + +A delegate is a program built into codeaf that takes a whole task and works on it inside your conversation's time and spending limits. The delegates are senior-dev, sec and review. Give one a brief with its slash command or `codeaf `. Each delegate's own manual page describes its work and limits. + ## What a program codeaf carries is — a delegate, another coding agent, an agent of its own for a whole task, which programs there are codeaf carries programs of its own that take one whole task and do it alone, for as diff --git a/internal/manual/chat/home.md b/internal/manual/chat/home.md index c51f8bb47e..5944781e34 100644 --- a/internal/manual/chat/home.md +++ b/internal/manual/chat/home.md @@ -315,14 +315,20 @@ what the row offers, described in the next two sections. ## How fast do chats on other machines update — live changes and polling -When the relay offers a live connection, home keeps one open and a change made on -another machine shows on home within a second or two. The list is also fetched once a -minute as a safety net, and at once when you open home or come back to the window. When the -relay does not offer a live connection (an older relay), or the connection is -down, home asks for the list instead. It asks only while home is on screen and the terminal -has your attention (its focus, or a key pressed in the last ten minutes). It asks every few -seconds while the list is changing and about once a minute while it is not. Each row's age is -counted from its last saved turn, so it keeps growing between fetches. +On a native relay with persistent storage, an open paired home joins as a learner. +Home follows its local copy of the committed directory, and its list, roster and safety +refresh read that same local hint. A caught-up idle home sends no recurring directory +requests or peer keepalives. A committed change wakes delivery; a change made on another +machine normally shows within a second or two. Continuing a chat and every ownership +decision still require fresh majority reads. + +On the hosted relay, home keeps a live connection when one is offered. While the follower +is healthy, the list has a five-minute safety refresh; opening home or returning to the +window refreshes at once. Without a live connection, home asks for the list only while it +is on screen and the terminal has your attention (its focus, or a key pressed in the last +ten minutes). It asks every few seconds while the list is changing and about once a minute +while it is not. Each row's age is counted from its last saved turn, so it keeps growing +between refreshes. If the other machines cannot be reached, the last list stays on screen and the line `other machines unreachable` stands under it. With no connection set up at all (sync off, see "Sync your chats between machines" in running-from-the-terminal), nothing diff --git a/internal/manual/chat/running-on-another-machine.md b/internal/manual/chat/running-on-another-machine.md index 1c3a3f7514..365524317c 100644 --- a/internal/manual/chat/running-on-another-machine.md +++ b/internal/manual/chat/running-on-another-machine.md @@ -1,6 +1,6 @@ # Running on another machine -## Can I run it on another computer — what --host is, running this on my dev box over ssh +## Can I run it on another computer? `--host` runs the chat surface on the machine you are sitting at, and runs the conversation on another machine, over ssh. @@ -10,6 +10,8 @@ it runs, the model key, the session file. Your terminal keeps the screen, the ke your draft. The two halves talk to each other with a small line-based protocol over the ssh pipes; you never see it. +## What --host is, running this on my dev box over ssh + ``` codeaf chat --host devbox codeaf resume --host devbox diff --git a/internal/manual/chat/worker-harness.md b/internal/manual/chat/worker-harness.md index 610707f1da..b372d9b31a 100644 --- a/internal/manual/chat/worker-harness.md +++ b/internal/manual/chat/worker-harness.md @@ -298,7 +298,7 @@ room's head says the task is working, how long it has run and what it has cost, task room's head does. When the command ends the store clears the live step and the next read draws it as an ordinary call, with what came back behind it. -## A program's task page is the actions it took, not steps — a delegate's page: open it, leave it, no tab of its own, no note box, what the box says +## What does the delegate's task page show — a program's actions, open it, leave it, no tab of its own, no note box, what the box says A task handed to a program codeaf carries (`/ `, such as `/senior-dev`) opens **inside the conversation's own tab**, as any task does: from its row on the side diff --git a/internal/manual/coordination_test.go b/internal/manual/coordination_test.go new file mode 100644 index 0000000000..a796e6f7c4 --- /dev/null +++ b/internal/manual/coordination_test.go @@ -0,0 +1,33 @@ +package manual + +import ( + "strings" + "testing" +) + +// R9/R23/R27 keep the moved coordination material reachable while device pages +// keep their own vocabulary and the existing retrieval probes stay intact. +func TestR9R23R27LearnerAndIdleAnswersRemainReachable(t *testing.T) { + for _, q := range []string{ + "joined computers learner capacity outdated build", + "native relay open idle home vault polling", + "self hosted codeaf relay persistent computer", + "hosted relay directory upgrade older clients", + } { + found := false + for _, section := range Chat().Search(q, DefaultResults) { + if section.Page == "coordination-and-workers" { + found = true + } + } + if !found { + t.Errorf("R9 coordination answer unreachable: %q", q) + } + } + page, _ := Chat().Page("coordination-and-workers") + for _, claim := range []string{"durable learner", "The three anchors remain the only voters", "Leadership is not offered yet on any relay", "hosted relay has no learner door"} { + if !strings.Contains(page, claim) { + t.Errorf("R9 missing current capability: %q", claim) + } + } +} From 0b6d6d58d7f098260f4676814987baab47b42f9a Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 18:21:59 -0400 Subject: [PATCH 060/103] Check that the public relay config declares the Raft entry, build, binding and migration. The private deployment configs and the test that checked them are no longer in the repository. The base config still has to send fresh installs and upgrades to the Raft reactor and the three anchor stores, and it must only append the anchor migration so that existing namespace IDs stay in place. This test keeps those assertions for wrangler.toml alone. Co-Authored-By: Claude Opus 5.5 (1M context) --- relay/hosted/test/config.test.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 relay/hosted/test/config.test.js diff --git a/relay/hosted/test/config.test.js b/relay/hosted/test/config.test.js new file mode 100644 index 0000000000..2cc437d7b8 --- /dev/null +++ b/relay/hosted/test/config.test.js @@ -0,0 +1,19 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; + +// wrangler.toml is the base every deployment starts from. Fresh installs and +// upgrades must both reach the Raft reactor entry and the three anchor stores, +// and the anchors may only be appended so that existing namespace IDs, and the +// saved chats behind them, stay where they are. +test('R30 the base config uses the Raft entry and an append-only v3-raft-members migration', () => { + const text = readFileSync(new URL('../wrangler.toml', import.meta.url), 'utf8'); + assert.match(text, /^main = "src\/raft-worker\.js"$/m); + assert.match(text, /name = "RAFT_MEMBERS"\s+class_name = "RaftMemberDO"/); + assert.match(text, /\[build\]\s+command = "sh \.\.\/\.\.\/scripts\/raft-hosted-build\.sh"/); + const tags = [...text.matchAll(/^tag = "([^"]+)"/gm)].map((m) => m[1]); + assert.deepEqual(tags, ['v1', 'v2', 'v3-raft-members']); + assert.match(text, /tag = "v3-raft-members"\s+new_sqlite_classes = \["RaftMemberDO"\]/); + assert.doesNotMatch(text, /deleted_classes|renamed_classes/); + assert.match(text, /name = "IDENTITY"\s+class_name = "IdentityDO"/); +}); From 47d957365034926c0ad5f41257e9c2dc7dce87d5 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 18:35:46 -0400 Subject: [PATCH 061/103] Describe the coordination code by its behaviour rather than how it was built. Comments and package notes cited internal planning ids, development steps and private documents that mean nothing to a reader of the public tree. State what each component does, what it leaves to its host and what is not available yet, keeping every limit, invariant and refusal intact. Test comments keep the contract item they prove. The hosted relay notes now say plainly that personal-member learners and leadership requests are not offered there yet, and source backups are described as lasting until the identity is retired. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/codeaf-raft-probe/actor.go | 2 +- cmd/codeaf-raft-probe/measure.go | 4 +- cmd/codeaf-raft-probe/rows.go | 4 +- docs/relay.md | 12 +- internal/clusterstate/README.md | 30 ++--- internal/clusterstate/effects_test.go | 4 +- internal/clusterstate/imports_test.go | 2 +- internal/clusterstate/membership.go | 2 +- internal/clusterstate/records.go | 4 +- internal/clusterstate/types.go | 4 +- internal/clusterstate/worker.go | 2 +- internal/clusterstate/worker_test.go | 2 +- internal/directory/client.go | 2 +- .../directory/directorytest/conditional.go | 10 +- internal/directory/directorytest/lease.go | 2 +- internal/directory/directorytest/watch.go | 2 +- internal/directory/raft.go | 2 +- internal/dirwatch/stream.go | 4 +- .../manual/chat/coordination-and-workers.md | 2 +- internal/raftabi/README.md | 112 ++++++++-------- internal/raftabi/probe.go | 6 +- internal/raftcore/README.md | 120 ++++++++--------- internal/raftcore/core.go | 2 +- internal/raftcore/types.go | 2 +- internal/raftdirectory/README.md | 121 +++++++++--------- internal/raftdirectory/directory_test.go | 2 +- internal/raftdirectory/routing_test.go | 2 +- internal/raftsim/README.md | 51 ++++---- internal/raftsim/application.go | 8 +- internal/raftsim/campaign_test.go | 2 +- internal/raftsim/model/model.go | 4 +- internal/raftsim/simulator_test.go | 2 +- internal/raftsnapshot/README.md | 5 +- internal/raftstore/README.md | 65 +++++----- internal/raftstore/native_test.go | 2 +- internal/raftstore/sqlite_test.go | 4 +- internal/rafttransport/README.md | 74 ++++++----- internal/relayserve/cluster_test.go | 2 +- internal/relayserve/raft_cutover_test.go | 6 +- relay/hosted/README.md | 4 +- relay/hosted/src/raft/README.md | 95 +++++++------- relay/hosted/src/raft/actor.js | 2 +- relay/hosted/src/raft/member.js | 2 +- relay/hosted/src/tenant.js | 2 +- relay/hosted/test/raft/directory.test.js | 2 +- relay/hosted/test/raft/smoke.test.js | 4 +- relay/hosted/test/raft/storage-worker.js | 4 +- relay/hosted/test/raft/worker.js | 2 +- scripts/raft-proof-footprint.sh | 2 +- third_party/raft-patch/README.md | 6 +- third_party/raft-patch/verify.py | 2 +- 51 files changed, 421 insertions(+), 394 deletions(-) diff --git a/cmd/codeaf-raft-probe/actor.go b/cmd/codeaf-raft-probe/actor.go index 38e406501e..ddfa6393ff 100644 --- a/cmd/codeaf-raft-probe/actor.go +++ b/cmd/codeaf-raft-probe/actor.go @@ -293,7 +293,7 @@ func (a *probeActor) call(r actorRequest, dir string) (any, error) { } // actorProof uses actual native SQLite stores beside hosted actors. The only -// application is the explicit development lease probe, never production policy. +// application is the narrow lease probe, never production policy. func actorProof(dir string) error { if err := os.MkdirAll(dir, 0700); err != nil { return err diff --git a/cmd/codeaf-raft-probe/measure.go b/cmd/codeaf-raft-probe/measure.go index 772f5931fd..9e8f14699b 100644 --- a/cmd/codeaf-raft-probe/measure.go +++ b/cmd/codeaf-raft-probe/measure.go @@ -23,9 +23,9 @@ import ( "github.com/coder/websocket/wsjson" ) -// measureRows runs the R27/R28 memory gate without retaining the Node consensus +// measureRows runs the memory gate without retaining the Node consensus // test coordinator. Six production handles and real disk-backed DOs remain in -// workerd; only the bulk driver changes. Inspector sampling never forces GC. +// workerd; the bulk driver streams input. Inspector sampling never forces GC. func measureRows(out, binaryPath, schema string) error { ctx, cancel := context.WithTimeout(context.Background(), 110*time.Second) defer cancel() diff --git a/cmd/codeaf-raft-probe/rows.go b/cmd/codeaf-raft-probe/rows.go index aad8785c6a..bb75cbf678 100644 --- a/cmd/codeaf-raft-probe/rows.go +++ b/cmd/codeaf-raft-probe/rows.go @@ -18,10 +18,10 @@ import ( "github.com/Agent-Field/codeaf/internal/raftabi" ) -// rowProof keeps the oversized R14/R27/R28 fixture in a bounded native driver. +// rowProof keeps the oversized-row memory fixture in a bounded native driver. // The Worker still evaluates every row and its actual heap is sampled without // forcing collection. Keeping HTTP/JSON copies out of the Node coordinator is -// necessary for the shared-laptop budget, not an alternative application rule. +// necessary for the combined RSS budget, not an alternative application rule. func rowProof(origin string, result io.Writer) error { u, err := url.Parse(origin) if err != nil || u.Scheme != "http" || u.Hostname() != "127.0.0.1" || u.Port() == "" || u.Path != "" { diff --git a/docs/relay.md b/docs/relay.md index 1019c5e101..e051bfad69 100644 --- a/docs/relay.md +++ b/docs/relay.md @@ -57,12 +57,12 @@ private and back up its provisioning file together with its databases. ## Upgrade -Stop it, replace the `codeaf` binary, start it with the same `--store`. Back up first. -There is no migration command or feature switch. First use freezes an existing identity, -imports the same rows into three local anchors, checks a fresh majority and publishes -one durable owner. Interrupted imports resume with the same UUID, keys and ceiling; -they never reopen the legacy writer. The original source stays as a backup until an -explicit owner cleanup or retirement. +Stop it, replace the `codeaf` binary, start it with the same `--store`. Back up +first. There is no migration command or feature switch. First use freezes an +existing identity, imports the same rows into three local anchors, checks a +fresh majority and publishes one durable owner. Interrupted imports resume with +the same UUID, keys and ceiling; they never reopen the legacy writer. The +original source stays as a backup until the identity is retired. All directory, rotation and blob authority requests use consensus. Three local voters still share one host and disk: losing that host loses service. This setup does not diff --git a/internal/clusterstate/README.md b/internal/clusterstate/README.md index 72182d84c3..013fdb7ca0 100644 --- a/internal/clusterstate/README.md +++ b/internal/clusterstate/README.md @@ -1,13 +1,11 @@ # Pure coordination boundary -`clusterstate` is shared by native members and the future WASI reactor. It has +`clusterstate` is shared by native members and the WASI reactor. It has no directory, HTTP, database, identity-key, wall-clock or random-source imports. `directory` keeps aliases to its records/errors and forwards its existing rules. The old v1 JSON representation and pure rule behavior remain unchanged. -P01 defines schema 1 for commands, results and individual application rows. This -is internal development scaffolding, not a production routing change. Later -packets add their typed operation/row variants before the single feature ships. +Schema 1 covers commands, results and individual application rows. An older member must reject unknown versions and variants and stop applying; it cannot turn an unknown committed entry into a successful empty operation. @@ -17,8 +15,8 @@ it cannot turn an unknown committed entry into a successful empty operation. verify a signature, validate upload receipts, implement deduplication, or apply a mutation. The host still authenticates the original signed request, and apply must check committed enrollment, revocation, rotation, dependencies and scope. -`RequestDigest` and `Signature` are original authentication/session inputs; P07 -and P09 supply their cryptographic and deduplication validation. +`RequestDigest` and `Signature` are original authentication/session inputs; +ingress and apply supply their cryptographic and deduplication validation. The v2 canonical encoding has these rules: @@ -44,17 +42,17 @@ read by the targeted hosted codec test. The existing relay rule vectors are consumed unchanged by the pure Go test and the hosted vector test. `Row` holds one record, so a host can supply bounded read-sets and stream sorted -rows at a fixed applied index. It is not a whole-identity snapshot format. P04/P05 +rows at a fixed applied index. It is not a whole-identity snapshot format. Hosts must atomically persist applied index, visible version, logical time, dedup and membership metadata alongside rows. They must not use `Listing` JSON as a state export: its visible version deliberately lives outside the v1 body. -`raftsim/model` is an independent sequential ownership/vault skeleton. It imports -no production types or apply functions. P06 adds the Porcupine adapter, unknown -timeout outcomes and replay/history machinery; later assignment packets extend -the independent domain. P01 does not claim simulator or consensus proof. +`raftsim/model` is an independent sequential ownership/vault model. It imports +no production types or apply functions. The simulator uses Porcupine, unknown +timeout outcomes and replay/history machinery; a separate assignment model +extends the independent domain. Encoding tests alone do not prove consensus. -Worker delegation (P15) is a fourth exclusive Authority role. Personal pairing +Worker delegation is a fourth exclusive Authority role. Personal pairing Cert remains unchanged; WorkerCertificate uses its own signed domain and binds the root public identity, cluster UUID, worker key, sorted selected cell IDs, exclusive expiry and explicit learner consent. EnrollWorker commits only sealed @@ -74,15 +72,15 @@ or renew a chat. RenewHolds and upload receipts bind worker scopes/incarnations. A delegated worker may beat/publish/release an already held selected chat, close its request session, register, heartbeat or drain itself. It cannot use ordinary Acquire, create/archive, pairing, vault, rotation or membership/leadership doors. -P16 supplies the conditional assignment claim and must extend the allowlist and -assignment dependencies there; it must bind an assignment's claimant incarnation -before granting execution, even when a chat fence survives a worker restart. +Conditional assignment claims require their own allowlist and declared +dependencies. They bind an assignment's claimant incarnation before granting +execution, even when a chat fence survives a worker restart. ReadWorkerCell returns one selected chat and its sealed grant after the host commits current ingress time and passes a fresh quorum ReadIndex barrier. The native AuthorizedWorkerRead and hosted workerCell adapters have the SAME host prerequisite, not a follower/local read path. Identity-wide AuthorizedRead still -refuses workers. The public setup/client/provisioning lifecycle remains P17. +refuses workers. Public worker setup and provisioning are not available yet. Scoped API workers never receive the identity log. Explicit learner consent permits its routing metadata, never content keys outside selected sealed grants. diff --git a/internal/clusterstate/effects_test.go b/internal/clusterstate/effects_test.go index 2623761f7d..ab89e08840 100644 --- a/internal/clusterstate/effects_test.go +++ b/internal/clusterstate/effects_test.go @@ -73,7 +73,7 @@ func TestR21UploadReceiptBeforeHeadAndFenceBeforeReceipt(t *testing.T) { } // R16/R17/R21 enumerate authorization orderings with independently selected -// expected outcomes. Every seed honors the wave's 2,000-seed soak lever. +// expected outcomes. The soak supports 2,000 seeds through RAFTSIM_SEEDS. func TestR16R17R21EffectsAuthorizationCampaign(t *testing.T) { for seed := 0; seed < seedCount(t); seed++ { t.Run(fmt.Sprintf("seed-%04d", seed), func(t *testing.T) { @@ -147,7 +147,7 @@ func TestR16PairingCertificateAndStickyRevocation(t *testing.T) { // R25 preserves real old-client requests that cannot name an encrypted head // RID, while R21 requires evidence at the distinct v2 command door. Ingress -// supplies protocol provenance; P17 must never copy compatibility flags from v2. +// supplies protocol provenance; v2 input must never supply compatibility flags. func TestR21R25LegacyProtocolKeepsExistingUploadOrdering(t *testing.T) { a := newApplication(t) m := cs.Mutation{Create: &cs.CreateCell{Cell: "chat", Init: cs.CellInit{Head: "legacy"}, V1: true}} diff --git a/internal/clusterstate/imports_test.go b/internal/clusterstate/imports_test.go index 5b1a089599..40d885830f 100644 --- a/internal/clusterstate/imports_test.go +++ b/internal/clusterstate/imports_test.go @@ -11,7 +11,7 @@ import ( // R14 needs the same application package in the native and wasm builds. Keep // its import boundary explicit so a storage/auth/HTTP helper cannot silently -// enter the pure graph when another packet adds a record or rule. +// enter the pure graph when records or rules change. func TestR14PureImportBoundary(t *testing.T) { allowed := map[string]bool{ "io": true, "bytes": true, "encoding/json": true, "errors": true, "fmt": true, diff --git a/internal/clusterstate/membership.go b/internal/clusterstate/membership.go index f9aa29edac..1ebd542c46 100644 --- a/internal/clusterstate/membership.go +++ b/internal/clusterstate/membership.go @@ -490,7 +490,7 @@ func (h Membership) Validate() error { return nil } -// EnrollmentMessage is shared by later ingress adapters and the manual. The +// EnrollmentMessage is shared by ingress adapters and the manual. The // protocol refusal distinguishes an old build from missing saved content. func EnrollmentMessage(code Code) string { switch code { diff --git a/internal/clusterstate/records.go b/internal/clusterstate/records.go index 46028c7d88..6cccec31b1 100644 --- a/internal/clusterstate/records.go +++ b/internal/clusterstate/records.go @@ -28,8 +28,8 @@ type Caps struct { OS string `json:"os"` // runtime.GOOS Arch string `json:"arch"` // runtime.GOARCH Sandbox *string `json:"sandbox"` // "seatbelt" | "landlock" | null - Container *string `json:"container"` // null in Stage 1 - GPU *string `json:"gpu"` // null in Stage 1 + Container *string `json:"container"` // null if unreported + GPU *string `json:"gpu"` // null if unreported Cow string `json:"cow"` // "apfs" | "btrfs" | "xfs-reflink" | "overlayfs" | "none" } diff --git a/internal/clusterstate/types.go b/internal/clusterstate/types.go index 0b0bed7431..b2d5c63738 100644 --- a/internal/clusterstate/types.go +++ b/internal/clusterstate/types.go @@ -43,8 +43,8 @@ type Command struct { Signature []byte `json:"signature"` } -// Mutation is a closed union: exactly one field is present. Future packets add -// their typed operations here; unknown fields fail decoding on older members. +// Mutation is a closed union: exactly one typed operation field is present. +// Unknown fields fail decoding on older members. // There is no arbitrary JSON payload in which task text or secrets can hide. type Mutation struct { Assign *Assign `json:"assign,omitempty"` diff --git a/internal/clusterstate/worker.go b/internal/clusterstate/worker.go index 547ba07b2b..19a7374c88 100644 --- a/internal/clusterstate/worker.go +++ b/internal/clusterstate/worker.go @@ -396,7 +396,7 @@ func publicCaps(c Caps) bool { return allowed(c.OS, "", "linux", "darwin", "windows", "ios", "android", "other") && allowed(c.Arch, "", "amd64", "arm64", "386", "arm", "wasm", "other") && pointer(c.Sandbox, "seatbelt", "landlock") && pointer(c.Container, "none", "container") && pointer(c.GPU, "none", "present") && allowed(c.Cow, "", "apfs", "btrfs", "xfs-reflink", "overlayfs", "none") } -// WorkerCanClaim is the common P16 prerequisite at the claim's log position. +// WorkerCanClaim checks worker eligibility at the claim's log position. // A registry row, socket or downloaded grant alone never starts work. func WorkerCanClaim(cluster, worker, cell string, incarnation uint64, now int64, identity IdentityRec, a Authority, d Device, w WorkerRecord) Code { if code := workerAuthority(cluster, worker, now, identity, a, d); code != OK { diff --git a/internal/clusterstate/worker_test.go b/internal/clusterstate/worker_test.go index 2599291a03..9289960dc3 100644 --- a/internal/clusterstate/worker_test.go +++ b/internal/clusterstate/worker_test.go @@ -123,7 +123,7 @@ func TestR15R16R18WorkerScopeAndAuthorityDoors(t *testing.T) { } }) } - // The worker may save/release a fence obtained through P16's future claim. + // The worker may save/release a fence obtained through an assignment claim. f.a.put(cs.CellKey("chat"), cs.Created(cs.CellInit{Head: "old"}, f.id, 1000)) receipt := cs.UploadReceipt{Device: f.id, Cell: "chat", WorkerIncarnation: 1, Head: "new", HeadRID: "opaque-rid"} f.a.put(cs.UploadKey(cs.UploadID(receipt)), receipt) diff --git a/internal/directory/client.go b/internal/directory/client.go index 661ebdf406..93bff63921 100644 --- a/internal/directory/client.go +++ b/internal/directory/client.go @@ -85,7 +85,7 @@ var ( ErrNotFound = clusterstate.ErrNotFound ErrExists = clusterstate.ErrExists ErrLeaseHeld = clusterstate.ErrLeaseHeld - ErrFenceStale = clusterstate.ErrFenceStale // L7 + ErrFenceStale = clusterstate.ErrFenceStale ErrHeadMoved = clusterstate.ErrHeadMoved ErrCAS = clusterstate.ErrCAS ErrRevoked = wireauth.ErrRevoked // every verb of a revoked device; the wires share this one sentinel diff --git a/internal/directory/directorytest/conditional.go b/internal/directory/directorytest/conditional.go index 26fe767508..a6bb8145ee 100644 --- a/internal/directory/directorytest/conditional.go +++ b/internal/directory/directorytest/conditional.go @@ -31,21 +31,21 @@ func conditionalForcedTakeRace(t *testing.T, e env) { } } -// Legacy stores still prove their historical blind-force behavior until P11 -// freezes them. The consensus suite below proves R25's safe normalization. +// Legacy stores retain blind-force behavior until migration freezes them. +// The consensus suite below proves safe normalization of old requests. func conditionalForcedTakeOneWins(t *testing.T, e env) { conditionalForcedTakeRace(t, e) _, err := e.as(devA).Acquire(ctx, cell, directory.AcquireOpts{Force: true}) must(t, err) } -// RunConsensus uses every existing verb case, with the R25 compatibility -// contract at the one place where blind legacy force deliberately changes. +// RunConsensus uses every existing verb case, with the compatibility +// rule at the one place where blind legacy force deliberately changes. func RunConsensus(t *testing.T, factory Factory) { RunConsensusRigs(t, func(t *testing.T) Rig { clock := NewFakeClock(); return Rig{Clock: clock, Devices: factory(t, clock)} }) } -// RunConsensusRigs retains every wire case while asserting R25 at blind force. +// RunConsensusRigs retains every wire case while refusing blind force on a live lease. func RunConsensusRigs(t *testing.T, factory RigFactory) { for name, fn := range cases { t.Run(name, func(t *testing.T) { diff --git a/internal/directory/directorytest/lease.go b/internal/directory/directorytest/lease.go index 9b9c6b7df5..1cf8b49e72 100644 --- a/internal/directory/directorytest/lease.go +++ b/internal/directory/directorytest/lease.go @@ -349,7 +349,7 @@ func (w watchEnv) wantHoldQuery(query string, status int) { wantRefusal(w.t, conn, resp, err, status, "bad_request") } -// R22 supersedes the old socket-authority advertisement on consensus routes. +// Committed receipts replace the old socket-authority advertisement on consensus routes. // The same suite still exercises the original legacy directory protocol. func upgradeAnnouncesVouching(w watchEnv) { want := "1" diff --git a/internal/directory/directorytest/watch.go b/internal/directory/directorytest/watch.go index b03eed1d91..13fae5532b 100644 --- a/internal/directory/directorytest/watch.go +++ b/internal/directory/directorytest/watch.go @@ -16,7 +16,7 @@ import ( // act as a stranger, and to know how many sockets the relay lets one identity // hold. type WatchRig struct { - // Committed selects R22's receipt protocol for a consensus backend. Legacy + // Committed selects the receipt protocol for a consensus backend. Legacy // in-process directory fixtures retain their original socket-vouch contract. Committed bool Rig diff --git a/internal/directory/raft.go b/internal/directory/raft.go index 668427c411..e0c4e4edb1 100644 --- a/internal/directory/raft.go +++ b/internal/directory/raft.go @@ -12,7 +12,7 @@ import ( // await durable application, and recheck committed authorization at that view. // The host authenticates the original ingress and preserves its request ID; // forwarding through this facade never chooses a new expectation or retries a -// tool. P11 installs this boundary only after the legacy writer is frozen. +// tool. Install this boundary only after the legacy writer is frozen. type RaftBackend interface { Change(context.Context, string, clusterstate.Mutation) (clusterstate.Result, error) Listing(context.Context, string) (Listing, error) diff --git a/internal/dirwatch/stream.go b/internal/dirwatch/stream.go index 0bf2a6708f..24756f0114 100644 --- a/internal/dirwatch/stream.go +++ b/internal/dirwatch/stream.go @@ -24,7 +24,8 @@ type Stream interface { Close() } -// Frame is what arrived: the directory's version, an event, or only a pong. +// HoldReceipt is a committed renewal receipt for one exact lease: its cell, +// fence and log index, with its validity window. type HoldReceipt struct { Cell string `json:"cell"` Fence uint64 `json:"fence,string"` @@ -36,6 +37,7 @@ type HoldReceipt struct { // HoldIdentity identifies the exact lease named by a committed receipt. type HoldIdentity interface{ HoldIdentity() (string, uint64) } +// Frame is what arrived: the directory's version, an event, or only a pong. type Frame struct { Holds []HoldReceipt Version uint64 diff --git a/internal/manual/chat/coordination-and-workers.md b/internal/manual/chat/coordination-and-workers.md index 6b812a8e62..beaa403cdc 100644 --- a/internal/manual/chat/coordination-and-workers.md +++ b/internal/manual/chat/coordination-and-workers.md @@ -50,7 +50,7 @@ The coordination service accepts an explicit selected chat and target worker. In Parallel tasks use distinct child chats, each with its own holder. Finishing a child records its already published result; incorporating that result into the parent still requires the parent's current holder and fence. The worker uploads the complete saved result before publishing its head. Leadership needs neither those files nor their keys. -Worker setup and assignment execution are not available as a terminal command yet. The internal worker adapter supports a small scripted worker; phone controls and creating remote machines come later. +Worker setup and assignment execution are not available as a terminal command yet. The internal worker adapter supports a small scripted worker; phone controls and creating remote machines are not available yet. ## Will an interrupted worker replay tools after a crash or reassignment? diff --git a/internal/raftabi/README.md b/internal/raftabi/README.md index eb666f068e..12ad1ff51c 100644 --- a/internal/raftabi/README.md +++ b/internal/raftabi/README.md @@ -1,10 +1,10 @@ # Worker reactor ABI, version 1 -P03 is an internal feasibility boundary. Neither the local proof Worker nor the -stdio peer is a production directory or a new codeaf command. P08 owns the hosted -actor/queue/storage adapter; P09 owns complete paged application evaluation, +`raftabi` defines the internal reactor boundary. Neither the local proof Worker +nor the stdio peer is a production directory or a new codeaf command. Host and +application adapters supply actor queues, storage, complete paged evaluation, authorization, monotonic time, dedup and conditional transaction policy. No -JavaScript lease or consensus rules are added here. +JavaScript lease or consensus rules are implemented here. Build/run the bounded local proof from the repository root: @@ -44,7 +44,7 @@ a core. Handles are monotonic uint32 values and are never reused within a runtime; a new runtime discards all old completions. The JS guard stops the whole instance if linear memory exceeds 64 MiB or a call -traps. This is a fail-closed guard, not a substitute for P08's preallocation +traps. This is a fail-closed guard, not a substitute for the host's preallocation admission/cache/eviction policy. `GOMEMLIMIT=48MiB`, `GOGC=50` and `GOMAXPROCS=1` are the entire runtime environment. Empty args, unsupported fd/ socket operations, runtime-only clock/entropy and discarded failure output are @@ -58,8 +58,8 @@ counter/node ID/index/seed is little-endian uint64 (BigInt in JS). A blob is uint32 length followed by exact bytes; a string is a UTF-8 blob. A protobuf field is an optional deterministic protobuf blob, with zero length for absent. An entry list is count plus one blob per entry. Bytes cross this boundary -without JSON/base64 wrapping. Application command/row blobs retain P01's -canonical schema inside the binary envelope. +without JSON/base64 wrapping. Application command/row blobs retain the +canonical application schema inside the binary envelope. Every request starts with ABI version and operation. Every response starts with ABI version and an error string; empty means success and the remaining @@ -93,41 +93,42 @@ inventory hash and wasm SHA-256. It also pins portable snapshot schema 1 and the SHA-256 of `internal/raftsnapshot/format.go`. Operation 6 exposes the core's SnapshotNeeded maintenance hint without changing ABI-1 effect framing; tag 11 completes a host's durable local checkpoint/compaction. Actual hosted chunk -storage and bidirectional native/wasm installation are P08's gate; the P03 opaque -checkpoint fixture alone does not establish that interoperability. +storage and bidirectional native/wasm installation need separate verification; +the opaque checkpoint fixture alone does not establish that interoperability. ## Oversized legacy representation decision -Keep each existing canonical row in host storage and pass only one immutable -row (at most the existing 1 MiB legacy envelope) to Go. Inventory/import and +Keep each existing canonical row in host storage and pass only one immutable row +(at most the existing 1 MiB legacy envelope) to Go. Inventory/import and snapshot creation enumerate sorted rows at a fixed applied index, with no whole-identity listing or JSON string in wasm. Persist the imported byte ceiling `max(8 MiB, imported application bytes)` outside the reactor alongside the applied metadata. Preserve every row, sealed title/key and frame plan. Do not -truncate to make import fit. P05 supplies chunk/checksum/install generations; -P09/P11 supply complete read-sets, atomic row/byte accounting and growth policy. +truncate to make import fit. Snapshot and application hosts supply +chunk/checksum/ install generations, complete read-sets, atomic row/byte +accounting and growth policy. The proof uses 129 separately persisted rows totaling over 8 MiB, including one near-limit row and 128 smaller pages. A small native driver compares every row with wasm and verifies every stored byte on reread, then evaluates a real release over the large row. Six production handles remain resident while the -engine is sampled through its loopback inspector. This establishes representation/runtime feasibility; it is -not a claim that migration, snapshots, complete policy or production admission -have already shipped. A legacy row over 1 MiB remains a migration refusal until -an owner-approved streaming row extension exists, as the design requires. +engine is sampled through its loopback inspector. This establishes +representation/runtime feasibility; it does not establish migration, snapshots, +complete policy or production admission. A legacy row over 1 MiB remains a +migration refusal; a streaming row extension is not part of this build. -## P03 stop/go measurement +## Local runtime measurements -The final local proof on Go 1.26.5 and pinned workerd 1.20260926.1 passed. The -full existing relay plus reactor/shim/storage fixture/manifest is 2,248,258 bytes -compressed (3 MiB gate); wasm is 9,285,637 bytes raw. All 26 distinct imports are -covered by the closed shim. The mixed phase compared 743 effect responses +A local proof on Go 1.26.5 and pinned workerd 1.20260926.1 passed. The full +existing relay plus reactor/shim/storage fixture/manifest is 2,248,258 bytes +compressed (3 MiB gate); wasm is 9,285,637 bytes raw. All 26 distinct imports +are covered by the closed shim. The mixed phase compared 743 effect responses byte-for-byte, including RNG state/draws, committed a real create, passed a -quorum ReadIndex, refused a partitioned read, fell back to native peers and recovered -from an actual workerd process kill with disk-backed DO state. +quorum ReadIndex, refused a partitioned read, fell back to native peers and +recovered from an actual workerd process kill with disk-backed DO state. -The final mixed phase took 5.85 s, observed 230,334,464 bytes combined RSS and +The mixed phase took 5.85 s, observed 230,334,464 bytes combined RSS and 12,949,979 bytes of isolate components, and cold first call was 32.33 ms. The sequential six-handle/two-identity memory phase took 3.75 s, observed 207,298,560 bytes combined RSS and 39,200,796 bytes of isolate components, @@ -136,14 +137,12 @@ workerd startup was 145.90 ms. The 129 rows contain 9,205,245 bytes; largest row is 786,665 bytes. These are observed local samples, not production allocation admission guarantees or cloud percentiles. -This is a **go** for the measured shared-Go Worker boundary; no Go-host topology -alternative needs an owner decision at P03. P04/P05 still supply real member -stores and snapshot/compaction, P08 supplies production actors/admission and -serialization, and P09/P11 supply complete policy and migration. P08/P18 must -repeat the size/memory gate with those later components. Application rows are -paged; no production route or feature toggle was added by this proof. +The measured shared-Go Worker boundary meets the local size and memory gates. +Repeat those gates with real member stores, snapshot/compaction, production +actors/admission/serialization, complete policy and migration. Application rows +are paged; the proof exposes no production route or feature toggle. -P08 adds stateless host codecs without changing the existing operations: op 7 +Stateless host codecs preserve the existing operations: op 7 inspects Entry/HardState/Snapshot/peer Message (kind 1/2/3/4); op 8 hashes a bounded block with an explicit serialized SHA-256 state and returns the next state/digest; op 9 validates a canonical portable manifest and constructs its @@ -152,14 +151,15 @@ flags for transport completion. These helpers provide framing and integrity, never authorization or application policy. The same codecs run in native proof peers and the bundled reactor. -P11 adds op 12 (canonical LegacyConfig, kind, ID, original v1 JSON and sampled -hold-until; returns canonical RecordWrite), op 13 (validated LegacyManifest; -returns the shared genesis Entry/HardState/ConfState protobuf bytes), and op 14 -(validated public certificates/effects authority rows). These calls convert -bounded bytes and establish no request authentication or successful coordination -receipt. Source/target hosts own the irrevocable migration gate and durability. +Migration uses op 12 (canonical LegacyConfig, kind, ID, original v1 JSON and +sampled hold-until; returns canonical RecordWrite), op 13 (validated +LegacyManifest; returns the shared genesis Entry/HardState/ConfState protobuf +bytes), and op 14 (validated public certificates/effects authority rows). These +calls convert bounded bytes and establish no request authentication or +successful coordination receipt. Source/target hosts own the irrevocable +migration gate and durability. -P13 adds op 21 (node handle, canonical Membership header, canonical target +Leadership uses op 21 (node handle, canonical Membership header, canonical target LeadershipEvidence; returns a canonical LeadershipPlan) and op 22 (node handle, canonical Membership header; returns a consumed current-term LeadershipReceipt). These are internal host calls, never public configuration or readiness inputs. @@ -174,7 +174,7 @@ of the unchanged 256 MiB combined RSS gate. The same six hosted handles, native process death, both personal-leader roles and all snapshot/configuration steps remain in the workload; production isolate/core ceilings are unchanged. -## Ranked election timing (R7/R14) +## Ranked election timing ABI 1 create optionally appends an anchor count (exactly three) and three uint64 IDs after the applied index. Old mechanism frames may omit it. Hosts obtain the @@ -184,18 +184,20 @@ list, which changes when C becomes a standby. The core uses election timing version 2 in native and WASI. Rank survives snapshots and restarts as registry state; absent provisioning identifies a personal learner, never an invented rank. -P15 adds operation 27: cluster/worker/cell strings, incarnation/committed-time -uint64s, then five blobs in WorkerReadKeys order (empty means absent). It returns -one canonical WorkerRead blob. The host must pass its fresh current-term quorum -barrier before returning these bytes; the ABI supplies policy, not consensus. -Operation 16 accepts an optional tail: committed time uint64, then identity, -authority and worker-record blobs. It is required for delegated peers; the old -metadata-only form returns them revoked. Personal/anchor encodings are unchanged. - -P16 adds operation 28: cluster/worker strings, incarnation/committed-time uint64s, -canonical AssignmentRef blob, then six blobs in AssignmentReadKeys order. It -returns one canonical AssignmentRead blob. Like operation 27, it runs under the -member queue after current ingress time commits and a fresh quorum barrier -passes; the ABI never supplies a local success path. Assignments, per-cell active -slots and incarnation-bound holds use the existing state namespace and survive -ordinary native/hosted snapshots. No storage format or old ABI tag changes. +Selected worker reads use operation 27: cluster/worker/cell strings, +incarnation/committed-time uint64s, then five blobs in WorkerReadKeys order +(empty means absent). It returns one canonical WorkerRead blob. The host must +pass its fresh current-term quorum barrier before returning these bytes; the ABI +supplies policy, not consensus. Operation 16 accepts an optional tail: committed +time uint64, then identity, authority and worker-record blobs. It is required +for delegated peers; the old metadata-only form returns them revoked. +Personal/anchor encodings are unchanged. + +Selected assignment reads use operation 28: cluster/worker strings, +incarnation/committed-time uint64s, canonical AssignmentRef blob, then six blobs +in AssignmentReadKeys order. It returns one canonical AssignmentRead blob. Like +operation 27, it runs under the member queue after current ingress time commits +and a fresh quorum barrier passes; the ABI never supplies a local success path. +Assignments, per-cell active slots and incarnation-bound holds use the existing +state namespace and survive ordinary native/hosted snapshots. No storage format +or old ABI tag changes. \ No newline at end of file diff --git a/internal/raftabi/probe.go b/internal/raftabi/probe.go index f011aa3e28..a3383e172f 100644 --- a/internal/raftabi/probe.go +++ b/internal/raftabi/probe.go @@ -8,9 +8,9 @@ import ( ) // ProbeApply exercises real lease rules over one immutable host-supplied row. -// It is a feasibility seam, NOT an authorized directory writer: P09 must add -// the complete dependency read-set, committed policy, dedup and atomic writes. -// Keeping this probe narrow avoids treating prototype policy as production. +// It is a feasibility seam. An authorized directory writer also needs the +// complete dependency read-set, committed policy, dedup and atomic writes. +// Keeping this probe narrow avoids treating fixture policy as production. func ProbeApply(command, row []byte) ([]byte, error) { if len(command) > raftcore.MaxMessageBytes || len(row) > 1<<20 { return nil, errFrame diff --git a/internal/raftcore/README.md b/internal/raftcore/README.md index 84c527e964..06c8d26c55 100644 --- a/internal/raftcore/README.md +++ b/internal/raftcore/README.md @@ -1,11 +1,12 @@ # Deterministic consensus boundary -`raftcore` is an internal development seam shared by native and WASI hosts. It +`raftcore` is an internal consensus boundary shared by native and WASI hosts. It has no production routing, goroutine, filesystem, timer, network, or ambient random input. It owns RawNode and a copied in-memory view of recovered log -bytes. It imports P01's canonical command decoder, not directory/HTTP/SQLite. -Both targets must build through the root module's patched Raft replacement; -see `third_party/raft-patch/README.md` for pinned source and licenses. +bytes. It imports the canonical application command decoder, not +directory/HTTP/SQLite. Both targets must build through the root module's patched +Raft replacement; see `third_party/raft-patch/README.md` for pinned source and +licenses. ## One call, explicit effects @@ -52,27 +53,28 @@ committed command schemas/operations stop participation rather than becoming successful no-ops. Invariant panics are caught and stop the handle with a constant diagnostic; no command bytes are logged. -## Host and later packet responsibilities +## Host responsibilities `PeerMessage` accepts only network Raft replication/election types. It rejects local triggers, storage completions, forwarded proposals, peer authority reads, -and peer-initiated transfers. P07 authenticates original envelope signatures, -cluster/protocol, enrollment and revocation generations before this call. Local -`Propose` validates canonical schema/cluster/size but does not authenticate the -signature or apply policy. P09 must recheck committed authorization and produce -application results. Normal commands are bounded by the shared 64 KiB batch -budget; uncommitted payload is bounded at 1 MiB and inflight append batches at 8. -Authenticated transport and per-identity snapshot byte admission remain P07/P08/P09. +and peer-initiated transfers. The transport authenticates original envelope +signatures, cluster/protocol, enrollment and revocation generations before this +call. Local `Propose` validates canonical schema/cluster/size but does not +authenticate the signature or apply policy. The application must recheck +committed authorization and produce application results. Normal commands are +bounded by the shared 64 KiB batch budget; uncommitted payload is bounded at 1 +MiB and inflight append batches at 8. Authenticated transport and per-identity +snapshot byte admission belong to the host. `Configure` and `Transfer` are mechanisms for authorized host intents, not public APIs. Transfer requires a caught-up incoming voter outside joint state. Request admission is not transfer completion: the host must observe the target -as leader with a current-term commit and a new read barrier. P12/P13 own the +as leader with a current-term commit and a new read barrier. Host policy owns the registry, single membership intent, joint swaps, configuration revision, owner/persistent-storage eligibility, fallback and policy. No topology or leadership-reacquisition policy is implemented here. -P04 supplies actual durable storage. Recovered suffixes must be contiguous and +The host supplies actual durable storage. Recovered suffixes must be contiguous and include uncommitted entries; ConfState is the most recent DURABLY APPLIED one. The snapshot supplied at construction must already be fully installed in the application (Applied >= its included index). If a crash left a snapshot pointer @@ -84,47 +86,50 @@ new handle receives explicit election seed input; it does not restore volatile leadership, timers or RNG state. Replay records each such seed. The current in-memory log view is not a persistent database or a complete -production memory policy. P05 must bound it through explicit snapshot/cache and +production memory policy. Hosts must bound it through explicit snapshot/cache and compaction effects, stream snapshot data and preserve two durable generations. -Do not use this package to load a full application identity into wasm: P01 rows -and P09 paged apply remain outside the consensus cache. Quiescence scheduling, +Do not use this package to load a full application identity into wasm: application +rows and paged apply remain outside the consensus cache. Quiescence scheduling, clock samples, retry deadlines and application logical time are explicit host -inputs in later packets. `NeedNextTick` currently means every non-stopped handle +inputs. `NeedNextTick` currently means every non-stopped handle needs ticks unless the explicit committed quiescence protocol below stops them. ## Replay contract Election timing version 2 gives provisioned anchors A/B/C disjoint randomized -windows of 6–7/8–9/10–11 ticks; a personal voter uses 12–13. Rank is the index in -the immutable, sorted `Membership.Anchors` manifest, not current ConfState voter -order. Before registry initialization the same source-provisioned bootstrap -manifest supplies it. Native recovery supplies `RecoveredState.Anchors` from the -checksummed applied registry, including installed snapshots. Hosted create frames -carry that same durable list. Conflicting host/recovered manifests are refused. -An unprovisioned learner uses the personal window. Heartbeats remain every tick; -CheckQuorum and PreVote leader leases retain their six-tick interval. Hosts currently -schedule 5-second ticks (P14); no wall duration is read here. SplitMix64's exact -64-bit constants, overflow and rejection sampling are specified in random.go; -seed zero is valid. Every version-2 timeout reset consumes one bounded draw of width two and adds +windows of 6–7/8–9/10–11 ticks; a personal voter uses 12–13. Rank is the index +in the immutable, sorted `Membership.Anchors` manifest, not current ConfState +voter order. Before registry initialization the same source-provisioned +bootstrap manifest supplies it. Native recovery supplies +`RecoveredState.Anchors` from the checksummed applied registry, including +installed snapshots. Hosted create frames carry that same durable list. +Conflicting host/recovered manifests are refused. An unprovisioned learner uses +the personal window. Heartbeats remain every tick; CheckQuorum and PreVote +leader leases retain their six-tick interval. Hosts currently schedule 5-second +ticks; no wall duration is read here. SplitMix64's exact 64-bit constants, +overflow and rejection sampling are specified in random.go; seed zero is valid. +Every version-2 timeout reset consumes one bounded draw of width two and adds twice the immutable rank through the existing provider. The library's separate `ElectionRandomBound=8` admits these offsets without changing `ElectionTick`. Version 1 remains available only as explicit internal replay compatibility for -old recorded inputs; production defaults and the ABI always use version 2. -The simulator writes its timing version and reads unversioned old recordings as +old recorded inputs; production defaults and the ABI always use version 2. The +simulator writes its timing version and reads unversioned old recordings as version 1, preserving their exact event digests. Every timeout reset uses this -per-node provider and -never upstream global randomness. RNG state/draws appear only in diagnostics. -Wire protobuf marshaling is deterministic. RawNode already orders meaningful -peer traversal; preserve its per-peer message order. Completed read contexts -sort before emission. P03 versions the binary ABI and checks actual wasm/native -parity; a JSON encoding of Go diagnostic structs is not that ABI. - -Focused proof: `go test -p=1 ./internal/raftcore -run '^TestR' -count=1` at the -wave's nice/CPU/pressure limits. The tests cover R3/R4/R5/R6/R12/R14 mechanisms, -not the later simulator, persistent power-loss, joint-swap, cross-runtime or -real-client release proofs. The initial trace SHA-256 is a regression fixture; -SplitMix64 vectors are independently published values. Full fault checking is -P06 and the actual Worker stop/go measurement is P03. +per-node provider and never upstream global randomness. RNG state/draws appear +only in diagnostics. Wire protobuf marshaling is deterministic. RawNode already +orders meaningful peer traversal; preserve its per-peer message order. Completed +read contexts sort before emission. `raftabi` versions the binary ABI and checks +actual wasm/native parity; a JSON encoding of Go diagnostic structs is not that +ABI. + +Focused proof: run `go test -p=1 ./internal/raftcore -run '^TestR' -count=1` +through `scripts/raft-proof-footprint.sh`. The tests cover persistence, quorum +reads, PreVote, leadership transfer, configuration recovery and deterministic +replay. Simulator faults, persistent power loss, joint swaps, cross-runtime +parity and real-client behavior need separate proofs. The initial trace SHA-256 +is a regression fixture; SplitMix64 vectors are independently published values. +`raftsim` supplies full fault checking, and the local Worker proof measures the +runtime. ## Applied checkpoints and bounded history @@ -154,7 +159,7 @@ constructing a core; no application rows or chunks belong in its log cache. ## Committed learner intents -P12's `PlanMembership` accepts only the durable `clusterstate.Membership` header +`PlanMembership` accepts only the durable `clusterstate.Membership` header under the member serialization lock. `InitMembership` records exactly the three already-applied anchors and their protocol public keys; source provisioning must supply that SAME manifest to the bootstrap transport. Personal enrollment must @@ -174,8 +179,9 @@ A removal immediately retires transport authority, but its configuration change still completes through Raft. IDs and tombstones survive snapshots and are never reused. New IDs increase monotonically. Live learners are limited separately from retained records; exhaustion refuses enrollment without deleting anything. -P13 owns promotion, joint swaps, transfer completion and anchor fallback. No -P12 policy command removes/promotes a voter. Configuration/provisioning APIs are +The leadership workflow owns promotion, joint swaps, transfer completion and +anchor fallback. Learner policy commands never remove or promote a voter. +Configuration/provisioning APIs are internal host seams, never unauthenticated client endpoints. Provisioned anchor keys receive a restricted committed authority that can only @@ -184,7 +190,7 @@ remove, read directory content, mutate chats, pair or rotate. Thus a crashed requester need not return for an already authorized admission to finish. The settlement still requires the exact applied configuration and pending revision. -P13 leadership hosts reconcile `Membership.Leadership` under their actor queue: +Leadership hosts reconcile `Membership.Leadership` under their actor queue: obtain authenticated target `LeadershipEvidence` from its durable store at the intent barrier; call `PlanLeadership`; durably apply each explicit joint enter and separate leave; commit `SettleLeadership`; then execute the planned transfer. @@ -203,10 +209,10 @@ settlement. An interrupted joint state is finished before reversing. Concurrent fallback requests conflict on the same revision. Returning P stays a learner; only another personal request may promote it. Public handlers accept the typed signed request and never expose Configure or host-supplied readiness evidence. -P14 schedules ticks/wake/housekeeping, and P17 assembles these host/control seams +The host schedules ticks/wake/housekeeping and assembles these control seams with the personal member lifecycle and the one production directory route. -## Committed liveness and idle scheduling (P14) +## Committed liveness and idle scheduling `Schedule` takes monotonic elapsed milliseconds, conservative host activity, its durable `clusterstate.Quiescence` row and authenticated applied voter @@ -233,13 +239,13 @@ cover a sample; the authoritative read plus shared `RenewalCovered` helper skips that item before logging. Successful receipt indexes escape only after durable application. Presence pongs alone do not suppress client heartbeats. -The router's timer driver must resume the existing P12/P13 committed workflow -and P10 outboxes before idle preparation. It must authenticate target evidence, -finish joint stages, settle, transfer, obtain the elected target's fresh read, -commit completion and obtain a new receipt. Returning completed intents never -regain leadership automatically. `QuiescenceCoordinator` sequences the idle -part without adding another coordination writer. P17 supplies the sole product -host and the outbound router assembly; no deployment or rollout switch exists. +The router's timer driver must resume the committed membership/leadership +workflow and outboxes before idle preparation. It must authenticate target +evidence, finish joint stages, settle, transfer, obtain the elected target's +fresh read, commit completion and obtain a new receipt. Returning completed +intents never regain leadership automatically. `QuiescenceCoordinator` sequences +the idle part without adding another coordination writer. The production host +supplies the outbound router assembly; no deployment or rollout switch exists. Portable snapshot metadata keeps the manifest's exact protobuf field presence. The pinned library fills omitted default fields such as `auto_leave:false` in diff --git a/internal/raftcore/core.go b/internal/raftcore/core.go index 7b480c1c79..94f4e3c440 100644 --- a/internal/raftcore/core.go +++ b/internal/raftcore/core.go @@ -118,7 +118,7 @@ func copyEntries(entries []*pb.Entry) []*pb.Entry { // New refuses inconsistent recovery rather than manufacturing missing committed // bytes. The caller validates durable checksums and cluster/format manifests -// before supplying this state; those storage formats belong to P04/P05. +// before supplying this state; storage formats are the host adapters' responsibility. func New(cfg Config, recovered RecoveredState) (core *Core, err error) { defer func() { if recover() != nil { diff --git a/internal/raftcore/types.go b/internal/raftcore/types.go index cea6dedca8..d703c65321 100644 --- a/internal/raftcore/types.go +++ b/internal/raftcore/types.go @@ -97,7 +97,7 @@ type Tick struct{} type PeerMessage struct{ Bytes []byte } type Propose struct{ Command []byte } -// Configure is a mechanism for later authorized membership intents, not a +// Configure is a mechanism for committed authorized membership intents, not a // public enrollment door. The bytes encode a ConfChangeV2. type Configure struct{ Change []byte } type Read struct{ RequestID string } diff --git a/internal/raftdirectory/README.md b/internal/raftdirectory/README.md index 08c4871b7a..8b29e73548 100644 --- a/internal/raftdirectory/README.md +++ b/internal/raftdirectory/README.md @@ -1,11 +1,12 @@ # Committed directory application -`clusterstate.Evaluate` is the single directory application evaluator for native and -WASI members. It takes a signed canonical command, immutable application metadata -and the complete declared dependency read-set. Missing and unread records differ. -The evaluator verifies the original signature against committed personal enrollment, -checks revocation at that log position, then applies the directory rules. Metadata -upserts cannot create signing authority. Delegated worker policy remains P15. +`clusterstate.Evaluate` is the single directory application evaluator for native +and WASI members. It takes a signed canonical command, immutable application +metadata and the complete declared dependency read-set. Missing and unread +records differ. The evaluator verifies the original signature against committed +personal enrollment, checks revocation at that log position, then applies the +directory rules. Metadata upserts cannot create signing authority. Delegated +worker policy uses a separate role and scope. Every accepted session keeps its exact last result until an acknowledged sequence advance or explicit close; closed IDs remain tombstoned. There are 128 live sessions @@ -27,13 +28,13 @@ an authority response. The integration harness proves this through `directory.Ra and its real HTTP handler; the backend interface is an assembly boundary. The hosted member chooses `directoryEvaluator`, which calls Go through additive -ABI-1 operations 11 (dependencies) and 10 (evaluation). JS copies immutable bytes, -persists the returned writes and advances the core after durable completion. It -contains no lease, CAS, revocation, clock or sequence decisions. `RaftDirectoryFacade` -forwards original signed bytes to a host's commit and barrierRead operations. The -old hosted directory is still solely the pre-migration path; P11 must irrevocably -freeze it before routing a namespace to this facade. No production cutover or -feature toggle is introduced here. +ABI-1 operations 11 (dependencies) and 10 (evaluation). JS copies immutable +bytes, persists the returned writes and advances the core after durable +completion. It contains no lease, CAS, revocation, clock or sequence decisions. +`RaftDirectoryFacade` forwards original signed bytes to a host's commit and +barrierRead operations. The old hosted directory is solely the pre-migration +path; migration must irrevocably freeze it before routing a namespace to this +facade. The facade itself neither performs cutover nor selects a feature toggle. The Client facade exposes all existing directory verbs. Its v1 acquire normalization never manufactures an observed tuple: a missing expectation disables force, and a @@ -41,21 +42,24 @@ required conditional take without an observation refuses. v1 fences cannot excee the exact JSON-integer limit; direct v2 commands retain full uint64 fences and refuse overflow. Pure rules retain their legacy behavior for pre-cutover stores. -Pairing commands require an existing committed request dependency, atomically write -the device metadata and sealed decision, and retain the first decision. The native -and hosted tests provision genesis/request dependencies explicitly. The effects adapter below supplies -external request/certificate admission, authorization installation, outbox delivery, -blob receipts and v2 upload-before-head. P11 owns actual genesis/import; P12 owns peer -registry; P14 owns watch presence and committed renewal scheduling. No socket can -extend a lease through this evaluator. The v1 adapter below retains original HTTP authentication proof; production -member/client routing still requires the complete namespace assembly. These boundaries are internal, not new manual claims. - -P10 adds `Effects` and the equivalent hosted `RaftEffectsFacade`. Provision a -separate `Authority{Effects:true}` protocol key; it must not be personal. It can -only record upload evidence, reserve a join dependency, or complete a delivery. -These commands are idempotent by immutable effect keys and do not consume client -sessions. All effect rows count against the identity's application ceiling; -capacity refuses further growth without removing receipts, decisions or blobs. +Pairing commands require an existing committed request dependency, atomically +write the device metadata and sealed decision, and retain the first decision. +The native and hosted tests provision genesis/request dependencies explicitly. +The effects adapter below supplies external request/certificate admission, +authorization installation, outbox delivery, blob receipts and v2 +upload-before-head. Separate host adapters supply genesis/import, peer registry, +watch presence and committed renewal scheduling. No socket can extend a lease +through this evaluator. The v1 adapter below retains original HTTP +authentication proof; production member/client routing requires the complete +namespace assembly. + +`Effects` and the equivalent hosted `RaftEffectsFacade` bridge external +delivery. Provision a separate `Authority{Effects:true}` protocol key; it must +not be personal. It can only record upload evidence, reserve a join dependency, +or complete a delivery. These commands are idempotent by immutable effect keys +and do not consume client sessions. All effect rows count against the identity's +application ceiling; capacity refuses further growth without removing receipts, +decisions or blobs. Before issuing a receipt, call `Effects.Upload` with the local head ID, its **different encrypted remote object ID**, and the existing incremental frame @@ -64,15 +68,17 @@ confirmation awaits R2, its complete index and the supplied storage durability boundary. A client still uploads its entire encrypted closure before requesting this receipt: a relay cannot decrypt or traverse it, and an empty frame hint never proves an empty closure. The receipt binds actor, both head IDs and the -exact frame hint. V2 Create/Publish require the committed receipt, and still report -stale fences/heads before missing upload evidence. The existing Client facade explicitly identifies v1 Create/Publish, as it -already identifies v1 Acquire. Old clients cannot supply the encrypted head RID; -they retain the existing upload-before-publish client ordering and their existing -safe API semantics. This is protocol compatibility in the same evaluator, never -a second writer. V1Directory selects this marker only from the original authenticated v1 -ingress; ClientCommand rejects compatibility markers at a public v2 boundary. A v2 body cannot become -v1 merely by requesting it. The production writer remains unchanged until namespace migration and host -assembly install the adapter. No internal facade activates that route by itself. Never derive an encrypted remote +exact frame hint. V2 Create/Publish require the committed receipt, and still +report stale fences/heads before missing upload evidence. The existing Client +facade explicitly identifies v1 Create/Publish, as it already identifies v1 +Acquire. Old clients cannot supply the encrypted head RID; they retain the +existing upload-before-publish client ordering and their existing safe API +semantics. This is protocol compatibility in the same evaluator, never a second +writer. V1Directory selects this marker only from the original authenticated v1 +ingress; ClientCommand rejects compatibility markers at a public v2 boundary. A +v2 body cannot become v1 merely by requesting it. Namespace migration and host +assembly must install the adapter before it serves production requests. No +internal facade activates that route by itself. Never derive an encrypted remote object ID from a local turn ID. Receipts admit at most 512 declared frame hints; a frame plan remains optional and partial. @@ -81,22 +87,22 @@ other identities and legacy decision paths. Failed admission retains it until the ordinary public request expires. Approval verifies the original existing certificate under `IdentityRec.SigningKey`, checks that its device key matches the bound request, and installs personal authorization, device metadata and the -sealed decision atomically. P11 must populate that **public** root key from -authenticated genesis/import. An upsert never installs authorization. Peer/node -membership is still P12, and delegated workers remain P15. +sealed decision atomically. Genesis/import must populate that **public** root key +from authenticated input. An upsert never installs authorization. Peer/node +membership and delegated workers use separate committed policies. Replay scans at most 64 retained outbox rows per call, from a quorum-authorized view. Deliver reads the decision through another fresh barrier, sends its hash as the destination's idempotency key, then commits `CompleteJoin`. A crash at any boundary repeats only that same decision. A public request that expired records `gone`; a lost destination reply retries without changing its grant. -Restarting a scan at the beginning is safe. P11/P14 must schedule these bounded +Restarting a scan at the beginning is safe. The host must schedule these bounded pages on activation/wake; no periodic network loop starts here. Delivery of a grant approved before revocation cannot restore authority after it: every subsequent operation still checks the committed device record. Already delivered sealed bytes cannot be recalled, as with existing personal pairing. -P16 supplies Assign/Reassign/ClaimAssignment/CancelAssignment/FinishAssignment in +Assign/Reassign/ClaimAssignment/CancelAssignment/FinishAssignment run in the shared Go evaluator. AssignmentRef declares every routing dependency before application. Only an explicit current scoped worker/incarnation with recent committed availability can claim; claim and ordinary fenced acquire commit in @@ -114,14 +120,15 @@ pending run is permanently unclaimable and its raw history remains retained. An owner can explicitly cancel it to reuse the per-cell slot. syncsetup.WorkerAdapter accepts scoped content dependencies and a selected -quorum-backed directory. It uses handoff.Taker for stage/claim/reconcile/install, -saves original signed claim/finish requests and durably records execution intent -before its callback. Completed attempts return locally; a recorded finish retries -only coordination. Interrupted attempts and reassigned revisions return incomplete -without invoking tools. Execute must use the ordinary call recorder and -cellsync.Publisher; it returns only an already published head. The adapter never -loads a personal identity, discovers a relay, provisions a box or adds a public -command. P17/P18 retain public client assembly and operator journeys. +quorum-backed directory. It uses handoff.Taker for +stage/claim/reconcile/install, saves original signed claim/finish requests and +durably records execution intent before its callback. Completed attempts return +locally; a recorded finish retries only coordination. Interrupted attempts and +reassigned revisions return incomplete without invoking tools. Execute must use +the ordinary call recorder and cellsync.Publisher; it returns only an already +published head. The adapter never loads a personal identity, discovers a relay, +provisions a box or adds a public command. Public worker setup and assignment +execution commands are not available yet. The native `V1Directory` implements `directory.RequestDirectory`. The HTTP handler first authenticates, then binds the exact request to its client. The @@ -139,11 +146,9 @@ rotation view passes another barrier without repeating the mutation. The adapter checks current root/device authority inside that callback. Missing request proof is refused; no in-process `For` call can bypass authentication. -This adapter is an integration boundary, not the production cutover. It does -not yet supply watches, personal-member lifecycle, worker/v2 HTTP ingress, or -the wake/outbox coordinator. Encoded commands remain bounded at 64 KiB, so -large legacy HTTP bodies need a supported representation before full legacy -compatibility can be claimed. The production relay contract test intentionally -remains red until persistent namespace assembly routes its real HTTP doors -through consensus. Do not enable a second writer or weaken a quorum to make -that test pass. +The host supplies watches, personal-member lifecycle, worker/v2 HTTP ingress +and the wake/outbox coordinator around this translation adapter. Encoded +commands remain bounded at 64 KiB, so large legacy HTTP bodies need a supported +representation before full legacy compatibility can be claimed. Persistent +namespace assembly must route the real HTTP doors through consensus, with one +writer and an unchanged quorum. diff --git a/internal/raftdirectory/directory_test.go b/internal/raftdirectory/directory_test.go index a7dcf59ca9..5c87e4befc 100644 --- a/internal/raftdirectory/directory_test.go +++ b/internal/raftdirectory/directory_test.go @@ -366,7 +366,7 @@ func TestR3R13R17DirectoryRetryAfterSnapshotAndNativeRestart(t *testing.T) { } // R30 covers the remaining Client verbs at the actual committed adapter door. -// External LinkRequest delivery and certificate admission remain P10's gate. +// External LinkRequest delivery and certificate admission have separate effects tests. func TestR16R30PresenceAndPairDecisionDependencies(t *testing.T) { clock := directorytest.NewFakeClock() g := newGroup(t, clock.Now) diff --git a/internal/raftdirectory/routing_test.go b/internal/raftdirectory/routing_test.go index e6c262f051..ff99ac5221 100644 --- a/internal/raftdirectory/routing_test.go +++ b/internal/raftdirectory/routing_test.go @@ -7,7 +7,7 @@ import ( ) // R30 forbids another lease writer on the authoritative hosted path. Existing -// pre-cutover legacy rules remain tested until P11 freezes that source. +// pre-cutover legacy rules remain tested until migration freezes that source. func TestR30HostedDirectoryRoutesOnlyThroughGoPolicy(t *testing.T) { b, err := os.ReadFile("../../relay/hosted/src/raft/directory.js") must(t, err) diff --git a/internal/raftsim/README.md b/internal/raftsim/README.md index a7e4073c9f..94097127de 100644 --- a/internal/raftsim/README.md +++ b/internal/raftsim/README.md @@ -1,12 +1,12 @@ # Deterministic fault gate -This development package runs 3–5 **production** `raftcore` instances on one +This fault-testing package runs 3–5 **production** `raftcore` instances on one virtual-time event queue. It adds no product route, environment switch, socket, -clock read or background consensus loop. P09 owns the authorized production -application adapter; this packet's `ProbeApplication` exercises the real pure -lease rules and a small session-policy fixture. The independent oracle in -`model` uses neither those rules nor production state types. Assignment policy -extends that oracle in P16; it is not advertised as implemented here. +clock read or background consensus loop. `ProbeApplication` exercises the real +pure lease rules and a small session-policy fixture, separate from the authorized +production application adapter. The independent oracle in `model` uses neither +those rules nor production state types. A separate assignment model extends the +independent domain. ## Run the bounded gate @@ -43,13 +43,13 @@ anchors and 12–13 for personal voters). Split votes and omitted messages can require several attempts; the campaign allows eight maximum timeout windows while retaining the 2,000-event ceiling. A separate seeded check asserts one delivered tick per live node per round and each election -anchor attempt within 6–12 ticks. The separate R7 campaign measures the +anchor attempt within 6–12 ticks. The separate leadership campaign measures the phone-fallback deadline through a committed conditional operation. Lost-reply retries first wait for the original request's durable session result without submitting a second proposal. An absent result after a timeout remains unknown; it cannot be compared to a later successful retry as an empty result. Portable replay fixtures retain the original split-vote and uncommitted-timeout -prefixes that exposed these two harness mistakes. +prefixes for regression coverage. To keep exact negative counterexamples, pass the simulator package alone: @@ -93,20 +93,20 @@ against its flush, and every state transition checks leader-per-term, committed log agreement, monotonic indexes/fences and application equivalence. The disk model is a fault model, not another durable storage implementation. -P04/P05 prove SQLite process cuts and actual WAL flushes. The native parity test -replays the simulator's exact production batches into real SQLite stores and -compares recovery bytes, application state and portable snapshots; simulated -snapshot chunks are also installed through the real store's API. The simulated -one-row application stays below the portable 1 MiB row envelope; it is not the -paged production directory evaluator or a migration representation. +Native store tests prove SQLite process cuts and actual WAL flushes. The native +parity test replays the simulator's exact production batches into real SQLite +stores and compares recovery bytes, application state and portable snapshots; +simulated snapshot chunks are also installed through the real store's API. The +simulated one-row application stays below the portable 1 MiB row envelope; it is +not the paged production directory evaluator or a migration representation. -## Constraints subsequent packets must keep +## Host constraints Create a new checkpoint **after committed learner admission** when catch-up requires a compacted prefix. etcd refuses an older snapshot whose configuration omits the recipient. Wait until the learner applies the advertised barrier -before proposing its promotion. This test host supplies the mechanism; P12 owns -the durable enrollment/intent and policy workflow. +before proposing its promotion. This test host supplies the mechanism; the +production host owns the durable enrollment/intent and policy workflow. Snapshot transfer completion is separate from Raft installation. Chunks stage without changing application authority, complete streams are checksummed, @@ -117,18 +117,17 @@ so RawNode can retry. A transport receipt alone is never an append acknowledgmen The fixture session contract keeps one exact result per live session, requires an acknowledgment to advance, retains closed-ID tombstones, refuses conflicting -digests and caps live sessions at 128. Acks/close are replay-catalog inputs until -P09 supplies their real schema; no public protocol for them is invented here. -Authentication, durable upload receipts, complete directory verbs, assignments, -learner quotas/transport queues, hosted actors and client journeys remain the -packets that own them. R1's displayed sentence, unsent local branches and the -other X/J requirements are not claimed by a metadata simulator. +digests and caps live sessions at 128. Acks/close are replay-catalog inputs, +not a public protocol. Authentication, durable upload receipts, complete +directory verbs, assignments, learner quotas/transport queues, hosted actors +and client journeys require separate tests. A metadata simulator does not prove +visible refusal wording, unsent local branches or chat/workspace behavior. Porcupine v1.1.0 is MIT licensed and only imported by the simulator checker. Neither this dependency nor the simulator is linked into `bin/codeaf`. -P12 extends the bounded topology to three voters plus at most 32 learner cores. +The bounded topology supports three voters plus at most 32 learner cores. Its full-fanout case admits the learners in one internal configuration batch, sleeps all of them, commits writes, compacts, restores a lagging learner through a fresh snapshot and removes it with delayed responses. The unchanged 2,000-event @@ -143,10 +142,10 @@ recordings selects their original version 1, including exact RNG consumption and event digests. This compatibility is internal replay input, never a product feature switch. New groups always default to version 2 in both runtimes. -R7 keeps all 190 original split-vote seeds in testdata/leadership. The seeded +All 190 original split-vote seeds remain in testdata/leadership. The seeded leadership campaign measures through a real committed conditional acquire and reports p50/p95/max; it never accepts an elected role alone as successful fallback. The B+C campaign separately requires a conditional operation within 50 seconds after A and learner P disappear in restored anchor topology. If C is still a learner and the voter set is A+B+P, simultaneous P+A loss remains -unavailable under R8: no election window can replace a missing majority. +unavailable: no election window can replace a missing majority. diff --git a/internal/raftsim/application.go b/internal/raftsim/application.go index 1238a15e1f..d7d69eaf1d 100644 --- a/internal/raftsim/application.go +++ b/internal/raftsim/application.go @@ -1,5 +1,5 @@ // Package raftsim drives production consensus cores with explicit virtual time, -// durability and faults. It is a development gate, never a directory writer. +// durability and faults. It is a fault-testing harness, never a directory writer. package raftsim import ( @@ -12,9 +12,9 @@ import ( "github.com/Agent-Field/codeaf/internal/raftsim/model" ) -// ProbeApplication exercises real pure lease rules while P09 supplies the full -// authorized writer. Session policy here is a contract fixture, not a second -// production implementation. The checker uses its own rules and state types. +// ProbeApplication exercises real pure lease rules without the full authorized +// writer. Session policy here is a contract fixture, not a second production +// implementation. The checker uses its own rules and state types. type ProbeApplication struct { Coordination map[string][]byte `json:",omitempty"` Now int64 diff --git a/internal/raftsim/campaign_test.go b/internal/raftsim/campaign_test.go index 6db668dc67..b9d5c8e377 100644 --- a/internal/raftsim/campaign_test.go +++ b/internal/raftsim/campaign_test.go @@ -79,7 +79,7 @@ func TestR17LostReplyReplayWaitsForOriginalCommit(t *testing.T) { finish(t, s) } -// R4/R5 and design §7 bound each election timeout at 6–12 ticks. Check the +// R4/R5 bound each election timeout at 6–12 ticks. Check the // observable election messages, separately from eventual election of a leader. func TestR4R5SeededElectionTimeoutAndTickPacing(t *testing.T) { for seed := 0; seed < seedCount(t); seed++ { diff --git a/internal/raftsim/model/model.go b/internal/raftsim/model/model.go index b93d4e0357..6ff7c4e298 100644 --- a/internal/raftsim/model/model.go +++ b/internal/raftsim/model/model.go @@ -1,7 +1,7 @@ // Package model is the independent sequential ownership/vault oracle for the // fault simulator. It deliberately imports no production state types or rules. -// Histories include timed-out requests as unknown outcomes; assignments extend -// this domain when their production adapter lands. +// Histories include timed-out requests as unknown outcomes. The assignment +// model covers assignment policy separately. package model // State describes observable coordination, independent of Raft internals. diff --git a/internal/raftsim/simulator_test.go b/internal/raftsim/simulator_test.go index 2d7fbc2387..db34842e59 100644 --- a/internal/raftsim/simulator_test.go +++ b/internal/raftsim/simulator_test.go @@ -311,7 +311,7 @@ func TestR5HundredMinorityFlapsKeepHealthyTerm(t *testing.T) { t.Logf("100 flaps, %d events, stable term %d", s.Events, term) } -// These deliberate faults live only in the development harness. The same +// These deliberate faults live only in the fault harness. The same // checker/ordering monitor must reject each, otherwise the gate proves little. func TestR1R3R4BrokenFenceReadAndFlushAreDetected(t *testing.T) { for _, variant := range []string{"fence", "read", "flush"} { diff --git a/internal/raftsnapshot/README.md b/internal/raftsnapshot/README.md index f3155edcdd..804f44a62c 100644 --- a/internal/raftsnapshot/README.md +++ b/internal/raftsnapshot/README.md @@ -3,8 +3,9 @@ This pure package is shared by native Go and the WASI core. SQLite and DO hosts own chunk staging, durable generation publication and installation. The format contains coordination only, with no plaintext chat content or private keys. It -copies every opaque application row without interpreting its policy. P09 owns -complete authorization/evaluation; P08 owns the hosted storage implementation. +copies every opaque application row without interpreting its policy. The +application supplies complete authorization/evaluation; host adapters supply +storage. `raftpb.Snapshot.Data` is a canonical manifest, at most 16 KiB. Its metadata index, term and complete ConfState must match that manifest. It never embeds chunks or diff --git a/internal/raftstore/README.md b/internal/raftstore/README.md index 3c04e07d93..1e3e366925 100644 --- a/internal/raftstore/README.md +++ b/internal/raftstore/README.md @@ -42,12 +42,13 @@ never permission to acknowledge a client or send a Raft append response. Applied reaching that barrier. The host serializes all core work and storage completions; the database mutex alone does not serialize a whole actor. -P09 owns signatures, authorization, declared read dependencies, deterministic -evaluation, response encoding, dedup sequencing and admission policy. This adapter -does not infer these from stored bytes. It accepts opaque records with a 1 MiB -per-record and aggregate read-set envelope. A missing key is absent from the map; -an existing empty record remains present. `Update.Delete` is for an explicitly -authorized application change; no consensus maintenance path deletes records. +The application owns signatures, authorization, declared read dependencies, +deterministic evaluation, response encoding, dedup sequencing and admission +policy. This adapter does not infer these from stored bytes. It accepts opaque +records with a 1 MiB per-record and aggregate read-set envelope. A missing key +is absent from the map; an existing empty record remains present. +`Update.Delete` is for an explicitly authorized application change; no consensus +maintenance path deletes records. ## Format and corruption checks @@ -75,8 +76,8 @@ here that deletes chats, frames, branches, saved heads or legacy original stores Storage format 2 adds a checksummed `meta.snapshot` pointer, `snapshots` manifests and `snapshot_chunks`, with full-width big-endian chunk numbers. Format-1 stores -from the earlier development packet are refused, retained and never upgraded in -place; this is an internal pre-cutover format, not a legacy directory migration. +are refused, retained and never upgraded in place; this is an internal +pre-cutover format, not a legacy directory migration. The cross-runtime stream is format 1 in `internal/raftsnapshot/README.md`. For a local checkpoint, serialize the actor, call `CreateSnapshot`, then @@ -97,7 +98,7 @@ consensus bytes. A completed transfer awaiting installation keeps that slot. application hash and returns the portable protobuf pointer. This publishes a complete transfer, not application authority. The active pointer remains old. The host must enforce the imported/application byte ceiling and transport quota -before staging; this adapter does not choose P09/P11's admission policy. +before staging; this adapter does not choose application/import admission policy. Only `Persist` of the real incoming Ready atomically replaces application rows, header/ConfState, active snapshot pointer, log prefix and hard state. Successful @@ -124,13 +125,13 @@ existing policy boundary. No frame garbage collection is introduced here. ## Focused proof -Run through the wave's pressure/CPU wrapper: +Run through `scripts/raft-proof-footprint.sh` for pressure and CPU limits: ```sh sh scripts/raft-proof-footprint.sh go test -p=1 ./internal/raftstore ./internal/raftcore -run '^TestR' -count=1 -timeout=120s -v ``` -The R3/R12/R17 transaction tests terminate subprocesses without Close/rollback at +The transaction tests terminate subprocesses without Close/rollback at every Ready/application statement and both sides of COMMIT. They recover copies of only the database and WAL, excluding process memory and the shared-memory index. The Linux `strace` probe asserts successful WAL fsync/fdatasync between invocation @@ -141,30 +142,32 @@ Physical power cuts or hardware that lies about flushes are not simulated. The bounded three-native-node fixture runs real cores/stores and real create/ publish lease rules, then loses a leader and recovers another member from durable bytes before a quorum read. It also recovers and leaves real joint membership. -Its one-cell evaluator is test-only, not P09's authorized directory writer or the -independent P06 history oracle. Full simulator fault schedules, hosted snapshot interoperability, transport, -membership policy, cross-runtime storage and real-client journeys remain later -packets; these mechanism tests do not claim complete R1–R30 release proof. +Its one-cell evaluator is test-only, separate from the authorized directory +writer and independent history oracle. Full simulator fault schedules, hosted +snapshot interoperability, transport, membership policy, cross-runtime storage +and real-client journeys need separate proofs. -The P05 R13 tests cut real subprocesses at staging/chunk/completion, each +The snapshot tests cut real subprocesses at staging/chunk/completion, each source/install row and created chunk, application/pointer/log publication, compaction and both sides of COMMIT. They retain database/WAL only. Missing and corrupt chunk tests cover every chunk in a near-limit record, plus canonical -manifest refusals. The R11 native fixture demotes a member to learner, disconnects -it, commits more than the 4 MiB trigger through the majority, compacts both stores -and caches, and then installs/replays/restarts the learner. The pure format tests -pin full-width integer and binary framing vectors independently of the encoder. -These are the P05 mechanism gates; P06/P08 own simulator/cross-runtime campaigns. +manifest refusals. The native learner fixture demotes a member to learner, +disconnects it, commits more than the 4 MiB trigger through the majority, +compacts both stores and caches, and then installs/replays/restarts the learner. +The pure format tests pin full-width integer and binary framing vectors +independently of the encoder. These storage mechanism tests complement the +separate simulator/cross-runtime campaigns. ## Legacy directory import -P11 freezes the source with `directory.SQLite.FreezeLegacy`, waits for admitted -legacy writes and pairing decisions, samples hold evidence, then inventories the -original rows through the shared Go converter. `InventoryLegacy` is resumable; -`ExportLegacy` streams canonical rows in namespace/key order. Unknown fields or -rows above the existing one-row envelope refuse the import and retain the frozen -source. It never truncates records or rewrites original JSON. Erased namespaces -remain refused by the existing namespace tombstone before opening the source. +The migration host freezes the source with `directory.SQLite.FreezeLegacy`, +waits for admitted legacy writes and pairing decisions, samples hold evidence, +then inventories the original rows through the shared Go converter. +`InventoryLegacy` is resumable; `ExportLegacy` streams canonical rows in +namespace/key order. Unknown fields or rows above the existing one-row envelope +refuse the import and retain the frozen source. It never truncates records or +rewrites original JSON. Erased namespaces remain refused by the existing +namespace tombstone before opening the source. The source allocates ONE UUID/generation and ordered three-anchor configuration. Each target calls `ImportLegacy` on its own fresh store, streams the same export, @@ -180,11 +183,11 @@ The `membership/legacy-import` row retains provenance and the byte ceiling at least two distinct configured anchors; the host obtains them through a fresh current-term quorum barrier. Only then does `MigrationRoute` return the Raft facade. Freeze, inventory and ownership never reopen the old writer, including -stale handles and restarts. The full production host/client routing is assembled -in P17. No API caller may submit import receipts or allocate replacement UUIDs. +stale handles and restarts. Production host/client routing uses this migration +boundary. No API caller may submit import receipts or allocate replacement UUIDs. Public root and original authenticated device certificates install authority; metadata alone cannot enroll a device. Previously unseen personal devices must present their root-signed certificate at the later enrollment/ingress door. Legacy public link decisions drain before freeze and remain in their original -request store through its existing TTL; pending codes enter P10's reserved +request store through its existing TTL; pending codes enter the reserved outbox after cutover. No grant is regenerated or automatically replayed as work. diff --git a/internal/raftstore/native_test.go b/internal/raftstore/native_test.go index 7781e762f1..a54a7e951f 100644 --- a/internal/raftstore/native_test.go +++ b/internal/raftstore/native_test.go @@ -17,7 +17,7 @@ import ( // nativeGroup is a bounded, socket-free host fixture using REAL per-member // SQLite databases and production cores. Its one-cell rule evaluator is not -// P09's authorized directory writer or P06's independent history oracle. +// the authorized directory writer or the independent history oracle. type nativeGroup struct { t *testing.T nodes [3]*raftcore.Core diff --git a/internal/raftstore/sqlite_test.go b/internal/raftstore/sqlite_test.go index a81a43b5ca..5646911555 100644 --- a/internal/raftstore/sqlite_test.go +++ b/internal/raftstore/sqlite_test.go @@ -554,8 +554,8 @@ func TestR3ShorterConflictingSuffixSurvivesRestart(t *testing.T) { } } -// R13's snapshot lifecycle belongs to P05. Until then a snapshot cannot receive -// a false success that would permit dependent replication acknowledgments. +// R13 forbids acknowledging a snapshot before its staged chunks are complete; +// a false success would permit dependent replication acknowledgments. func TestR13SnapshotInputFailsClosed(t *testing.T) { s := openTest(t, filepath.Join(t.TempDir(), "member.sqlite")) seed(t, s) diff --git a/internal/rafttransport/README.md b/internal/rafttransport/README.md index 653768994b..dbecbf54ce 100644 --- a/internal/rafttransport/README.md +++ b/internal/rafttransport/README.md @@ -29,18 +29,19 @@ configuration changes, tombstones and registry persistence. The routing inbox is expendable memory. Before restoring service after loss of that memory, increment and durably store the view's router `Generation`, then -install it. That fences all old delivery cursors and sockets. Member incarnations -also come from durable enrollment. Do not replace either number with a random -connection ID. A router has no automatic bootstrap, singleton fallback or authority -read. P12 must enforce a single durable epoch for each identity; registering two -unrelated scopes is not migration or permission to run two writable clusters. +install it. That fences all old delivery cursors and sockets. Member +incarnations also come from durable enrollment. Do not replace either number +with a random connection ID. A router has no automatic bootstrap, singleton +fallback or authority read. Membership policy must enforce a single durable +epoch for each identity; registering two unrelated scopes is not migration or +permission to run two writable clusters. ## Exchanges and sockets `POST /v2/cluster/exchange` takes one canonical `Request` and returns `Response`. `Client.Exchange` makes one bounded outbound request, refuses HTTP redirects and starts no background work, listeners, retries or discovery. Hosts enforce paired -and sync-off policy **before** creating/using this client; P14/P17 own that wiring. +and sync-off policy **before** creating/using this client. Use HTTPS with the configured relay's authenticated TLS endpoint (loopback HTTP is used by the hermetic tests). Keep protocol admission separate from the public application-command limiter, and configure an operator ingress rate/connection @@ -64,7 +65,7 @@ committed peer key and incarnation, its expected identity/cluster/epoch/protocol and its own destination/incarnation. Remote hosts must not trust a stale cached registry to authorize traffic. Recheck after asynchronous waits; discard callbacks from an old actor generation. Only step a message once the receiver is available -for it. Keep/retry `ErrBusy` input. Keep all P04 persistence/application boundaries: +for it. Keep/retry `ErrBusy` input. Keep all persistence/application boundaries: `Persist -> Persisted -> Apply -> Applied`; a relay receipt completes none of them. ## Wire version 1 @@ -76,14 +77,16 @@ are 1–128 ASCII letters, digits, hyphen, underscore or dot. Digests are lowerc SHA-256 hex. Strict decoding requires exact re-encoding: unknown/duplicate fields, extra array elements, alternate numbers, whitespace and trailing bytes are refused. -Envelope field order is `scope,from,to,incarnation,destination,sequence,chunk,digest, -payload,signature`; scope is `identity,cluster,epoch,protocol`; a non-null chunk is -`generation,offset,hash`. Request order is `scope,node,incarnation,generation, -messages,signature,ack`. Signing replaces signature with JSON null and prepends -`codeaf/raft/envelope/1\n` or `codeaf/raft/exchange/1\n` respectively. No JSON object -key sorting or JS Number conversion is allowed. The committed independently -Node-signed `testdata/exchange-v1.json` is checked by Go and the targeted hosted -Node test. P08 can use that vector before implementing its host-side codec. +Envelope field order is +`scope,from,to,incarnation,destination,sequence,chunk,digest, +payload,signature`; scope is `identity,cluster,epoch,protocol`; a non-null chunk +is `generation,offset,hash`. Request order is +`scope,node,incarnation,generation, messages,signature,ack`. Signing replaces +signature with JSON null and prepends `codeaf/raft/envelope/1\n` or +`codeaf/raft/exchange/1\n` respectively. No JSON object key sorting or JS Number +conversion is allowed. The committed independently Node-signed +`testdata/exchange-v1.json` is checked by Go and the targeted hosted Node test. +Hosted codecs must preserve that same wire representation. ## Bounds, priority and retry @@ -116,35 +119,38 @@ seeded jitter and a 60-second ceiling. Buffers expire after 30 seconds of monoto sampled host time, lazily on activity, requiring no idle alarm. Raft retransmits; lease renewal and snapshots retain their own durable semantics. -Default operator admission is 1,024 scopes, 4,096 retained enrollment records per -scope, at most 32 live learners and six live voter IDs (room for both sides of a -three-voter joint change). `Limits` makes scope/retained-record/voter ceilings -explicit operator choices; exceeding one refuses growth. Larger topology policy -belongs to P12, not transport. Tombstones and application records are not deleted -to regain admission. `Stats` reports counts/bytes/socket usage only, never payloads. +Default operator admission is 1,024 scopes, 4,096 retained enrollment records +per scope, at most 32 live learners and six live voter IDs (room for both sides +of a three-voter joint change). `Limits` makes scope/retained-record/voter +ceilings explicit operator choices; exceeding one refuses growth. Larger +topology policy belongs to membership policy, not transport. Tombstones and +application records are not deleted to regain admission. `Stats` reports +counts/bytes/socket usage only, never payloads. -## Snapshots and next hosts +## Snapshot host responsibilities -A `MsgSnap` contains P05's small portable manifest pointer. Chunk envelopes carry +A `MsgSnap` contains a small portable manifest pointer. Chunk envelopes carry that manifest's generation ID, zero-based chunk number and SHA-256. The receiver -must verify generation, offset, expected length and manifest totals before P05's +must verify generation, offset, expected length and manifest totals before `BeginSnapshot/PutSnapshotChunk/CompleteSnapshot`. A queued chunk or completed stream is **not** installation: host success is reported through `raftcore.TransportResult{Snapshot:true,Success:true}` only after the actual snapshot Ready has installed durably. A timeout/interruption reports failure so Raft can retry. Do not feed `MsgSnap` into the core before its chunks are ready. -P08 owns those hosted actor/storage operations and native/wasm lifecycle proof. +Hosted actor/storage adapters own those operations and native/wasm lifecycle +verification. -P07's tests use real cores and three SQLite stores behind the actual relay route, -with only outbound member HTTP, intentional duplicate/reorder, create, ReadIndex, -leader death/restart, joint demotion and writes with an absent learner. Transport -unit tests pin auth, generation, socket revocation and all queue reservations. -P06/P06b's 2,000-seed simulator soak remains a required regression gate. Full -R10 enrollment/worker journeys, R11 churn/snapshot interoperability, application -revocation and the R15 full-state sentinel campaign remain later packet gates. +The transport tests use real cores and three SQLite stores behind the actual +relay route, with only outbound member HTTP, intentional duplicate/reorder, +create, ReadIndex, leader death/restart, joint demotion and writes with an +absent learner. Transport unit tests pin auth, generation, socket revocation and +all queue reservations. The 2,000-seed simulator soak remains a required +regression gate. Full enrollment/worker journeys, churn/snapshot +interoperability, application revocation and full-state sentinel campaigns +require separate verification. -P12's `CommittedView` derives protocol authority from the applied registry and +`CommittedView` derives protocol authority from the applied registry and current device revocations. Hold the actor lock while reading and installing that view, before acknowledging the command. Use the applied index as its revision, so a device revoke updates routing even without a membership edit. @@ -154,7 +160,7 @@ An authenticated source's three provisioned anchor keys may bootstrap protocol traffic before registry commitment. A personal key requires committed enrollment; never accept a caller's arbitrary view as an enrollment grant. -P15 adds `CommittedViewAt` for delegated worker learners. Commit current ingress +`CommittedViewAt` supports delegated worker learners. Commit current ingress time and supply the applied configuration/read-set under the actor lock, then Install before acknowledging the result. It checks the worker's explicit learner consent, exclusive certificate expiry, revocation, rotation and current process diff --git a/internal/relayserve/cluster_test.go b/internal/relayserve/cluster_test.go index a04211efe3..c04168aa4e 100644 --- a/internal/relayserve/cluster_test.go +++ b/internal/relayserve/cluster_test.go @@ -24,7 +24,7 @@ import ( // clusterFixture drives actual native stores and cores. The only listener is // the relay; members can only make outbound exchange requests. Application is -// P03's narrow real-rule probe, never the future authorized directory writer. +// the narrow real-rule probe, never the authorized directory writer. type clusterFixture struct { t *testing.T ctx context.Context diff --git a/internal/relayserve/raft_cutover_test.go b/internal/relayserve/raft_cutover_test.go index 1e05f71101..08abd95d72 100644 --- a/internal/relayserve/raft_cutover_test.go +++ b/internal/relayserve/raft_cutover_test.go @@ -18,8 +18,8 @@ import ( ) // R25/R30 pin the real persistent relay door, independently of an adapter or -// fixture backend. This gate must stay red until the sole production route -// actually commits requests through the migrated namespace. +// fixture backend. The sole production route must commit requests through +// the migrated namespace. func TestR25R30ProductionRelayRefusesLegacyBlindForce(t *testing.T) { raw, err := os.ReadFile("../raftdirectory/testdata/v1-requests.json") if err != nil { @@ -69,7 +69,7 @@ func TestR25R30ProductionRelayRefusesLegacyBlindForce(t *testing.T) { } } // A legacy writer could be taught only the blind-force refusal. The durable - // source ownership receipt independently requires the actual P11 cutover. + // source ownership receipt independently requires the actual migration cutover. root, err := base64.RawURLEncoding.DecodeString(f.Cases[0].Headers[reqsign.HeaderIdentity]) if err != nil { t.Fatal(err) diff --git a/relay/hosted/README.md b/relay/hosted/README.md index a841da2da7..b20838461d 100644 --- a/relay/hosted/README.md +++ b/relay/hosted/README.md @@ -29,11 +29,11 @@ it, checks the device is not revoked, spends the caller's rate budget, and serve puts in flight (`src/flight.js`, a port of `internal/blobstore/flight.go`) and fails with 503 after 30 s rather than say no. A caller with no cert of the identity cannot hold a put open. - Directory and authority decisions live in three separately addressed `RaftMemberDO` SQLite - stores. `src/raft/hosted.js` freezes the original source on first use, streams P11's same + stores. `src/raft/hosted.js` freezes the original source on first use, streams the same genesis to A/B/C and publishes ownership only after a fresh quorum read. Commands use the original v1 proof and the shared Go state machine; JavaScript has no authoritative lease writer. `src/directory.js` and its old rules remain only for immutable source conversion and - legacy fixture tests. Source backups are retained until explicit owner cleanup or retirement. + legacy fixture tests. Source backups are retained until the identity is retired. An absent quorum returns retryable 503 and never reopens the old writer. - Frames live in R2 at `/f/` (`src/store.js`), each validated (magics, header, `cell_key_id`) before it is stored. The index that finds an object inside its frame is rows in the diff --git a/relay/hosted/src/raft/README.md b/relay/hosted/src/raft/README.md index e67e77204e..fefbcaa64b 100644 --- a/relay/hosted/src/raft/README.md +++ b/relay/hosted/src/raft/README.md @@ -4,8 +4,8 @@ `src/raft-worker.js` statically imports the generated `core.wasm` and installs one shared reactor per isolate. Wrangler's local build command compiles that artifact from the pinned modified Go Raft source. The bindings add independent -SQL objects; they do not route the legacy identity directory to a new writer. -P09 owns the application adapter and P11 the irrevocable migration gate. +SQL objects. The application adapter and irrevocable migration gate route +identity coordination to those members. The internal namespace provisions an immutable cluster/node/bootstrap/seed configuration before bootstrap writes. All anchors use the same initial voter @@ -21,14 +21,15 @@ reply or outbound dependent message. Failures stop participation and preserve the database. Initialization/recovery is serialized too. Recovery starts as a follower and never serves an authority read from remembered status. -Application writes and index/configuration/version/time commit atomically. -The evaluator is synchronous with an immutable applied view and bounded 1 MiB +Application writes and index/configuration/version/time commit atomically. The +evaluator is synchronous with an immutable applied view and bounded 1 MiB read/write sets. It must implement committed policy, declared dependencies and request/session semantics; storage supplies atomicity, not that policy. Rows cover state, sessions, membership, enrollment, tombstones and outboxes. The persisted application ceiling defaults to 8 MiB, including framed keys/values; -an explicit larger imported ceiling is preserved. Pressure refuses growth, -never deletes application rows. P11 owns actual inventory/grandfathering. +an explicit larger imported ceiling is preserved. Pressure refuses growth, never +deletes application rows. The migration host inventories and grandfathers +imported rows. Snapshots use `internal/raftsnapshot` format 1 without a second JS schema. Rows stream in SQL binary order into 256 KiB chunks. The shared Go codec @@ -41,23 +42,25 @@ stream signed chunks, then retry MsgSnap. Only its durable Ready install may return `installed: true`. A transport sender must report failure/timeout to its core if delivery is interrupted. The receiver can resume after eviction. -Every peer envelope is the exact P07 wire/signature domain. Current committed -source/destination incarnations and revocation are checked after cryptographic -awaits and again inside the actor queue. Queues reserve voter/learner/snapshot -slots (4/2/2 per member) and encoded bytes across an identity (512/128/384 KiB). -They do not coalesce votes, appends or read barriers. There is no durable cursor -in this direct internal delivery path: a lost call is failed delivery and Raft -retransmits. P12 supplies durable enrollment; public outbound exchange/router -assembly reuses P07's authenticated generation-fenced bridge. +Every peer envelope uses the exact shared transport wire/signature domain. +Current committed source/destination incarnations and revocation are checked +after cryptographic awaits and again inside the actor queue. Queues reserve +voter/learner/snapshot slots (4/2/2 per member) and encoded bytes across an +identity (512/128/384 KiB). They do not coalesce votes, appends or read +barriers. There is no durable cursor in this direct internal delivery path: a +lost call is failed delivery and Raft retransmits. Durable enrollment supplies +peer authority; public outbound exchange/router assembly reuses the +authenticated generation-fenced bridge. Each active handle reserves 6 MiB before creation; total reservations plus -observed reactor memory must fit 64 MiB. Retained core log cache is capped at -2 MiB, with lossless checkpoints at 1 MiB or 1,024 applied entries. This earlier -local checkpoint keeps the cache below the shared 4 MiB retained-log ceiling. -A leader is never evicted to meet admission. Only an explicitly quiescent, -nonleader, drained actor may close; the P14 scheduler coordinates quiescence/wake -and alarm scheduling. `drainRecovery` exposes recovered dependent effects for -an internal router to deliver rather than silently dropping them. +observed reactor memory must fit 64 MiB. Retained core log cache is capped at 2 +MiB, with lossless checkpoints at 1 MiB or 1,024 applied entries. This earlier +local checkpoint keeps the cache below the shared 4 MiB retained-log ceiling. A +leader is never evicted to meet admission. Only an explicitly quiescent, +nonleader, drained actor may close; the shared scheduler coordinates +quiescence/wake and alarm scheduling. `drainRecovery` exposes recovered +dependent effects for an internal router to deliver rather than silently +dropping them. Run the bounded local proof with `sh relay/hosted/test/raft/actors-run.sh /tmp/codeaf-p08`. It exercises real workerd actors and native SQLite peers, @@ -67,7 +70,7 @@ Unit tests cover durability waits, queue bounds, all snapshot namespaces, partial/corrupt generations, revocation during an await and native wire bytes. This is internal host proof, not the final client journeys or a deployment. -P11 adds `LegacyAdmission` around full legacy IdentityDO requests, so freeze +`LegacyAdmission` wraps full legacy IdentityDO requests, so freeze waits for admitted pairing/blob awaits. `Tenant.migration` supplies a `LegacySource`: freeze once, inventory original SQL rows through Go ABI op 12, then stream one converted row at a time. Sampled watch renewals are stored apart @@ -83,17 +86,17 @@ checkpoint, then removes only redundant staging rows and clears the pending marker durably. The stored ceiling includes all grandfathered rows. One import request per member can run at once, preventing page buffers stacking in memory. After fresh quorum readback, `LegacySource.own` publishes `raft-owned`; `route` -then resolves only that UUID. There is no rollback to a legacy writer. P17 -assembles the complete facade, receipt ingress and existing-client doors; P14 -owns continuing watch renewal/scheduling. Source backup cleanup is explicit. - - -P12 adds membership-aware evaluation (ABI op 15), paged peer derivation (16), -core reconciliation (17), local learner catch-up (18) and provisioned-anchor -manifest validation (19). Original ABI operations remain unchanged. Native and -hosted application adapters supply the entry index and its exact configuration -as trusted host metadata. Operation 20 replays each committed configuration -entry with the pinned library changer, so batching cannot alter policy. No request may supply those fields. +then resolves only that UUID. There is no rollback to a legacy writer. The host +assembles the complete facade, receipt ingress, existing-client doors and +continuing watch renewal/scheduling. Source backup cleanup is explicit. + + Membership uses evaluation (ABI op 15), paged peer derivation (16), core +reconciliation (17), local learner catch-up (18) and provisioned-anchor manifest +validation (19). Original ABI operations remain unchanged. Native and hosted +application adapters supply the entry index and its exact configuration as +trusted host metadata. Operation 20 replays each committed configuration entry +with the pinned library changer, so batching cannot alter policy. No request may +supply those fields. Provision `config.anchors` with the same three public protocol records and IDs used by the source manifest/InitMembership. Only these anchors can bootstrap @@ -106,8 +109,9 @@ reports whether a signed settlement may be proposed. It never synthesizes that settlement signature or declares a join successful. `joinStatus(barrier)` additionally checks current authorization and the learner's own applied index. -P17 still assembles authenticated client enrollment, registry publication and -quiet-mode guards. P13 owns voter swaps; P15 owns delegated worker certificates. +These are internal membership methods. Client enrollment, registry publication, +quiet-mode guards, voter swaps and delegated worker certificates each need their +own host door before a client can use them. Do not advertise a public leader/worker command from these internal methods. Provisioned anchor keys receive a restricted committed authority that can only @@ -138,7 +142,7 @@ The DO alarm handler retains at most one bounded durable transport batch, with a monotonic batch ID. `scheduled` rereads that same batch after a lost reply; `acknowledgeScheduled` removes only the matching ID. A stalled router backpressures further turns with bounded retry delay. Draining this batch is transport delivery, -never append or application success. The sole P17 router must consume those +never append or application success. The sole namespace router must consume those original peer bytes through the authenticated exchange bridge, propagate wake to current voters, and supply queued assignment/outbox activity conservatively. No host listener, provider service or alternate directory writer is added here. @@ -146,13 +150,13 @@ No host listener, provider service or alternate directory writer is added here. ## Hosted public assembly `HostedIdentity` now runs at every authenticated public identity request. The -source object keeps blob/index/quota/socket admission and its irreversible P11 -gate; all directory, authority, rotation and blob authorization goes through -three separately addressed `RaftMemberDO` actors. Imports advance bounded pages, -reuse source UUID/keys/ceiling and require an actual quorum ReadIndex before -ownership publication. After publication, only stores with durable matching -provenance can participate. A missing quorum refuses; it never reimports a lost -voter under its former ID or reopens the source writer. +source object keeps blob/index/quota/socket admission and its irreversible +migration gate; all directory, authority, rotation and blob authorization goes +through three separately addressed `RaftMemberDO` actors. Imports advance +bounded pages, reuse source UUID/keys/ceiling and require an actual quorum +ReadIndex before ownership publication. After publication, only stores with +durable matching provenance can participate. A missing quorum refuses; it never +reimports a lost voter under its former ID or reopens the source writer. ABI 30 prepares restricted effects using the source's protocol seed (never a command field), 31 projects typed canonical rows to legacy numeric JSON, and 32 @@ -164,6 +168,7 @@ normalized before original-proof comparison, preserving replacement semantics. Public source serialization spans coordination, while external blob I/O leaves that queue and passes a new authority barrier afterward. Public watch sockets carry presence without the legacy socket-vouch promise; v1 clients keep ordinary -beats. P17C retains personal/worker transport, leadership maintenance and the full -committed-hold/quiescence assembly. Explicit retirement has its own retained +beats. The hosted relay does not offer personal-member learners or leadership +requests yet; its committed holds and idle maintenance follow the shared Go +policy. Explicit retirement has its own retained all-replica erasure journal and never serves as pressure cleanup. diff --git a/relay/hosted/src/raft/actor.js b/relay/hosted/src/raft/actor.js index 0afe3f2364..f36331b01a 100644 --- a/relay/hosted/src/raft/actor.js +++ b/relay/hosted/src/raft/actor.js @@ -11,7 +11,7 @@ const active = new Set(); // A compiled module and runtime are shared by every object in this isolate. // Reservations precede allocation; pressure refuses activation, never evicts a -// serving leader. Quiescent eviction is an explicit later scheduler decision. +// serving leader. Quiescent eviction requires an explicit scheduler decision. export function installRaftModule(compiled) { if (module && module !== compiled) throw new Error('raft member: core module already installed'); module = compiled; diff --git a/relay/hosted/src/raft/member.js b/relay/hosted/src/raft/member.js index 056d2c239c..0bb8dc3756 100644 --- a/relay/hosted/src/raft/member.js +++ b/relay/hosted/src/raft/member.js @@ -15,7 +15,7 @@ export { installRaftModule, runtimeMetrics } from './actor.js'; // Only internal namespace calls can initialize a provisioned member. The // public Worker routes identity coordination through its authenticated source. -// Enrollment is derived from the durable Go registry; P09 supplies evaluation. +// Enrollment and evaluation use the durable Go registry and shared Go policy. export class RaftMemberDO extends DurableObject { constructor(ctx, env) { super(ctx, env); this.actor = null; this.loading = null; this.ctx = ctx; this.env = env; } evaluator(config) { diff --git a/relay/hosted/src/tenant.js b/relay/hosted/src/tenant.js index 4d9ea4ca48..f527b8cf73 100644 --- a/relay/hosted/src/tenant.js +++ b/relay/hosted/src/tenant.js @@ -160,6 +160,6 @@ export class Tenant { // This facade is assembled only for a Raft-owned namespace after migration. // Keeping it separate prevents constructing or invoking the legacy SQL writer -// while the integration harness exercises the authoritative application path. +// on the authoritative application path. export { RaftDirectoryFacade } from './raft/directory.js'; export { RaftEffectsFacade, RaftBlobTenant } from './raft/effects.js'; diff --git a/relay/hosted/test/raft/directory.test.js b/relay/hosted/test/raft/directory.test.js index cf78488a60..39b1484ac7 100644 --- a/relay/hosted/test/raft/directory.test.js +++ b/relay/hosted/test/raft/directory.test.js @@ -57,7 +57,7 @@ test('R1 R2 R4 R14 R16 R17 R21 hosted facade uses committed Go policy and native try { for (let i = 0; i < 3; i++) { const host = new Host(); hostStores.push(host); - // P11 owns genesis import. This test-only genesis is supplied at the + // This test-only genesis is supplied at the // first committed application entry, identically on all three stores. const evaluator = (raw, read, app) => { let base = []; diff --git a/relay/hosted/test/raft/smoke.test.js b/relay/hosted/test/raft/smoke.test.js index cf95b414e9..31f00446ba 100644 --- a/relay/hosted/test/raft/smoke.test.js +++ b/relay/hosted/test/raft/smoke.test.js @@ -47,7 +47,7 @@ class Native { } // Protobuf inspection is transport/storage framing only. Application meaning -// is evaluated by Go, and the independent consensus checker remains P06. +// is evaluated by Go; raftsim supplies the independent consensus checker. function fields(bytes) { let at = 0; const result = new Map(); const vint = () => { let n=0n, shift=0n; for(let i=0;i<10;i++){assert(at Date: Sat, 24 Oct 2026 20:28:19 -0400 Subject: [PATCH 062/103] Let joined personal computers coordinate the native cluster safely. Forward original application requests over the enrolled member connection while peer persistence stays live. Complete explicit joint leadership transfer with fresh receipts, retain saved requests across retries, and restore the anchors after personal departure. Preserve quorum safety, bounded snapshot flow and quiet idle homes, and document the native capability with production lifecycle proofs. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/codeaf/cluster.go | 7 +- cmd/codeaf/syncdrive_test.go | 7 +- internal/clusterctl/control.go | 7 +- internal/clusterctl/forward.go | 61 +++ internal/clusterctl/forward_test.go | 37 ++ .../continuing-a-chat-on-another-computer.md | 2 +- .../manual/chat/coordination-and-workers.md | 23 +- internal/manual/coordination_test.go | 4 +- internal/raftdirectory/control.go | 3 +- internal/raftdirectory/forward.go | 297 ++++++++++++ .../raftdirectory/forward_internal_test.go | 59 +++ internal/raftdirectory/ingress.go | 5 +- internal/raftdirectory/maintenance.go | 124 ++++- internal/raftdirectory/native.go | 232 ++++++++-- internal/raftdirectory/personal.go | 27 +- internal/raftdirectory/personal_lead.go | 176 +++++++ internal/raftdirectory/snapshot_receive.go | 70 +++ internal/raftdirectory/transport.go | 47 +- internal/rafttransport/README.md | 73 ++- internal/rafttransport/backpressure_test.go | 161 +++++++ internal/rafttransport/envelope.go | 21 +- internal/rafttransport/exchange.go | 16 +- internal/rafttransport/router.go | 37 ++ internal/relayserve/control.go | 9 +- internal/relayserve/personal.go | 12 +- internal/relayserve/service.go | 1 + internal/syncsetup/cluster.go | 22 +- internal/syncsetup/leadership_ingress_test.go | 436 +++++++++++++++++ internal/syncsetup/leadership_native_test.go | 381 +++++++++++++++ internal/syncsetup/leadership_process_test.go | 132 ++++++ internal/syncsetup/member.go | 217 ++++++--- internal/syncsetup/member_client_test.go | 27 +- internal/syncsetup/member_forward.go | 437 ++++++++++++++++++ internal/syncsetup/member_snapshot_send.go | 108 +++++ 34 files changed, 3111 insertions(+), 167 deletions(-) create mode 100644 internal/clusterctl/forward.go create mode 100644 internal/clusterctl/forward_test.go create mode 100644 internal/raftdirectory/forward.go create mode 100644 internal/raftdirectory/forward_internal_test.go create mode 100644 internal/raftdirectory/personal_lead.go create mode 100644 internal/raftdirectory/snapshot_receive.go create mode 100644 internal/rafttransport/backpressure_test.go create mode 100644 internal/syncsetup/leadership_ingress_test.go create mode 100644 internal/syncsetup/leadership_native_test.go create mode 100644 internal/syncsetup/leadership_process_test.go create mode 100644 internal/syncsetup/member_forward.go create mode 100644 internal/syncsetup/member_snapshot_send.go diff --git a/cmd/codeaf/cluster.go b/cmd/codeaf/cluster.go index 590ca0437a..aa2b04fe52 100644 --- a/cmd/codeaf/cluster.go +++ b/cmd/codeaf/cluster.go @@ -10,14 +10,13 @@ import ( "github.com/Agent-Field/codeaf/internal/syncsetup" "io" "os" - "time" ) const clusterHelp = `codeaf cluster status Show this joined computer, learners, voters and the current coordinator. codeaf cluster lead - Leadership is not offered yet; your saved chats are still here. - A paired home on a self-hosted relay with storage joins as a learner. + Ask this joined computer to coordinate through a self-hosted relay. + Persistent storage is required; hosted leadership is not available yet. Three voters tolerate one unavailable member. ` @@ -44,7 +43,7 @@ func runCluster(args []string) error { if !ok { return errors.New("sync is off on this computer") } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), clusterctl.ForwardTimeout) defer cancel() return clusterCommand(ctx, s, args[0], os.Stdout) } diff --git a/cmd/codeaf/syncdrive_test.go b/cmd/codeaf/syncdrive_test.go index 177686f3f8..3a437a2c28 100644 --- a/cmd/codeaf/syncdrive_test.go +++ b/cmd/codeaf/syncdrive_test.go @@ -365,7 +365,12 @@ func TestAChatTakenAndTakenBackGetsADriveSideOfItsOwn(t *testing.T) { if err != nil || !ok { t.Fatalf("Open(%s) = %v, %v", dir, ok, err) } - if _, err := s.Dir.Acquire(context.Background(), c.ID, directory.AcquireOpts{Force: true}); err != nil { + // R25/C7 require the tuple the person observed; blind force is refused. + view, err := s.Dir.Cell(context.Background(), c.ID) + if err != nil { + t.Fatal(err) + } + if _, err := s.Dir.Acquire(context.Background(), c.ID, directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: view.Cell.Lease.Device, Fence: view.Cell.Lease.Fence}}); err != nil { t.Fatal(err) } } diff --git a/internal/clusterctl/control.go b/internal/clusterctl/control.go index 0c09c2f63b..6dd547dcf1 100644 --- a/internal/clusterctl/control.go +++ b/internal/clusterctl/control.go @@ -11,7 +11,6 @@ import ( "io" "net/http" "strings" - "time" ) const Path = "/v2/cluster/control" @@ -129,12 +128,12 @@ func (c Client) request(ctx context.Context, method string, body []byte, out any return wireauth.ErrUnauthorized } c.Sign(req, body) - hc := http.Client{Timeout: 30 * time.Second} + hc := http.Client{Timeout: ForwardTimeout} if c.HTTP != nil { hc = *c.HTTP } - if hc.Timeout <= 0 || hc.Timeout > 30*time.Second { - hc.Timeout = 30 * time.Second + if hc.Timeout <= 0 || hc.Timeout > ForwardTimeout { + hc.Timeout = ForwardTimeout } hc.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } res, err := hc.Do(req) diff --git a/internal/clusterctl/forward.go b/internal/clusterctl/forward.go new file mode 100644 index 0000000000..a4957b1ef0 --- /dev/null +++ b/internal/clusterctl/forward.go @@ -0,0 +1,61 @@ +package clusterctl + +import ( + "crypto/ed25519" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/rafttransport" +) + +const ForwardPath = "/v2/cluster/forward" +const ForwardTimeout = 30 * time.Second + +// Forward is application ingress on a SEPARATE outbound connection. Original +// commands never become peer envelopes or untrusted Raft proposal messages. +// Each direction signs its complete scope, request ID and body independently. +type Forward struct { + Scope rafttransport.Scope `json:"scope"` + Generation uint64 `json:"generation,string"` + From uint64 `json:"from,string"` + To uint64 `json:"to,string"` + ID string `json:"id"` + Action string `json:"action"` + Command []byte `json:"command,omitempty"` + Revision uint64 `json:"revision,omitempty"` + Barrier uint64 `json:"barrier,omitempty"` + Schedule *raftcore.ScheduleInput `json:"schedule,omitempty"` + Now int64 `json:"now,omitempty"` + Result *cs.Result `json:"result,omitempty"` + Evidence *raftcore.LeadershipEvidence `json:"evidence,omitempty"` + Receipt *cs.LeadershipReceipt `json:"receipt,omitempty"` + Plan *raftcore.SchedulePlan `json:"plan,omitempty"` + Membership *raftcore.MembershipPlan `json:"membership,omitempty"` + SleepAck *raftcore.SleepAck `json:"sleep_ack,omitempty"` + Status ForwardStatus `json:"status"` + Applied uint64 `json:"applied,omitempty"` + Error string `json:"error,omitempty"` + Signature []byte `json:"signature"` +} + +func (f Forward) bytes() []byte { + f.Signature = nil + b, _ := rafttransport.Encode(f) + return append([]byte("codeaf/cluster/forward/1\n"), b...) +} +func (f *Forward) Sign(sign func([]byte) []byte) { f.Signature = sign(f.bytes()) } +func (f Forward) Verify(key ed25519.PublicKey) bool { + return len(key) == ed25519.PublicKeySize && f.ID != "" && len(f.ID) <= raftcore.MaxRequestIDBytes && ed25519.Verify(key, f.bytes(), f.Signature) +} + +// ForwardStatus contains only routing hints; quorum replies carry their own barrier. +type ForwardStatus struct { + Leader uint64 + Term uint64 + Applied uint64 +} + +func PeerStatus(s raftcore.Status) ForwardStatus { + return ForwardStatus{Leader: s.Leader, Term: s.Term, Applied: s.Applied} +} diff --git a/internal/clusterctl/forward_test.go b/internal/clusterctl/forward_test.go new file mode 100644 index 0000000000..16feafabb0 --- /dev/null +++ b/internal/clusterctl/forward_test.go @@ -0,0 +1,37 @@ +package clusterctl_test + +import ( + "crypto/ed25519" + "testing" + + ctl "github.com/Agent-Field/codeaf/internal/clusterctl" + "github.com/Agent-Field/codeaf/internal/rafttransport" +) + +// R6/R15 bind original application ingress and its fresh read ID to a single +// enrolled scope and direction, independently of the peer-envelope domain. +func TestR6R15ForwardSignaturesBindOriginalRequestAndScope(t *testing.T) { + key := ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize)) + sign := func(b []byte) []byte { return ed25519.Sign(key, b) } + frame := ctl.Forward{Scope: rafttransport.Scope{Identity: "identity", Cluster: "cluster", Epoch: 1, Protocol: 1}, Generation: 2, From: 1, To: 4, ID: "read-17", Action: "submit", Command: []byte("original signed command")} + frame.Sign(sign) + raw, err := rafttransport.Encode(frame) + if err != nil { + t.Fatal(err) + } + var copy ctl.Forward + if rafttransport.Decode(raw, ©) != nil || !copy.Verify(key.Public().(ed25519.PublicKey)) { + t.Fatal("R6 original frame did not round trip") + } + mutations := []func(*ctl.Forward){func(f *ctl.Forward) { f.ID = "read-18" }, func(f *ctl.Forward) { f.Generation++ }, func(f *ctl.Forward) { f.Scope.Cluster = "other" }, func(f *ctl.Forward) { f.From, f.To = f.To, f.From }, func(f *ctl.Forward) { f.Action = "read" }, func(f *ctl.Forward) { f.Command = []byte("replacement command") }, func(f *ctl.Forward) { f.Barrier = 99 }} + for i, mutate := range mutations { + changed := frame + mutate(&changed) + if changed.Verify(key.Public().(ed25519.PublicKey)) { + t.Fatalf("R6 substitution %d kept authority", i) + } + } + if ed25519.Verify(key.Public().(ed25519.PublicKey), append([]byte("codeaf/raft/exchange/1\n"), raw...), frame.Signature) { + t.Fatal("R15 application signature authorized a peer request") + } +} diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index 5f4e60b299..d6d55e839f 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -251,7 +251,7 @@ chat first; this does not discard the work saved on your computer. ## Which computer coordinates: cluster status and cluster lead -On a self-hosted service with persistent storage, an open paired home joins as a learner. Three persistent voters keep coordination available when your laptops sleep. `codeaf cluster status` shows this computer after joining. Leadership is not offered yet; continue chats through the ordinary continue door. See the coordination and workers page for command refusals and setup details. +On a self-hosted service with persistent storage, an open paired home joins as a learner. Three persistent voters keep coordination available when your laptops sleep. `codeaf cluster status` shows this computer after joining. Continue chats through the ordinary continue door. See the coordination and workers page for command refusals and setup details. ## Can a worker access all my chats or approve pairing? diff --git a/internal/manual/chat/coordination-and-workers.md b/internal/manual/chat/coordination-and-workers.md index beaa403cdc..76824040c5 100644 --- a/internal/manual/chat/coordination-and-workers.md +++ b/internal/manual/chat/coordination-and-workers.md @@ -7,7 +7,7 @@ A relay directory upgrade can briefly pause pairing and chat moves. Retry after ## Joined computers, learner capacity and an outdated build -An open, paired home with sync on joins a self-hosted `codeaf relay --store ` as a durable learner. Its store lives under the home, and reopening reuses its node. Losing that store enrolls a fresh node and retires the old authority. The three anchors remain the only voters, so neither laptop needs to stay awake. Joining starts while the paired home is open, or when you ask for `codeaf cluster status`. +An open, paired home with sync on joins a self-hosted `codeaf relay --store ` as a durable learner. Its store lives under the home, and reopening reuses its node. Losing that store enrolls a fresh node and retires the old authority. Three anchors vote until a personal computer is explicitly asked to lead, so pairing alone never requires either laptop to stay awake. Joining starts while the paired home is open, or when you ask for `codeaf cluster status`. If joining has no new data to apply, the client keeps its store for the next retry round instead of polling continuously. A status activation can say `this computer is still joining the cluster; retry shortly; your saved chats are still here`. @@ -15,14 +15,31 @@ The membership protocol admits at most 32 live learners and retains retired reco ## Which computer coordinates: cluster status and cluster lead -`codeaf cluster status` on a self-hosted `codeaf relay` with storage reports the committed voters and current leader after a fresh majority read. A paired computer joins as a learner with persistent storage before status shows its node. Readiness requires its own durable application through the advertised join position. Status names `this computer`, `learning computers` and `ready computers` when known, alongside `voting computers` and the coordinator. `codeaf cluster lead` on an unjoined computer says `this computer must join the cluster with persistent storage before it can lead`. +`codeaf cluster status` on a self-hosted `codeaf relay` with storage reports the committed voters and current leader after a fresh majority read. A paired computer joins with persistent storage as a learner before status shows its node. Readiness requires its own durable application through the advertised join position. Status names `this computer`, `learning computers`, `ready computers` and `voting computers` when known. `codeaf cluster lead` on an unjoined computer says `this computer must join the cluster with persistent storage before it can lead`. -Leadership is not offered yet on any relay. A direct leadership request to the self-hosted relay is refused without changing membership. The hosted relay has no cluster control door and says `this relay does not offer cluster leadership yet; your saved chats are still here`. Ordinary chat sync remains available. +On a self-hosted `codeaf relay` with storage, run `codeaf cluster lead` on the joined computer that should coordinate. It first catches up, replaces the third anchor’s vote through a joint change, durably finishes that change with the third anchor still learning, and transfers leadership. The voters are then two anchors and this computer. Keep codeaf open to keep its member running; a command used with no open terminal runs its member only for that command. Leadership moves coordination metadata, never chat content or chat ownership. + +The hosted relay has no cluster control door and says `this relay does not offer cluster leadership yet; your saved chats are still here`. Ordinary hosted chat sync remains available. Do not treat pairing or a saved leadership request as a successful transfer. Success requires the selected computer to be elected, commit its current term and pass a fresh majority read; its receipt names the computer, term, revision and read barrier. Retrying `codeaf cluster lead` preserves an already saved signed request. Continue chats through the ordinary continue door. Three voters tolerate one unavailable member. Losing a majority pauses coordination; saved chats remain. The refusal is `this cluster cannot coordinate right now; reconnect enough of its voting computers and try again; your saved chats are still here`. +## Pending leadership request, retry and refusal + +A lost reply or restarted home keeps the original signed leadership request. Retry `codeaf cluster lead` to resume it; a pending request cannot choose a different computer. A completed refusal says `the leadership request could not complete; refresh cluster status and try again` and is acknowledged before another request is saved. Only one configuration change can run at a time. A different personal coordinator cannot directly replace an active personal coordinator; let the anchors resume, then request leadership again. + +## Sleeping coordinator, fallback timing and no automatic regain + +When a personal coordinator sleeps, exits or turns sync off, the two remaining anchor voters can elect a coordinator and restore the third anchor after it catches up. The missing computer need not approve its removal. A returning computer catches up as a learner and does not automatically regain leadership: request `codeaf cluster lead` again. + +Coordination uses five-second ticks. The provisioned anchors’ ranked election windows are six–seven, eight–nine and ten–eleven ticks (30–35, 40–45 and 50–55 seconds); other computers use twelve–thirteen ticks (60–65 seconds). Healthy fallback targets about 65 seconds. Scheduling delays or a partition can take longer. Fully idle groups stop periodic ticks only after every voter has applied the same idle barrier; a new operation or a leading computer’s disconnection wakes coordination and needs a fresh majority response. + +## Coordinator and anchor both unavailable, lost majority + +While a personal computer leads, the voters are that computer and two anchors. Losing that computer and either voting anchor leaves no majority, even though the third anchor retains its learning copy. The response is `this cluster cannot coordinate right now; reconnect enough of its voting computers and try again; your saved chats are still here`. codeaf preserves the existing cluster and saved work; it cannot make a writable minority or remove voters to manufacture a majority. + + ## Does an open connection keep my chat running on this computer? The sync connection carries presence hints and committed lease acknowledgments. An open socket or an answered ping alone cannot keep ownership of a chat. codeaf skips an ordinary heartbeat only while it has a recent acknowledgment for that exact chat and ownership number, with enough time left before expiry. Without that acknowledgment it keeps sending ordinary heartbeats. A delayed acknowledgment for an earlier owner cannot take the chat back, and a delayed renewal cannot undo a clean quit. diff --git a/internal/manual/coordination_test.go b/internal/manual/coordination_test.go index a796e6f7c4..a50c2a88df 100644 --- a/internal/manual/coordination_test.go +++ b/internal/manual/coordination_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -// R9/R23/R27 keep the moved coordination material reachable while device pages +// R6/R9/R23/R27 keep the moved coordination material reachable while device pages // keep their own vocabulary and the existing retrieval probes stay intact. func TestR9R23R27LearnerAndIdleAnswersRemainReachable(t *testing.T) { for _, q := range []string{ @@ -25,7 +25,7 @@ func TestR9R23R27LearnerAndIdleAnswersRemainReachable(t *testing.T) { } } page, _ := Chat().Page("coordination-and-workers") - for _, claim := range []string{"durable learner", "The three anchors remain the only voters", "Leadership is not offered yet on any relay", "hosted relay has no learner door"} { + for _, claim := range []string{"durable learner", "Three anchors vote until a personal computer is explicitly asked to lead", "run `codeaf cluster lead`", "does not automatically regain leadership", "hosted relay has no learner door"} { if !strings.Contains(page, claim) { t.Errorf("R9 missing current capability: %q", claim) } diff --git a/internal/raftdirectory/control.go b/internal/raftdirectory/control.go index 10376d54eb..c738703bec 100644 --- a/internal/raftdirectory/control.go +++ b/internal/raftdirectory/control.go @@ -53,7 +53,7 @@ func (g *Native) PersonalStatus(ctx context.Context, actor, certificate string) if err != nil || membership.Revision == 0 { return cs.ErrDependency } - out = clusterctl.Status{Cluster: g.manifest.Config.Cluster, Leader: g.status[g.leader()].NodeID, Membership: membership, Configuration: conf, LeadershipUnavailable: true} + out = clusterctl.Status{Cluster: g.manifest.Config.Cluster, Leader: g.leaderID(), Membership: membership, Configuration: conf, LeadershipUnavailable: true} if g.peers != nil { for _, id := range conf.Learners { raw, err := read(cs.MemberKey(id)) @@ -91,6 +91,7 @@ func (g *Native) PersonalStatus(ctx context.Context, actor, certificate string) if !member.Retired { out.Self = member.Node + out.LeadershipUnavailable = false } return nil }) diff --git a/internal/raftdirectory/forward.go b/internal/raftdirectory/forward.go new file mode 100644 index 0000000000..7695920b1b --- /dev/null +++ b/internal/raftdirectory/forward.go @@ -0,0 +1,297 @@ +package raftdirectory + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "fmt" + "net/http" + + "github.com/Agent-Field/codeaf/internal/clusterctl" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/rafttransport" + "github.com/coder/websocket" +) + +type forwardLink struct { + send chan clusterctl.Forward + pending map[string]chan clusterctl.Forward + conn *websocket.Conn +} + +func (g *Native) signal() { + if g.changed != nil { + close(g.changed) + } + g.changed = make(chan struct{}) +} + +// waitProgress releases ONLY the member queue. Public callers retain opMu, +// while authenticated peer callbacks finish persistence and application. +func (g *Native) waitProgress(ctx context.Context) error { + if g.changed == nil { + g.changed = make(chan struct{}) + } + changed := g.changed + g.mu.Unlock() + select { + case <-ctx.Done(): + g.mu.Lock() + return directory.ErrUnreachable + case <-changed: + } + g.mu.Lock() + if g.closed || g.failed != nil { + return directory.ErrUnreachable + } + return nil +} +func (g *Native) replica() int { + if i := g.leader(); i >= 0 { + return i + } + best := -1 + for i, s := range g.status { + if g.nodes[i] != nil && (best < 0 || s.Applied > g.status[best].Applied) { + best = i + } + } + return best +} +func (g *Native) leaderID() uint64 { + if i := g.leader(); i >= 0 { + return g.status[i].NodeID + } + i := g.replica() + if i < 0 { + return 0 + } + return g.status[i].Leader +} +func (g *Native) forward(ctx context.Context, node uint64, work clusterctl.Forward) (clusterctl.Forward, error) { + var empty clusterctl.Forward + if work.ID == "" { + g.serial++ + work.ID = fmt.Sprintf("forward-%d", g.serial) + } + empty.ID = work.ID + link := g.forwarders[node] + if link == nil || len(link.pending) >= rafttransport.MaxInflight { + return empty, directory.ErrUnreachable + } + + work.Scope = g.peers.view.Scope + work.Generation = g.peers.view.Generation + work.From = g.manifest.Config.Anchors[0] + work.To = node + work.Sign(func(b []byte) []byte { return ed25519.Sign(g.peers.keys[0], b) }) + result := make(chan clusterctl.Forward, 1) + link.pending[work.ID] = result + defer delete(link.pending, work.ID) + select { + case link.send <- work: + default: + return empty, directory.ErrUnreachable + } + g.mu.Unlock() + select { + case empty = <-result: + case <-ctx.Done(): + } + g.mu.Lock() + if ctx.Err() != nil { + return empty, directory.ErrUnreachable + } + if empty.ID != work.ID || empty.Error != "" { + return empty, directory.ErrUnreachable + } + return empty, nil +} + +// ForwardHTTP authorizes every frame against CURRENT committed enrollment. +// The socket belongs to the computer dialing out; the relay never dials P. +func (g *Native) ForwardHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.RawQuery != "" { + http.Error(w, "invalid member", http.StatusBadRequest) + return + } + raw, err := base64.RawURLEncoding.DecodeString(r.Header.Get(rafttransport.SocketAuthHeader)) + var auth rafttransport.Request + if err != nil || len(raw) > rafttransport.MaxExchangeBytes || rafttransport.Decode(raw, &auth) != nil || len(auth.Messages) != 0 { + http.Error(w, "invalid member", 403) + return + } + g.mu.Lock() + valid := g.forwardAuthorized(auth) + if valid && g.forwarders[auth.Node] != nil { + valid = false + } + g.mu.Unlock() + if !valid { + http.Error(w, "member authorization refused", 403) + return + } + conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{CompressionMode: websocket.CompressionDisabled}) + if err != nil { + return + } + defer conn.CloseNow() + conn.SetReadLimit(rafttransport.MaxExchangeBytes) + link := &forwardLink{send: make(chan clusterctl.Forward, rafttransport.MaxInflight), pending: map[string]chan clusterctl.Forward{}, conn: conn} + g.mu.Lock() + if !g.forwardAuthorized(auth) || g.forwarders[auth.Node] != nil { + g.mu.Unlock() + return + } + if g.forwarders == nil { + g.forwarders = map[uint64]*forwardLink{} + } + g.forwarders[auth.Node] = link + g.signal() + g.mu.Unlock() + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + done := make(chan struct{}) + go func() { + defer close(done) + defer cancel() + for { + select { + case <-ctx.Done(): + return + case f := <-link.send: + g.mu.Lock() + valid := g.forwardAuthorized(auth) + g.mu.Unlock() + if !valid { + return + } + b, _ := rafttransport.Encode(f) + write, c := context.WithTimeout(ctx, clusterctl.ForwardTimeout) + err := conn.Write(write, websocket.MessageBinary, b) + c() + if err != nil { + return + } + } + } + }() + defer func() { + cancel() + conn.CloseNow() + <-done + g.mu.Lock() + defer g.mu.Unlock() + if g.forwarders[auth.Node] == link { + delete(g.forwarders, auth.Node) + } + for _, ch := range link.pending { + select { + case ch <- clusterctl.Forward{}: + default: + } + } + g.signal() + }() + for { + kind, b, err := conn.Read(ctx) + if err != nil { + return + } + var f clusterctl.Forward + if kind != websocket.MessageBinary || rafttransport.Decode(b, &f) != nil { + return + } + g.mu.Lock() + peer, ok := g.peer(auth.Node) + valid := g.forwardAuthorized(auth) && ok && f.Scope == auth.Scope && f.Generation == auth.Generation && f.From == auth.Node && f.To == g.manifest.Config.Anchors[0] && f.Verify(peer.Key) + if valid { + if ch := link.pending[f.ID]; ch != nil { + select { + case ch <- f: + default: + } + } + } + g.mu.Unlock() + if !valid { + return + } + } +} +func (g *Native) forwardAuthorized(auth rafttransport.Request) bool { + if g.closed || g.peers == nil || auth.Scope != g.peers.view.Scope || auth.Generation != g.peers.view.Generation { + return false + } + peer, ok := g.peer(auth.Node) + return ok && peer.Incarnation == auth.Incarnation && auth.Verify(peer.Key) == nil +} + +// appliedReplica waits for the relay's OWN durable application through the +// exact forwarded admission/read index. A cached result alone cannot reply. +func (g *Native) appliedReplica(ctx context.Context, index uint64) (int, raftstore.Application, error) { + for { + i := g.replica() + if i < 0 || index == 0 { + return -1, raftstore.Application{}, directory.ErrUnreachable + } + app, _, err := g.stores[i].ApplicationView(ctx) + if err != nil { + return i, app, err + } + if app.Applied >= index { + return i, app, nil + } + if err = g.waitProgress(ctx); err != nil { + return i, app, err + } + } +} + +// boundedCall refuses a caller already gone BEFORE admission. Once admitted, +// its existing control deadline bounds the wait, while disk completion outlives +// a lost HTTP reply. Expired queued calls must not create more durable work. +func boundedCall(ctx context.Context) (context.Context, context.CancelFunc) { + if ctx.Err() != nil { + return context.WithCancel(ctx) + } + return context.WithTimeout(context.WithoutCancel(ctx), clusterctl.ForwardTimeout) +} + +// PersonalDeparted treats a leading voter's outbound connection closing as +// activity, including a graceful sync-off. It wakes the ordinary election +// clock; it grants no vote, removal authority or successful quorum response. +func (g *Native) PersonalDeparted(node uint64) bool { + g.mu.Lock() + defer g.mu.Unlock() + if g.closed || g.peers == nil { + return false + } + peer, ok := g.peer(node) + if !ok || !peer.Voter { + return false + } + g.peers.sleeping = false + for _, core := range g.nodes { + if core != nil { + _, _ = core.Schedule(raftcore.ScheduleInput{Busy: true}) + } + } + g.signal() + return true +} + +// refreshHint polls only already hosted cores. The resulting address remains a +// hint: a retry must still commit the original request or finish a fresh read. +func (g *Native) refreshHint(ctx context.Context) error { + for i, node := range g.nodes { + if node != nil { + if err := g.drive(ctx, i, raftcore.Poll{}); err != nil { + return err + } + } + } + return g.drain(ctx) +} diff --git a/internal/raftdirectory/forward_internal_test.go b/internal/raftdirectory/forward_internal_test.go new file mode 100644 index 0000000000..a207635bd9 --- /dev/null +++ b/internal/raftdirectory/forward_internal_test.go @@ -0,0 +1,59 @@ +package raftdirectory + +import ( + "context" + "crypto/ed25519" + "path/filepath" + "sync/atomic" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftstore" +) + +// R3/R6 require canceled callers waiting outside the actor to leave no new +// durable work behind. A later fresh read must still reach the healthy quorum. +func TestR3R6CanceledQueuedCallDoesNotEnterTheDurableActor(t *testing.T) { + ctx := t.Context() + root := ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize)).Public().(ed25519.PublicKey) + effectSeed := make([]byte, ed25519.SeedSize) + effectSeed[0] = 4 + effects := ed25519.NewKeyFromSeed(effectSeed).Public().(ed25519.PublicKey) + now := time.UnixMilli(1700000000000) + source, err := directory.OpenSQLite(filepath.Join(t.TempDir(), "source.db"), func() time.Time { return now }) + if err != nil { + t.Fatal(err) + } + defer source.Close() + cfg := cs.LegacyConfig{Cluster: "cancelled-queue", Generation: 1, Identity: cs.IdentityID(root), SigningKey: root, EffectsKey: effects, Anchors: []uint64{1, 2, 3}, Now: now.UnixMilli()} + if err = source.FreezeLegacy(ctx, cfg); err != nil { + t.Fatal(err) + } + manifest, err := source.InventoryLegacy(ctx) + if err != nil { + t.Fatal(err) + } + host, err := OpenNative(ctx, t.TempDir(), source, manifest) + if err != nil { + t.Fatal(err) + } + defer host.Close() + var entered atomic.Bool + inspect := func(context.Context, *raftstore.SQLite, raftstore.Application) error { entered.Store(true); return nil } + cancelled, cancel := context.WithCancel(ctx) + host.opMu.Lock() + started := make(chan struct{}) + done := make(chan error, 1) + go func() { close(started); done <- host.Inspect(cancelled, inspect) }() + <-started + cancel() + host.opMu.Unlock() + if err = <-done; err == nil || entered.Load() { + t.Fatal("R3/R6 canceled queued call entered the actor", err) + } + if err = host.Inspect(ctx, inspect); err != nil || !entered.Load() { + t.Fatal("R3/R6 fresh quorum read failed", err) + } +} diff --git a/internal/raftdirectory/ingress.go b/internal/raftdirectory/ingress.go index 7dd1b1965d..789c22caa7 100644 --- a/internal/raftdirectory/ingress.go +++ b/internal/raftdirectory/ingress.go @@ -12,7 +12,10 @@ import ( // read. Caller cancellation cannot interrupt an admitted durability transaction. // The restricted effects witness advances time; it cannot authorize the caller. func (g *Native) ClusterAPI(ctx context.Context, request cs.ClusterRequest, clock cs.Command) (cs.ClusterResponse, error) { - ctx = context.WithoutCancel(ctx) + g.opMu.Lock() + defer g.opMu.Unlock() + ctx, cancel := boundedCall(ctx) + defer cancel() g.mu.Lock() defer g.mu.Unlock() var out cs.ClusterResponse diff --git a/internal/raftdirectory/maintenance.go b/internal/raftdirectory/maintenance.go index b622dab13c..8cfb8b9026 100644 --- a/internal/raftdirectory/maintenance.go +++ b/internal/raftdirectory/maintenance.go @@ -4,6 +4,8 @@ import ( "context" "crypto/ed25519" "fmt" + "github.com/Agent-Field/codeaf/internal/clusterctl" + "slices" cs "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/directory" @@ -15,7 +17,10 @@ import ( // ordinary log. It never edits a voter list or invents missing authoritative // stores. Existing committed registry rows are left intact on every activation. func (g *Native) InitializeMembership(ctx context.Context, key ed25519.PrivateKey) error { - ctx = context.WithoutCancel(ctx) + g.opMu.Lock() + defer g.opMu.Unlock() + ctx, cancel := boundedCall(ctx) + defer cancel() g.mu.Lock() defer g.mu.Unlock() i, app, err := g.barrier(ctx) @@ -52,7 +57,10 @@ func (g *Native) InitializeMembership(ctx context.Context, key ed25519.PrivateKe // sleep acknowledgment is a local durable marker, and an incomplete delivery // keeps ordinary ticks. Reads still need their own fresh quorum on wake. func (g *Native) Maintain(ctx context.Context, elapsed, now int64, key ed25519.PrivateKey) (int64, error) { - ctx = context.WithoutCancel(ctx) + g.opMu.Lock() + defer g.opMu.Unlock() + ctx, cancel := boundedCall(ctx) + defer cancel() g.mu.Lock() defer g.mu.Unlock() after := cs.TickEveryMillis @@ -78,34 +86,77 @@ func (g *Native) Maintain(ctx context.Context, elapsed, now int64, key ed25519.P return after, err } leader := g.leader() + remote := uint64(0) + var remotePlan raftcore.SchedulePlan + var remoteStatus clusterctl.ForwardStatus + var remoteAck *raftcore.SleepAck + scheduleRemote := func(acks []raftcore.SleepAck) (raftcore.SchedulePlan, error) { + reply, err := g.forward(ctx, remote, clusterctl.Forward{Action: "schedule", Now: now, Schedule: &raftcore.ScheduleInput{Now: elapsed, Acks: acks}}) + if err != nil || reply.Plan == nil { + return raftcore.SchedulePlan{}, directory.ErrUnreachable + } + remoteStatus = reply.Status + remoteAck = reply.SleepAck + return *reply.Plan, nil + } if leader < 0 { - return after, directory.ErrUnreachable + remote = g.leaderID() + leader = g.replica() + if remote == 0 || leader < 0 || g.forwarders[remote] == nil { + return after, directory.ErrUnreachable + } + var err error + remotePlan, err = scheduleRemote(nil) + if err != nil { + return after, err + } } + if g.peers != nil { g.peers.now = max(g.peers.now, now) if err := g.reconcileMembership(ctx); err != nil { return after, err } + if _, err := g.reconcileLeadership(ctx); err != nil { + return after, err + } } plan := plans[leader] + if remote != 0 { + plan = remotePlan + } if plan.Barrier { id := fmt.Sprintf("idle-%d", g.serial) g.serial++ - if err := g.drive(ctx, leader, raftcore.IdleRead{RequestID: id}); err != nil { - return after, err - } - if err := g.drain(ctx); err != nil { - return after, err - } - ok := g.reads[id] - delete(g.reads, id) - if !ok { - return after, directory.ErrUnreachable - } - var err error - plan, err = Schedule(ctx, g.stores[leader], g.nodes[leader], elapsed, now, false, nil) - if err != nil { - return after, err + if remote != 0 { + reply, err := g.forward(ctx, remote, clusterctl.Forward{Action: "idle-read", ID: id}) + if err != nil || reply.Barrier == 0 { + return after, directory.ErrUnreachable + } + if _, _, err = g.appliedReplica(ctx, reply.Barrier); err != nil { + return after, err + } + plan, err = scheduleRemote(nil) + if err != nil { + return after, err + } + } else { + if err := g.drive(ctx, leader, raftcore.IdleRead{RequestID: id}); err != nil { + return after, err + } + if err := g.drain(ctx); err != nil { + return after, err + } + ok := g.reads[id] + delete(g.reads, id) + if !ok { + return after, directory.ErrUnreachable + } + var err error + plan, err = Schedule(ctx, g.stores[leader], g.nodes[leader], elapsed, now, false, nil) + if err != nil { + return after, err + } } } if plan.Prepare { @@ -113,7 +164,12 @@ func (g *Native) Maintain(ctx context.Context, elapsed, now int64, key ed25519.P if err != nil { return after, err } - c, err := cs.Sign(cs.Command{Schema: cs.SchemaVersion, Cluster: g.manifest.Config.Cluster, Actor: cs.AnchorActor(g.manifest.Config.Anchors[0]), Session: "sleep", Sequence: 1, Now: now, Change: cs.Mutation{PrepareQuiescence: &cs.PrepareQuiescence{Barrier: app.Applied, Term: g.status[leader].Term, Voters: raftcore.Configuration(app.ConfState).Voters}}}, key) + c, err := cs.Sign(cs.Command{Schema: cs.SchemaVersion, Cluster: g.manifest.Config.Cluster, Actor: cs.AnchorActor(g.manifest.Config.Anchors[0]), Session: "sleep", Sequence: 1, Now: now, Change: cs.Mutation{PrepareQuiescence: &cs.PrepareQuiescence{Barrier: app.Applied, Term: func() uint64 { + if remote != 0 { + return remoteStatus.Term + } + return g.status[leader].Term + }(), Voters: raftcore.Configuration(app.ConfState).Voters}}}, key) if err != nil { return after, err } @@ -140,16 +196,23 @@ func (g *Native) Maintain(ctx context.Context, elapsed, now int64, key ed25519.P } // Deliver the committed marker to every reachable follower before collecting // durable acknowledgments. Missing voters remain active and prevent sleep. - if err = g.drive(ctx, leader, raftcore.Tick{}); err != nil { - return after, err + if remote == 0 { + if err = g.drive(ctx, leader, raftcore.Tick{}); err != nil { + return after, err + } } if err = g.drain(ctx); err != nil { return after, err } } acks := []raftcore.SleepAck{} + app, _, err := g.stores[leader].ApplicationView(ctx) + if err != nil { + return after, err + } + voters := raftcore.Configuration(app.ConfState).Voters for i, store := range g.stores { - if store == nil { + if store == nil || !slices.Contains(voters, g.manifest.Config.Anchors[i]) { continue } app, _, err := store.ApplicationView(ctx) @@ -166,7 +229,22 @@ func (g *Native) Maintain(ctx context.Context, elapsed, now int64, key ed25519.P acks = append(acks, raftcore.SleepAck{Node: g.manifest.Config.Anchors[i], Index: m.Index, Term: m.Term}) } } + if remote != 0 { + if _, err := scheduleRemote(nil); err != nil { + return after, err + } + if remoteAck != nil { + acks = append(acks, *remoteAck) + } + } allSleep := true + if remote != 0 { + p, err := scheduleRemote(acks) + if err != nil { + return after, err + } + allSleep = p.Sleeping + } for i, node := range g.nodes { if node == nil { allSleep = false @@ -194,7 +272,7 @@ func (g *Native) Maintain(ctx context.Context, elapsed, now int64, key ed25519.P func (g *Native) PendingEffects(ctx context.Context) (bool, error) { g.mu.Lock() defer g.mu.Unlock() - i := g.leader() + i := g.replica() if i < 0 { return false, directory.ErrUnreachable } diff --git a/internal/raftdirectory/native.go b/internal/raftdirectory/native.go index fc95b52690..94554e9bc4 100644 --- a/internal/raftdirectory/native.go +++ b/internal/raftdirectory/native.go @@ -1,6 +1,7 @@ package raftdirectory import ( + "bytes" "context" "crypto/rand" "encoding/binary" @@ -10,6 +11,7 @@ import ( "path/filepath" "sync" + "github.com/Agent-Field/codeaf/internal/clusterctl" "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/directory" "github.com/Agent-Field/codeaf/internal/raftcore" @@ -20,24 +22,35 @@ import ( "google.golang.org/protobuf/proto" ) -// Native owns three independent durable actors on one host. The queue covers -// each persistence completion, authenticated local delivery and read callback; -// no caller can inspect a replica between its barrier and its application read. -// Peer bytes reach a core only inside this queue, after committed enrollment -// authenticates external input. Public operations never wait on a learner. +type appliedCommand struct { + result clusterstate.Result + index uint64 +} + +// Native owns three independently durable anchors and bridges external members. +// opMu orders public operations; mu serializes persistence, application and +// authenticated delivery. Waiting callers release only mu so peers can finish +// their quorum. An authority callback retains mu from its fresh barrier through +// its application read, preventing an intervening state change. type Native struct { - mu sync.Mutex - peers *nativePeers - manifest clusterstate.LegacyManifest - stores [3]*raftstore.SQLite - nodes [3]*raftcore.Core - status [3]raftcore.Status - queue []raftcore.Outbound - reads map[string]bool - results [3][]clusterstate.Result - serial uint64 - failed error - closed bool + mu sync.Mutex + opMu sync.Mutex + changed chan struct{} + forwarders map[uint64]*forwardLink + pendingCommand []byte + pendingAdmission uint64 + outcome *appliedCommand + peers *nativePeers + manifest clusterstate.LegacyManifest + stores [3]*raftstore.SQLite + nodes [3]*raftcore.Core + status [3]raftcore.Status + queue []raftcore.Outbound + reads map[string]bool + results [3][]clusterstate.Result + serial uint64 + failed error + closed bool // Changed projects durable rows into advisory notifications under the queue. Changed func(*raftstore.SQLite, raftstore.Application) error } @@ -184,6 +197,10 @@ func (g *Native) Close() error { g.mu.Lock() defer g.mu.Unlock() g.closed = true + g.signal() + for _, link := range g.forwarders { + link.conn.CloseNow() + } var err error for _, s := range g.stores { if s != nil { @@ -204,12 +221,23 @@ func (g *Native) leader() int { return -1 } func (g *Native) drive(ctx context.Context, i int, input raftcore.Input) error { + // An admitted Ready finishes atomically even if the waiting HTTP deadline + // expires. Only an actual storage error may stop this durable actor. + ctx = context.WithoutCancel(ctx) + defer g.signal() for n := 0; n < 10000; n++ { e, err := g.nodes[i].Step(input) if err != nil { return err } g.status[i] = e.Status + if proposal, ok := input.(raftcore.Propose); ok && bytes.Equal(proposal.Command, g.pendingCommand) && e.Persist != nil { + for _, entry := range e.Persist.Entries { + if bytes.Equal(entry.GetData(), g.pendingCommand) { + g.pendingAdmission = entry.GetIndex() + } + } + } g.queue = append(g.queue, e.Send...) for _, r := range e.Reads { app, _, err := g.stores[i].ApplicationView(ctx) @@ -233,8 +261,27 @@ func (g *Native) drive(ctx context.Context, i int, input raftcore.Input) error { _, _ = g.nodes[i].Step(raftcore.StorageFailed{BatchID: e.Apply.ID}) return fmt.Errorf("raftdirectory: apply member %d index %d: %w", g.manifest.Config.Anchors[i], e.Apply.Index, err) } - g.results[i] = append(g.results[i], results...) + g.results[i] = results + resultIndex := 0 + for _, entry := range e.Apply.Entries { + if entry.GetType() != pb.EntryNormal || len(entry.GetData()) == 0 { + continue + } + if len(g.pendingCommand) > 0 && bytes.Equal(entry.GetData(), g.pendingCommand) && entry.GetIndex() == g.pendingAdmission && g.pendingAdmission != 0 && g.outcome == nil { + g.outcome = &appliedCommand{result: results[resultIndex], index: entry.GetIndex()} + } + resultIndex++ + } input = raftcore.Applied{BatchID: e.Apply.ID} + } else if e.SnapshotNeeded != nil { + snapshot, err := g.stores[i].CreateSnapshot(ctx) + if err != nil { + return err + } + if err = g.stores[i].Compact(ctx, snapshot.GetMetadata().GetIndex()); err != nil { + return err + } + input = raftcore.CompactSnapshot{Snapshot: snapshot} } else { return nil } @@ -242,6 +289,7 @@ func (g *Native) drive(ctx context.Context, i int, input raftcore.Input) error { return directory.ErrUnreachable } func (g *Native) drain(ctx context.Context) error { + ctx = context.WithoutCancel(ctx) for n := 0; len(g.queue) > 0; n++ { if n >= 10000 { return directory.ErrUnreachable @@ -334,18 +382,62 @@ func (g *Native) stageSnapshot(ctx context.Context, target int, message *pb.Mess } func (g *Native) submit(ctx context.Context, c clusterstate.Command) (clusterstate.Result, error) { + if ctx.Err() != nil { + return clusterstate.Result{}, directory.ErrUnreachable + } if g.peers != nil { g.peers.sleeping = false g.peers.now = max(g.peers.now, c.Now) } - i := g.leader() - if i < 0 { - return clusterstate.Result{}, directory.ErrUnreachable - } + ctx, cancel := context.WithTimeout(ctx, clusterctl.ForwardTimeout) + defer cancel() raw, err := clusterstate.EncodeCommand(c) if err != nil { return clusterstate.Result{}, err } + i := g.leader() + if i < 0 { + node := g.leaderID() + reply, e := g.forward(ctx, node, clusterctl.Forward{Action: "submit", Command: raw}) + if e != nil { + if err := g.refreshHint(ctx); err != nil { + return clusterstate.Result{}, err + } + if g.leader() >= 0 { + return g.submit(ctx, c) + } + // A stale hint retries the SAME signed request and forwarding ID. + newer := g.leaderID() + if newer != 0 && newer != node { + reply, e = g.forward(ctx, newer, clusterctl.Forward{ID: reply.ID, Action: "submit", Command: raw}) + } + } + if e != nil || reply.Result == nil { + return clusterstate.Result{}, directory.ErrUnreachable + } + if _, _, e = g.appliedReplica(ctx, reply.Applied); e != nil { + return clusterstate.Result{}, e + } + if g.peers != nil { + if e = g.installPeers(ctx); e != nil { + return clusterstate.Result{}, e + } + } + if g.Changed != nil { + j, app, e := g.appliedReplica(ctx, reply.Applied) + if e != nil { + return clusterstate.Result{}, e + } + if e = g.Changed(g.stores[j], app); e != nil { + return clusterstate.Result{}, e + } + } + return *reply.Result, nil + } + g.pendingCommand = raw + g.pendingAdmission = 0 + g.outcome = nil + defer func() { g.pendingCommand = nil; g.pendingAdmission = 0; g.outcome = nil }() for j := range g.results { g.results[j] = nil } @@ -356,10 +448,22 @@ func (g *Native) submit(ctx context.Context, c clusterstate.Command) (clustersta g.failed = err return clusterstate.Result{}, err } - if len(g.results[i]) != 1 { - return clusterstate.Result{}, directory.ErrUnreachable + for g.outcome == nil { + if g.leader() != i { + return g.submit(ctx, c) + } + if g.peers == nil || len(g.forwarders) == 0 { + return clusterstate.Result{}, directory.ErrUnreachable + } + if err = g.waitProgress(ctx); err != nil { + return clusterstate.Result{}, err + } + } + r := g.outcome.result + if _, _, err := g.appliedReplica(ctx, g.outcome.index); err != nil { + return clusterstate.Result{}, err } - r := g.results[i][0] + ctx = context.WithoutCancel(ctx) // Compaction follows complete local delivery so every local voter has the // checkpoint's prefix. It removes only redundant consensus history. for j, s := range g.stores { @@ -408,12 +512,49 @@ func (g *Native) submit(ctx context.Context, c clusterstate.Command) (clustersta return r, nil } func (g *Native) barrier(ctx context.Context) (int, raftstore.Application, error) { + if ctx.Err() != nil { + return -1, raftstore.Application{}, directory.ErrUnreachable + } if g.peers != nil { g.peers.sleeping = false } + ctx, cancel := context.WithTimeout(ctx, clusterctl.ForwardTimeout) + defer cancel() i := g.leader() if i < 0 { - return i, raftstore.Application{}, directory.ErrUnreachable + node := g.leaderID() + g.serial++ + id := fmt.Sprintf("read-%d", g.serial) + reply, e := g.forward(ctx, node, clusterctl.Forward{ID: id, Action: "read"}) + if e != nil { + if err := g.refreshHint(ctx); err != nil { + return i, raftstore.Application{}, err + } + if local := g.leader(); local >= 0 { + if err := g.drive(ctx, local, raftcore.Read{RequestID: id}); err != nil { + return local, raftstore.Application{}, err + } + if err := g.drain(ctx); err != nil { + return local, raftstore.Application{}, err + } + defer delete(g.reads, id) + for !g.reads[id] { + if err := g.waitProgress(ctx); err != nil { + return local, raftstore.Application{}, err + } + } + app, _, err := g.stores[local].ApplicationView(ctx) + return local, app, err + } + newer := g.leaderID() + if newer != 0 && newer != node { + reply, e = g.forward(ctx, newer, clusterctl.Forward{ID: id, Action: "read"}) + } + } + if e != nil || reply.Barrier == 0 { + return i, raftstore.Application{}, directory.ErrUnreachable + } + return g.appliedReplica(ctx, reply.Barrier) } g.serial++ id := fmt.Sprintf("native-%d", g.serial) @@ -424,22 +565,39 @@ func (g *Native) barrier(ctx context.Context) (int, raftstore.Application, error g.failed = err return i, raftstore.Application{}, err } - ok := g.reads[id] - delete(g.reads, id) - if !ok { - return i, raftstore.Application{}, directory.ErrUnreachable + defer delete(g.reads, id) + for !g.reads[id] { + if node := g.leaderID(); node != 0 && node != g.manifest.Config.Anchors[i] { + reply, err := g.forward(ctx, node, clusterctl.Forward{ID: id, Action: "read"}) + if err != nil || reply.Barrier == 0 { + return i, raftstore.Application{}, directory.ErrUnreachable + } + return g.appliedReplica(ctx, reply.Barrier) + } + if g.peers == nil || len(g.forwarders) == 0 { + return i, raftstore.Application{}, directory.ErrUnreachable + } + if err := g.waitProgress(ctx); err != nil { + return i, raftstore.Application{}, err + } } app, _, err := g.stores[i].ApplicationView(ctx) return i, app, err } func (g *Native) CommitV1(ctx context.Context, c clusterstate.Command) (clusterstate.Result, error) { - ctx = context.WithoutCancel(ctx) + g.opMu.Lock() + defer g.opMu.Unlock() + ctx, cancel := boundedCall(ctx) + defer cancel() g.mu.Lock() defer g.mu.Unlock() return g.submit(ctx, c) } func (g *Native) ReadV1(ctx context.Context, c clusterstate.Command, committed bool, inspect func(context.Context, *raftstore.SQLite, raftstore.Application) error) error { - ctx = context.WithoutCancel(ctx) + g.opMu.Lock() + defer g.opMu.Unlock() + ctx, cancel := boundedCall(ctx) + defer cancel() g.mu.Lock() defer g.mu.Unlock() if !committed { @@ -461,7 +619,10 @@ func (g *Native) ReadV1(ctx context.Context, c clusterstate.Command, committed b return inspect(ctx, g.stores[i], app) } func (g *Native) CommitEffect(ctx context.Context, c clusterstate.Command) (clusterstate.Result, error) { - ctx = context.WithoutCancel(ctx) + g.opMu.Lock() + defer g.opMu.Unlock() + ctx, cancel := boundedCall(ctx) + defer cancel() g.mu.Lock() defer g.mu.Unlock() return g.submit(ctx, c) @@ -471,7 +632,10 @@ func (g *Native) CommitEffect(ctx context.Context, c clusterstate.Command) (clus // quorum barrier and gives the callback its actor-owned context. A disconnected // HTTP request cannot turn a healthy durable read into a storage failure. func (g *Native) Inspect(ctx context.Context, inspect func(context.Context, *raftstore.SQLite, raftstore.Application) error) error { - ctx = context.WithoutCancel(ctx) + g.opMu.Lock() + defer g.opMu.Unlock() + ctx, cancel := boundedCall(ctx) + defer cancel() g.mu.Lock() defer g.mu.Unlock() i, app, err := g.barrier(ctx) diff --git a/internal/raftdirectory/personal.go b/internal/raftdirectory/personal.go index cae252093a..fe119a15c3 100644 --- a/internal/raftdirectory/personal.go +++ b/internal/raftdirectory/personal.go @@ -6,6 +6,7 @@ import ( "errors" "strconv" + "github.com/Agent-Field/codeaf/internal/clusterctl" "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/directory" "github.com/Agent-Field/codeaf/internal/raftcore" @@ -38,7 +39,7 @@ func recordReader(ctx context.Context, s *raftstore.SQLite, app raftstore.Applic } } func (g *Native) membership(ctx context.Context) (int, raftstore.Application, clusterstate.Membership, clusterstate.ReadRecord, error) { - i := g.leader() + i := g.replica() if i < 0 { return i, raftstore.Application{}, clusterstate.Membership{}, nil, directory.ErrUnreachable } @@ -89,7 +90,22 @@ func (g *Native) reconcileMembership(ctx context.Context) error { if h.Pending == nil { return g.installPeers(ctx) } - plan, err := g.nodes[i].PlanMembership(h) + var plan raftcore.MembershipPlan + if g.leader() < 0 { + reply, e := g.forward(ctx, g.leaderID(), clusterctl.Forward{Action: "membership"}) + if reply.Membership == nil { + return directory.ErrUnreachable + } + if e != nil { + return nil + } + plan = *reply.Membership + if len(plan.Change) > 0 { + return nil + } + } else { + plan, err = g.nodes[i].PlanMembership(h) + } if errors.Is(err, raftcore.ErrBusy) { return nil } @@ -111,7 +127,7 @@ func (g *Native) reconcileMembership(ctx context.Context) error { if _, err = g.controlChange(ctx, clusterstate.Mutation{SettleMembership: &clusterstate.SettleMembership{Revision: h.Revision, Node: h.Pending.Node}}, app.LogicalTime); err != nil { return err } - if plan.Checkpoint { + if plan.Checkpoint && g.leader() >= 0 { p, err := g.stores[i].CreateSnapshot(ctx) if err != nil { return err @@ -130,7 +146,10 @@ func (g *Native) reconcileMembership(ctx context.Context) error { // EnrollPersonal serializes allocation, replacement and the original signed // proof with all other member input. No public call waits on the learner. func (g *Native) EnrollPersonal(ctx context.Context, proof clusterstate.V1Request) (PersonalEnrollment, error) { - ctx = context.WithoutCancel(ctx) + g.opMu.Lock() + defer g.opMu.Unlock() + ctx, cancel := boundedCall(ctx) + defer cancel() g.mu.Lock() defer g.mu.Unlock() var out PersonalEnrollment diff --git a/internal/raftdirectory/personal_lead.go b/internal/raftdirectory/personal_lead.go new file mode 100644 index 0000000000..ae83a2870f --- /dev/null +++ b/internal/raftdirectory/personal_lead.go @@ -0,0 +1,176 @@ +package raftdirectory + +import ( + "context" + "errors" + + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftcore" +) + +// reconcileLeadership enacts only the committed policy. Every configuration +// entry finishes durably before another plan; fallback uses A+B, never P's +// consent or a connectivity-based forced voter removal. +func (g *Native) reconcileLeadership(ctx context.Context) (bool, error) { + i, app, h, _, err := g.membership(ctx) + if err != nil { + return false, err + } + if h.Leadership == nil || g.leader() < 0 { + return false, nil + } + l := h.Leadership + _, plan, err := PlanLeadership(ctx, g.stores[i], g.nodes[i], raftcore.LeadershipEvidence{}) + if err == nil && plan.Fallback { + _, err = g.controlChange(ctx, cs.Mutation{RestoreAnchors: &cs.RestoreAnchors{Revision: h.Revision}}, app.LogicalTime) + return true, err + } + target := l.Node + if l.Restore { + target = l.Standby + } + var evidence raftcore.LeadershipEvidence + if !h.StableTopology(raftcore.Configuration(app.ConfState)) || l.Settled && !l.Restore && !l.Completed { + local := -1 + for j, id := range h.Anchors { + if id == target { + local = j + } + } + if local >= 0 && g.stores[local] != nil { + evidence, err = DurableLeadershipEvidence(ctx, g.stores[local], target, h.Revision, l.Barrier) + } else { + var reply clusterctl.Forward + reply, err = g.forward(ctx, target, clusterctl.Forward{Action: "evidence", Revision: h.Revision, Barrier: l.Barrier}) + if err == nil && reply.Evidence != nil { + evidence = *reply.Evidence + } else { + err = raftcore.ErrTransfer + } + } + if err != nil { + return false, nil + } + } + if g.leader() != i { + return true, nil + } + _, plan, err = PlanLeadership(ctx, g.stores[i], g.nodes[i], evidence) + if errors.Is(err, raftcore.ErrBusy) || errors.Is(err, raftcore.ErrTransfer) { + return false, nil + } + if err != nil { + return false, err + } + if plan.Configure { + before := g.status[i].Applied + if err = g.drive(ctx, i, raftcore.Configure{Change: plan.Change}); err != nil { + return false, err + } + if err = g.drain(ctx); err != nil { + return false, err + } + for g.status[i].Applied <= before { + if err = g.waitProgress(ctx); err != nil { + return false, err + } + } + return true, g.installPeers(ctx) + } + if plan.Settle { + _, err = g.controlChange(ctx, cs.Mutation{SettleLeadership: &cs.SettleLeadership{Revision: h.Revision}}, app.LogicalTime) + return true, err + } + if plan.Transfer != 0 { + if err = g.drive(ctx, i, raftcore.Transfer{Target: plan.Transfer}); err != nil { + return false, err + } + return false, g.drain(ctx) + } + return false, nil +} + +func (g *Native) PersonalLead(ctx context.Context, c cs.Command, certificate string) (cs.LeadershipReceipt, error) { + var empty cs.LeadershipReceipt + g.opMu.Lock() + defer g.opMu.Unlock() + ctx, cancel := boundedCall(ctx) + defer cancel() + g.mu.Lock() + defer g.mu.Unlock() + i, app, err := g.barrier(ctx) + if err != nil { + return empty, err + } + read := recordReader(ctx, g.stores[i], app) + raw, err := read(cs.IdentityKey()) + if err != nil { + return empty, err + } + var identity cs.IdentityRec + if cs.Decode(raw, &identity) != nil || cs.PersonalCertificate(identity, c.Actor, certificate) != nil { + return empty, directory.ErrUnauthorized + } + raw, err = read(cs.MemberDeviceKey(c.Actor)) + if err != nil { + return empty, err + } + var mapping cs.DeviceMember + if raw == nil || cs.Decode(raw, &mapping) != nil || mapping.Node == 0 { + return empty, clusterctl.ErrUnsupported + } + r, err := g.submit(ctx, c) + if err != nil { + return empty, err + } + if r.Code != cs.OK { + return empty, clusterctl.CommittedRefusal{Code: r.Code, Sequence: c.Sequence} + } + for ctx.Err() == nil { + _, _, h, _, err := g.membership(ctx) + if err != nil { + return empty, err + } + l := h.Leadership + if l == nil || l.Restore || l.Node != c.Change.RequestLeadership.Node { + return empty, clusterctl.CommittedRefusal{Code: cs.Conflict, Sequence: c.Sequence} + } + if g.leaderID() == l.Node && g.leader() < 0 && l.Settled { + // Completion persists before the FINAL fresh current-term receipt. A retry + // of this exact request performs another read instead of reusing a hint. + if !l.Completed { + reply, e := g.forward(ctx, l.Node, clusterctl.Forward{Action: "receipt"}) + if e != nil || reply.Receipt == nil { + return empty, directory.ErrUnreachable + } + if _, _, e = g.appliedReplica(ctx, reply.Barrier); e != nil { + return empty, e + } + if _, e = g.controlChange(ctx, cs.Mutation{CompleteLeadership: &cs.CompleteLeadership{Revision: h.Revision}}, app.LogicalTime); e != nil { + return empty, e + } + } + reply, e := g.forward(ctx, l.Node, clusterctl.Forward{Action: "receipt"}) + if e != nil || reply.Receipt == nil || reply.Receipt.Node != l.Node || reply.Receipt.Revision != h.Revision { + return empty, directory.ErrUnreachable + } + if _, _, e = g.appliedReplica(ctx, reply.Barrier); e != nil { + return empty, e + } + return *reply.Receipt, nil + } + before := g.changed + changed, err := g.reconcileLeadership(ctx) + if err != nil { + return empty, err + } + if !changed && before == g.changed { + if err = g.waitProgress(ctx); err != nil { + return empty, err + } + } + } + return empty, directory.ErrUnreachable +} diff --git a/internal/raftdirectory/snapshot_receive.go b/internal/raftdirectory/snapshot_receive.go new file mode 100644 index 0000000000..aa05311047 --- /dev/null +++ b/internal/raftdirectory/snapshot_receive.go @@ -0,0 +1,70 @@ +package raftdirectory + +import ( + "context" + + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/rafttransport" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +type incomingSnapshot struct { + message *pb.Message + chunks map[uint64]bool +} + +// receivePeer stages the actual sender's portable chunks before the native +// actor may Step its pointer. A delivery acknowledgment is never installation. +func (g *Native) receivePeer(ctx context.Context, i int, e rafttransport.Envelope) error { + if e.AppendChunk() { + e.Chunk = nil + } + if e.Chunk != nil { + transfer := g.peers.incoming[i] + if transfer == nil || transfer.message.GetFrom() != e.From || raftsnapshot.ID(transfer.message.GetSnapshot().GetData()) != e.Chunk.Generation { + return nil + } + if err := g.stores[i].PutSnapshotChunk(ctx, e.Chunk.Generation, e.Chunk.Offset, e.Payload, e.Chunk.Hash); err != nil { + return err + } + transfer.chunks[e.Chunk.Offset] = true + manifest, err := raftsnapshot.Decode(transfer.message.GetSnapshot().GetData()) + if err != nil { + return err + } + if uint64(len(transfer.chunks)) != manifest.Chunks { + return nil + } + pointer, err := g.stores[i].CompleteSnapshot(ctx, e.Chunk.Generation) + if err != nil { + return err + } + if !proto.Equal(pointer, transfer.message.GetSnapshot()) { + return raftstore.ErrSnapshot + } + raw, err := proto.Marshal(transfer.message) + if err != nil { + return err + } + delete(g.peers.incoming, i) + return g.drive(ctx, i, raftcore.PeerMessage{Bytes: raw}) + } + message := new(pb.Message) + if proto.Unmarshal(e.Payload, message) != nil { + return raftcore.ErrPeer + } + if message.GetType() == pb.MsgSnap && message.GetSnapshot().GetMetadata().GetIndex() > g.status[i].Applied { + transfer := g.peers.incoming[i] + if transfer == nil || !proto.Equal(transfer.message.GetSnapshot(), message.GetSnapshot()) { + if _, err := g.stores[i].BeginSnapshot(ctx, message.GetSnapshot().GetData()); err != nil { + return err + } + g.peers.incoming[i] = &incomingSnapshot{message: message, chunks: map[uint64]bool{}} + } + return nil + } + return g.drive(ctx, i, raftcore.PeerMessage{Bytes: e.Payload}) +} diff --git a/internal/raftdirectory/transport.go b/internal/raftdirectory/transport.go index e4f368b6dd..f05205daab 100644 --- a/internal/raftdirectory/transport.go +++ b/internal/raftdirectory/transport.go @@ -4,8 +4,10 @@ import ( "context" "crypto/ed25519" "errors" + "fmt" "reflect" + "github.com/Agent-Field/codeaf/internal/clusterctl" "github.com/Agent-Field/codeaf/internal/directory" "github.com/Agent-Field/codeaf/internal/raftcore" "github.com/Agent-Field/codeaf/internal/raftsnapshot" @@ -23,6 +25,7 @@ type snapshotTransfer struct { started int64 } type nativePeers struct { + incoming map[int]*incomingSnapshot router *rafttransport.Router view rafttransport.View keys [3]ed25519.PrivateKey @@ -49,7 +52,7 @@ func (g *Native) AttachTransport(ctx context.Context, generation uint64, keys [3 return nil, rafttransport.ErrAuth } } - g.peers = &nativePeers{router: router, keys: keys, view: rafttransport.View{Scope: rafttransport.Scope{Identity: g.manifest.Config.Identity, Cluster: g.manifest.Config.Cluster, Epoch: g.manifest.Config.Generation, Protocol: raftcore.ProtocolVersion}, Generation: generation}, applied: map[uint64]uint64{}, snapshots: map[uint64]*snapshotTransfer{}} + g.peers = &nativePeers{router: router, keys: keys, view: rafttransport.View{Scope: rafttransport.Scope{Identity: g.manifest.Config.Identity, Cluster: g.manifest.Config.Cluster, Epoch: g.manifest.Config.Generation, Protocol: raftcore.ProtocolVersion}, Generation: generation}, incoming: map[int]*incomingSnapshot{}, applied: map[uint64]uint64{}, snapshots: map[uint64]*snapshotTransfer{}} if err = g.installPeers(ctx); err != nil { return nil, err } @@ -79,6 +82,35 @@ func (g *Native) installPeers(ctx context.Context) error { return err } p.view = v + // Installing a changed voter class discards expendable deliveries. Start a + // real read round so append progress is retried immediately, including while + // a public transfer owns opMu and the periodic maintenance turn is waiting. + if i := g.leader(); i >= 0 { + g.serial++ + id := fmt.Sprintf("routing-%d", g.serial) + if err = g.drive(ctx, i, raftcore.IdleRead{RequestID: id}); err != nil { + return err + } + if err = g.drain(ctx); err != nil { + return err + } + if err = g.drive(ctx, i, raftcore.CancelRead{RequestID: id}); err != nil { + return err + } + delete(g.reads, id) + } else if target := g.leaderID(); target != 0 { + if link := g.forwarders[target]; link != nil { + g.serial++ + work := clusterctl.Forward{Scope: p.view.Scope, Generation: p.view.Generation, From: g.manifest.Config.Anchors[0], To: target, ID: fmt.Sprintf("routing-%d", g.serial), Action: "poke"} + work.Sign(func(b []byte) []byte { return ed25519.Sign(p.keys[0], b) }) + // A pump must never await this application callback. The elected member + // starts a real read round to retry deliveries discarded by the new view. + select { + case link.send <- work: + default: + } + } + } // Registry changes discard expendable inboxes. Report an unfinished snapshot // as failed so Raft can restart it under the new committed transport view. for to, transfer := range p.snapshots { @@ -129,13 +161,11 @@ func (g *Native) queueExternal(from int, to uint64, payload []byte, chunk *raftt } func (g *Native) sendExternal(ctx context.Context, m raftcore.Outbound) error { p := g.peers - target, ok := g.peer(m.To) + _, ok := g.peer(m.To) if !ok { return nil } - if target.Voter { - return directory.ErrUnreachable - } + if p.sleeping { return nil } @@ -207,7 +237,8 @@ func (g *Native) pumpPeers(request rafttransport.Request, now int64) error { if err != nil { return err } - if request.Applied > app.Applied { + peer, _ := g.peer(request.Node) + if !peer.Voter && request.Applied > app.Applied { return rafttransport.ErrWire } p.applied[request.Node] = max(p.applied[request.Node], request.Applied) @@ -233,11 +264,11 @@ func (g *Native) pumpPeers(request rafttransport.Request, now int64) error { for _, delivery := range response.Messages { e := delivery.Envelope member, ok := g.peer(e.From) - if !ok || e.Scope != p.view.Scope || e.Destination != 1 || e.Verify(member.Key) != nil || e.Chunk != nil { + if !ok || e.Scope != p.view.Scope || e.Destination != 1 || e.Verify(member.Key) != nil { return rafttransport.ErrAuth } if !g.idleLearnerResponse(e) { - if err = g.drive(ctx, i, raftcore.PeerMessage{Bytes: e.Payload}); err != nil { + if err = g.receivePeer(ctx, i, e); err != nil { return err } } diff --git a/internal/rafttransport/README.md b/internal/rafttransport/README.md index dbecbf54ce..a6e48589b1 100644 --- a/internal/rafttransport/README.md +++ b/internal/rafttransport/README.md @@ -114,7 +114,19 @@ A batch with an invalid member/signature admits no outbound message. Backpressur admits no part of an otherwise valid batch; valid incoming acks still drain that caller's inbox to avoid a circular queue-full stall. Copy retained bytes before returning. `ErrCapacity`/HTTP 429 means retain/retry or report failed transport to -the core, never a successful append. Retry with host-owned exponential backoff, +the core, never a successful append. Socket responses carry optional `accepted` +and `rejected` arrays of canonical decimal envelope sequences. They describe +transient routing admission only, never Raft persistence. A rejected batch leaves +the socket usable for other voters. Senders retain at most eight outstanding +socket sends until that response and report failure to the core on rejection. +Optional `chunks` credit lists the target node and canonical envelope sequence +after that target acknowledges staging a chunk. A personal snapshot sender keeps +one chunk outstanding per target and advances on this pushed credit, so a large +snapshot cannot overrun the unchanged chunk reservation. Credit is transient +flow control; only the actual durable snapshot append reply completes Raft +snapshot delivery. A partially staged joining member keeps its actor while +subscribing passively for the next chunk; readiness still requires application. +Retry with host-owned exponential backoff, seeded jitter and a 60-second ceiling. Buffers expire after 30 seconds of monotonic sampled host time, lazily on activity, requiring no idle alarm. Raft retransmits; lease renewal and snapshots retain their own durable semantics. @@ -216,7 +228,7 @@ wakes the ordinary quorum path. The codeaf client starts its personal learner only inside the existing terminal state-sync lifetime, or a scoped status activation, after current quiet guards. Open remains local. Hosted relays offer neither enrollment nor control. Worker -setup and leadership remain unavailable. +setup remains unavailable. Native personal leadership uses the separate application ingress below; hosted leadership remains unavailable. The native host advances its checksummed `transport-generation` file with atomic rename and directory fsync before opening its router; it never mutates the source @@ -226,3 +238,60 @@ is installed. A maximum-sized legal command can acquire protobuf framing larger than the normal peer budget. In that case the host checkpoints only the applied prefix and uses snapshot chunks for catch-up, keeping both existing byte limits and successful voter operations intact. + + +## Native personal leaders and application forwarding + +A native personal member also opens `/v2/cluster/forward` outward. This is a +separate application connection, never a Raft peer envelope. The upgrade uses +the same enrolled, signed scope/node/incarnation/router-generation proof. Every +application frame signs its scope, generation, source/destination, request ID, +action and complete body under `codeaf/cluster/forward/1`. Only the immutable +provisioned anchor keys route work; each reply uses the actual personal key. +Current committed revocation/generation is checked on every send and receive. +The existing eight-inflight and 512 KiB frame limits bound this channel, with +the ordinary 30-second control deadline and no idle keepalive. + +The relay forwards the original signed command unchanged or a request-specific +ReadIndex operation. The elected personal core persists/applies it normally and +returns its actual admission/read index; the relay independently waits for its +own durable application through that index. A prior dedup result alone cannot +acknowledge a newly forwarded proposal. Public calls retain their operation +serialization but release the member queue while waiting, so authenticated peer +traffic and per-member persistence stay live. Calls canceled while still queued +are refused before admission; an admitted disk transaction finishes despite a +lost HTTP reply. A changed leader hint retries the +same original request, never a cached successful response. + +Native leadership reconciles only committed owner intent through explicit joint +entry, durable leave, settlement and actual transfer. Target evidence comes from +its own durable store and actual recent reverse replication. Success follows a +current-term commit, fresh read, durable completion and another fresh receipt. +A+B restore C using the same planner after winning a new term; P is never asked +for removal consent and never automatically regains leadership. Disconnecting a +voting member wakes the ordinary scheduler, including after all-voter sleep. +Personal snapshot senders stream the same portable manifest/chunks to anchors +and learners; receivers stage and checksum chunks before stepping the pointer. +Committed revocation may precede removal from the Raft configuration. Senders +discard traffic lacking a currently authorized destination, preserving other +voters rather than failing the whole member. +The member and both connections share the existing terminal/scoped lifetime and +all use-time quiet guards. No content route or new daemon is involved. + +A maximum-sized legal proposal can acquire protobuf framing larger than the +normal peer envelope. A personal leader sends that one `MsgApp` in the existing +chunk lane: offset zero and generation equal to the payload hash identify the +complete framed append, independently of any snapshot manifest. The envelope +checks its actual inner from/to/type and only permits `MsgApp` in this mode. +Snapshot generations remain hashes of manifests, not chunk payloads. Recipients +step the authenticated append normally, and only actual Raft persistence and +application acknowledgments count; routing/chunk credit cannot commit it. The +64 KiB proposal/normal-envelope, 256 KiB chunk and all queue caps are unchanged. + +The personal store reserves an outbound sequence range in its existing durable +per-open identity write before any frame can escape. A process restart skips the +unused remainder, so an old unacknowledged learner delivery cannot collide with +new bytes under the same node/incarnation. No fsync per peer frame is needed. +Final sleep also waits for the member's bounded sends to be admitted. After the +all-voter marker, obsolete nonvoter append/heartbeat responses cannot wake the +leader; actual work, read requests and higher-term messages still do. diff --git a/internal/rafttransport/backpressure_test.go b/internal/rafttransport/backpressure_test.go new file mode 100644 index 0000000000..ffaae681b0 --- /dev/null +++ b/internal/rafttransport/backpressure_test.go @@ -0,0 +1,161 @@ +package rafttransport + +import ( + "bytes" + "context" + "crypto/ed25519" + "encoding/base64" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + + "github.com/coder/websocket" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// R6/R11/R27 require a full sleeping learner's inbox to reject delivery while +// the SAME outward connection continues carrying normal voter traffic. +func TestR6R11R27SocketBackpressurePreservesVoterConnection(t *testing.T) { + r := router(t) + server := httptest.NewServer(Handler(r, nil)) + defer server.Close() + ctx, cancel := context.WithTimeout(t.Context(), socketTimeout) + defer cancel() + auth, err := Encode(request(t, 1)) + must(t, err) + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+SocketPath, &websocket.DialOptions{HTTPHeader: http.Header{SocketAuthHeader: []string{base64.RawURLEncoding.EncodeToString(auth)}}}) + must(t, err) + defer conn.CloseNow() + read := func() Response { + t.Helper() + kind, raw, err := conn.Read(ctx) + must(t, err) + if kind != websocket.MessageBinary { + t.Fatal("R6 changed framing") + } + var response Response + must(t, Decode(raw, &response)) + return response + } + read() + send := func(packet Envelope) Response { + t.Helper() + raw, err := Encode(request(t, 1, packet)) + must(t, err) + must(t, conn.Write(ctx, websocket.MessageBinary, raw)) + return read() + } + for sequence := uint64(1); sequence <= 3; sequence++ { + response := send(message(t, 1, 4, sequence, pb.MsgHeartbeat, 0)) + if sequence < 3 { + if !slices.Equal(response.Accepted, []string{decimal(sequence)}) || len(response.Rejected) != 0 { + t.Fatal("R11 legal learner reservation refused", response) + } + } else { + if !slices.Equal(response.Rejected, []string{"3"}) || len(response.Accepted) != 0 { + t.Fatal("R11 overflow was acknowledged", response) + } + } + } + packet := message(t, 1, 2, 4, pb.MsgHeartbeat, 0) + response := send(packet) + if !slices.Equal(response.Accepted, []string{"4"}) || len(response.Rejected) != 0 { + t.Fatal("R6 blocked voter behind sleeping learner", response) + } + delivered := exchange(t, r, request(t, 2)) + if len(delivered.Messages) != 1 || !sameEnvelope(delivered.Messages[0].Envelope, packet) { + t.Fatal("R6 voter traffic was lost", delivered) + } + sum := 0 + for _, n := range r.Stats(testScope, 1000).Bytes { + sum += n + } + if sum > MaxQueuedBytes { + t.Fatal("R27 queue cap changed") + } +} + +// R6/R13/R27 allow a multi-chunk snapshot to advance only when the target +// releases its bounded staging reservation, without polling the source socket. +func TestR6R13R27SnapshotChunkCreditPushesOnTheSameSocket(t *testing.T) { + r := router(t) + server := httptest.NewServer(Handler(r, nil)) + defer server.Close() + ctx, cancel := context.WithTimeout(t.Context(), socketTimeout) + defer cancel() + auth, err := Encode(request(t, 1)) + must(t, err) + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+SocketPath, &websocket.DialOptions{HTTPHeader: http.Header{SocketAuthHeader: []string{base64.RawURLEncoding.EncodeToString(auth)}}}) + must(t, err) + defer conn.CloseNow() + read := func() Response { + t.Helper() + _, raw, err := conn.Read(ctx) + must(t, err) + var response Response + must(t, Decode(raw, &response)) + return response + } + read() + for sequence := uint64(1); sequence <= 2; sequence++ { + payload := bytes.Repeat([]byte{byte(sequence)}, MaxChunkBytes) + packet := Envelope{Scope: testScope, From: 1, To: 4, Incarnation: 1, Destination: 1, Sequence: sequence, Payload: payload, Chunk: &Chunk{Generation: digest([]byte("snapshot")), Offset: sequence - 1, Hash: digest(payload)}} + must(t, packet.Sign(key(1))) + raw, err := Encode(request(t, 1, packet)) + must(t, err) + must(t, conn.Write(ctx, websocket.MessageBinary, raw)) + admitted := read() + if !slices.Equal(admitted.Accepted, []string{decimal(sequence)}) || len(admitted.Rejected) != 0 { + t.Fatal("R13 chunk refused after target credit", admitted) + } + delivered := exchange(t, r, request(t, 4)) + if len(delivered.Messages) != 1 || !sameEnvelope(delivered.Messages[0].Envelope, packet) { + t.Fatal("R13 lost snapshot chunk", delivered) + } + ack := request(t, 4) + ack.Ack[Snapshot] = delivered.Messages[0].Cursor + must(t, ack.Sign(key(4))) + exchange(t, r, ack) + credited := read() + if !slices.Equal(credited.Chunks, []ChunkAck{{Node: 4, Sequence: sequence}}) { + t.Fatal("R13 no target credit without source poll", credited) + } + } + if r.Stats(testScope, 1000).Queued != 0 { + t.Fatal("R27 snapshot reservation retained") + } +} + +// R6/R13/R16/R27 bind an oversized framed append to its actual sender and type +// while retaining BOTH the normal proposal budget and existing chunk ceiling. +func TestR6R13R16R27FramedAppendUsesExistingChunkCaps(t *testing.T) { + wrap := func(kind pb.MessageType, from uint64) Envelope { + raw, err := proto.Marshal(&pb.Message{Type: kind.Enum(), From: proto.Uint64(from), To: proto.Uint64(2), Term: proto.Uint64(1), Context: bytes.Repeat([]byte("x"), MaxBatchBytes)}) + must(t, err) + hash := digest(raw) + return Envelope{Scope: testScope, From: 1, To: 2, Incarnation: 1, Destination: 1, Sequence: 1, Payload: raw, Chunk: &Chunk{Generation: hash, Hash: hash}} + } + e := wrap(pb.MsgApp, 1) + must(t, e.Sign(key(1))) + if !e.AppendChunk() || len(e.Payload) <= MaxBatchBytes || len(e.Payload) > MaxChunkBytes { + t.Fatal("R27 framed append missed the two existing bounds") + } + must(t, e.Verify(key(1).Public().(ed25519.PublicKey))) + raw, err := Encode(e) + must(t, err) + var round Envelope + must(t, Decode(raw, &round)) + must(t, round.Verify(key(1).Public().(ed25519.PublicKey))) + for _, bad := range []Envelope{wrap(pb.MsgProp, 1), wrap(pb.MsgReadIndex, 1), wrap(pb.MsgApp, 3)} { + if bad.Sign(key(1)) == nil { + t.Fatal("R16 framed append bypassed peer sender/type checks") + } + } + e.Chunk = nil + if e.Sign(key(1)) == nil { + t.Fatal("R27 normal peer cap was raised") + } +} diff --git a/internal/rafttransport/envelope.go b/internal/rafttransport/envelope.go index 66028354ec..352bf10ccc 100644 --- a/internal/rafttransport/envelope.go +++ b/internal/rafttransport/envelope.go @@ -130,6 +130,14 @@ func (e Envelope) Verify(key ed25519.PublicKey) error { } return nil } + +// AppendChunk carries one framed append whose command already fits the normal +// proposal budget. The payload hash as generation distinguishes it from a +// snapshot manifest; both retain the existing chunk byte and queue limits. +func (e Envelope) AppendChunk() bool { + return e.Chunk != nil && e.Chunk.Offset == 0 && e.Chunk.Generation == e.Chunk.Hash +} + func (e Envelope) validate() error { if !e.Scope.valid() || !node(e.From) || !node(e.To) || e.From == e.To || e.Incarnation == 0 || e.Destination == 0 || e.Sequence == 0 || e.Digest != digest(e.Payload) { return ErrWire @@ -138,15 +146,22 @@ func (e Envelope) validate() error { if len(e.Payload) == 0 || len(e.Payload) > MaxChunkBytes || !hash(e.Chunk.Generation) || e.Chunk.Hash != e.Digest { return ErrWire } - return nil - } - if len(e.Payload) == 0 || len(e.Payload) > MaxBatchBytes { + if !e.AppendChunk() { + return nil + } + if len(e.Payload) <= MaxBatchBytes { + return ErrWire + } + } else if len(e.Payload) == 0 || len(e.Payload) > MaxBatchBytes { return ErrWire } m := &pb.Message{} if proto.Unmarshal(e.Payload, m) != nil || m.Type == nil || m.GetFrom() != e.From || m.GetTo() != e.To || m.GetTerm() == 0 || len(m.GetResponses()) != 0 || len(m.ProtoReflect().GetUnknown()) != 0 { return ErrWire } + if e.AppendChunk() && m.GetType() != pb.MsgApp { + return ErrWire + } switch m.GetType() { case pb.MsgApp, pb.MsgAppResp, pb.MsgVote, pb.MsgVoteResp, pb.MsgSnap, pb.MsgHeartbeat, pb.MsgHeartbeatResp, pb.MsgPreVote, pb.MsgPreVoteResp, pb.MsgTimeoutNow: return nil diff --git a/internal/rafttransport/exchange.go b/internal/rafttransport/exchange.go index cf7de24a6f..715128e480 100644 --- a/internal/rafttransport/exchange.go +++ b/internal/rafttransport/exchange.go @@ -8,6 +8,7 @@ import ( "encoding/json" "io" "net/http" + "slices" "strconv" "time" @@ -310,6 +311,7 @@ func serveSocket(ctx context.Context, conn *websocket.Conn, router *Router, init return } arm(response, true) + lastChunks := response.Chunks for { receiveOnly := false select { @@ -335,15 +337,27 @@ func serveSocket(ctx context.Context, conn *websocket.Conn, router *Router, init return } response, err = router.exchange(req, now().UnixMilli(), receiveOnly) + rejected := err == ErrCapacity && !receiveOnly + if rejected { + response, err = router.exchange(req, now().UnixMilli(), true) + for _, envelope := range req.Messages { + response.Rejected = append(response.Rejected, decimal(envelope.Sequence)) + } + } else if err == nil && !receiveOnly { + for _, envelope := range req.Messages { + response.Accepted = append(response.Accepted, decimal(envelope.Sequence)) + } + } if err != nil { return } - if receiveOnly && len(response.Messages) == 0 { + if receiveOnly && len(response.Messages) == 0 && slices.Equal(lastChunks, response.Chunks) { continue } if !writeSocket(ctx, conn, response) { return } + lastChunks = response.Chunks arm(response, !receiveOnly) } } diff --git a/internal/rafttransport/router.go b/internal/rafttransport/router.go index 3622fcb236..8c41c66430 100644 --- a/internal/rafttransport/router.go +++ b/internal/rafttransport/router.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/ed25519" "math" + "slices" "sync" ) @@ -61,6 +62,7 @@ type inbox struct { next, issued, acked [laneCount]uint64 sockets int changed chan struct{} + chunks map[uint64]uint64 } type group struct { view View @@ -148,6 +150,12 @@ func (r *Router) Install(v View) error { if v.Generation == old.view.Generation && m.Incarnation == old.members[id].Incarnation { box.next, box.issued, box.acked = prev.next, prev.issued, prev.acked box.sockets = prev.sockets + box.chunks = map[uint64]uint64{} + for destination, sequence := range prev.chunks { + if peer, ok := members[destination]; ok && !peer.Revoked { + box.chunks[destination] = sequence + } + } } } } @@ -206,7 +214,17 @@ type Delivery struct { Cursor uint64 `json:"cursor,string"` Envelope Envelope `json:"envelope"` } +type ChunkAck struct { + Node uint64 `json:"node,string"` + Sequence uint64 `json:"sequence,string"` +} + type Response struct { + Chunks []ChunkAck `json:"chunks,omitempty"` + // Accepted/Rejected describe SOCKET admission only. Neither means a peer + // persisted an append; a busy learner must not close its sender's voter path. + Accepted []string `json:"accepted,omitempty"` + Rejected []string `json:"rejected,omitempty"` Generation uint64 `json:"generation,string"` Messages []Delivery `json:"messages"` } @@ -304,6 +322,17 @@ func (r *Router) route(req Request, now int64, receiveOnly bool) (Response, erro keep = append(keep, q) } else { g.bytes[lane] -= q.size + if q.Envelope.Chunk != nil { + if source := g.inboxes[q.Envelope.From]; source != nil { + if source.chunks == nil { + source.chunks = map[uint64]uint64{} + } + if q.Envelope.Sequence > source.chunks[req.Node] { + source.chunks[req.Node] = q.Envelope.Sequence + signal(source) + } + } + } } } clear(box.queues[lane][len(keep):]) @@ -361,6 +390,14 @@ func (r *Router) route(req Request, now int64, receiveOnly bool) (Response, erro signal(target) } response := Response{Generation: g.view.Generation, Messages: []Delivery{}} + destinations := make([]uint64, 0, len(box.chunks)) + for id := range box.chunks { + destinations = append(destinations, id) + } + slices.Sort(destinations) + for _, id := range destinations { + response.Chunks = append(response.Chunks, ChunkAck{Node: id, Sequence: box.chunks[id]}) + } size := 0 normal := 0 snapshot := false diff --git a/internal/relayserve/control.go b/internal/relayserve/control.go index 973a99d0db..99c19d74d9 100644 --- a/internal/relayserve/control.go +++ b/internal/relayserve/control.go @@ -39,8 +39,9 @@ func (b nativeControl) Status(ctx context.Context, actor string) (clusterctl.Sta return b.owner.host.PersonalStatus(ctx, actor, certificate) } -// Lead cannot create an intent until the production personal-member lifecycle -// can finish it. In particular, refusal never proposes a configuration change. -func (b nativeControl) Lead(context.Context, cs.Command) (cs.LeadershipReceipt, error) { - return cs.LeadershipReceipt{}, clusterctl.ErrUnsupported +// Lead runs the native committed promotion workflow with the original proof. +func (b nativeControl) Lead(ctx context.Context, c cs.Command) (cs.LeadershipReceipt, error) { + b.owner.armMembers() + certificate, _ := ctx.Value(controlCertificateKey{}).(string) + return b.owner.host.PersonalLead(ctx, c, certificate) } diff --git a/internal/relayserve/personal.go b/internal/relayserve/personal.go index 7f13251276..ddc573cfe4 100644 --- a/internal/relayserve/personal.go +++ b/internal/relayserve/personal.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" + "github.com/Agent-Field/codeaf/internal/clusterctl" cs "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/raftdirectory" "github.com/Agent-Field/codeaf/internal/rafttransport" @@ -69,7 +70,7 @@ func (n *namespaces) enrollPersonal(w http.ResponseWriter, r *http.Request) { func (n *namespaces) exchangePersonal(w http.ResponseWriter, r *http.Request) { var raw []byte var err error - if r.URL.Path == rafttransport.SocketPath { + if r.URL.Path == rafttransport.SocketPath || r.URL.Path == clusterctl.ForwardPath { raw, err = base64.RawURLEncoding.DecodeString(r.Header.Get(rafttransport.SocketAuthHeader)) } else { raw, err = io.ReadAll(http.MaxBytesReader(w, r.Body, rafttransport.MaxExchangeBytes)) @@ -94,7 +95,14 @@ func (n *namespaces) exchangePersonal(w http.ResponseWriter, r *http.Request) { http.Error(w, "member authorization refused", 403) return } - d.peers.ServeHTTP(w, r) + if r.URL.Path == clusterctl.ForwardPath { + d.host.ForwardHTTP(w, r) + if d.host.PersonalDeparted(request.Node) { + d.armMembers() + } + } else { + d.peers.ServeHTTP(w, r) + } // Merely polling an idle learner does not arm recurring work. Actual peer // ingress wakes the same scheduler as every other consensus input. if len(request.Messages) > 0 { diff --git a/internal/relayserve/service.go b/internal/relayserve/service.go index d43e1ed302..c675938503 100644 --- a/internal/relayserve/service.go +++ b/internal/relayserve/service.go @@ -125,6 +125,7 @@ func New(cfg Config) *Service { if cfg.Cluster == nil { mux.HandleFunc(rafttransport.ExchangePath, svc.ns.exchangePersonal) mux.HandleFunc(rafttransport.SocketPath, svc.ns.exchangePersonal) + mux.HandleFunc(clusterctl.ForwardPath, svc.ns.exchangePersonal) } mux.Handle(clusterctl.Path, clusterctl.Handler(svc.ns.controlAuthenticate, svc.ns.controlBackend)) auth := noting(svc.ns.authenticate) diff --git a/internal/syncsetup/cluster.go b/internal/syncsetup/cluster.go index 0aa24487b8..b7a7a70120 100644 --- a/internal/syncsetup/cluster.go +++ b/internal/syncsetup/cluster.go @@ -12,11 +12,10 @@ import ( "net/http" "os" "path/filepath" - "time" ) func (s *Sync) clusterClient() clusterctl.Client { - return clusterctl.Client{Base: s.Relay, Sign: reqsign.SignFor(deviceSigner{s.Identity, s.Device}, s.now), HTTP: s.ClusterHTTP(&http.Client{Timeout: 30 * time.Second})} + return clusterctl.Client{Base: s.Relay, Sign: reqsign.SignFor(deviceSigner{s.Identity, s.Device}, s.now), HTTP: s.ClusterHTTP(&http.Client{Timeout: clusterctl.ForwardTimeout})} } func (s *Sync) ClusterStatus(ctx context.Context) (clusterctl.Status, error) { if !s.memberAllowed() { @@ -94,7 +93,7 @@ func (s *Sync) LeadCluster(ctx context.Context, node uint64) (cs.LeadershipRecei if saved.Sequence == ^uint64(0) { return receipt, errors.New("this computer's leadership request sequence is full") } - c := cs.Command{Schema: cs.SchemaVersion, Cluster: status.Cluster, Actor: s.Device.ID(), Session: "leadership", Sequence: saved.Sequence + 1, Ack: saved.Sequence, Now: time.Now().UnixMilli(), Change: cs.Mutation{RequestLeadership: &cs.RequestLeadership{Revision: status.Membership.Revision, Node: node, Device: s.Device.ID()}}} + c := cs.Command{Schema: cs.SchemaVersion, Cluster: status.Cluster, Actor: s.Device.ID(), Session: "leadership", Sequence: saved.Sequence + 1, Ack: saved.Sequence, Now: s.now().UnixMilli(), Change: cs.Mutation{RequestLeadership: &cs.RequestLeadership{Revision: status.Membership.Revision, Node: node, Device: s.Device.ID()}}} // A personal device may request another enrolled target, but must bind its // committed device ID rather than guessing it from a node number. if node != status.Self { @@ -116,6 +115,23 @@ func (s *Sync) LeadCluster(ctx context.Context, node uint64) (cs.LeadershipRecei } else if node != 0 && saved.Pending.Change.RequestLeadership.Node != node { return receipt, errors.New("a leadership request is still pending; retry it before choosing another computer") } + // A closed terminal may use a scoped lifetime over its existing member store. + // Another open window already owns that store and keeps serving independently. + var scoped *personalMember + if s.displayMember() == nil { + if _, e := os.Stat(filepath.Join(s.memberPath(), "identity.json")); e == nil { + scoped, e = s.openMember(ctx) + if e != nil && !filelock.IsBusy(e) { + return receipt, e + } + if scoped != nil { + defer scoped.close() + if e = scoped.start(ctx); e != nil { + return receipt, e + } + } + } + } receipt, err = client.Lead(ctx, *saved.Pending) if err != nil { var refused clusterctl.CommittedRefusal diff --git a/internal/syncsetup/leadership_ingress_test.go b/internal/syncsetup/leadership_ingress_test.go new file mode 100644 index 0000000000..f7a5551e77 --- /dev/null +++ b/internal/syncsetup/leadership_ingress_test.go @@ -0,0 +1,436 @@ +package syncsetup + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/blobstore" + "github.com/Agent-Field/codeaf/internal/clusterapi" + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/reqsign" +) + +// T2/R6/R15/R18 cover original signed v2/scoped ingress and committed watches +// while P leads. P never requests or stores the independent content blob. +func TestR6R15R18PersonalLeaderOriginalWorkerIngressAndThinStore(t *testing.T) { + r := newMemberRig(t) + p := r.home(nil) + var blobRequests atomic.Uint64 + signed, _ := http.NewRequest(http.MethodGet, r.server.URL, nil) + reqsign.Sign(signed, nil, deviceSigner{p.Identity, p.Device}, r.clock.Now()) + certP := signed.Header.Get(reqsign.HeaderCert) + originalHandler := r.server.Config.Handler + r.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, q *http.Request) { + if strings.HasPrefix(q.URL.Path, "/v1/store/") && q.Header.Get(reqsign.HeaderCert) == certP { + blobRequests.Add(1) + } + originalHandler.ServeHTTP(w, q) + }) + personalLeader(t, r, p) + id := p.Identity + b := r.home(&id) + const cell = "01J0000000000000000000000A" + head := strings.Repeat("1", 64) + _, err := b.Dir.Create(t.Context(), cell, directory.CellInit{Head: head}) + mustNil(t, err) + mustNil(t, b.Dir.Release(t.Context(), cell, 1)) + status, err := p.clusterClient().Status(t.Context()) + mustNil(t, err) + owner := b.ClusterAPI(status.Cluster) + owner.Now = r.clock.Now + sequence := uint64(0) + signOwner := func(change cs.Mutation) cs.Command { + sequence++ + c, err := b.SignClusterCommand(cs.Command{Schema: cs.SchemaVersion, Cluster: status.Cluster, Session: "external-ingress", Sequence: sequence, Ack: sequence - 1, Now: r.clock.Now().UnixMilli(), Change: change}) + mustNil(t, err) + return c + } + submit := func(client clusterapi.Client, c cs.Command, want cs.Code) cs.Result { + t.Helper() + result, err := client.Submit(t.Context(), c) + mustNil(t, err) + if result.Code != want { + t.Fatalf("R6 original proposal: want %s got %+v", want, result) + } + return result + } + _, key, err := ed25519.GenerateKey(rand.Reader) + mustNil(t, err) + cert, err := identity.IssueWorkerCert(id, key.Public().(ed25519.PublicKey), status.Cluster, []string{cell}, r.clock.Now(), r.clock.Now().Add(time.Hour), false) + mustNil(t, err) + worker := cert.DeviceID() + incarnation := uint64(0) + wire := clusterapi.Client{Base: r.server.URL, HTTP: r.server.Client(), Now: r.clock.Now, Allowed: func() bool { return true }, Sign: func(b []byte) []byte { return ed25519.Sign(key, b) }, Request: func() cs.ClusterRequest { + return cs.ClusterRequest{IdentityKey: id.PublicKey(), Cluster: status.Cluster, Worker: &cert, Incarnation: incarnation} + }} + n := uint64(0) + signWorker := func(change cs.Mutation) cs.Command { + n++ + c, err := cs.Sign(cs.Command{Schema: cs.SchemaVersion, Cluster: status.Cluster, Actor: worker, Session: "external-worker", Sequence: n, Ack: n - 1, WorkerIncarnation: incarnation, Now: r.clock.Now().UnixMilli(), Change: change}, key) + mustNil(t, err) + return c + } + original := signOwner(cs.Mutation{EnrollWorker: &cs.EnrollWorker{Certificate: cert, Grants: map[string][]byte{cell: {0x81, 0x82}}}}) + raw, err := cs.EncodeCommand(original) + mustNil(t, err) + submit(owner, original, cs.OK) + submit(wire, signWorker(cs.Mutation{RegisterWorker: &cs.RegisterWorker{Incarnation: 1}}), cs.OK) + incarnation = 1 + view, grant, err := wire.Cell(t.Context(), cell) + mustNil(t, err) + if view.Cell.Head != head || !bytes.Equal(grant, []byte{0x81, 0x82}) { + t.Fatal("R18 scoped read lost grant", view, grant) + } + if _, _, err = wire.Cell(t.Context(), "outside"); err == nil { + t.Fatal("R18 worker escaped its grant") + } + hb := cs.WorkerHeartbeat{Worker: worker, Incarnation: 1, Sequence: 1, SeenAt: r.clock.Now().UnixMilli()} + hraw, err := cs.WorkerHeartbeatBytes(status.Cluster, hb) + mustNil(t, err) + hb.Signature = ed25519.Sign(key, hraw) + submit(wire, signWorker(cs.Mutation{WorkerBeatBatch: &cs.WorkerBeatBatch{Items: []cs.WorkerHeartbeat{hb}}}), cs.OK) + assignment := cs.Assign{ID: "external-run", Revision: 1, Cell: cell, Expect: cs.LeaseExpectation{Device: b.Device.ID(), Fence: 1}, Target: worker, Incarnation: 1, Expires: r.clock.Now().Add(time.Minute).UnixMilli(), SealedPayload: []byte{0x91}} + submit(owner, signOwner(cs.Mutation{Assign: &assignment}), cs.OK) + ref := (cs.Assignment{Spec: assignment}).Ref() + current, _, err := wire.Current(t.Context(), ref) + mustNil(t, err) + if current.Status != "pending" { + t.Fatal(current) + } + claim := signWorker(cs.Mutation{ClaimAssignment: &cs.ClaimAssignment{Ref: ref, Expect: assignment.Expect}}) + result := submit(wire, claim, cs.OK) + if result.View == nil || result.View.Cell.Lease.Fence != 2 { + t.Fatal(result) + } + submit(wire, claim, cs.OK) + _, err = b.Dir.Acquire(t.Context(), cell, directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: worker, Fence: 2}}) + mustNil(t, err) + current, view, err = wire.Current(t.Context(), ref) + mustNil(t, err) + if current.Fence != 2 || view.Cell.Lease.Fence != 3 { + t.Fatal("R18 cached claim became current authority", current, view) + } + watch, err := b.Dir.(directory.HoldWatcher).WatchHolding(t.Context(), []directory.Hold{{Cell: cell, Fence: 3}}) + mustNil(t, err) + defer watch.Close() + ctx, cancel := context.WithTimeout(t.Context(), requestTimeout) + defer cancel() + _, err = watch.Next(ctx) + mustNil(t, err) + r.clock.Advance(time.Duration(cs.RenewEveryMillis) * time.Millisecond) + mustNil(t, watch.Ping(ctx)) + receipt := false + for !receipt { + frame, err := watch.Next(ctx) + mustNil(t, err) + for _, hold := range frame.Holds { + if hold.Cell == cell && hold.Fence == 3 && hold.Index > 0 { + receipt = true + } + } + } + // C1/C3 decisions use the same original public write/read path while P + // leads. Pairing itself never adds a vote or a peer on the new device. + requests := directory.NewLinkHTTP(r.server.URL, r.server.Client()) + for _, approve := range []bool{true, false} { + seed, e := identity.NewSeed() + mustNil(t, e) + opened, e := requests.CreateRequest(t.Context(), directory.NewRequest{Pubkey: base64.RawURLEncoding.EncodeToString(seed.Public()), X25519: base64.RawURLEncoding.EncodeToString(bytes.Repeat([]byte{0x37}, 32)), NameSealed: base64.RawURLEncoding.EncodeToString(bytes.Repeat([]byte{0x81}, 48)), Platform: "linux"}) + mustNil(t, e) + if approve { + certRaw, e := json.Marshal(identity.IssueCert(id, seed.Public(), r.clock.Now())) + mustNil(t, e) + approval := directory.Approval{Device: directory.Device{V: 1, Name: "c2VhbGVk", Platform: "linux"}, Cert: base64.RawURLEncoding.EncodeToString(certRaw), Grant: base64.RawURLEncoding.EncodeToString(bytes.Repeat([]byte{0x87}, 64))} + mustNil(t, b.Dir.ApproveRequest(t.Context(), opened.Code, approval)) + mustNil(t, b.Dir.ApproveRequest(t.Context(), opened.Code, approval)) + } else { + mustNil(t, b.Dir.DenyRequest(t.Context(), opened.Code)) + } + decision, e := requests.GetRequest(t.Context(), opened.Code, 0) + mustNil(t, e) + if approve && (decision.State != directory.RequestApproved || decision.Grant == nil) || !approve && decision.State != directory.RequestDenied { + t.Fatal("R6 pairing decision did not commit", decision) + } + } + // A content object exists at the relay; only its uploader may request it. + + sentinel := []byte("P-MUST-NOT-STORE-THIS-CONTENT-5f99") + frame, err := blobstore.Encode(strings.Repeat("0", 32), []blobstore.Object{{RID: strings.Repeat("a", 64), Bytes: append([]byte("AGEO\x01"), sentinel...)}}) + mustNil(t, err) + _, err = b.Store.PutFrame(t.Context(), frame) + mustNil(t, err) + _, err = b.Dir.List(t.Context()) + mustNil(t, err) + waitMember(t, p, func(m *personalMember) bool { return m.idle }) + p.member.mu.Lock() + recovered, err := p.member.store.Recover(t.Context()) + p.member.mu.Unlock() + mustNil(t, err) + found := false + for _, entry := range recovered.Core.Entries { + found = found || bytes.Equal(entry.GetData(), raw) + } + if !found { + t.Fatal("R6 relay replaced the original signed proposal") + } + err = filepath.WalkDir(p.memberPath(), func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + raw, err := os.ReadFile(path) + if err != nil { + return err + } + if bytes.Contains(raw, sentinel) { + t.Error("R15 P stored content", path) + } + } + return nil + }) + mustNil(t, err) + if blobRequests.Load() != 0 { + t.Fatal("R15 P fetched content") + } +} + +// T3/R6 consumes a committed refusal before issuing the next owner request. +func TestR6PersonalLeadershipCommittedRefusalAcknowledged(t *testing.T) { + r := newMemberRig(t) + s := r.home(nil) + readyMember(t, s) + status, err := s.clusterClient().Status(t.Context()) + mustNil(t, err) + c, err := s.SignClusterCommand(cs.Command{Schema: cs.SchemaVersion, Cluster: status.Cluster, Session: "leadership", Sequence: 1, Now: r.clock.Now().UnixMilli(), Change: cs.Mutation{RequestLeadership: &cs.RequestLeadership{Revision: 0, Node: status.Self, Device: s.Device.ID()}}}) + mustNil(t, err) + path := filepath.Join(s.Home, "cluster-leadership-"+s.Device.ID()+".json") + mustNil(t, saveLeadership(path, leadershipSession{Pending: &c})) + _, err = s.LeadCluster(t.Context(), 0) + var refused clusterctl.CommittedRefusal + if !errors.As(err, &refused) || refused.Sequence != 1 { + t.Fatal("R6 refusal was not committed", err) + } + raw, err := os.ReadFile(path) + mustNil(t, err) + var saved leadershipSession + mustNil(t, json.Unmarshal(raw, &saved)) + if saved.Pending != nil || saved.Sequence != 1 { + t.Fatal("R6 refusal was not acknowledged", saved) + } + receipt, err := s.LeadCluster(t.Context(), 0) + mustNil(t, err) + if receipt.Node != status.Self { + t.Fatal(receipt) + } +} + +// T2/R6/R9/R13 keep admission and streamed catch-up working when the actual +// leader is outside the relay and has compacted its coordination log. +func TestR6R9R13PersonalLeaderAdmitsAndSnapshotsAnotherHome(t *testing.T) { + r := newMemberRig(t) + p := r.home(nil) + receipt := personalLeader(t, r, p) + mustNil(t, p.Dir.PutDevice(t.Context(), p.Device.ID(), directory.Device{V: 1, Name: "c2VhbGVk"})) + frames := make([]string, 400) + for i := range frames { + frames[i] = fmt.Sprintf("%064x", i+1) + } + for i := 0; i < 12; i++ { + _, err := p.Dir.Create(t.Context(), fmt.Sprintf("%026d", i+1), directory.CellInit{Head: strings.Repeat("1", 64), Frames: frames}) + mustNil(t, err) + } + p.member.mu.Lock() + err := p.member.checkpoint(t.Context()) + recovered, recoverErr := p.member.store.Recover(t.Context()) + p.member.mu.Unlock() + mustNil(t, err) + mustNil(t, recoverErr) + manifest, err := raftsnapshot.Decode(recovered.Core.Snapshot.GetData()) + mustNil(t, err) + if manifest.Chunks < 2 { + t.Fatal("R13 fixture did not require multiple snapshot chunks", manifest.Chunks) + } + id := p.Identity + joined := r.home(&id) + // Retry only when the actual leader reports local progress, preserving the + // same admission token. There is no real-time polling or fixed retry budget. + deadline := time.NewTimer(requestTimeout) + defer deadline.Stop() + for { + p.member.mu.Lock() + changed := p.member.changed + p.member.mu.Unlock() + err = joined.memberRound(t.Context()) + if err == nil && joined.member != nil { + break + } + if err != nil && !errors.Is(err, errMemberWaiting) { + t.Fatal(err) + } + joined.memberRetry = time.Time{} + select { + case <-changed: + case <-deadline.C: + t.Fatal("R9 learner did not join while P led", err) + } + } + waitMember(t, joined, func(m *personalMember) bool { return m.idle && m.status.Applied >= m.record.Grant.Barrier }) + status, err := joined.clusterClient().Status(t.Context()) + mustNil(t, err) + if status.Leader != receipt.Node || !slices.Contains(status.Ready, nodeOf(joined)) || !slices.Contains(status.Configuration.Learners, nodeOf(joined)) { + t.Fatal("R6/R9 external leader did not serve catch-up", status) + } +} + +// T3/R6 resumes the saved bytes after the member and its Sync owner restart. +// A durable fallback supersedes that request; it must be acknowledged BEFORE +// a later explicit request can choose leadership again. +func TestR6PersonalLeadershipRestartResumesSavedRequest(t *testing.T) { + r := newMemberRig(t) + s := r.home(nil) + original := r.server.Config.Handler + var mu sync.Mutex + var requests [][]byte + lost := false + r.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, q *http.Request) { + if q.URL.Path != clusterctl.Path || q.Method != http.MethodPost { + original.ServeHTTP(w, q) + return + } + raw, err := io.ReadAll(q.Body) + if err != nil { + panic(err) + } + q.Body = io.NopCloser(bytes.NewReader(raw)) + record := httptest.NewRecorder() + original.ServeHTTP(record, q) + mu.Lock() + defer mu.Unlock() + requests = append(requests, raw) + if record.Code == 200 && !lost { + lost = true + http.Error(w, "reply lost", 503) + return + } + for key, values := range record.Header() { + w.Header()[key] = values + } + w.WriteHeader(record.Code) + w.Write(record.Body.Bytes()) + }) + readyMember(t, s) + if _, err := s.LeadCluster(t.Context(), 0); err == nil { + t.Fatal("R6 lost reply claimed success") + } + path := filepath.Join(s.Home, "cluster-leadership-"+s.Device.ID()+".json") + raw, err := os.ReadFile(path) + mustNil(t, err) + var pending leadershipSession + mustNil(t, json.Unmarshal(raw, &pending)) + if pending.Pending == nil { + t.Fatal("R6 unknown outcome discarded request") + } + s.stopMember() + for tick := 0; tick < 22; tick++ { + r.clock.Advance(DefaultInterval) + } + again := buildAt(s.Home, s.Relay, s.Identity, s.Device, s.Interval, r.clock.Now) + _, err = again.LeadCluster(t.Context(), 0) + var refused clusterctl.CommittedRefusal + if !errors.As(err, &refused) || refused.Sequence != pending.Pending.Sequence { + t.Fatal("R6 restart did not acknowledge superseded intent", err) + } + mu.Lock() + same := len(requests) == 2 && bytes.Equal(requests[0], requests[1]) + mu.Unlock() + if !same { + t.Fatal("R6 restart allocated or changed its saved request") + } + raw, err = os.ReadFile(path) + mustNil(t, err) + var saved leadershipSession + mustNil(t, json.Unmarshal(raw, &saved)) + if saved.Pending != nil || saved.Sequence != pending.Pending.Sequence { + t.Fatal("R6 refusal not acknowledged", saved) + } + status, err := again.clusterClient().Status(t.Context()) + mustNil(t, err) + if status.Membership.Leadership != nil || !slices.Equal(status.Configuration.Voters, []uint64{1, 2, 3}) { + t.Fatal("R6 saved retry automatically chose leadership", status) + } +} + +// R6/R13/R27 keep the largest admitted public command usable while P leads. +// Protobuf framing must consume the existing bounded chunk lane, not a larger cap. +func TestR6R13R27PersonalLeaderKeepsMaximumPublicCommand(t *testing.T) { + r := newMemberRig(t) + p := r.home(nil) + personalLeader(t, r, p) + status, err := p.clusterClient().Status(t.Context()) + mustNil(t, err) + encodedSize := func(n int) int { + t.Helper() + body, _ := json.Marshal(directory.Device{V: 1, Name: strings.Repeat("x", n)}) + request, err := http.NewRequest(http.MethodPut, r.server.URL+"/v1/dir/devices/"+p.Device.ID(), bytes.NewReader(body)) + mustNil(t, err) + reqsign.Sign(request, body, deviceSigner{p.Identity, p.Device}, r.clock.Now()) + proof, err := reqsign.Capture(request, body) + mustNil(t, err) + command, err := cs.V1Command(status.Cluster, r.clock.Now().UnixMilli(), proof) + mustNil(t, err) + raw, err := cs.EncodeCommand(command) + mustNil(t, err) + return len(raw) + } + low, high := 0, cs.MaxCommandBytes + for low < high { + middle := (low + high + 1) / 2 + if encodedSize(middle) <= cs.MaxCommandBytes { + low = middle + } else { + high = middle - 1 + } + } + if encodedSize(low) > cs.MaxCommandBytes || encodedSize(low) < cs.MaxCommandBytes-4 { + t.Fatal("R27 fixture missed the legal command boundary") + } + mustNil(t, p.Dir.PutDevice(t.Context(), p.Device.ID(), directory.Device{V: 1, Name: strings.Repeat("x", low)})) + _, err = p.Dir.List(t.Context()) + mustNil(t, err) + p.member.mu.Lock() + app, _, err := p.member.store.ApplicationView(t.Context()) + key := raftstore.Key{Namespace: raftstore.State, Key: cs.DeviceKey(p.Device.ID()).Key} + var rows map[raftstore.Key][]byte + if err == nil { + rows, _, err = p.member.store.Read(t.Context(), app.Applied, []raftstore.Key{key}) + } + p.member.mu.Unlock() + mustNil(t, err) + var saved cs.Device + mustNil(t, cs.Decode(rows[key], &saved)) + if saved.Name != strings.Repeat("x", low) { + t.Fatal("R6 maximum public command was not applied on P") + } +} diff --git a/internal/syncsetup/leadership_native_test.go b/internal/syncsetup/leadership_native_test.go new file mode 100644 index 0000000000..099267eb85 --- /dev/null +++ b/internal/syncsetup/leadership_native_test.go @@ -0,0 +1,381 @@ +package syncsetup + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/directory/directorytest" + "github.com/Agent-Field/codeaf/internal/dirwatch" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/relayserve" + "github.com/Agent-Field/codeaf/internal/reqsign" + "go.etcd.io/raft/v3" +) + +func personalLeader(t *testing.T, r *memberRig, s *Sync) cs.LeadershipReceipt { + t.Helper() + readyMember(t, s) + receipt, err := s.LeadCluster(t.Context(), 0) + mustNil(t, err) + waitMember(t, s, func(m *personalMember) bool { + return m.status.Role == raft.StateLeader && m.status.Term == receipt.Term && m.status.Applied >= receipt.Barrier + }) + status, err := s.clusterClient().Status(t.Context()) + mustNil(t, err) + if status.Leader != receipt.Node || !slices.Equal(status.Configuration.Voters, []uint64{1, 2, receipt.Node}) || status.Configuration.Joint() || !slices.Contains(status.Configuration.Learners, 3) || status.Membership.Leadership == nil || !status.Membership.Leadership.Completed { + t.Fatalf("R6 transfer lacks durable settlement/elected leader: %+v %+v", receipt, status) + } + return receipt +} + +// T1/T2/R6 run the existing directory contract against the REAL external +// leader. The relay is only ingress, storage and A/B/C; P has its own core. +func TestR6PersonalLeaderDirectoryConformance(t *testing.T) { + directorytest.RunConsensusRigs(t, func(t *testing.T) directorytest.Rig { + r := newMemberRig(t) + p := r.home(nil) + personalLeader(t, r, p) + id := p.Identity + type apiDevice struct { + client directory.Client + id string + } + devices := map[string]apiDevice{} + var mu sync.Mutex + get := func(name string) apiDevice { + mu.Lock() + defer mu.Unlock() + if d, ok := devices[name]; ok { + return d + } + seed, err := identity.NewSeed() + mustNil(t, err) + dev, err := seed.Join(identity.IssueCert(id, seed.Public(), r.clock.Now())) + mustNil(t, err) + d := apiDevice{client: directory.NewHTTP(r.server.URL, reqsign.SignFor(deviceSigner{id, dev}, r.clock.Now), r.server.Client()), id: dev.ID()} + devices[name] = d + return d + } + return directorytest.Rig{Clock: r.clock, Devices: func(name string) directory.Client { return get(name).client }, ID: func(name string) string { return get(name).id }} + }) +} + +// T3/R6 retains the exact saved signature after a committed reply is lost. +// A new client resumes it and cannot choose another target or sequence. +func TestR6PersonalLeadershipLostReplyResumesSignedRequest(t *testing.T) { + r := newMemberRig(t) + s := r.home(nil) + original := r.server.Config.Handler + var mu sync.Mutex + commands := []cs.Command{} + lost := false + r.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, q *http.Request) { + if q.URL.Path != clusterctl.Path || q.Method != http.MethodPost { + original.ServeHTTP(w, q) + return + } + record := httptest.NewRecorder() + original.ServeHTTP(record, q) + mu.Lock() + defer mu.Unlock() + // The client file is the independently persisted request sent before HTTP. + raw, e := os.ReadFile(filepath.Join(s.Home, "cluster-leadership-"+s.Device.ID()+".json")) + if e != nil { + panic(e) + } + var saved leadershipSession + if json.Unmarshal(raw, &saved) != nil || saved.Pending == nil { + panic("missing saved request") + } + commands = append(commands, *saved.Pending) + if record.Code == 200 && !lost { + lost = true + http.Error(w, "lost reply", 503) + return + } + for key, values := range record.Header() { + w.Header()[key] = values + } + w.WriteHeader(record.Code) + w.Write(record.Body.Bytes()) + }) + readyMember(t, s) + if _, err := s.LeadCluster(t.Context(), 0); err == nil { + t.Fatal("R6 lost reply claimed success") + } + again := buildAt(s.Home, s.Relay, s.Identity, s.Device, s.Interval, r.clock.Now) + if _, err := again.LeadCluster(t.Context(), nodeOf(s)+1); err == nil { + t.Fatal("R6 pending request changed target") + } + receipt, err := again.LeadCluster(t.Context(), 0) + mustNil(t, err) + mu.Lock() + defer mu.Unlock() + if !lost || len(commands) != 2 || !reflect.DeepEqual(commands[0], commands[1]) || receipt.Node != nodeOf(s) { + t.Fatalf("R6 retry changed request: %+v", commands) + } + raw, err := os.ReadFile(filepath.Join(s.Home, "cluster-leadership-"+s.Device.ID()+".json")) + mustNil(t, err) + var saved leadershipSession + mustNil(t, json.Unmarshal(raw, &saved)) + if saved.Pending != nil || saved.Sequence != commands[0].Sequence { + t.Fatal("R6 completed request not acknowledged") + } +} + +// T4/R7 exercises process death and a use-time quiet decision with virtual +// healthy ticks. A+B must decide before restoring C, without asking P. +func TestR7PersonalLeaderFallbackAndNoAutomaticRegain(t *testing.T) { + for _, mode := range []string{"death", "death-after-idle", "sync-off", "unpaired"} { + t.Run(mode, func(t *testing.T) { + r := newMemberRig(t) + s := r.home(nil) + receipt := personalLeader(t, r, s) + id := s.Identity + observer := r.home(&id) + if mode == "death-after-idle" { + for tick := 0; tick < 20; tick++ { + r.clock.Advance(DefaultInterval) + waitMember(t, s, func(m *personalMember) bool { return m.idle }) + } + waitMember(t, s, func(m *personalMember) bool { return m.sleeping }) + } + if mode == "death" || mode == "death-after-idle" { + s.stopMember() + if mode == "death-after-idle" { + select { + case <-r.departures: + case <-time.After(requestTimeout): + t.Fatal("R7 server did not observe the closed leader connection") + } + } + } else { + attempts := s.relayAttempts.Load() + s.clusterFrames.Load() + if mode == "sync-off" { + t.Setenv(URLVar, "") + mustNil(t, SaveRelayURL(s.Home, "off")) + } else { + mustNil(t, os.Remove(filepath.Join(s.Home, identity.File))) + } + mustNil(t, s.memberRound(t.Context())) + if s.member != nil || s.relayAttempts.Load()+s.clusterFrames.Load() != attempts { + t.Fatal("R23 quiet did not stop its leading member locally") + } + if mode == "sync-off" { + mustNil(t, SaveRelayURL(s.Home, r.server.URL)) + } else { + _, err := os.Stat(filepath.Join(s.Home, identity.File)) + if !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + } + } + // Local elapsed time, not real sleeps, drives the healthy ranked windows. + for tick := 0; tick < 13; tick++ { + r.clock.Advance(time.Duration(cs.TickEveryMillis) * time.Millisecond) + } + status, err := observer.clusterClient().Status(t.Context()) + mustNil(t, err) + if status.Leader == receipt.Node || status.Leader == 0 { + t.Fatal("R7 A+B did not elect within 65 virtual seconds", status) + } + _, err = observer.Dir.Create(t.Context(), "01J0000000000000000000FAL", directory.CellInit{Head: strings.Repeat("1", 64)}) + mustNil(t, err) + _, err = observer.Dir.Acquire(t.Context(), "01J0000000000000000000FAL", directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: observer.Device.ID(), Fence: 1}}) + mustNil(t, err) + for tick := 0; tick < 8; tick++ { + r.clock.Advance(time.Duration(cs.TickEveryMillis) * time.Millisecond) + } + status, err = observer.clusterClient().Status(t.Context()) + mustNil(t, err) + if !slices.Equal(status.Configuration.Voters, []uint64{1, 2, 3}) || status.Configuration.Joint() || status.Membership.Leadership != nil || !slices.Contains(status.Configuration.Learners, receipt.Node) { + t.Fatalf("R7 C was not restored: %+v", status) + } + if mode == "unpaired" { + _, err = identity.Adopt(s.Home, id, true) + mustNil(t, err) + } + readyMember(t, s) + status, err = s.clusterClient().Status(t.Context()) + mustNil(t, err) + if status.Leader == receipt.Node || !slices.Contains(status.Configuration.Learners, receipt.Node) { + t.Fatal("R7 returning P regained leadership", status) + } + }) + } +} + +// T5/R8 retains the SAME three-voter cluster with P+A absent. Surviving B and +// learner C cannot serve a writable minority or force removal. +func TestR8PersonalLeaderAndAnchorLossNeverCreatesSecondCluster(t *testing.T) { + r := newMemberRig(t) + s := r.home(nil) + receipt := personalLeader(t, r, s) + status, err := s.clusterClient().Status(t.Context()) + mustNil(t, err) + s.stopMember() + mustNil(t, r.svc.Close()) + path := filepath.Join(r.storage, "raft", s.Identity.ID(), "1.db") + mustNil(t, os.Rename(path, path+".asleep")) + r.svc = relayserve.New(relayserve.Config{Store: r.storage, Now: r.clock.Now, After: r.clock.After}) + _, err = s.clusterClient().Status(t.Context()) + if err == nil || err.Error() != cs.LeadershipUnavailable { + t.Fatal("R8 minority status", err) + } + if _, err = s.Dir.Create(t.Context(), "01J0000000000000000000MIN", directory.CellInit{Head: strings.Repeat("1", 64)}); err == nil { + t.Fatal("R8 minority write succeeded") + } + mustNil(t, r.svc.Close()) + mustNil(t, os.Rename(path+".asleep", path)) + r.svc = relayserve.New(relayserve.Config{Store: r.storage, Now: r.clock.Now, After: r.clock.After}) + after, err := s.clusterClient().Status(context.Background()) + mustNil(t, err) + if after.Cluster != status.Cluster || receipt.Node == 0 { + t.Fatal("R8 recovered a different cluster") + } +} + +// T6/R23/R26 repeats the whole open-Home measurement with actual P leadership +// and with P asleep after fallback. Both wire sides AND local attempts count. +func TestR6R7R23R26PersonalLeaderWholeHomeIdle(t *testing.T) { + for _, mode := range []string{"leading", "asleep-after-fallback"} { + t.Run(mode, func(t *testing.T) { + r := newMemberRig(t) + a := r.home(nil) + id := a.Identity + b := r.home(&id) + readyMember(t, b) + personalLeader(t, r, a) + live := []*Sync{a, b} + if mode == "asleep-after-fallback" { + a.stopMember() + live = []*Sync{b} + for tick := 0; tick < 22; tick++ { + r.clock.Advance(DefaultInterval) + } + status, err := b.clusterClient().Status(t.Context()) + mustNil(t, err) + if status.Membership.Leadership != nil || !slices.Equal(status.Configuration.Voters, []uint64{1, 2, 3}) { + t.Fatal("R7 fallback not settled", status) + } + } + ticks := []func(){} + screens := []*Sync{} + followers := []dirwatch.Follower{} + streams := []dirwatch.Stream{} + for _, s := range live { + mustNil(t, s.SyncState(t.Context())) + ctx, cancel := context.WithCancel(t.Context()) + ch := make(chan time.Time) + done := make(chan struct{}) + go func() { s.runState(ctx, ch); close(done) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(requestTimeout): + t.Error("R23 state owner did not stop") + } + }) + tick := func() { + select { + case ch <- r.clock.Now(): + case <-time.After(requestTimeout): + t.Fatal("R26 owner stalled") + } + } + ticks = append(ticks, func() { tick(); tick() }) + screen := buildAt(s.Home, s.Relay, s.Identity, s.Device, s.Interval, r.clock.Now) + screens = append(screens, screen) + follower := screen.Follow() + followers = append(followers, follower) + defer follower.Close() + stream, err := screen.followStream(t.Context(), screen.Dir.(directory.Watcher)) + mustNil(t, err) + streams = append(streams, stream) + defer stream.Close() + } + round := func() { + for _, tick := range ticks { + tick() + } + for _, s := range live { + waitMember(t, s, func(m *personalMember) bool { return m.idle }) + } + } + for tick := 0; tick < 24; tick++ { + r.clock.Advance(DefaultInterval) + round() + } + attempts := func() uint64 { + n := a.relayAttempts.Load() + a.clusterFrames.Load() + b.relayAttempts.Load() + b.clusterFrames.Load() + for _, s := range screens { + n += s.relayAttempts.Load() + s.clusterFrames.Load() + } + return n + } + httpBefore, framesBefore, attemptBefore := r.requests.Load(), r.in.count.Load()+r.out.count.Load(), attempts() + for tick := 0; tick < 120; tick++ { + r.clock.Advance(DefaultInterval) + round() + for i, screen := range screens { + mustNil(t, streams[i].Ping(t.Context())) + for { + frame, err := streams[i].Next(t.Context()) + mustNil(t, err) + if frame.Pong { + break + } + } + _, _, err := screen.RowsAt(t.Context()) + mustNil(t, err) + _, err = screen.Roster(t.Context()) + mustNil(t, err) + _, err = screen.FleetSize(t.Context()) + mustNil(t, err) + followers[i].Probe() + } + } + gotHTTP, gotFrames, gotAttempts := r.requests.Load()-httpBefore, r.in.count.Load()+r.out.count.Load()-framesBefore, attempts()-attemptBefore + t.Logf("R23/R26 %s ten-minute idle: relay HTTP=%d socket frames=%d client attempts=%d", mode, gotHTTP, gotFrames, gotAttempts) + if gotHTTP != 0 || gotFrames != 0 || gotAttempts != 0 { + t.Fatal("R23/R26 whole idle Home generated traffic") + } + _, err := b.Dir.List(t.Context()) + mustNil(t, err) + mustNil(t, b.Dir.PutDevice(t.Context(), b.Device.ID(), directory.Device{V: 1, Name: "c2VhbGVk"})) + }) + } +} + +// R6/R16 require the new leader to keep serving voters when its configuration +// still names a learner whose committed device revocation already denies use. +func TestR6R16PersonalLeaderSkipsRevokedLearner(t *testing.T) { + r := newMemberRig(t) + a := r.home(nil) + id := a.Identity + b := r.home(&id) + readyMember(t, a) + readyMember(t, b) + mustNil(t, b.Dir.PutDevice(t.Context(), b.Device.ID(), directory.Device{V: 1, Name: "c2VhbGVk"})) + mustNil(t, a.Dir.Revoke(t.Context(), b.Device.ID())) + personalLeader(t, r, a) + mustNil(t, a.Dir.PutDevice(t.Context(), a.Device.ID(), directory.Device{V: 1, Name: "c2VhbGVk"})) + if _, err := b.Dir.List(t.Context()); err == nil { + t.Fatal("R16 revoked learner retained public authority") + } +} diff --git a/internal/syncsetup/leadership_process_test.go b/internal/syncsetup/leadership_process_test.go new file mode 100644 index 0000000000..562d0f41f5 --- /dev/null +++ b/internal/syncsetup/leadership_process_test.go @@ -0,0 +1,132 @@ +package syncsetup + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "slices" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/cell/celltest" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" +) + +// R7's helper runs the production member in a separate address space so the +// parent can kill it without any graceful close or sleep notification. +func TestR7PersonalMemberProcessHelper(t *testing.T) { + home := os.Getenv("CODEAF_TEST_MEMBER_HOME") + if home == "" { + return + } + celltest.Available(t) + s, ok, err := Open(home) + mustNil(t, err) + if !ok { + t.Fatal("R7 member home unavailable") + } + readyMember(t, s) + receipt, err := s.LeadCluster(t.Context(), 0) + mustNil(t, err) + raw, err := json.Marshal(receipt) + mustNil(t, err) + fmt.Println("R7 elected " + string(raw)) + // The parent's SIGKILL is the only exit. The ordinary package deadline still + // bounds a failed parent; there is no test-created service or daemon. + select {} +} + +// T4/R7 kills the ACTUAL personal process after election and its read receipt. +// Surviving A+B perform a new conditional operation and restore C with P dead. +func TestR7PersonalLeaderProcessDeathRestoresAnchors(t *testing.T) { + r := newMemberRig(t) + s := r.home(nil) + cmd := exec.Command(os.Args[0], "-test.run=^TestR7PersonalMemberProcessHelper$", "-test.v", "-test.timeout=15m") + cmd.Env = append(os.Environ(), "CODEAF_TEST_MEMBER_HOME="+s.Home) + stdout, err := cmd.StdoutPipe() + mustNil(t, err) + var stderr bytes.Buffer + cmd.Stderr = &stderr + mustNil(t, cmd.Start()) + defer func() { + if cmd.ProcessState == nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + } + }() + elected := make(chan cs.LeadershipReceipt, 1) + failure := make(chan error, 1) + go func() { + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "R7 elected ") { + var receipt cs.LeadershipReceipt + err := json.Unmarshal([]byte(strings.TrimPrefix(line, "R7 elected ")), &receipt) + if err != nil { + failure <- err + return + } + elected <- receipt + return + } + } + failure <- fmt.Errorf("R7 child exited before election: %v", scanner.Err()) + }() + var receipt cs.LeadershipReceipt + select { + case receipt = <-elected: + case err := <-failure: + t.Fatal(err) + case <-time.After(requestTimeout): + t.Fatal("R7 child did not finish leadership") + } + before, err := s.clusterClient().Status(t.Context()) + mustNil(t, err) + if before.Leader != receipt.Node || receipt.Barrier == 0 { + t.Fatal("R7 child was not elected", before, receipt) + } + saved := recordOf(t, s) + mustNil(t, cmd.Process.Kill()) + if err = cmd.Wait(); err == nil { + t.Fatal("R7 child exited gracefully") + } + for tick := 0; tick < 13; tick++ { + r.clock.Advance(time.Duration(cs.TickEveryMillis) * time.Millisecond) + } + status, err := s.clusterClient().Status(t.Context()) + mustNil(t, err) + if status.Leader == receipt.Node || status.Leader == 0 { + t.Fatal("R7 anchors did not elect after SIGKILL", status, stderr.String()) + } + const cell = "01J000000000000000000KILL" + _, err = s.Dir.Create(t.Context(), cell, directory.CellInit{Head: strings.Repeat("1", 64)}) + mustNil(t, err) + _, err = s.Dir.Acquire(t.Context(), cell, directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: s.Device.ID(), Fence: 1}}) + mustNil(t, err) + for tick := 0; tick < 8; tick++ { + r.clock.Advance(time.Duration(cs.TickEveryMillis) * time.Millisecond) + } + status, err = s.clusterClient().Status(t.Context()) + mustNil(t, err) + if status.Membership.Leadership != nil || !slices.Equal(status.Configuration.Voters, []uint64{1, 2, 3}) { + t.Fatal("R7 C not restored with dead P", status) + } + readyMember(t, s) + if s.member.sequence < saved.SequenceCeiling { + t.Fatal("R6/R7 restart reused a transport sequence") + } + if nodeOf(s) != receipt.Node { + t.Fatal("R7 returning durable process received another identity") + } + status, err = s.clusterClient().Status(t.Context()) + mustNil(t, err) + if status.Leader == receipt.Node || !slices.Contains(status.Configuration.Learners, receipt.Node) { + t.Fatal("R7 dead process regained its vote or leadership", status) + } +} diff --git a/internal/syncsetup/member.go b/internal/syncsetup/member.go index 5056cac207..82215a7eb8 100644 --- a/internal/syncsetup/member.go +++ b/internal/syncsetup/member.go @@ -35,37 +35,51 @@ import ( var errMemberWaiting = errors.New("this computer is still joining the cluster; retry shortly; your saved chats are still here") -// memberRecord belongs to the store, rather than the device's pairing. Losing -// an initialized store must retire its old authority instead of bootstrapping -// another log under the same node. Pending admission keeps its token on retry. +// A durable reservation avoids fsync per frame while keeping sequences unique +// across process death, including deliveries still buffered for a sleeping peer. +const memberSequenceBlock uint64 = 1 << 32 + +// memberRecord belongs to the store rather than the device's pairing. Losing +// an initialized store retires its authority instead of bootstrapping another +// log under the same node. Pending admission keeps its token on retry. type memberRecord struct { - Store string - Grant raftdirectory.PersonalEnrollment - Initialized bool - Seed uint64 + SequenceCeiling uint64 + Store string + Grant raftdirectory.PersonalEnrollment + Initialized bool + Seed uint64 } type personalMember struct { - closed bool - changed chan struct{} - idle bool - mu sync.Mutex - owner *Sync - record memberRecord - path string - lock *os.File - store *raftstore.SQLite - core *raftcore.Core - status raftcore.Status - pending []rafttransport.Envelope - sequence uint64 - ack [3]uint64 - snapshot *pb.Message - chunks map[uint64]bool - conn *websocket.Conn - cancel context.CancelFunc - done chan struct{} - failure error + chunkWaiting map[uint64]uint64 + sleeping bool + outSnapshots map[uint64]*memberSnapshot + forwardConn *websocket.Conn + forwardDone chan struct{} + forwardReplies chan clusterctl.Forward + forwardPending map[string]clusterctl.Forward + forwardStarted map[string]time.Time + closed bool + changed chan struct{} + idle bool + mu sync.Mutex + owner *Sync + record memberRecord + path string + lock *os.File + store *raftstore.SQLite + core *raftcore.Core + status raftcore.Status + pending []rafttransport.Envelope + sent map[uint64]rafttransport.Envelope + sequence uint64 + ack [3]uint64 + snapshot *pb.Message + chunks map[uint64]bool + conn *websocket.Conn + cancel context.CancelFunc + done chan struct{} + failure error } func (s *Sync) memberPath() string { @@ -213,6 +227,11 @@ func (s *Sync) openMember(ctx context.Context) (_ *personalMember, err error) { return nil, err } m.record.Seed = binary.LittleEndian.Uint64(entropy[:]) + if m.record.SequenceCeiling > ^uint64(0)-memberSequenceBlock { + return nil, rafttransport.ErrGeneration + } + m.sequence = m.record.SequenceCeiling + m.record.SequenceCeiling += memberSequenceBlock if err = saveLeadership(filepath.Join(path, "identity.json"), m.record); err != nil { return nil, err } @@ -227,6 +246,9 @@ func (s *Sync) openMember(ctx context.Context) (_ *personalMember, err error) { beforeApplied, beforeAck := m.status.Applied, m.ack sent := len(m.pending) != 0 if err = m.exchange(ctx); err != nil { + if errors.Is(err, errMemberWaiting) && m.snapshot != nil { + break + } return nil, err } if m.status.Applied >= g.Barrier && len(m.pending) == 0 { @@ -239,6 +261,12 @@ func (s *Sync) openMember(ctx context.Context) (_ *personalMember, err error) { // An unavailable leader must not turn activation into an empty hot poll. // Retain the same durable store and resume at the owner's next retry round. if !sent && beforeApplied == m.status.Applied && beforeAck == m.ack && len(m.pending) == 0 { + // A staged snapshot may need the sender's next asynchronous chunk credit. + // Keep this actor and subscribe passively instead of losing its manifest + // between HTTP rounds. Readiness still requires durable application. + if m.snapshot != nil { + break + } return nil, errMemberWaiting } } @@ -255,6 +283,21 @@ func (m *personalMember) step(ctx context.Context, input raftcore.Input) error { return err } m.status = effects.Status + if proposal, ok := input.(raftcore.Propose); ok && effects.Persist != nil { + for id, work := range m.forwardPending { + if work.Action == "submit" && work.Barrier == 0 && bytes.Equal(work.Command, proposal.Command) { + for _, entry := range effects.Persist.Entries { + if bytes.Equal(entry.GetData(), proposal.Command) { + work.Barrier = entry.GetIndex() + } + } + m.forwardPending[id] = work + } + } + } + for _, read := range effects.Reads { + m.readForward(ctx, read.RequestID, read.Index) + } for _, out := range effects.Send { var destination uint64 for _, peer := range m.record.Grant.Members { @@ -262,8 +305,23 @@ func (m *personalMember) step(ctx context.Context, input raftcore.Input) error { destination = peer.Incarnation } } + // Revocation can precede configuration removal. Dropping that + // unreachable delivery must not stop replication to other voters. + if destination == 0 { + continue + } + if err = m.rememberSnapshot(out.To, out.Bytes); err != nil { + return err + } + if m.sequence == m.record.SequenceCeiling { + return rafttransport.ErrGeneration + } m.sequence++ e := rafttransport.Envelope{Scope: m.record.Grant.Scope, From: m.record.Grant.Node, To: out.To, Incarnation: m.record.Grant.Incarnation, Destination: destination, Sequence: m.sequence, Payload: out.Bytes} + if len(out.Bytes) > rafttransport.MaxBatchBytes { + hash := raftsnapshot.ID(out.Bytes) + e.Chunk = &rafttransport.Chunk{Generation: hash, Hash: hash} + } if err = e.SignWith(m.owner.Device.Sign); err != nil { return err } @@ -277,7 +335,20 @@ func (m *personalMember) step(ctx context.Context, input raftcore.Input) error { continue } if effects.Apply != nil { - if _, err = raftdirectory.Apply(ctx, m.store, *effects.Apply, m.record.Grant.Scope.Cluster, 0); err != nil { + results, applyErr := raftdirectory.Apply(ctx, m.store, *effects.Apply, m.record.Grant.Scope.Cluster, 0) + if err = applyErr; err != nil { + return err + } + entries := [][]byte{} + indices := []uint64{} + for _, entry := range effects.Apply.Entries { + if entry.GetType() == pb.EntryNormal && len(entry.GetData()) != 0 { + entries = append(entries, entry.GetData()) + indices = append(indices, entry.GetIndex()) + } + } + m.appliedForward(entries, indices, results) + if err = m.refreshPeers(ctx); err != nil { return err } input = raftcore.Applied{BatchID: effects.Apply.ID} @@ -316,6 +387,12 @@ func (m *personalMember) exchange(ctx context.Context) error { return err } response, err := (rafttransport.Client{URL: m.owner.Relay, HTTP: m.owner.ClusterHTTP(nil)}).Exchange(ctx, req) + // Startup retains its durable store and admission token on transient inbox + // pressure. The existing owner retries with bounded backoff instead of + // reporting a permanently unusable member or starting an empty hot poll. + if errors.Is(err, rafttransport.ErrCapacity) { + return errMemberWaiting + } if err != nil { return err } @@ -335,6 +412,9 @@ func (m *personalMember) receive(ctx context.Context, response rafttransport.Res if len(response.Messages) > rafttransport.MaxMessages { return rafttransport.ErrWire } + if err := m.socketAdmission(ctx, response); err != nil { + return err + } for _, delivery := range response.Messages { if delivery.Cursor == 0 { return rafttransport.ErrCursor @@ -374,9 +454,15 @@ func (m *personalMember) receive(ctx context.Context, response rafttransport.Res if e.Verify(source) != nil || e.Scope != m.record.Grant.Scope || e.To != m.record.Grant.Node || e.Destination != m.record.Grant.Incarnation { return rafttransport.ErrAuth } + if e.AppendChunk() { + e.Chunk = nil + } if e.Chunk != nil { if m.snapshot == nil || e.Chunk.Generation != raftsnapshot.ID(m.snapshot.GetSnapshot().GetData()) { - return rafttransport.ErrWire + // A reconnect may outlive its expendable snapshot receiver. Drain + // orphaned chunks; only a fresh signed manifest can start installation. + m.ack[delivery.Lane] = delivery.Cursor + continue } if err = m.store.PutSnapshotChunk(ctx, e.Chunk.Generation, e.Chunk.Offset, e.Payload, e.Chunk.Hash); err != nil { return err @@ -416,13 +502,38 @@ func (m *personalMember) receive(ctx context.Context, response rafttransport.Res m.snapshot = message m.chunks = map[uint64]bool{} } - } else if err = m.step(ctx, raftcore.PeerMessage{Bytes: e.Payload}); err != nil { - return err + } else { + obsolete := false + if m.sleeping && message.GetTerm() <= m.status.Term && (message.GetType() == pb.MsgAppResp || message.GetType() == pb.MsgHeartbeatResp) { + for _, peer := range peers { + if peer.Node == e.From && !peer.Voter { + obsolete = true + } + } + } + if !obsolete { + m.sleeping = false + if err = m.step(ctx, raftcore.PeerMessage{Bytes: e.Payload}); err != nil { + return err + } + } + } + } + if e.Chunk == nil { + message := new(pb.Message) + if proto.Unmarshal(e.Payload, message) == nil && message.GetType() == pb.MsgAppResp && !message.GetReject() { + if transfer := m.outSnapshots[e.From]; transfer != nil && message.GetIndex() >= transfer.manifest.Index { + delete(m.outSnapshots, e.From) + delete(m.chunkWaiting, e.From) + if err = m.step(ctx, raftcore.TransportResult{Peer: e.From, Snapshot: true, Success: true}); err != nil { + return err + } + } } } m.ack[delivery.Lane] = delivery.Cursor } - return nil + return m.finishSchedules(ctx) } func (m *personalMember) start(ctx context.Context) error { @@ -433,7 +544,8 @@ func (m *personalMember) start(ctx context.Context) error { if err != nil { return err } - // Upgrade authentication carries no outbound messages. Catch-up flushed them. + // Upgrade authentication carries no outbound messages. The peer writer + // flushes pending work after the initial subscription response. req.Messages = nil if err = req.SignWith(m.owner.Device.Sign); err != nil { return err @@ -470,38 +582,13 @@ func (m *personalMember) start(ctx context.Context) error { } // An empty push is only a subscription acknowledgment. Answering it would // create an endless empty frame loop and wake an otherwise idle group. - if err == nil && (len(response.Messages) > 0 || len(m.pending) > 0) { - for { - req, e := m.request() - if e != nil { - err = e - break - } - data, _ := rafttransport.Encode(req) - if !m.owner.memberAllowed() { - err = ErrQuiet - break - } - writeCtx, cancel := context.WithTimeout(ctx, requestTimeout) - m.owner.clusterFrames.Add(1) - err = conn.Write(writeCtx, websocket.MessageBinary, data) - cancel() - if err != nil { - break - } - if len(req.Messages) == 0 { - break - } - m.pending = m.pending[1:] - if len(m.pending) == 0 { - break - } - } + if err == nil && (len(response.Messages) > 0 || len(m.pending) > 0 || m.hasSnapshotWork()) { + err = m.flushPeer(ctx) } if err != nil { m.failure = err } - m.idle = err == nil && len(response.Messages) == 0 && len(m.pending) == 0 + m.idle = err == nil && len(response.Messages) == 0 && len(m.pending) == 0 && len(m.sent) == 0 close(m.changed) m.changed = make(chan struct{}) m.mu.Unlock() @@ -511,7 +598,7 @@ func (m *personalMember) start(ctx context.Context) error { } } }() - return nil + return m.startForward(ctx) } func (m *personalMember) close() { @@ -529,6 +616,12 @@ func (m *personalMember) close() { if m.conn != nil { m.conn.CloseNow() } + if m.forwardConn != nil { + m.forwardConn.CloseNow() + } + if m.forwardDone != nil { + <-m.forwardDone + } if m.done != nil { <-m.done } diff --git a/internal/syncsetup/member_client_test.go b/internal/syncsetup/member_client_test.go index 0f3d0615a3..cfbad9942f 100644 --- a/internal/syncsetup/member_client_test.go +++ b/internal/syncsetup/member_client_test.go @@ -36,6 +36,7 @@ import ( // peerFrameCounter observes actual socket bytes at the relay, independently of // client counters. It parses complete masked/unmasked frames across I/O splits. type peerFrameCounter struct { + total *peerFrameCounter mu sync.Mutex b []byte count atomic.Uint64 @@ -72,6 +73,9 @@ func (c *peerFrameCounter) add(b []byte) { return } c.count.Add(1) + if c.total != nil { + c.total.count.Add(1) + } c.b = c.b[header+int(n):] } } @@ -102,13 +106,16 @@ func (w peerCountWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { if e != nil { return nil, nil, e } - wrapped := peerCountConn{c, w.in, w.out} + // Each socket owns its framing buffer; independent streams can split frames + // concurrently while contributing to the same aggregate counters. + wrapped := peerCountConn{c, &peerFrameCounter{total: w.in}, &peerFrameCounter{total: w.out}} // The HTTP handshake already flushed through ResponseWriter before Hijack. rw.Writer.Reset(wrapped) return wrapped, rw, nil } type memberRig struct { + departures chan struct{} parkExchanges atomic.Bool storage string t *testing.T @@ -124,12 +131,20 @@ type memberRig struct { func newMemberRig(t *testing.T) *memberRig { t.Helper() t.Setenv(config.ProfileDirEnv, "") - r := &memberRig{t: t, clock: directorytest.NewFakeClock()} + r := &memberRig{t: t, clock: directorytest.NewFakeClock(), departures: make(chan struct{}, rafttransport.MaxInflight)} r.clock.Advance(time.Now().Sub(r.clock.Now())) r.storage = t.TempDir() r.svc = relayserve.New(relayserve.Config{Store: r.storage, Now: r.clock.Now, After: r.clock.After}) r.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, q *http.Request) { r.requests.Add(1) + if q.URL.Path == clusterctl.ForwardPath { + defer func() { + select { + case r.departures <- struct{}{}: + default: + } + }() + } if q.URL.Path == "/v2/cluster/enroll" && r.interrupt.Swap(false) { // Commit admission and lose its reply before the client's store is opened. recorder := httptest.NewRecorder() @@ -262,8 +277,12 @@ func TestR9PersonalHomesJoinAsDurableLearners(t *testing.T) { t.Fatal(e) } } - if _, e = a.LeadCluster(t.Context(), 0); !errors.Is(e, clusterctl.ErrUnsupported) { - t.Fatal("R6 leadership was offered", e) + // R6 now offers leadership to this caught-up home; the earlier slice's + // unsupported boundary must not contradict the production contract. + receipt, e := a.LeadCluster(t.Context(), 0) + mustNil(t, e) + if receipt.Node != nodeOf(a) || receipt.Barrier == 0 { + t.Fatal("R6 joined home did not receive its elected read receipt", receipt) } } diff --git a/internal/syncsetup/member_forward.go b/internal/syncsetup/member_forward.go new file mode 100644 index 0000000000..6f6dca326a --- /dev/null +++ b/internal/syncsetup/member_forward.go @@ -0,0 +1,437 @@ +package syncsetup + +import ( + "bytes" + "context" + "encoding/base64" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" + "net/http" + "strconv" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/rafttransport" + "github.com/coder/websocket" +) + +// startForward opens application ingress separately from the authenticated +// peer stream. Its lifetime is the SAME terminal/scoped member owner. +func (m *personalMember) startForward(ctx context.Context) error { + m.mu.Lock() + defer m.mu.Unlock() + if !m.owner.memberAllowed() { + return ErrQuiet + } + auth, err := m.request() + if err != nil { + return err + } + auth.Messages = nil + if err = auth.SignWith(m.owner.Device.Sign); err != nil { + return err + } + b, _ := rafttransport.Encode(auth) + headers := http.Header{} + headers.Set(rafttransport.SocketAuthHeader, base64.RawURLEncoding.EncodeToString(b)) + conn, resp, err := websocket.Dial(ctx, m.owner.Relay+clusterctl.ForwardPath, &websocket.DialOptions{HTTPClient: m.owner.ClusterHTTP(&http.Client{Timeout: requestTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}), HTTPHeader: headers, CompressionMode: websocket.CompressionDisabled}) + if resp != nil && err != nil && resp.Body != nil { + resp.Body.Close() + } + if err != nil { + return err + } + conn.SetReadLimit(rafttransport.MaxExchangeBytes) + m.forwardConn = conn + m.forwardDone = make(chan struct{}) + m.forwardReplies = make(chan clusterctl.Forward, rafttransport.MaxInflight) + m.forwardPending = map[string]clusterctl.Forward{} + m.forwardStarted = map[string]time.Time{} + go func() { + defer close(m.forwardDone) + defer func() { + m.mu.Lock() + if !m.closed && m.failure == nil && m.owner.memberAllowed() { + m.failure = rafttransport.ErrGeneration + } + close(m.changed) + m.changed = make(chan struct{}) + m.mu.Unlock() + }() + writeCtx, cancel := context.WithCancel(ctx) + defer cancel() + writerDone := make(chan struct{}) + go func() { + defer close(writerDone) + for { + select { + case <-writeCtx.Done(): + return + case reply := <-m.forwardReplies: + if !m.owner.memberAllowed() { + conn.CloseNow() + return + } + reply.Sign(m.owner.Device.Sign) + b, _ := rafttransport.Encode(reply) + deadline, c := context.WithTimeout(writeCtx, requestTimeout) + m.owner.clusterFrames.Add(1) + err := conn.Write(deadline, websocket.MessageBinary, b) + c() + if err != nil { + conn.CloseNow() + return + } + } + } + }() + defer func() { cancel(); conn.CloseNow(); <-writerDone }() + for { + kind, b, err := conn.Read(ctx) + if err != nil { + return + } + m.mu.Lock() + var work clusterctl.Forward + if !m.owner.memberAllowed() || kind != websocket.MessageBinary || rafttransport.Decode(b, &work) != nil { + m.mu.Unlock() + return + } + valid := work.Scope == m.record.Grant.Scope && work.Generation == m.record.Grant.Generation && work.To == m.record.Grant.Node + authorized := false + // Only immutable provisioned anchor keys may route application ingress. + h, e := raftdirectory.LeadershipState(ctx, m.store) + if e == nil { + for _, id := range h.Anchors { + if id == work.From { + for _, peer := range m.record.Grant.Members { + authorized = authorized || peer.Node == id && !peer.Revoked && work.Verify(peer.Key) + } + } + } + } + if !valid || !authorized { + m.mu.Unlock() + return + } + err = m.acceptForward(ctx, work) + if err == nil && (len(m.pending) > 0 || m.hasSnapshotWork()) { + err = m.flushPeer(ctx) + } + if err != nil { + m.failure = err + } + close(m.changed) + m.changed = make(chan struct{}) + m.mu.Unlock() + if err != nil { + return + } + } + }() + return nil +} +func (m *personalMember) answerForward(work clusterctl.Forward) { + work.From, work.To = work.To, work.From + work.Command = nil + work.Schedule = nil + work.Status = clusterctl.PeerStatus(m.status) + if work.Applied == 0 { + work.Applied = m.status.Applied + } + work.Signature = nil + select { + case m.forwardReplies <- work: + default: + m.failure = rafttransport.ErrCapacity + } +} +func (m *personalMember) acceptForward(ctx context.Context, work clusterctl.Forward) error { + for id, started := range m.forwardStarted { + if time.Since(started) < clusterctl.ForwardTimeout { + continue + } + pending, ok := m.forwardPending[id] + delete(m.forwardStarted, id) + delete(m.forwardPending, id) + if ok && (pending.Action == "read" || pending.Action == "receipt" || pending.Action == "idle-read" || pending.Action == "poke") { + if err := m.step(ctx, raftcore.CancelRead{RequestID: id}); err != nil { + return err + } + } + } + if _, ok := m.forwardPending[work.ID]; ok { + return nil + } + if len(m.forwardPending) >= rafttransport.MaxInflight { + work.Error = "busy" + m.answerForward(work) + return nil + } + var err error + switch work.Action { + case "submit": + m.sleeping = false + if _, err = cs.DecodeCommand(work.Command); err != nil { + break + } + m.forwardPending[work.ID] = work + m.forwardStarted[work.ID] = time.Now() + err = m.step(ctx, raftcore.Propose{Command: work.Command}) + case "read", "receipt", "idle-read", "poke": + if work.Action == "read" || work.Action == "receipt" { + m.sleeping = false + } + m.forwardPending[work.ID] = work + m.forwardStarted[work.ID] = time.Now() + if work.Action == "idle-read" || work.Action == "poke" { + err = m.step(ctx, raftcore.IdleRead{RequestID: work.ID}) + } else { + err = m.step(ctx, raftcore.Read{RequestID: work.ID}) + } + case "membership": + h, e := raftdirectory.LeadershipState(ctx, m.store) + var plan raftcore.MembershipPlan + if e == nil { + plan, e = m.core.PlanMembership(h) + } + if e == nil && len(plan.Change) > 0 { + e = m.step(ctx, raftcore.Configure{Change: plan.Change}) + } + if e == nil && plan.Checkpoint { + e = m.checkpoint(ctx) + } + work.Membership = &plan + m.answerForwardResult(work, e) + return nil + case "evidence": + var evidence raftcore.LeadershipEvidence + evidence, err = raftdirectory.DurableLeadershipEvidence(ctx, m.store, m.record.Grant.Node, work.Revision, work.Barrier) + if err == nil { + work.Evidence = &evidence + } + m.answerForwardResult(work, err) + return nil + case "schedule": + if work.Schedule != nil && len(work.Schedule.Acks) > 0 && (len(m.pending) > 0 || len(m.sent) > 0) { + m.forwardPending[work.ID] = work + m.forwardStarted[work.ID] = time.Now() + return nil + } + if work.Schedule == nil { + return rafttransport.ErrWire + } + var plan raftcore.SchedulePlan + plan, err = raftdirectory.Schedule(ctx, m.store, m.core, work.Schedule.Now, work.Now, work.Schedule.Busy, work.Schedule.Acks) + if err == nil && plan.Tick { + err = m.step(ctx, raftcore.Tick{}) + } + if err == nil { + work.Plan = &plan + m.sleeping = plan.Sleeping + app, _, e := m.store.ApplicationView(ctx) + if e != nil { + err = e + } else { + key := raftstore.Key{Namespace: raftstore.Membership, Key: cs.QuiescenceKey().Key} + rows, _, e := m.store.Read(ctx, app.Applied, []raftstore.Key{key}) + var marker cs.Quiescence + if e == nil && cs.Decode(rows[key], &marker) == nil && marker.Index == app.Applied && marker.Term == m.status.Term { + work.SleepAck = &raftcore.SleepAck{Node: m.record.Grant.Node, Index: marker.Index, Term: marker.Term} + } + } + } + m.answerForwardResult(work, err) + return nil + default: + return rafttransport.ErrWire + } + if err != nil { + delete(m.forwardPending, work.ID) + delete(m.forwardStarted, work.ID) + m.answerForwardResult(work, err) + } + return nil +} +func (m *personalMember) answerForwardResult(work clusterctl.Forward, err error) { + if err != nil { + work.Error = "retry" + } + m.answerForward(work) +} +func (m *personalMember) appliedForward(entries [][]byte, indices []uint64, results []cs.Result) { + for id, work := range m.forwardPending { + if work.Action != "submit" { + continue + } + for i, raw := range entries { + if bytes.Equal(raw, work.Command) && work.Barrier != 0 && indices[i] == work.Barrier { + work.Result = &results[i] + work.Applied = indices[i] + m.answerForward(work) + delete(m.forwardPending, id) + delete(m.forwardStarted, id) + break + } + } + } +} +func (m *personalMember) readForward(ctx context.Context, id string, index uint64) { + work, ok := m.forwardPending[id] + if !ok { + return + } + delete(m.forwardPending, id) + delete(m.forwardStarted, id) + work.Barrier = index + var err error + if work.Action == "receipt" { + var h cs.Membership + h, err = raftdirectory.LeadershipState(ctx, m.store) + if err == nil { + var r cs.LeadershipReceipt + r, err = m.core.LeadershipReceipt(h) + work.Receipt = &r + work.Barrier = r.Barrier + } + } + m.answerForwardResult(work, err) +} + +// flushPeer shares the peer writer's member queue, so forwarded ingress can +// immediately replicate without waiting for an unrelated inbound frame. +func (m *personalMember) flushPeer(ctx context.Context) error { + if err := m.nextSnapshot(ctx); err != nil { + return err + } + if m.conn == nil { + return nil + } + if m.sent == nil { + m.sent = map[uint64]rafttransport.Envelope{} + } + for len(m.sent) < rafttransport.MaxInflight { + req, err := m.request() + if err != nil { + return err + } + data, _ := rafttransport.Encode(req) + if !m.owner.memberAllowed() { + return ErrQuiet + } + deadline, cancel := context.WithTimeout(ctx, requestTimeout) + m.owner.clusterFrames.Add(1) + err = m.conn.Write(deadline, websocket.MessageBinary, data) + cancel() + if err != nil { + return err + } + if len(req.Messages) == 0 { + return nil + } + envelope := req.Messages[0] + m.sent[envelope.Sequence] = envelope + m.pending = m.pending[1:] + if len(m.pending) == 0 { + return nil + } + } + return nil +} + +// socketAdmission retires only transient sends. Backpressure reports actual +// transport failure to Raft and preserves the connection for other voters. +func (m *personalMember) socketAdmission(ctx context.Context, response rafttransport.Response) error { + // Chunk credit means only that the target accepted that staged input. Actual + // snapshot installation still needs its authenticated durable append reply. + for _, ack := range response.Chunks { + if sequence := m.chunkWaiting[ack.Node]; sequence != 0 && ack.Sequence >= sequence { + delete(m.chunkWaiting, ack.Node) + } + } + for _, text := range response.Accepted { + sequence, err := strconv.ParseUint(text, 10, 64) + if err != nil || strconv.FormatUint(sequence, 10) != text { + return rafttransport.ErrWire + } + delete(m.sent, sequence) + } + for _, text := range response.Rejected { + sequence, err := strconv.ParseUint(text, 10, 64) + if err != nil || strconv.FormatUint(sequence, 10) != text { + return rafttransport.ErrWire + } + envelope, ok := m.sent[sequence] + if !ok { + continue + } + delete(m.sent, sequence) + snapshot := envelope.Chunk != nil && !envelope.AppendChunk() + if !snapshot { + message := new(pb.Message) + if proto.Unmarshal(envelope.Payload, message) != nil { + return rafttransport.ErrWire + } + snapshot = message.GetType() == pb.MsgSnap + } + if snapshot { + delete(m.outSnapshots, envelope.To) + delete(m.chunkWaiting, envelope.To) + } + if err = m.step(ctx, raftcore.TransportResult{Peer: envelope.To, Snapshot: snapshot}); err != nil { + return err + } + } + return nil +} + +func (m *personalMember) refreshPeers(ctx context.Context) error { + app, _, err := m.store.ApplicationView(ctx) + if err != nil { + return err + } + if app.Applied < m.record.Grant.Barrier { + return nil + } + view, err := rafttransport.CommittedViewAt(m.record.Grant.Scope, m.record.Grant.Generation, app.Applied, raftcore.Configuration(app.ConfState), app.LogicalTime, func(k cs.RecordKey) ([]byte, error) { + key := raftstore.Key{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key} + rows, _, e := m.store.Read(ctx, app.Applied, []raftstore.Key{key}) + return rows[key], e + }) + if err == nil { + m.record.Grant.Members = view.Members + } + return err +} +func (m *personalMember) checkpoint(ctx context.Context) error { + snapshot, err := m.store.CreateSnapshot(ctx) + if err != nil { + return err + } + if err = m.store.Compact(ctx, snapshot.GetMetadata().GetIndex()); err != nil { + return err + } + return m.step(ctx, raftcore.CompactSnapshot{Snapshot: snapshot}) +} + +// Final sleep waits for this socket's bounded sends to be admitted, rather than +// leaving pre-sleep voter heartbeats in flight. Learner application is irrelevant. +func (m *personalMember) finishSchedules(ctx context.Context) error { + if len(m.pending) > 0 || len(m.sent) > 0 { + return nil + } + for id, work := range m.forwardPending { + if work.Action == "schedule" { + delete(m.forwardPending, id) + delete(m.forwardStarted, id) + if err := m.acceptForward(ctx, work); err != nil { + return err + } + } + } + return nil +} diff --git a/internal/syncsetup/member_snapshot_send.go b/internal/syncsetup/member_snapshot_send.go new file mode 100644 index 0000000000..4dc9012c0d --- /dev/null +++ b/internal/syncsetup/member_snapshot_send.go @@ -0,0 +1,108 @@ +package syncsetup + +import ( + "context" + "slices" + + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + "github.com/Agent-Field/codeaf/internal/rafttransport" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +type memberSnapshot struct { + manifest raftsnapshot.Manifest + generation string + offset uint64 + started int64 +} + +func (m *personalMember) rememberSnapshot(to uint64, raw []byte) error { + message := new(pb.Message) + if proto.Unmarshal(raw, message) != nil { + return raftcore.ErrPeer + } + if message.GetType() != pb.MsgSnap { + return nil + } + manifest, err := raftsnapshot.Decode(message.GetSnapshot().GetData()) + if err != nil { + return err + } + if m.outSnapshots == nil { + m.outSnapshots = map[uint64]*memberSnapshot{} + } + if m.chunkWaiting == nil { + m.chunkWaiting = map[uint64]uint64{} + } + delete(m.chunkWaiting, to) + m.outSnapshots[to] = &memberSnapshot{manifest: manifest, generation: raftsnapshot.ID(message.GetSnapshot().GetData()), started: m.owner.now().UnixMilli()} + return nil +} + +// nextSnapshot queues at most ONE bounded chunk after normal messages. The +// next signed exchange response drives another chunk, without an idle poll. +func (m *personalMember) nextSnapshot(ctx context.Context) error { + if len(m.pending) > 0 { + return nil + } + ids := make([]uint64, 0, len(m.outSnapshots)) + for id := range m.outSnapshots { + ids = append(ids, id) + } + slices.Sort(ids) + for _, id := range ids { + transfer := m.outSnapshots[id] + if m.owner.now().UnixMilli()-transfer.started >= rafttransport.RetentionMillis { + delete(m.outSnapshots, id) + delete(m.chunkWaiting, id) + if err := m.step(ctx, raftcore.TransportResult{Peer: id, Snapshot: true}); err != nil { + return err + } + continue + } + if transfer.offset >= transfer.manifest.Chunks || m.chunkWaiting[id] != 0 { + continue + } + chunk, err := m.store.ReadSnapshotChunk(ctx, transfer.generation, transfer.offset) + if err != nil { + delete(m.outSnapshots, id) + delete(m.chunkWaiting, id) + return m.step(ctx, raftcore.TransportResult{Peer: id, Snapshot: true}) + } + destination := uint64(0) + for _, peer := range m.record.Grant.Members { + if peer.Node == id && !peer.Revoked { + destination = peer.Incarnation + } + } + if destination == 0 { + delete(m.outSnapshots, id) + delete(m.chunkWaiting, id) + continue + } + if m.sequence == m.record.SequenceCeiling { + return rafttransport.ErrGeneration + } + m.sequence++ + e := rafttransport.Envelope{Scope: m.record.Grant.Scope, From: m.record.Grant.Node, To: id, Incarnation: m.record.Grant.Incarnation, Destination: destination, Sequence: m.sequence, Payload: chunk, Chunk: &rafttransport.Chunk{Generation: transfer.generation, Offset: transfer.offset, Hash: raftsnapshot.ID(chunk)}} + if err = e.SignWith(m.owner.Device.Sign); err != nil { + return err + } + m.pending = append(m.pending, e) + m.chunkWaiting[id] = e.Sequence + transfer.offset++ + return nil + } + return nil +} + +func (m *personalMember) hasSnapshotWork() bool { + for id, transfer := range m.outSnapshots { + if transfer.offset < transfer.manifest.Chunks && m.chunkWaiting[id] == 0 { + return true + } + } + return false +} From a219b833460cb15a584f71b842c52afa278968bc Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 22:19:02 -0400 Subject: [PATCH 063/103] Move the size cap by the coordination core's measured weight on the current dev base. On linux/amd64 the stripped binary built from this branch weighs 78,332,169 bytes and the dev base it sits on weighs 75,071,753, so the coordination core adds 3,260,416 bytes. The earlier cap was set from a smaller measurement on an older base and later coordination work used up its headroom, which left the branch 1,281 bytes over. The cap now moves from dev's 75,811,848 by the measured amount, keeping the 740,095 bytes of headroom dev has, and PERF.md states the new measurement. Co-Authored-By: Claude Opus 5.5 (1M context) --- PERF.md | 7 ++++--- SIZE-BUDGET | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/PERF.md b/PERF.md index eeed16d8ad..5039ba4873 100644 --- a/PERF.md +++ b/PERF.md @@ -111,6 +111,7 @@ the feature and its measurements; keep `SIZE-BUDGET` explicit and reviewable. Every build links the Raft coordination core: the modified Raft copy, protobuf, the signed state machine and its SQLite snapshot host. The relay and devices -always coordinate through it, so no build leaves it out. On linux/amd64 it added -2,519,040 bytes to the stripped binary (75,268,361 to 77,787,401), and the cap -moved by the same amount, from 75,811,848 to 78,330,888. +always coordinate through it, so no build leaves it out. On linux/amd64 it adds +3,260,416 bytes to the stripped binary (75,071,753 to 78,332,169 on the same dev +base), and the cap moves by the same amount, from 75,811,848 to 79,072,264, so +the 740,095 bytes of headroom the base had are kept. diff --git a/SIZE-BUDGET b/SIZE-BUDGET index 7f9f69cc47..3420c0cc28 100644 --- a/SIZE-BUDGET +++ b/SIZE-BUDGET @@ -1 +1 @@ -78330888 +79072264 From 9c5c878c7e945e8872857bd7d90bff7aa21b345a Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sat, 24 Oct 2026 22:32:58 -0400 Subject: [PATCH 064/103] State the personal-member lifecycle for the default hosted relay. The real codeaf client, pointed at the hosted relay running in the local Worker runtime, must join its identity's cluster as a durable learner beside the three hosted anchors and then lead it with an elected, committed and freshly read receipt. The Go test skips outside the local hosted proof runner; its launcher compiles the client tests before starting the Worker so linking cannot spend the journey's request deadlines. It fails at the status door until the hosted relay offers personal members. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/syncsetup/member_hosted_test.go | 46 ++++++++++++++++++++ relay/hosted/test/raft/hosted-member.test.js | 30 +++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 internal/syncsetup/member_hosted_test.go create mode 100644 relay/hosted/test/raft/hosted-member.test.js diff --git a/internal/syncsetup/member_hosted_test.go b/internal/syncsetup/member_hosted_test.go new file mode 100644 index 0000000000..bf2fc2ad42 --- /dev/null +++ b/internal/syncsetup/member_hosted_test.go @@ -0,0 +1,46 @@ +package syncsetup + +import ( + "context" + "net/url" + "os" + "slices" + "testing" +) + +// R6/R9/R23 hold on the default relay too: the REAL codeaf client, talking to +// the hosted relay inside the local Worker runtime, joins its identity's +// cluster as a durable learner beside the three hosted anchors, and leading +// returns only an elected, committed and freshly read receipt for itself. +func TestR6R9R23HostedPersonalMemberLifecycle(t *testing.T) { + endpoint := os.Getenv("CODEAF_HOSTED_PROOF_URL") + if endpoint == "" { + t.Skip("local hosted proof runner only") + } + target, err := url.Parse(endpoint) + if err != nil || target.Scheme != "http" || target.Hostname() != "127.0.0.1" { + t.Fatal("hosted proof requires loopback HTTP") + } + home := machine(t, endpoint) + client, ok, err := Open(home) + if err != nil || !ok { + t.Fatal(err) + } + status, err := client.ClusterStatus(context.Background()) + if err != nil { + t.Fatalf("R6/R9 personal activation/status door: %v", err) + } + if status.Self == 0 || len(status.Configuration.Voters) != 3 || len(status.Configuration.Learners) == 0 { + t.Fatalf("R6/R9 personal home did not join as a learner: %+v", status) + } + if !slices.Contains(status.Configuration.Learners, status.Self) || slices.Contains(status.Configuration.Voters, status.Self) { + t.Fatalf("R9 joining made this computer a voter instead of a learner: %+v", status) + } + receipt, err := client.LeadCluster(context.Background(), 0) + if err != nil { + t.Fatalf("R6 actual personal leadership: %v", err) + } + if receipt.Node != status.Self || receipt.Term == 0 || receipt.Barrier == 0 { + t.Fatalf("R6 pending intent presented as leadership: %+v", receipt) + } +} diff --git a/relay/hosted/test/raft/hosted-member.test.js b/relay/hosted/test/raft/hosted-member.test.js new file mode 100644 index 0000000000..039baf6d1d --- /dev/null +++ b/relay/hosted/test/raft/hosted-member.test.js @@ -0,0 +1,30 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { hostedFixture } from './hosted-fixture.js'; + +const root=new URL('../../../../',import.meta.url); +const footprint=new URL('../../../../scripts/raft-proof-footprint.sh',import.meta.url).pathname; +async function run(args,env={}){ + const child=spawn('sh',[footprint,...args],{cwd:root,env:{...process.env,...env},stdio:['ignore','pipe','pipe']}); + let output='';child.stdout.on('data',b=>{output+=b});child.stderr.on('data',b=>{output+=b}); + const code=await new Promise((resolve,reject)=>{child.once('error',reject);child.once('exit',resolve)}); + return {code,output}; +} +// Linking the client tests can leave a disk backlog, so it finishes before +// the Worker starts and cannot spend the journey's own request deadlines. +const compiled=await run(['go','test','-p=1','./internal/syncsetup','-run','^$','-count=1']); +console.log(compiled.output);assert.equal(compiled.code,0,compiled.output); + +// R6/R9/R23 run the real codeaf client against the production Worker entry, +// so the personal-member lifecycle is proved on the default relay itself. +test('R6 R9 R23 a real codeaf home joins the hosted cluster as a learner and leads it',{timeout:300000},async()=>{ + const h=await hostedFixture(); + try{ + await h.start();const url=await h.mf.ready;assert.equal(url.hostname,'127.0.0.1'); + const {code,output}=await run(['go','test','-p=1','./internal/syncsetup','-run','^TestR6R9R23HostedPersonalMemberLifecycle$','-count=1','-timeout=240s','-v'],{CODEAF_HOSTED_PROOF_URL:url.origin}); + console.log(output); + assert.doesNotMatch(output,/--- SKIP:/); + assert.equal(code,0,output);assert.match(output,/--- PASS: TestR6R9R23HostedPersonalMemberLifecycle/); + }finally{await h.close();} +}); From 43b905f399e95b28b12fc53cdeb57f1c685a5797 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 01:38:51 -0400 Subject: [PATCH 065/103] Share committed personal membership policy with the hosted core. Use the same admission stages and status projection for native and hosted relays so enrollment retries retain their node and signed proof. Authenticate native peer frames and encode client responses inside the Go core while preserving the WASI network boundary. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterstate/peer_limits.go | 9 + internal/clusterstate/personal.go | 3 +- internal/clusterstate/personal_host.go | 210 +++++++++++++++++ internal/raftabi/abi.go | 6 + internal/raftabi/application.go | 5 + internal/raftabi/hosted.go | 2 +- internal/raftabi/peer.go | 156 +++++++++++++ internal/raftabi/peer_wire.go | 303 +++++++++++++++++++++++++ internal/raftabi/personal.go | 226 ++++++++++++++++++ internal/raftabi/personal_test.go | 125 ++++++++++ internal/raftabi/scheduler.go | 19 +- internal/raftdirectory/control.go | 72 +----- internal/raftdirectory/personal.go | 90 ++------ internal/rafttransport/envelope.go | 4 +- internal/rafttransport/router.go | 4 +- 15 files changed, 1089 insertions(+), 145 deletions(-) create mode 100644 internal/clusterstate/peer_limits.go create mode 100644 internal/clusterstate/personal_host.go create mode 100644 internal/raftabi/peer.go create mode 100644 internal/raftabi/peer_wire.go create mode 100644 internal/raftabi/personal.go create mode 100644 internal/raftabi/personal_test.go diff --git a/internal/clusterstate/peer_limits.go b/internal/clusterstate/peer_limits.go new file mode 100644 index 0000000000..d6c354db1e --- /dev/null +++ b/internal/clusterstate/peer_limits.go @@ -0,0 +1,9 @@ +package clusterstate + +// Peer quotas bound expendable routing independently of retained application +// records. Both relay runtimes derive their admission and wire limits here. +const ( + MaxPeerMessages = 64 + MaxPeerQueuedBytes = 1 << 20 + PeerRetentionMillis int64 = 30_000 +) diff --git a/internal/clusterstate/personal.go b/internal/clusterstate/personal.go index e2aa338853..30e304f2e0 100644 --- a/internal/clusterstate/personal.go +++ b/internal/clusterstate/personal.go @@ -5,6 +5,7 @@ import ( "crypto/ed25519" "encoding/hex" "strconv" + "strings" ) const PersonalAdmissionPath = "/v2/cluster/enroll" @@ -21,7 +22,7 @@ type PersonalAdmission struct { // for this certificate under the currently committed identity root. func VerifyPersonalAdmission(p V1Request, identity IdentityRec, now int64) (PersonalAdmission, Authority, string, error) { var request PersonalAdmission - if p.Method != "POST" || p.URI != PersonalAdmissionPath || !bytes.Equal(p.IdentityKey, identity.SigningKey) || legacyBody(p.Body, &request, false) != nil { + if p.Method != "POST" || strings.TrimPrefix(p.URI, "/fabric") != PersonalAdmissionPath || !bytes.Equal(p.IdentityKey, identity.SigningKey) || legacyBody(p.Body, &request, false) != nil { return request, Authority{}, "", ErrCodec } token, err := hex.DecodeString(request.Store) diff --git a/internal/clusterstate/personal_host.go b/internal/clusterstate/personal_host.go new file mode 100644 index 0000000000..49aa587f7e --- /dev/null +++ b/internal/clusterstate/personal_host.go @@ -0,0 +1,210 @@ +package clusterstate + +import "bytes" + +// PersonalPlan describes one durable admission stage. The host commits it and +// reconciles configuration before asking again, so a lost reply cannot allocate +// another node or discard the original device proof. +type PersonalPlan struct { + Code Code `json:"code"` + Change *Mutation `json:"change,omitempty"` + Node uint64 `json:"node"` +} + +func personalAllowed(cluster, actor, certificate string, read ReadRecord) (Code, error) { + var identity IdentityRec + raw, err := read(IdentityKey()) + if err != nil { + return BadRequest, err + } + if Decode(raw, &identity) != nil || PersonalCertificate(identity, actor, certificate) != nil { + return Unauthorized, nil + } + raw, err = read(DeviceKey(actor)) + if err != nil { + return BadRequest, err + } + if raw != nil { + var d Device + if Decode(raw, &d) != nil || d.Revoked { + return Unauthorized, nil + } + } + raw, err = read(AuthorityKey(actor)) + if err != nil { + return BadRequest, err + } + if raw != nil { + var a Authority + if Decode(raw, &a) != nil || !a.ValidRole() || !a.Personal || a.Cluster != cluster { + return Unauthorized, nil + } + } + return OK, nil +} + +// PlanPersonalAdmission reads only the committed root, revocations and registry. +// The replicated evaluator repeats these checks when the original proof commits. +func PlanPersonalAdmission(cluster string, proof V1Request, now int64, read ReadRecord) (PersonalPlan, error) { + out := PersonalPlan{Code: Unauthorized} + var identity IdentityRec + raw, err := read(IdentityKey()) + if err != nil { + return out, err + } + if Decode(raw, &identity) != nil { + return out, ErrDependency + } + request, granted, actor, err := VerifyPersonalAdmission(proof, identity, now) + if err != nil { + return out, nil + } + if code, e := personalAllowed(cluster, actor, proof.Certificate, read); e != nil { + return out, e + } else if code != OK { + out.Code = code + return out, nil + } + raw, err = read(AuthorityKey(actor)) + if err != nil { + return out, err + } + if raw != nil { + var a Authority + if Decode(raw, &a) != nil { + return out, ErrDependency + } + if !bytes.Equal(a.Key, granted.Key) { + return out, nil + } + } + if request.Protocol != ClusterProtocol { + out.Code = UpdateRequired + return out, nil + } + var h Membership + raw, err = read(MembershipKey()) + if err != nil { + return out, err + } + if Decode(raw, &h) != nil || h.Validate() != nil { + return out, ErrDependency + } + var mapping DeviceMember + raw, err = read(MemberStoreKey(actor, request.Store)) + if err != nil { + return out, err + } + if raw == nil { + if len(h.Nodes) >= int(h.Limit) { + out.Code = Capacity + return out, nil + } + raw, err = read(MemberDeviceKey(actor)) + if err != nil { + return out, err + } + if raw != nil && Decode(raw, &mapping) != nil { + return out, ErrDependency + } + out.Code = OK + if mapping.Node != 0 { + out.Change = &Mutation{RemoveMember: &RemoveMember{Node: mapping.Node, Device: actor, Revision: h.Revision, Admission: &proof}} + return out, nil + } + if h.Live >= MaxLearners || !validNode(h.NextNode+1) { + out.Code = Capacity + return out, nil + } + out.Change = &Mutation{EnrollMember: &EnrollMember{Node: h.NextNode, Revision: h.Revision, Device: actor, Protocol: request.Protocol, Store: request.Store, Admission: &proof}} + return out, nil + } + if Decode(raw, &mapping) != nil { + return out, ErrDependency + } + raw, err = read(MemberKey(mapping.Node)) + if err != nil { + return out, err + } + var m MemberRecord + if Decode(raw, &m) != nil { + return out, ErrDependency + } + if m.Device != actor || m.Store != request.Store || m.Retired || m.Worker || m.Anchor { + return out, nil + } + out.Code = OK + out.Node = m.Node + return out, nil +} + +// PersonalControl is the shared status shape. All counters use the canonical +// directory encoding, including fields nested in membership and configuration. +type PersonalControl struct { + Ready []uint64 `json:"ready,omitempty"` + LeadershipUnavailable bool `json:"leadership_unavailable,omitempty"` + Cluster string `json:"cluster"` + Self uint64 `json:"self"` + Leader uint64 `json:"leader"` + Membership Membership `json:"membership"` + Configuration Configuration `json:"configuration"` +} + +func PersonalControlView(cluster, actor, certificate string, conf Configuration, applied, leader uint64, now int64, progress map[uint64]uint64, read ReadRecord) (PersonalControl, Code, error) { + out := PersonalControl{Cluster: cluster, Leader: leader, Configuration: conf, LeadershipUnavailable: true} + if code, e := personalAllowed(cluster, actor, certificate, read); e != nil { + return out, code, e + } else if code != OK { + return out, code, nil + } + h, peers, err := MembershipViewAt(read, conf, applied, now) + if err != nil { + return out, BadRequest, err + } + if h.Revision == 0 { + return out, BadRequest, ErrDependency + } + out.Membership = h + for _, peer := range peers { + if peer.Revoked || peer.Voter { + continue + } + raw, e := read(MemberKey(peer.Node)) + if e != nil { + return out, BadRequest, e + } + var m MemberRecord + if Decode(raw, &m) != nil { + return out, BadRequest, ErrDependency + } + if LearnerReady(m, conf, progress[m.Node], m.ReadyIndex) { + out.Ready = append(out.Ready, m.Node) + } + } + raw, err := read(MemberDeviceKey(actor)) + if err != nil { + return out, BadRequest, err + } + if raw == nil { + return out, OK, nil + } + var mapping DeviceMember + if Decode(raw, &mapping) != nil { + return out, BadRequest, ErrDependency + } + if mapping.Node == 0 { + return out, OK, nil + } + raw, err = read(MemberKey(mapping.Node)) + if err != nil { + return out, BadRequest, err + } + var m MemberRecord + if Decode(raw, &m) != nil || m.Node != mapping.Node || m.Device != actor || m.Anchor || m.Worker { + return out, BadRequest, ErrDependency + } + if !m.Retired { + out.Self = m.Node + } + return out, OK, nil +} diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index 9628be32dc..7b12ebf035 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -164,6 +164,12 @@ func (r *Runtime) Call(b []byte) (out []byte) { func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { e := encoder{} switch op { + case 36: + return hostedPersonal(d) + case 37: + return hostedPeer(d) + case 38: + return hostedPeerWire(d) case 35: return configurationView(d) case 34: diff --git a/internal/raftabi/application.go b/internal/raftabi/application.go index 87dec81c29..59c0b15cad 100644 --- a/internal/raftabi/application.go +++ b/internal/raftabi/application.go @@ -93,6 +93,11 @@ func registryCodec(d *decoder) ([]byte, error) { conf := new(pb.ConfState) d.message(conf) header, member, device := d.blob(), d.blob(), d.blob() + // A missing metadata row is not an empty committed record. Personal + // certificate admission does not require a prior device-name write. + if len(device) == 0 { + device = nil + } supplied := len(d.b) > 0 now := int64(0) var identity, authority, worker []byte diff --git a/internal/raftabi/hosted.go b/internal/raftabi/hosted.go index 932527f328..da91e92a73 100644 --- a/internal/raftabi/hosted.go +++ b/internal/raftabi/hosted.go @@ -114,7 +114,7 @@ func hostedAnchor(d *decoder) ([]byte, error) { if cs.Decode(raw, &change) != nil { return nil, errFrame } - if change.InitMembership == nil && change.PrepareQuiescence == nil && change.SettleMembership == nil && change.SettleLeadership == nil && change.CompleteLeadership == nil && change.RestoreAnchors == nil { + if (change.EnrollMember == nil || change.EnrollMember.Admission == nil) && (change.RemoveMember == nil || change.RemoveMember.Admission == nil) && change.InitMembership == nil && change.PrepareQuiescence == nil && change.SettleMembership == nil && change.SettleLeadership == nil && change.CompleteLeadership == nil && change.RestoreAnchors == nil { return nil, errFrame } sequence, ack, err := cs.NextSession(saved) diff --git a/internal/raftabi/peer.go b/internal/raftabi/peer.go new file mode 100644 index 0000000000..baf1835577 --- /dev/null +++ b/internal/raftabi/peer.go @@ -0,0 +1,156 @@ +package raftabi + +import ( + "bytes" + "encoding/json" + "strconv" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +type peerScopeState struct { + Identity string `json:"identity"` + Cluster string `json:"cluster"` + Epoch uint64 `json:"epoch"` + Protocol uint16 `json:"protocol"` +} +type peerView struct { + Scope peerScopeState `json:"scope"` + Generation uint64 `json:"generation"` + Members []cs.PeerEnrollment `json:"members"` + Applied uint64 `json:"applied"` + Term uint64 `json:"term"` + Sleeping bool `json:"sleeping"` +} + +// The core checks the complete native wire and committed keys before any +// routing side effect. Its result contains only dispatch and sleep hints. +func hostedPeer(d *decoder) ([]byte, error) { + raw, trusted := d.blob(), d.blob() + if d.end() != nil { + return nil, errFrame + } + var v peerView + if cs.Decode(trusted, &v) != nil { + return nil, errFrame + } + var req peerRequest + code := "ok" + obsolete := []string{} + if peerDecode(raw, &req) != nil || req.validate() != nil { + code = "invalid" + } else if req.Scope != peerScope(v.Scope) { + code = "unauthorized" + } else { + members := map[uint64]cs.PeerEnrollment{} + for _, m := range v.Members { + members[m.Node] = m + } + source, ok := members[req.Node] + if !ok || source.Revoked || source.Incarnation != req.Incarnation || req.Verify(source.Key) != nil { + code = "unauthorized" + } else if req.Generation != v.Generation { + code = "generation" + } else if !source.Voter && req.Applied > v.Applied { + code = "invalid" + } else { + for _, e := range req.Messages { + to, ok := members[e.To] + if e.Scope != req.Scope || e.From != req.Node || e.Incarnation != req.Incarnation || !ok || to.Revoked || to.Incarnation != e.Destination { + code = "unauthorized" + break + } + if err := e.Verify(source.Key); err != nil { + if err == peerErrWire { + code = "invalid" + } else { + code = "unauthorized" + } + break + } + if v.Sleeping && !source.Voter && e.Chunk == nil { + m := new(pb.Message) + if proto.Unmarshal(e.Payload, m) == nil && m.GetTerm() <= v.Term && (m.GetType() == pb.MsgAppResp || m.GetType() == pb.MsgHeartbeatResp) { + obsolete = append(obsolete, strconv.FormatUint(e.Sequence, 10)) + } + } + } + } + } + out, err := json.Marshal(struct { + Code string `json:"code"` + Obsolete []string `json:"obsolete"` + }{code, obsolete}) + if err != nil { + return nil, err + } + e := encoder{} + e.blob(out) + return e.b, nil +} + +type peerDelivery struct { + Lane uint8 `json:"lane"` + Cursor uint64 `json:"cursor,string"` + Envelope peerEnvelope `json:"envelope"` +} +type peerCredit struct { + Node uint64 `json:"node,string"` + Sequence uint64 `json:"sequence,string"` +} +type peerResponse struct { + Chunks []peerCredit `json:"chunks,omitempty"` + Accepted []string `json:"accepted,omitempty"` + Rejected []string `json:"rejected,omitempty"` + Generation uint64 `json:"generation,string"` + Messages []peerDelivery `json:"messages"` +} + +func hostedPeerWire(d *decoder) ([]byte, error) { + action := d.u32() + e := encoder{} + if action == 2 { + code := cs.Code(d.str()) + if d.end() != nil { + return nil, errFrame + } + e.str(cs.EnrollmentMessage(code)) + return e.b, nil + } + if action == 0 { + if d.end() != nil { + return nil, errFrame + } + raw, err := json.Marshal(struct { + Batch, Chunk, Exchange, Messages, Inflight, Retention int + Slots [3]int + Budget [3]int + }{peerMaxBatchBytes, peerMaxChunkBytes, peerMaxExchangeBytes, peerMaxMessages, peerMaxInflight, int(cs.PeerRetentionMillis), [3]int{peerMaxInflight / 2, peerMaxInflight / 4, peerMaxInflight / 4}, [3]int{3 * peerMaxQueuedBytes / 8, peerMaxQueuedBytes / 8, peerMaxQueuedBytes / 2}}) + if err != nil { + return nil, err + } + e.blob(raw) + return e.b, nil + } + raw := d.blob() + if d.end() != nil || action != 1 { + return nil, errFrame + } + var out peerResponse + j := json.NewDecoder(bytes.NewReader(raw)) + j.DisallowUnknownFields() + if j.Decode(&out) != nil || out.Generation == 0 || len(out.Messages) > peerMaxMessages { + return nil, errFrame + } + if out.Messages == nil { + out.Messages = []peerDelivery{} + } + b, err := json.Marshal(out) + if err != nil || len(b) > peerMaxExchangeBytes { + return nil, errFrame + } + e.blob(b) + return e.b, nil +} diff --git a/internal/raftabi/peer_wire.go b/internal/raftabi/peer_wire.go new file mode 100644 index 0000000000..4c136dc669 --- /dev/null +++ b/internal/raftabi/peer_wire.go @@ -0,0 +1,303 @@ +package raftabi + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + "go.etcd.io/raft/v3" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +const peerLaneCount = 3 +const ( + peerProtocol = raftcore.ProtocolVersion + peerMaxBatchBytes = raftcore.MaxMessageBytes + peerMaxChunkBytes = raftsnapshot.ChunkBytes + peerMaxInflight = raftcore.MaxInflightBatches + peerMaxLearners = clusterstate.MaxLearners + peerMaxMessages = clusterstate.MaxPeerMessages + peerMaxQueuedBytes = clusterstate.MaxPeerQueuedBytes + // Encoded envelopes include base64 payloads, signatures and bounded IDs. + peerMaxExchangeBytes = 2 * peerMaxChunkBytes +) + +var ( + peerErrAuth = errors.New("rafttransport: member authorization refused") + peerErrWire = errors.New("rafttransport: invalid peer envelope") + peerErrGeneration = errors.New("rafttransport: transport generation changed; reconnect") + peerErrCapacity = errors.New("rafttransport: transport is busy; retry with backoff") + peerErrCursor = errors.New("rafttransport: invalid delivery cursor") +) + +// peerScope binds every signature to an identity, cluster epoch and protocol. All +// uint64 fields cross JavaScript as canonical unsigned decimal strings. +type peerScope struct { + Identity string `json:"identity"` + Cluster string `json:"cluster"` + Epoch uint64 `json:"epoch,string"` + Protocol uint16 `json:"protocol"` +} + +func peerToken(s string) bool { + if len(s) == 0 || len(s) > 128 { + return false + } + for _, c := range s { + if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || strings.ContainsRune("-_.", c)) { + return false + } + } + return true +} +func (s peerScope) valid() bool { + return peerToken(s.Identity) && peerToken(s.Cluster) && s.Epoch != 0 && s.Protocol == peerProtocol +} +func peerNode(id uint64) bool { return id != 0 && !raft.IsLocalMsgTarget(id) } + +// peerChunk names one portable snapshot generation and zero-based chunk number. +// Hash is checked before routing; installation remains the receiving host's job. +type peerChunk struct { + Generation string `json:"generation"` + Offset uint64 `json:"offset,string"` + Hash string `json:"hash"` +} + +type peerEnvelope struct { + Scope peerScope `json:"scope"` + From uint64 `json:"from,string"` + To uint64 `json:"to,string"` + Incarnation uint64 `json:"incarnation,string"` + Destination uint64 `json:"destination,string"` + Sequence uint64 `json:"sequence,string"` + Chunk *peerChunk `json:"chunk"` + Digest string `json:"digest"` + Payload []byte `json:"payload"` + Signature []byte `json:"signature"` +} + +func peerDigest(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } +func peerHash(s string) bool { + b, e := hex.DecodeString(s) + return e == nil && len(b) == sha256.Size && s == strings.ToLower(s) +} + +// signingBytes uses a fixed JSON field order and ASCII-only metadata. Payloads +// are base64, so native and JavaScript hosts can sign exactly the same bytes. +func (e peerEnvelope) signingBytes() []byte { + e.Signature = nil + b, _ := json.Marshal(e) + return append([]byte("codeaf/raft/envelope/1\n"), b...) +} +func (e *peerEnvelope) Sign(key ed25519.PrivateKey) error { + if len(key) != ed25519.PrivateKeySize { + return peerErrAuth + } + return e.SignWith(func(b []byte) []byte { return ed25519.Sign(key, b) }) +} + +// SignWith keeps device private keys behind their existing signing boundary. +func (e *peerEnvelope) SignWith(sign func([]byte) []byte) error { + if sign == nil { + return peerErrAuth + } + e.Digest = peerDigest(e.Payload) + if err := e.validate(); err != nil { + return err + } + e.Signature = sign(e.signingBytes()) + if len(e.Signature) != ed25519.SignatureSize { + return peerErrAuth + } + return nil +} +func (e peerEnvelope) Verify(key ed25519.PublicKey) error { + if err := e.validate(); err != nil { + return err + } + if len(key) != ed25519.PublicKeySize || !ed25519.Verify(key, e.signingBytes(), e.Signature) { + return peerErrAuth + } + return nil +} + +// AppendChunk carries one framed append whose command already fits the normal +// proposal budget. The payload peerHash as generation distinguishes it from a +// snapshot manifest; both retain the existing chunk byte and queue limits. +func (e peerEnvelope) AppendChunk() bool { + return e.Chunk != nil && e.Chunk.Offset == 0 && e.Chunk.Generation == e.Chunk.Hash +} + +func (e peerEnvelope) validate() error { + if !e.Scope.valid() || !peerNode(e.From) || !peerNode(e.To) || e.From == e.To || e.Incarnation == 0 || e.Destination == 0 || e.Sequence == 0 || e.Digest != peerDigest(e.Payload) { + return peerErrWire + } + if e.Chunk != nil { + if len(e.Payload) == 0 || len(e.Payload) > peerMaxChunkBytes || !peerHash(e.Chunk.Generation) || e.Chunk.Hash != e.Digest { + return peerErrWire + } + if !e.AppendChunk() { + return nil + } + if len(e.Payload) <= peerMaxBatchBytes { + return peerErrWire + } + } else if len(e.Payload) == 0 || len(e.Payload) > peerMaxBatchBytes { + return peerErrWire + } + m := &pb.Message{} + if proto.Unmarshal(e.Payload, m) != nil || m.Type == nil || m.GetFrom() != e.From || m.GetTo() != e.To || m.GetTerm() == 0 || len(m.GetResponses()) != 0 || len(m.ProtoReflect().GetUnknown()) != 0 { + return peerErrWire + } + if e.AppendChunk() && m.GetType() != pb.MsgApp { + return peerErrWire + } + switch m.GetType() { + case pb.MsgApp, pb.MsgAppResp, pb.MsgVote, pb.MsgVoteResp, pb.MsgSnap, pb.MsgHeartbeat, pb.MsgHeartbeatResp, pb.MsgPreVote, pb.MsgPreVoteResp, pb.MsgTimeoutNow: + return nil + default: + return peerErrWire + } +} + +// peerEncode is the wire format shared by HTTP and socket frames. peerDecode rejects +// alternative encodings so duplicate keys cannot disagree across runtimes. +func peerEncode(v any) ([]byte, error) { return json.Marshal(v) } +func peerDecode(b []byte, v any) error { + if len(b) > peerMaxExchangeBytes { + return peerErrWire + } + d := json.NewDecoder(bytes.NewReader(b)) + d.DisallowUnknownFields() + if d.Decode(v) != nil { + return peerErrWire + } + if d.Decode(new(any)) != io.EOF { + return peerErrWire + } + canonical, err := peerEncode(v) + if err != nil || !bytes.Equal(canonical, b) { + return peerErrWire + } + return nil +} + +// peerRequest signs polls and acknowledgments as well as outbound digests. A +// stolen unsigned cursor must not allow a third party to drain a peer's inbox. +type peerRequest struct { + // Applied is signed evidence from the member after its application transaction. + // A transport acknowledgment or leader Match alone never makes it ready. + Applied uint64 `json:"applied,string,omitempty"` + Scope peerScope `json:"scope"` + Node uint64 `json:"node,string"` + Incarnation uint64 `json:"incarnation,string"` + Generation uint64 `json:"generation,string"` + Ack [peerLaneCount]uint64 `json:"ack"` + Messages []peerEnvelope `json:"messages"` + Signature []byte `json:"signature"` +} + +func peerDecimal(n uint64) string { return strconv.FormatUint(n, 10) } +func peerParseDecimal(s string) (uint64, error) { + n, err := strconv.ParseUint(s, 10, 64) + if err != nil || peerDecimal(n) != s { + return 0, peerErrWire + } + return n, nil +} + +// Ack is encoded through peerDecimal strings even though it is a fixed array. +func (r peerRequest) MarshalJSON() ([]byte, error) { + type raw peerRequest + return json.Marshal(struct { + raw + Ack [peerLaneCount]string `json:"ack"` + }{raw(r), [peerLaneCount]string{peerDecimal(r.Ack[0]), peerDecimal(r.Ack[1]), peerDecimal(r.Ack[2])}}) +} +func (r *peerRequest) UnmarshalJSON(b []byte) error { + type raw peerRequest + var v struct { + raw + Ack [peerLaneCount]string `json:"ack"` + } + d := json.NewDecoder(bytes.NewReader(b)) + d.DisallowUnknownFields() + if d.Decode(&v) != nil { + return peerErrWire + } + *r = peerRequest(v.raw) + for i, s := range v.Ack { + n, err := peerParseDecimal(s) + if err != nil { + return err + } + r.Ack[i] = n + } + return nil +} +func (r peerRequest) signingBytes() []byte { + r.Signature = nil + b, _ := peerEncode(r) + return append([]byte("codeaf/raft/exchange/1\n"), b...) +} +func (r *peerRequest) Sign(key ed25519.PrivateKey) error { + if len(key) != ed25519.PrivateKeySize { + return peerErrAuth + } + return r.SignWith(func(b []byte) []byte { return ed25519.Sign(key, b) }) +} + +// SignWith uses the same canonical bytes without exporting a device seed. +func (r *peerRequest) SignWith(sign func([]byte) []byte) error { + if sign == nil { + return peerErrAuth + } + if err := r.validate(); err != nil { + return err + } + r.Signature = sign(r.signingBytes()) + if len(r.Signature) != ed25519.SignatureSize { + return peerErrAuth + } + return nil +} +func (r peerRequest) Verify(key ed25519.PublicKey) error { + if err := r.validate(); err != nil { + return err + } + if len(key) != ed25519.PublicKeySize || !ed25519.Verify(key, r.signingBytes(), r.Signature) { + return peerErrAuth + } + return nil +} +func (r peerRequest) validate() error { + if !r.Scope.valid() || !peerNode(r.Node) || r.Incarnation == 0 || r.Generation == 0 || len(r.Messages) > peerMaxMessages { + return peerErrWire + } + normal, chunks := 0, 0 + for _, e := range r.Messages { + if e.Chunk != nil { + chunks++ + } else { + normal += len(e.Payload) + } + } + if normal > peerMaxBatchBytes || chunks > 1 || chunks != 0 && len(r.Messages) != 1 { + return peerErrWire + } + b, err := peerEncode(r) + if err != nil || len(b) > peerMaxExchangeBytes { + return peerErrWire + } + return nil +} diff --git a/internal/raftabi/personal.go b/internal/raftabi/personal.go new file mode 100644 index 0000000000..302545e71a --- /dev/null +++ b/internal/raftabi/personal.go @@ -0,0 +1,226 @@ +package raftabi + +import ( + "encoding/json" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + pb "go.etcd.io/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +type personalInput struct { + Action string `json:"action"` + Cluster string `json:"cluster"` + Identity string `json:"identity"` + Epoch uint64 `json:"epoch"` + Generation uint64 `json:"generation"` + Applied uint64 `json:"applied"` + Now int64 `json:"now"` + CommittedNow int64 `json:"committed_now"` + Leader uint64 `json:"leader"` + Actor string `json:"actor"` + Certificate string `json:"certificate"` + Proof cs.V1Request `json:"proof"` + Conf []byte `json:"conf"` + Progress []personalProgress `json:"progress"` + Rows []personalRow `json:"rows"` +} +type personalProgress struct { + Node uint64 `json:"node"` + Applied uint64 `json:"applied"` +} +type personalRow struct { + Namespace string `json:"namespace"` + Key string `json:"key"` + Data []byte `json:"data"` +} +type personalOutput struct { + Need []cs.RecordKey `json:"need,omitempty"` + Code cs.Code `json:"code"` + Message string `json:"message,omitempty"` + Plan *cs.PersonalPlan `json:"plan,omitempty"` + Wire []byte `json:"wire,omitempty"` +} + +// The host supplies immutable rows at a fresh quorum barrier. Missing reads +// request bounded dependencies instead of embedding a second policy in JavaScript. +func hostedPersonal(d *decoder) ([]byte, error) { + raw := d.blob() + if d.end() != nil { + return nil, errFrame + } + var in personalInput + if cs.Decode(raw, &in) != nil { + return nil, errFrame + } + rows := map[cs.RecordKey][]byte{} + size := 0 + for _, row := range in.Rows { + k := cs.RecordKey{Namespace: row.Namespace, Key: row.Key} + if _, exists := rows[k]; exists { + return nil, errFrame + } + rows[k] = row.Data + size += len(row.Data) + len(row.Key) + len(row.Namespace) + } + if size > cs.MaxEvaluationBytes { + return nil, errFrame + } + // Discover a complete bounded page before evaluation so each retained peer + // does not require re-decoding every earlier record in another host round. + need := []cs.RecordKey{} + want := func(k cs.RecordKey) { + if _, ok := rows[k]; !ok { + for _, old := range need { + if old == k { + return + } + } + need = append(need, k) + } + } + want(cs.IdentityKey()) + want(cs.MembershipKey()) + actor := in.Actor + if in.Action != "status" { + var identity cs.IdentityRec + if cs.Decode(rows[cs.IdentityKey()], &identity) == nil { + request, _, who, e := cs.VerifyPersonalAdmission(in.Proof, identity, in.Now) + if e == nil { + actor = who + want(cs.MemberStoreKey(who, request.Store)) + } + } + } + if actor != "" { + want(cs.DeviceKey(actor)) + want(cs.AuthorityKey(actor)) + want(cs.MemberDeviceKey(actor)) + var mapping cs.DeviceMember + if cs.Decode(rows[cs.MemberDeviceKey(actor)], &mapping) == nil && mapping.Node != 0 { + want(cs.MemberKey(mapping.Node)) + } + if in.Action != "status" { + var identity cs.IdentityRec + if cs.Decode(rows[cs.IdentityKey()], &identity) == nil { + request, _, _, e := cs.VerifyPersonalAdmission(in.Proof, identity, in.Now) + if e == nil && cs.Decode(rows[cs.MemberStoreKey(actor, request.Store)], &mapping) == nil && mapping.Node != 0 { + want(cs.MemberKey(mapping.Node)) + } + } + } + } + if in.Action == "status" || in.Action == "grant" { + var h cs.Membership + if cs.Decode(rows[cs.MembershipKey()], &h) == nil { + for _, node := range h.Nodes { + want(cs.MemberKey(node)) + var m cs.MemberRecord + if cs.Decode(rows[cs.MemberKey(node)], &m) == nil && m.Device != "" { + want(cs.DeviceKey(m.Device)) + if m.Worker { + want(cs.AuthorityKey(m.Device)) + want(cs.WorkerKey(m.Device)) + } + } + } + } + } + if len(need) > 0 { + out, _ := cs.Encode(personalOutput{Code: cs.OK, Need: need}) + e := encoder{} + e.blob(out) + return e.b, nil + } + missing := []cs.RecordKey{} + read := func(k cs.RecordKey) ([]byte, error) { + v, ok := rows[k] + if !ok { + missing = append(missing, k) + return nil, cs.ErrDependency + } + return v, nil + } + conf := new(pb.ConfState) + if proto.Unmarshal(in.Conf, conf) != nil { + return nil, errFrame + } + out := personalOutput{Code: cs.OK} + var err error + switch in.Action { + case "admission", "grant": + var plan cs.PersonalPlan + plan, err = cs.PlanPersonalAdmission(in.Cluster, in.Proof, in.Now, read) + out.Code = plan.Code + out.Plan = &plan + if err == nil && plan.Code == cs.OK && in.Action == "grant" && plan.Node != 0 { + var m cs.MemberRecord + v, e := read(cs.MemberKey(plan.Node)) + if e != nil { + err = e + } else if cs.Decode(v, &m) != nil { + err = cs.ErrDependency + } else { + _, peers, e := cs.MembershipViewAt(read, raftcore.Configuration(conf), in.Applied, in.CommittedNow) + err = e + if err == nil { + out.Wire, err = json.Marshal(struct { + Scope peerScope `json:"scope"` + Generation uint64 `json:"generation,string"` + Node uint64 `json:"node,string"` + Incarnation uint64 `json:"incarnation,string"` + Barrier uint64 `json:"barrier,string"` + Members []personalWireMember `json:"members"` + }{peerScope{in.Identity, in.Cluster, in.Epoch, raftcore.ProtocolVersion}, in.Generation, m.Node, m.Incarnation, m.ReadyIndex, personalMembers(peers)}) + } + } + } + case "status": + progress := map[uint64]uint64{} + for _, p := range in.Progress { + progress[p.Node] = p.Applied + } + var status cs.PersonalControl + status, out.Code, err = cs.PersonalControlView(in.Cluster, in.Actor, in.Certificate, raftcore.Configuration(conf), in.Applied, in.Leader, in.CommittedNow, progress, read) + if err == nil && out.Code == cs.OK { + out.Wire, err = cs.Encode(status) + } + default: + return nil, errFrame + } + if len(missing) > 0 { + out = personalOutput{Need: missing, Code: cs.OK} + err = nil + } + if err != nil { + return nil, err + } + out.Message = cs.EnrollmentMessage(out.Code) + if out.Code == cs.Unauthorized || out.Code == cs.Revoked { + out.Message = "this computer is not allowed to join this cluster" + } + b, err := cs.Encode(out) + if err != nil { + return nil, err + } + e := encoder{} + e.blob(b) + return e.b, nil +} + +type personalWireMember struct { + Node uint64 + Incarnation uint64 + Key []byte + Voter bool + Revoked bool +} + +func personalMembers(peers []cs.PeerEnrollment) []personalWireMember { + out := make([]personalWireMember, 0, len(peers)) + for _, p := range peers { + out = append(out, personalWireMember{p.Node, p.Incarnation, p.Key, p.Voter, p.Revoked}) + } + return out +} diff --git a/internal/raftabi/personal_test.go b/internal/raftabi/personal_test.go new file mode 100644 index 0000000000..5e48d34b3c --- /dev/null +++ b/internal/raftabi/personal_test.go @@ -0,0 +1,125 @@ +package raftabi + +import ( + "bytes" + "crypto/ed25519" + "encoding/json" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + rt "github.com/Agent-Field/codeaf/internal/rafttransport" +) + +// R23 preserves the native authenticated wire, including signed applied and +// decimal cursors. Alternate JSON and hostile inner messages fail closed. +func TestR23HostedPeerWireMatchesNative(t *testing.T) { + key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{21}, 32)) + pub := key.Public().(ed25519.PublicKey) + scope := rt.Scope{Identity: "identity", Cluster: "cluster", Epoch: 1, Protocol: 1} + req := rt.Request{Scope: scope, Node: 4, Incarnation: 1, Generation: 7, Applied: 20} + if e := req.Sign(key); e != nil { + t.Fatal(e) + } + raw, _ := rt.Encode(req) + var copied peerRequest + if e := peerDecode(raw, &copied); e != nil { + t.Fatal(e) + } + encoded, _ := peerEncode(copied) + if !bytes.Equal(encoded, raw) { + t.Fatal("R23 request wire differs") + } + trusted := peerView{Scope: peerScopeState{"identity", "cluster", 1, 1}, Generation: 7, Applied: 20, Members: []cs.PeerEnrollment{{Node: 1, Incarnation: 1, Key: pub, Voter: true}, {Node: 4, Incarnation: 1, Key: pub}}} + runtime := New() + check := func(raw []byte, want string) { + t.Helper() + view, e := cs.Encode(trusted) + if e != nil { + t.Fatal(e) + } + f := request(37) + f.blob(raw) + f.blob(view) + d := answer(t, runtime.Call(f.b)) + var out struct { + Code string `json:"code"` + Obsolete []string `json:"obsolete"` + } + if json.Unmarshal(d.blob(), &out) != nil || out.Code != want { + t.Fatalf("R23 got %+v want %s", out, want) + } + } + check(raw, "ok") + check(append(bytes.Clone(raw), ' '), "invalid") + trusted.Generation++ + check(raw, "generation") + trusted.Generation-- + trusted.Members[1].Revoked = true + check(raw, "unauthorized") + trusted.Members[1].Revoked = false + req.Applied = 21 + _ = req.Sign(key) + raw, _ = rt.Encode(req) + check(raw, "invalid") + req.Applied = 20 + for _, kind := range []byte{0, 1, 2, 11, 15, 16, 19, 20, 21, 22} { + envelope := rt.Envelope{Scope: scope, From: 4, To: 1, Incarnation: 1, Destination: 1, Sequence: 1, Payload: []byte{8, kind, 16, 1, 24, 4, 32, 1}} + envelope.Digest = peerDigest(envelope.Payload) + b, _ := rt.Encode(envelope) + envelope.Signature = ed25519.Sign(key, append([]byte("codeaf/raft/envelope/1\n"), b...)) + req.Messages = []rt.Envelope{envelope} + if e := req.Sign(key); e != nil { + t.Fatal(e) + } + raw, _ = rt.Encode(req) + check(raw, "invalid") + } +} + +// R23 keeps response fields byte-compatible with the unchanged transport client. +func TestR23HostedPeerResponseMatchesNative(t *testing.T) { + native := rt.Response{Generation: 7, Messages: []rt.Delivery{}, Accepted: []string{"4"}, Rejected: []string{"5"}, Chunks: []rt.ChunkAck{{Node: 4, Sequence: 9}}} + raw, _ := rt.Encode(native) + f := request(38) + f.u32(1) + f.blob(raw) + d := answer(t, New().Call(f.b)) + got := d.blob() + if !bytes.Equal(raw, got) { + t.Fatalf("R23 response differs: %s != %s", got, raw) + } +} + +// R9/R12 personal certificates can enroll before any optional device-name row. +func TestR9HostedPersonalPeerWithoutDeviceMetadata(t *testing.T) { + key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{8}, 32)) + h := cs.Membership{Revision: 2, Index: 10, NextNode: 5, Anchors: []uint64{1, 2, 3}, Nodes: []uint64{1, 2, 3, 4}, Live: 1, Limit: 4096, Pending: &cs.MembershipIntent{Node: 4, Revision: 2}} + m := cs.MemberRecord{Node: 4, Device: "personal", Key: key.Public().(ed25519.PublicKey), Incarnation: 1, Protocol: 1} + header, _ := cs.Encode(h) + member, _ := cs.Encode(m) + f := request(16) + f.u64(4) + f.u64(10) + f.blob(nil) + f.blob(header) + f.blob(member) + f.blob(nil) + d := answer(t, New().Call(f.b)) + var peer cs.PeerEnrollment + if cs.Decode(d.blob(), &peer) != nil || peer.Node != 4 || peer.Revoked || peer.Voter { + t.Fatal("R9 personal peer without optional metadata was refused") + } +} + +// R9 keeps capacity and protocol refusals identical across relay runtimes. +func TestR9HostedEnrollmentRefusalSentences(t *testing.T) { + for _, code := range []cs.Code{cs.Capacity, cs.UpdateRequired} { + f := request(38) + f.u32(2) + f.str(string(code)) + d := answer(t, New().Call(f.b)) + if d.str() != cs.EnrollmentMessage(code) || d.end() != nil { + t.Fatal("R9 refusal differs from shared admission policy") + } + } +} diff --git a/internal/raftabi/scheduler.go b/internal/raftabi/scheduler.go index 29a4e35777..9ad81920aa 100644 --- a/internal/raftabi/scheduler.go +++ b/internal/raftabi/scheduler.go @@ -50,15 +50,16 @@ func coordinationPolicy(d *decoder) ([]byte, error) { return nil, err } b, err := clusterstate.Encode(struct { - TickEvery int64 `json:"tick_every"` - RenewEvery int64 `json:"renew_every"` - PresenceEvery int64 `json:"presence_every"` - EvidenceWindow int64 `json:"evidence_window"` - IdleAfter int64 `json:"idle_after"` - MaxRenewHolds int `json:"max_renew_holds"` - MaxMessageBytes int `json:"max_message_bytes"` - MaxMemberRecords uint32 `json:"max_member_records"` - }{clusterstate.TickEveryMillis, clusterstate.RenewEveryMillis, clusterstate.PresenceEveryMillis, clusterstate.EvidenceWindowMillis, clusterstate.IdleAfterMillis, clusterstate.MaxRenewHolds, clusterstate.MaxCommandBytes, clusterstate.MaxMemberRecords}) + TickEvery int64 `json:"tick_every"` + RenewEvery int64 `json:"renew_every"` + PresenceEvery int64 `json:"presence_every"` + EvidenceWindow int64 `json:"evidence_window"` + IdleAfter int64 `json:"idle_after"` + MaxRenewHolds int `json:"max_renew_holds"` + MaxMessageBytes int `json:"max_message_bytes"` + MaxMemberRecords uint32 `json:"max_member_records"` + SequenceReservation int `json:"sequence_reservation"` + }{clusterstate.TickEveryMillis, clusterstate.RenewEveryMillis, clusterstate.PresenceEveryMillis, clusterstate.EvidenceWindowMillis, clusterstate.IdleAfterMillis, clusterstate.MaxRenewHolds, clusterstate.MaxCommandBytes, clusterstate.MaxMemberRecords, peerMaxMessages * peerMaxInflight}) if err != nil { return nil, err } diff --git a/internal/raftdirectory/control.go b/internal/raftdirectory/control.go index c738703bec..f037313a8a 100644 --- a/internal/raftdirectory/control.go +++ b/internal/raftdirectory/control.go @@ -20,79 +20,19 @@ func (g *Native) PersonalStatus(ctx context.Context, actor, certificate string) rows, _, err := store.Read(ctx, app.Applied, []raftstore.Key{key}) return rows[key], err } - var identity cs.IdentityRec - raw, err := read(cs.IdentityKey()) - if err != nil { - return err - } - if cs.Decode(raw, &identity) != nil || cs.PersonalCertificate(identity, actor, certificate) != nil { - return directory.ErrUnauthorized - } - raw, err = read(cs.DeviceKey(actor)) - if err != nil { - return err - } - if raw != nil { - var device cs.Device - if cs.Decode(raw, &device) != nil || device.Revoked { - return directory.ErrUnauthorized - } - } - raw, err = read(cs.AuthorityKey(actor)) - if err != nil { - return err - } - if raw != nil { - var authority cs.Authority - if cs.Decode(raw, &authority) != nil || !authority.ValidRole() || !authority.Personal || authority.Cluster != g.manifest.Config.Cluster { - return directory.ErrUnauthorized - } - } - conf := raftcore.Configuration(app.ConfState) - membership, _, err := cs.MembershipView(read, conf, app.Applied) - if err != nil || membership.Revision == 0 { - return cs.ErrDependency - } - out = clusterctl.Status{Cluster: g.manifest.Config.Cluster, Leader: g.leaderID(), Membership: membership, Configuration: conf, LeadershipUnavailable: true} + progress := map[uint64]uint64{} if g.peers != nil { - for _, id := range conf.Learners { - raw, err := read(cs.MemberKey(id)) - if err != nil { - return err - } - var record cs.MemberRecord - if cs.Decode(raw, &record) != nil { - return cs.ErrDependency - } - if peer, ok := g.peer(id); ok && !peer.Revoked && cs.LearnerReady(record, conf, g.peers.applied[id], record.ReadyIndex) { - out.Ready = append(out.Ready, id) - } - } - } - raw, err = read(cs.MemberDeviceKey(actor)) - if err != nil || raw == nil { - return err - } - var mapping cs.DeviceMember - if cs.Decode(raw, &mapping) != nil { - return cs.ErrDependency - } - if mapping.Node == 0 { - return nil + progress = g.peers.applied } - raw, err = read(cs.MemberKey(mapping.Node)) + status, code, err := cs.PersonalControlView(g.manifest.Config.Cluster, actor, certificate, raftcore.Configuration(app.ConfState), app.Applied, g.leaderID(), app.LogicalTime, progress, read) if err != nil { return err } - var member cs.MemberRecord - if cs.Decode(raw, &member) != nil || member.Node != mapping.Node || member.Device != actor || member.Anchor || member.Worker { - return cs.ErrDependency + if code != cs.OK { + return directory.ErrUnauthorized } - if !member.Retired { + out = clusterctl.Status{Ready: status.Ready, LeadershipUnavailable: status.Self == 0, Cluster: status.Cluster, Self: status.Self, Leader: status.Leader, Membership: status.Membership, Configuration: status.Configuration} - out.Self = member.Node - out.LeadershipUnavailable = false - } return nil }) if err != nil { diff --git a/internal/raftdirectory/personal.go b/internal/raftdirectory/personal.go index fe119a15c3..adb9307325 100644 --- a/internal/raftdirectory/personal.go +++ b/internal/raftdirectory/personal.go @@ -2,7 +2,6 @@ package raftdirectory import ( "context" - "crypto/ed25519" "errors" "strconv" @@ -162,90 +161,51 @@ func (g *Native) EnrollPersonal(ctx context.Context, proof clusterstate.V1Reques if err := g.reconcileMembership(ctx); err != nil { return out, err } - _, app, h, read, err := g.membership(ctx) - if err != nil { - return out, err - } - var identity clusterstate.IdentityRec - raw, err := read(clusterstate.IdentityKey()) - if err != nil || clusterstate.Decode(raw, &identity) != nil { - return out, clusterstate.ErrDependency - } at, _ := strconv.ParseInt(proof.At, 10, 64) - now := max(at, app.LogicalTime) - request, granted, actor, err := clusterstate.VerifyPersonalAdmission(proof, identity, now) - if err != nil { - return out, directory.ErrUnauthorized - } - var device clusterstate.Device - raw, err = read(clusterstate.DeviceKey(actor)) - if err != nil { - return out, err - } - if raw != nil && (clusterstate.Decode(raw, &device) != nil || device.Revoked) { - return out, directory.ErrUnauthorized - } - var authority clusterstate.Authority - raw, err = read(clusterstate.AuthorityKey(actor)) - if err != nil { - return out, err - } - if raw != nil && (clusterstate.Decode(raw, &authority) != nil || !authority.ValidRole() || !authority.Personal || authority.Cluster != g.manifest.Config.Cluster || !ed25519.PublicKey(authority.Key).Equal(ed25519.PublicKey(granted.Key))) { - return out, directory.ErrUnauthorized - } - if request.Protocol != raftcore.ProtocolVersion { - return out, EnrollmentRefusal{clusterstate.UpdateRequired} - } - var mapping clusterstate.DeviceMember - raw, err = read(clusterstate.MemberStoreKey(actor, request.Store)) - if err != nil { - return out, err - } - if raw != nil { - if clusterstate.Decode(raw, &mapping) != nil { - return out, clusterstate.ErrDependency - } - } else { - if len(h.Nodes) >= int(h.Limit) { - return out, EnrollmentRefusal{clusterstate.Capacity} - } - raw, err = read(clusterstate.MemberDeviceKey(actor)) + var node uint64 + for stage := 0; stage < 3; stage++ { + _, app, _, read, err := g.membership(ctx) if err != nil { return out, err } - if raw != nil && clusterstate.Decode(raw, &mapping) != nil { - return out, clusterstate.ErrDependency + plan, err := clusterstate.PlanPersonalAdmission(g.manifest.Config.Cluster, proof, max(at, app.LogicalTime), read) + if err != nil { + return out, err } - if mapping.Node != 0 { - if _, err = g.controlChange(ctx, clusterstate.Mutation{RemoveMember: &clusterstate.RemoveMember{Node: mapping.Node, Device: actor, Revision: h.Revision, Admission: &proof}}, now); err != nil { - return out, err - } - if err = g.reconcileMembership(ctx); err != nil { - return out, err - } - _, app, h, read, err = g.membership(ctx) - if err != nil { - return out, err + if plan.Code != clusterstate.OK { + if plan.Code == clusterstate.Capacity || plan.Code == clusterstate.UpdateRequired { + return out, EnrollmentRefusal{plan.Code} } + return out, directory.ErrUnauthorized } - r, err := g.controlChange(ctx, clusterstate.Mutation{EnrollMember: &clusterstate.EnrollMember{Node: h.NextNode, Revision: h.Revision, Device: actor, Protocol: request.Protocol, Store: request.Store, Admission: &proof}}, now) + node = plan.Node + if plan.Change == nil { + break + } + result, err := g.controlChange(ctx, *plan.Change, max(at, app.LogicalTime)) if err != nil { return out, err } - mapping.Node = r.Membership.Node + node = result.Membership.Node + if plan.Change.EnrollMember != nil { + break + } + if err = g.reconcileMembership(ctx); err != nil { + return out, err + } } // Configuration is a separate maintenance stage. The enrollment reply can // survive a crash with Barrier=0; activation repairs that committed intent. - _, app, _, read, err = g.membership(ctx) + _, _, _, read, err := g.membership(ctx) if err != nil { return out, err } - raw, err = read(clusterstate.MemberKey(mapping.Node)) + raw, err := read(clusterstate.MemberKey(node)) if err != nil { return out, err } var member clusterstate.MemberRecord - if clusterstate.Decode(raw, &member) != nil || member.Device != actor || member.Store != request.Store || member.Retired || member.Worker || member.Anchor { + if clusterstate.Decode(raw, &member) != nil || member.Retired || member.Worker || member.Anchor { return out, directory.ErrUnauthorized } out = PersonalEnrollment{Scope: g.peers.view.Scope, Generation: g.peers.view.Generation, Node: member.Node, Incarnation: member.Incarnation, Barrier: member.ReadyIndex, Members: g.peers.view.Members} diff --git a/internal/rafttransport/envelope.go b/internal/rafttransport/envelope.go index 352bf10ccc..a37daa8e3d 100644 --- a/internal/rafttransport/envelope.go +++ b/internal/rafttransport/envelope.go @@ -26,8 +26,8 @@ const ( MaxChunkBytes = raftsnapshot.ChunkBytes MaxInflight = raftcore.MaxInflightBatches MaxLearners = clusterstate.MaxLearners - MaxMessages = 64 - MaxQueuedBytes = 1 << 20 + MaxMessages = clusterstate.MaxPeerMessages + MaxQueuedBytes = clusterstate.MaxPeerQueuedBytes // Encoded envelopes include base64 payloads, signatures and bounded IDs. MaxExchangeBytes = 2 * MaxChunkBytes ) diff --git a/internal/rafttransport/router.go b/internal/rafttransport/router.go index 8c41c66430..0cb3e19c9d 100644 --- a/internal/rafttransport/router.go +++ b/internal/rafttransport/router.go @@ -6,6 +6,8 @@ import ( "math" "slices" "sync" + + "github.com/Agent-Field/codeaf/internal/clusterstate" ) // Lane reservations are charged by encoded envelope bytes, including metadata. @@ -17,7 +19,7 @@ const ( Learner Snapshot laneCount - RetentionMillis int64 = 30_000 + RetentionMillis int64 = clusterstate.PeerRetentionMillis ) // Slot reservations bound the whole member inbox to eight outstanding From 1567ec0aa75672c33bbc4cedd09976379551840c Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 01:38:52 -0400 Subject: [PATCH 066/103] Serve durable personal learners through the hosted relay. Expose enrollment, peer sockets, snapshot catch-up and authorized status while keeping learners outside quorum and hosted leadership unavailable. Fence recovered routing and outbound sequences durably, and cover real home lifetimes, compaction and bounded delivery through the production Worker. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../continuing-a-chat-on-another-computer.md | 2 +- .../manual/chat/coordination-and-workers.md | 8 +- internal/manual/hosted_personal_test.go | 39 ++ internal/rafttransport/README.md | 31 +- .../syncsetup/member_hosted_learner_test.go | 334 ++++++++++++++++++ relay/hosted/src/front.js | 16 +- relay/hosted/src/identity.js | 53 ++- relay/hosted/src/raft/README.md | 50 ++- relay/hosted/src/raft/hosted.js | 93 ++++- relay/hosted/src/raft/inbox.js | 4 +- relay/hosted/src/raft/member.js | 30 +- relay/hosted/src/raft/peers.js | 188 ++++++++++ relay/hosted/src/watch.js | 39 +- relay/hosted/test/raft/hosted-control.test.js | 26 +- .../test/raft/hosted-learner-go.test.js | 26 ++ .../hosted/test/raft/hosted-personal.test.js | 133 +++++++ relay/hosted/test/raft/hosted-worker.js | 35 +- 17 files changed, 1033 insertions(+), 74 deletions(-) create mode 100644 internal/manual/hosted_personal_test.go create mode 100644 internal/syncsetup/member_hosted_learner_test.go create mode 100644 relay/hosted/src/raft/peers.js create mode 100644 relay/hosted/test/raft/hosted-learner-go.test.js create mode 100644 relay/hosted/test/raft/hosted-personal.test.js diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index d6d55e839f..4828894bfc 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -251,7 +251,7 @@ chat first; this does not discard the work saved on your computer. ## Which computer coordinates: cluster status and cluster lead -On a self-hosted service with persistent storage, an open paired home joins as a learner. Three persistent voters keep coordination available when your laptops sleep. `codeaf cluster status` shows this computer after joining. Continue chats through the ordinary continue door. See the coordination and workers page for command refusals and setup details. +On the hosted service, or a self-hosted service with persistent storage, an open paired home joins as a learner. Three persistent voters keep coordination available when your laptops sleep. `codeaf cluster status` shows this computer after joining. Continue chats through the ordinary continue door. See the coordination and workers page for command refusals and setup details. ## Can a worker access all my chats or approve pairing? diff --git a/internal/manual/chat/coordination-and-workers.md b/internal/manual/chat/coordination-and-workers.md index 76824040c5..15cb076406 100644 --- a/internal/manual/chat/coordination-and-workers.md +++ b/internal/manual/chat/coordination-and-workers.md @@ -7,7 +7,7 @@ A relay directory upgrade can briefly pause pairing and chat moves. Retry after ## Joined computers, learner capacity and an outdated build -An open, paired home with sync on joins a self-hosted `codeaf relay --store ` as a durable learner. Its store lives under the home, and reopening reuses its node. Losing that store enrolls a fresh node and retires the old authority. Three anchors vote until a personal computer is explicitly asked to lead, so pairing alone never requires either laptop to stay awake. Joining starts while the paired home is open, or when you ask for `codeaf cluster status`. +An open, paired home with sync on joins the hosted relay, or a self-hosted `codeaf relay --store `, as a durable learner. Its store lives under the home, and reopening reuses its node. Losing that store enrolls a fresh node and retires the old authority. Three anchors vote until a personal computer is explicitly asked to lead, so pairing alone never requires either laptop to stay awake. Joining starts while the paired home is open, or when you ask for `codeaf cluster status`. If joining has no new data to apply, the client keeps its store for the next retry round instead of polling continuously. A status activation can say `this computer is still joining the cluster; retry shortly; your saved chats are still here`. @@ -15,11 +15,11 @@ The membership protocol admits at most 32 live learners and retains retired reco ## Which computer coordinates: cluster status and cluster lead -`codeaf cluster status` on a self-hosted `codeaf relay` with storage reports the committed voters and current leader after a fresh majority read. A paired computer joins with persistent storage as a learner before status shows its node. Readiness requires its own durable application through the advertised join position. Status names `this computer`, `learning computers`, `ready computers` and `voting computers` when known. `codeaf cluster lead` on an unjoined computer says `this computer must join the cluster with persistent storage before it can lead`. +`codeaf cluster status` on the hosted relay, or a self-hosted `codeaf relay` with storage, reports the committed voters and current leader after a fresh majority read. A paired computer joins with persistent storage as a learner before status shows its node. Readiness requires its own durable application through the advertised join position. Status names `this computer`, `learning computers`, `ready computers` and `voting computers` when known. `codeaf cluster lead` on an unjoined computer says `this computer must join the cluster with persistent storage before it can lead`. On a self-hosted `codeaf relay` with storage, run `codeaf cluster lead` on the joined computer that should coordinate. It first catches up, replaces the third anchor’s vote through a joint change, durably finishes that change with the third anchor still learning, and transfers leadership. The voters are then two anchors and this computer. Keep codeaf open to keep its member running; a command used with no open terminal runs its member only for that command. Leadership moves coordination metadata, never chat content or chat ownership. -The hosted relay has no cluster control door and says `this relay does not offer cluster leadership yet; your saved chats are still here`. Ordinary hosted chat sync remains available. +The hosted relay admits paired computers as learners and shows them in `codeaf cluster status`. Leadership is not offered there yet: `codeaf cluster lead` says `this relay does not offer cluster leadership yet; your saved chats are still here`. Ordinary hosted chat sync remains available. Do not treat pairing or a saved leadership request as a successful transfer. Success requires the selected computer to be elected, commit its current term and pass a fresh majority read; its receipt names the computer, term, revision and read barrier. Retrying `codeaf cluster lead` preserves an already saved signed request. Continue chats through the ordinary continue door. @@ -46,7 +46,7 @@ The sync connection carries presence hints and committed lease acknowledgments. ## Does syncing keep sending requests when nothing is happening? -A home that has never paired sends no sync traffic. `CODEAF_SYNC_URL=off` also blocks new sync requests from clients that were opened before the setting changed. Presence pings on an open view are hints; they do not write chat ownership. On the native relay, an open idle home uses its learner’s applied vault pointer to notice shared memory and preference changes. Local edits are detected locally and pushed on the next state round. A caught-up idle home sends no recurring state polls or peer keepalives. Home also follows its local applied directory for display changes; its roster and backstop read that same local hint. Quiet neighbours’ presence can age out without forcing a network ping. Continuing a chat and all ownership decisions still use fresh majority reads. A committed change wakes peer delivery and the next state round pulls the changed vault. The hosted relay has no learner door and keeps its existing five-second state polling. On the consensus directory, a group with no work or held chats can stop its periodic coordination timer after an idle barrier has reached every voter. The first operation wakes coordination and must obtain a fresh majority response. A saved leader address cannot answer that operation by itself. Losing the majority leaves syncing unavailable and keeps local work intact. +A home that has never paired sends no sync traffic. `CODEAF_SYNC_URL=off` also blocks new sync requests from clients that were opened before the setting changed. Presence pings on an open view are hints; they do not write chat ownership. On the native relay, an open idle home uses its learner’s applied vault pointer to notice shared memory and preference changes. Local edits are detected locally and pushed on the next state round. A caught-up idle home sends no recurring state polls or peer keepalives. Home also follows its local applied directory for display changes; its roster and backstop read that same local hint. Quiet neighbours’ presence can age out without forcing a network ping. Continuing a chat and all ownership decisions still use fresh majority reads. A committed change wakes peer delivery and the next state round pulls the changed vault. A paired open home on the hosted relay also joins as a learner and keeps its connection open. If an older hosted relay has no learner door, the home keeps its existing five-second state polling. On the consensus directory, a group with no work or held chats can stop its periodic coordination timer after an idle barrier has reached every voter. The first operation wakes coordination and must obtain a fresh majority response. A saved leader address cannot answer that operation by itself. Losing the majority leaves syncing unavailable and keeps local work intact. ## Can a worker access all my chats or approve pairing? diff --git a/internal/manual/hosted_personal_test.go b/internal/manual/hosted_personal_test.go new file mode 100644 index 0000000000..639542ea8c --- /dev/null +++ b/internal/manual/hosted_personal_test.go @@ -0,0 +1,39 @@ +package manual + +import ( + "strings" + "testing" +) + +// R9/R23 keep current hosted enrollment and status distinct from the older +// relay fallback, while leaving leadership availability explicit. +func TestR9R23HostedLearnerManualTruth(t *testing.T) { + for _, query := range []string{"hosted relay learner cluster status", "paired open home hosted learner leadership"} { + found := false + for _, section := range Chat().Search(query, DefaultResults) { + found = found || section.Page == "coordination-and-workers" + } + if !found { + t.Errorf("R9 hosted learner answer unreachable: %q", query) + } + } + page, ok := Chat().Page("coordination-and-workers") + if !ok { + t.Fatal("R9 coordination page is missing") + } + for _, claim := range []string{ + "joins the hosted relay", + "hosted relay admits paired computers as learners", + "`codeaf cluster status`", + "this relay does not offer cluster leadership yet; your saved chats are still here", + } { + if !strings.Contains(page, claim) { + t.Errorf("R9/R23 current hosted capability missing: %q", claim) + } + } + for _, obsolete := range []string{"The hosted relay has no learner door", "The hosted relay has no cluster control door"} { + if strings.Contains(page, obsolete) { + t.Errorf("R9 obsolete current-hosted denial: %q", obsolete) + } + } +} diff --git a/internal/rafttransport/README.md b/internal/rafttransport/README.md index a6e48589b1..7cf36b7e5e 100644 --- a/internal/rafttransport/README.md +++ b/internal/rafttransport/README.md @@ -4,7 +4,7 @@ This is the authenticated peer bridge, separate from application ingress. The persistent native relay assembles its router from committed membership and serves personal admission, HTTP exchange and the outbound push socket on its ordinary listener. `relayserve.Config.Cluster` is the internal custom-host handler seam. -The hosted relay's member/control doors remain absent, and a paired codeaf home runs its durable learner inside the terminal state-sync lifetime. No environment switch selects another writer. +The hosted relay serves the same personal admission, member transport and control status doors. A paired codeaf home runs its durable learner inside the terminal state-sync lifetime. No environment switch selects another writer. ## Host ordering and authorization @@ -185,7 +185,7 @@ cannot resurrect a retired peer: remove/re-enroll with a fresh node ID. The persistent native relay now owns its committed router and mounts enrollment, exchange and socket routes on its ordinary listener. `Config.Cluster` remains the internal custom-host assembly seam; the default native service installs its own -router. Hosted member/control routes remain absent. +router. The hosted relay serves these member and control routes through its identity coordinator. `POST /v2/cluster/enroll` uses the existing `reqsign` headers, original paired personal certificate and body `{"store":"<64 lowercase hex characters>","protocol":1}`. @@ -227,7 +227,7 @@ wakes the ordinary quorum path. The codeaf client starts its personal learner only inside the existing terminal state-sync lifetime, or a scoped status activation, after current quiet guards. -Open remains local. Hosted relays offer neither enrollment nor control. Worker +Open remains local. Hosted relays offer enrollment and control status. Worker setup remains unavailable. Native personal leadership uses the separate application ingress below; hosted leadership remains unavailable. The native host advances its checksummed `transport-generation` file with atomic @@ -295,3 +295,28 @@ new bytes under the same node/incarnation. No fsync per peer frame is needed. Final sleep also waits for the member's bounded sends to be admitted. After the all-voter marker, obsolete nonvoter append/heartbeat responses cannot wake the leader; actual work, read requests and higher-term messages still do. + +## Hosted personal learners + +The hosted coordinator publishes routing only from committed anchor records. +Its generation is durably advanced before restoring expendable inboxes, so old +cursors and socket proofs cannot survive coordinator memory loss. Hibernating +peer and passive application sockets retain separate tags and attachments; +watch presence, broadcasts and watcher admission never count them. A stale +connection closes and the same durable home refreshes its enrollment grant. +The relay never opens an outbound connection to a personal member. + +Admission and status use shared Go policy through the reactor. A committed +admission survives a lost reply before configuration; activation or maintenance +reconciles only pending intent. The original signed device request remains in +the enrollment command. A fresh quorum status reports readiness only after the +enrolled key signs durable application through its advertised join barrier. +Leadership remains unavailable on the hosted relay, and application sockets +stay passive. Peer ingress progresses independently of the public request queue. + +Snapshots send a manifest followed by chunks on their reserved transport lane. +Each target has one outstanding chunk and advances only on signed delivery +credit; core snapshot success still waits for durable target application. +Learners never count toward quorum or hold back compaction. All-voter sleep +clears expendable learner deliveries and partial transfers; returning learners +catch up from the applied checkpoint and log. diff --git a/internal/syncsetup/member_hosted_learner_test.go b/internal/syncsetup/member_hosted_learner_test.go new file mode 100644 index 0000000000..ae64e1035d --- /dev/null +++ b/internal/syncsetup/member_hosted_learner_test.go @@ -0,0 +1,334 @@ +package syncsetup + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/url" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/reqsign" +) + +func hostedLearnerEndpoint(t *testing.T) string { + t.Helper() + endpoint := os.Getenv("CODEAF_HOSTED_PROOF_URL") + if endpoint == "" { + t.Skip("local hosted proof runner only") + } + u, e := url.Parse(endpoint) + if e != nil || u.Scheme != "http" || u.Hostname() != "127.0.0.1" { + t.Fatal("hosted proof requires loopback HTTP") + } + return endpoint +} +func hostedOwner(t *testing.T, s *Sync) func() { + t.Helper() + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan struct{}) + go func() { s.RunState(ctx); close(done) }() + stop := func() { + cancel() + select { + case <-done: + case <-time.After(requestTimeout): + t.Error("R9 terminal owner did not stop") + } + } + t.Cleanup(stop) + deadline := time.Now().Add(requestTimeout) + for time.Now().Before(deadline) { + s.memberMu.Lock() + m := s.member + var e error + ready := false + if m != nil { + m.mu.Lock() + e = m.failure + ready = m.conn != nil && m.forwardConn != nil && m.status.Applied >= m.record.Grant.Barrier + m.mu.Unlock() + } + s.memberMu.Unlock() + if e != nil { + t.Fatalf("R9 hosted member failed: %v", e) + } + if ready { + return stop + } + time.Sleep(time.Millisecond) + } + t.Fatal("R9 RunState did not join the hosted cluster") + return stop +} +func hostedReady(t *testing.T, s *Sync) uint64 { + t.Helper() + deadline := time.Now().Add(requestTimeout) + for time.Now().Before(deadline) { + status, e := s.clusterClient().Status(t.Context()) + mustNil(t, e) + if len(status.Configuration.Voters) != 3 || !status.LeadershipUnavailable { + t.Fatalf("R9 hosted voter/leadership policy: %+v", status) + } + if status.Self != 0 && slices.Contains(status.Ready, status.Self) && slices.Contains(status.Configuration.Learners, status.Self) { + return status.Self + } + time.Sleep(time.Millisecond) + } + t.Fatal("R9 member was not ready after durable application") + return 0 +} + +// R9/R23 exercise the terminal's real lifetime, store reuse and wiped-store +// retirement through the production Worker, without a client-side substitute. +func TestR9R23HostedLearnerRunStateReopenAndWipe(t *testing.T) { + endpoint := hostedLearnerEndpoint(t) + home := machine(t, endpoint) + a, ok, e := Open(home) + mustNil(t, e) + if !ok { + t.Fatal("open") + } + stop := hostedOwner(t, a) + node := hostedReady(t, a) + record := recordOf(t, a) + other := machine(t, endpoint) + _, e = identity.Adopt(other, a.Identity, true) + mustNil(t, e) + b, ok, e := Open(other) + mustNil(t, e) + if !ok { + t.Fatal("second home") + } + stopB := hostedOwner(t, b) + otherNode := hostedReady(t, b) + if node == otherNode { + t.Fatal("R9 two stores share a node") + } + status, e := b.clusterClient().Status(t.Context()) + mustNil(t, e) + if !slices.Contains(status.Configuration.Learners, node) || !slices.Contains(status.Configuration.Learners, otherNode) { + t.Fatal("R9 both homes must remain learners") + } + _, e = a.LeadCluster(t.Context(), 0) + if !errors.Is(e, clusterctl.ErrUnsupported) { + t.Fatalf("R9 leadership refusal: %v", e) + } + if _, e = os.Stat(filepath.Join(home, "cluster-leadership-"+a.Device.ID()+".json")); !errors.Is(e, os.ErrNotExist) { + t.Fatal("R9 unsupported lead wrote an intent", e) + } + stop() + reopened, _, e := Open(home) + mustNil(t, e) + stopAgain := hostedOwner(t, reopened) + if hostedReady(t, reopened) != node || recordOf(t, reopened).Store != record.Store { + t.Fatal("R9 reopen changed durable identity") + } + stopAgain() + mustNil(t, os.RemoveAll(reopened.memberPath())) + wiped, _, e := Open(home) + mustNil(t, e) + hostedOwner(t, wiped) + replacement := hostedReady(t, wiped) + if replacement <= node || recordOf(t, wiped).Store == record.Store { + t.Fatal("R9 wiped store reused authority") + } + status, e = wiped.clusterClient().Status(t.Context()) + mustNil(t, e) + if slices.Contains(status.Configuration.Learners, node) { + t.Fatal("R9 old learner was not retired") + } + stopB() +} + +// R23 requires zero attempts for off and never-paired homes, including the +// same owner that starts peer and application sockets on paired homes. +func TestR23HostedQuietHomesMakeNoClusterRequests(t *testing.T) { + endpoint := hostedLearnerEndpoint(t) + for _, off := range []bool{false, true} { + t.Run(map[bool]string{false: "unpaired", true: "off"}[off], func(t *testing.T) { + home := t.TempDir() + _, e := identity.EnsureSolo(home) + mustNil(t, e) + t.Setenv(URLVar, endpoint) + if off { + _, e = identity.Ensure(home) + mustNil(t, e) + mustNil(t, identity.EndSolo(home)) + } + s, ok, e := Open(home) + mustNil(t, e) + if !ok { + t.Fatal("open") + } + if off { + t.Setenv(URLVar, "off") + } + for n := 0; n < 3; n++ { + _, e = s.stateRound(t.Context()) + mustNil(t, e) + } + if s.member != nil || s.relayAttempts.Load() != 0 || s.clusterFrames.Load() != 0 { + t.Fatal("R23 quiet home made network attempts") + } + if _, e = os.Stat(s.memberPath()); !errors.Is(e, os.ErrNotExist) { + t.Fatal("R23 quiet home created a member store") + } + }) + } +} + +// R9/R23 catch up a real store through multiple bounded chunks, then apply a +// later log entry. Transport credit never substitutes for durable installation. +func TestR9HostedLearnerSnapshotChunksAndLog(t *testing.T) { + endpoint := hostedLearnerEndpoint(t) + home := machine(t, endpoint) + a, ok, e := Open(home) + mustNil(t, e) + if !ok { + t.Fatal("open") + } + for n := 0; n < 18; n++ { + other := machine(t, endpoint) + _, e = identity.Adopt(other, a.Identity, true) + mustNil(t, e) + writer, _, e := Open(other) + mustNil(t, e) + mustNil(t, writer.Dir.PutDevice(t.Context(), writer.Device.ID(), directory.Device{V: 1, Name: strings.Repeat("e", 20000)})) + } + stop := hostedOwner(t, a) + node := hostedReady(t, a) + a.memberMu.Lock() + m := a.member + m.mu.Lock() + recovered, e := m.store.Recover(t.Context()) + m.mu.Unlock() + a.memberMu.Unlock() + mustNil(t, e) + if recovered.Core.Snapshot == nil { + t.Fatal("R9 joining store received no snapshot") + } + manifest, e := raftsnapshot.Decode(recovered.Core.Snapshot.GetData()) + mustNil(t, e) + if manifest.Chunks < 2 { + t.Fatalf("R9 expected multiple chunks: %+v", manifest) + } + mustNil(t, a.Dir.PutDevice(t.Context(), a.Device.ID(), directory.Device{V: 1, Name: "after snapshot"})) + waitMember(t, a, func(m *personalMember) bool { + app, _, err := m.store.ApplicationView(t.Context()) + if err != nil { + return false + } + k := raftstore.Key{Namespace: raftstore.State, Key: cs.DeviceKey(a.Device.ID()).Key} + rows, _, err := m.store.Read(t.Context(), app.Applied, []raftstore.Key{k}) + var d cs.Device + return err == nil && cs.Decode(rows[k], &d) == nil && d.Name == "after snapshot" + }) + if hostedReady(t, a) != node { + t.Fatal("R9 log catch-up changed node") + } + t.Logf("R9 durable hosted snapshot installed: chunks=%d bytes=%d", manifest.Chunks, manifest.Bytes) + stop() + // R9/R27 repeatedly overwrite bounded metadata while the learner is absent. + // The retained log crosses the automatic checkpoint trigger independently + // of that learner's old progress, without adding application records. + var latest string + for n := 0; n < 32; n++ { + latest = strings.Repeat("p", 20000) + strconv.Itoa(n) + mustNil(t, a.Dir.PutDevice(t.Context(), a.Device.ID(), directory.Device{V: 1, Name: latest})) + } + reopened, _, e := Open(home) + mustNil(t, e) + hostedOwner(t, reopened) + waitMember(t, reopened, func(m *personalMember) bool { + app, _, err := m.store.ApplicationView(t.Context()) + if err != nil { + return false + } + k := raftstore.Key{Namespace: raftstore.State, Key: cs.DeviceKey(a.Device.ID()).Key} + rows, _, err := m.store.Read(t.Context(), app.Applied, []raftstore.Key{k}) + var d cs.Device + return err == nil && cs.Decode(rows[k], &d) == nil && d.Name == latest + }) + reopened.memberMu.Lock() + m = reopened.member + m.mu.Lock() + returned, e := m.store.Recover(t.Context()) + m.mu.Unlock() + reopened.memberMu.Unlock() + mustNil(t, e) + if returned.Core.Snapshot == nil || returned.Core.Snapshot.GetMetadata().GetIndex() <= manifest.Index { + t.Fatal("R27 absent learner pinned compaction or did not receive a fresh snapshot") + } + if hostedReady(t, reopened) != node { + t.Fatal("R9 returning learner changed its durable identity") + } + +} + +// R9/R27 a legal maximum public command must commit and reach a personal +// learner without increasing the normal peer or chunk byte budgets. +func TestR9R27HostedLearnerMaximumCommand(t *testing.T) { + endpoint := hostedLearnerEndpoint(t) + a, ok, e := Open(machine(t, endpoint)) + mustNil(t, e) + if !ok { + t.Fatal("open") + } + hostedOwner(t, a) + hostedReady(t, a) + status, e := a.clusterClient().Status(t.Context()) + mustNil(t, e) + encodedSize := func(n int) int { + body, _ := json.Marshal(directory.Device{V: 1, Name: strings.Repeat("x", n)}) + req, e := http.NewRequest(http.MethodPut, a.Relay+"/v1/dir/devices/"+a.Device.ID(), bytes.NewReader(body)) + mustNil(t, e) + reqsign.Sign(req, body, deviceSigner{a.Identity, a.Device}, a.now()) + proof, e := reqsign.Capture(req, body) + mustNil(t, e) + command, e := cs.V1Command(status.Cluster, a.now().UnixMilli(), proof) + mustNil(t, e) + raw, e := cs.EncodeCommand(command) + mustNil(t, e) + return len(raw) + } + low, high := 0, cs.MaxCommandBytes + for low < high { + middle := (low + high + 1) / 2 + if encodedSize(middle) <= cs.MaxCommandBytes { + low = middle + } else { + high = middle - 1 + } + } + if encodedSize(low) > cs.MaxCommandBytes || encodedSize(low) < cs.MaxCommandBytes-4 { + t.Fatal("R27 fixture missed the admitted command boundary") + } + name := strings.Repeat("x", low) + mustNil(t, a.Dir.PutDevice(t.Context(), a.Device.ID(), directory.Device{V: 1, Name: name})) + _, e = a.Dir.List(t.Context()) + mustNil(t, e) + waitMember(t, a, func(m *personalMember) bool { + app, _, err := m.store.ApplicationView(t.Context()) + if err != nil { + return false + } + k := raftstore.Key{Namespace: raftstore.State, Key: cs.DeviceKey(a.Device.ID()).Key} + rows, _, err := m.store.Read(t.Context(), app.Applied, []raftstore.Key{k}) + var d cs.Device + return err == nil && cs.Decode(rows[k], &d) == nil && d.Name == name + }) +} diff --git a/relay/hosted/src/front.js b/relay/hosted/src/front.js index de0ca5eed3..1dad964b44 100644 --- a/relay/hosted/src/front.js +++ b/relay/hosted/src/front.js @@ -1,9 +1,10 @@ // The front door's routing, kept apart from the Worker entry so a test can drive it without the // Durable Object classes (which only load inside the Workers runtime). +import { peerPath,peerLimits } from './raft/peers.js'; import { stripBase } from './base.js'; import { clusterRequestLimit } from './raft/ingress.js'; import { sharedRuntime } from './raft/actor.js'; -import { checkDeclared } from './body.js'; +import { checkDeclared,readBody } from './body.js'; import { serveLink } from './link/front.js'; import { isPagePath, servePage } from './page.js'; import { servePair } from './pair/front.js'; @@ -14,9 +15,16 @@ import { notFound, respond } from './wire.js'; /** forward hands an identity's request to its Durable Object, after refusing what needs no object to refuse. */ async function forward(request, env) { - const route = new URL(request.url).pathname==='/v2/cluster/api' ? {limit:clusterRequestLimit(sharedRuntime()),over:"too_large"} : matchRoute(request.method, new URL(request.url).pathname); + const path=new URL(request.url).pathname; + if(peerPath(path)){ + if(request.method!==(path.endsWith('/exchange')?'POST':'GET'))return new Response(null,{status:405}); + let scope;try{const limit=peerLimits(sharedRuntime()).Exchange,auth=request.headers.get('Codeaf-Cluster-Auth')??'';if(auth.length>limit)return new Response('invalid peer request',{status:400});const raw=path.endsWith('/exchange')?new TextDecoder().decode(await readBody(request.clone(),limit,'too_large')):atob(auth.replace(/-/g,'+').replace(/_/g,'/'));scope=JSON.parse(raw).scope;}catch{return new Response('invalid peer request',{status:400});} + if(!scope||typeof scope.identity!=='string'||!/^id_[a-f0-9]{32}$/.test(scope.identity))return new Response('member authorization refused',{status:403}); + return env.IDENTITY.get(env.IDENTITY.idFromName(scope.identity)).fetch(request); + } + const route = ['/v2/cluster/control','/v2/cluster/enroll'].includes(path)?{limit:clusterRequestLimit(sharedRuntime()),over:'too_large'}:new URL(request.url).pathname==='/v2/cluster/api' ? {limit:clusterRequestLimit(sharedRuntime()),over:"too_large"} : matchRoute(request.method, new URL(request.url).pathname); checkDeclared(request, route.limit, route.over); - const identity = await identityOf(request.headers.get('codeaf-identity')); + let identity;try{identity=await identityOf(request.headers.get('codeaf-identity'));}catch(error){if(path.startsWith('/v2/cluster/')&&path!=='/v2/cluster/api')return new Response(path.endsWith('/control')?'this computer is not allowed to coordinate this cluster':'this computer is not allowed to join this cluster',{status:403});throw error;} return env.IDENTITY.get(env.IDENTITY.idFromName(identity)).fetch(request); } @@ -28,6 +36,6 @@ export function dispatch(outer, env) { const { pathname } = new URL(request.url); if (pathname.startsWith('/v1/link/')) return serveLink(request, env); if (pathname.startsWith('/v1/pair')) return servePair(request, env); - if (pathname.startsWith('/v1/') || pathname==='/v2/cluster/api') return forward(request, env); + if (pathname.startsWith('/v1/') || pathname.startsWith('/v2/cluster/')) return forward(request, env); throw notFound(); } diff --git a/relay/hosted/src/identity.js b/relay/hosted/src/identity.js index 3ed8d50f0d..14288d2ae1 100644 --- a/relay/hosted/src/identity.js +++ b/relay/hosted/src/identity.js @@ -3,6 +3,7 @@ // none of its legacy directory methods are a production writer. import { DurableObject } from 'cloudflare:workers'; import { LegacyAdmission } from './raft/legacy.js'; +import { peerPath,peerSocket,peerRefusal } from './raft/peers.js'; import { HostedIdentity } from './raft/hosted.js'; import { clusterIngress,clusterRequestLimit } from './raft/ingress.js'; import { sharedRuntime } from './raft/actor.js'; @@ -54,19 +55,20 @@ export class IdentityDO extends DurableObject { return respond(() => this.#serve(request)); } - /** webSocketMessage runs only for a client frame that is not `ping` (the platform answers that without waking us); such a frame is ignored (contract 21.3). */ - webSocketMessage() { + /** Peer frames drive replication; the platform answers watch keepalives without waking this object. */ + async webSocketMessage(ws,raw) { + if(peerSocket(ws)){try{const host=await this.#peerHost();await host.peers.frame(ws,raw);}catch{try{ws.close(1012,'reconnecting');}catch{}}return;} console.warn('watch: message'); } /** webSocketClose completes the close handshake the client started, and notes when a device's last socket went. */ async webSocketClose(ws, code) { finishClose(ws, code); - await this.#noteLeft(ws); + if(!peerSocket(ws))await this.#noteLeft(ws); } webSocketError(ws) { - return this.#noteLeft(ws); + if(!peerSocket(ws))return this.#noteLeft(ws); } /** #noteLeft records last_seen of a device that holds no socket now; the offline frame follows 15 s later, by alarm. */ @@ -78,6 +80,35 @@ export class IdentityDO extends DurableObject { async #serve(request) { const url = new URL(request.url); + if(peerPath(url.pathname)){ + try{ + if(url.search)return peerRefusal({peerCode:'invalid'}); + const socket=url.pathname!=='/v2/cluster/exchange'; + if(request.method!==(socket?'GET':'POST'))return new Response(null,{status:405}); + const host=await this.#peerHost(); + const raw=socket?fromBase64Url(request.headers.get('Codeaf-Cluster-Auth')):await readBody(request,host.peers.limits.Exchange,'too_large'); + if(!raw)return peerRefusal({peerCode:'invalid'}); + if(socket)return await host.peers.accept(request,raw,url.pathname.endsWith('/forward')?'forward':'peer'); + const result=await host.peers.exchange(raw);if(result.rejected)return peerRefusal({peerCode:'busy'}); + return new Response(result.raw,{headers:{'content-type':'application/json','cache-control':'no-store'}}); + }catch(error){return peerRefusal(error);} + } + if(['/v2/cluster/control','/v2/cluster/enroll'].includes(url.pathname)){ + const enrolling=url.pathname.endsWith('/enroll'); + if(url.search||!(enrolling?request.method==='POST':['GET','POST'].includes(request.method)))return new Response(null,{status:404}); + let who,body; + try{const caller=await checkCert(signedHeaders(request));body=await readBody(request,clusterRequestLimit(sharedRuntime()),'too_large');who=await checkRequest(caller,{method:request.method,uri:signedUri(this.env,url)},body,Date.now());} + catch{return new Response(enrolling?'this computer is not allowed to join this cluster':'this computer is not allowed to coordinate this cluster',{status:403});} + await this.#refuseGone();await this.#admitted.get(ipOf(request,this.env)); + if(request.method==='GET'&&body.length)return new Response('unexpected request body',{status:400}); + const tenant=this.#tenantOf(who.identity);this.#admit(tenant,who.device); + this.host??=new HostedIdentity(this.ctx,this.env,tenant,this.legacyAdmission,at=>this.#arm(at)); + const reply=await this.host.serial(async()=>{ + await this.host.ensure(fromBase64Url(request.headers.get('codeaf-identity'))); + await this.host.wakeMaintenance(); + return enrolling?this.host.enroll(this.host.proof(request,body)):this.host.control(request.method,who.device,request.headers.get('codeaf-cert')); + });this.watchers=this.host.watchers;reply.headers.set('cache-control','no-store');return reply; + } if(url.pathname==='/v2/cluster/api'){ if(request.method!=='POST'||url.search)throw new Wire('bad_request',400); const raw=await readBody(request,clusterRequestLimit(sharedRuntime()),'too_large'); @@ -111,6 +142,19 @@ export class IdentityDO extends DurableObject { } } + // Peer ingress can restore an existing namespace, but cannot provision one. + // Check durable provenance before constructing a Tenant, which creates tables. + async #peerHost(){ + await this.#refuseGone(); + const provision=await this.ctx.storage.get('raft-provision'),identity=await this.ctx.storage.get('identity'); + if(!provision||!identity)throw Object.assign(new Error('member authorization refused'),{peerCode:'unauthorized'}); + if(!this.host?.ready){ + const tenant=this.#tenantOf(identity);this.host??=new HostedIdentity(this.ctx,this.env,tenant,this.legacyAdmission,at=>this.#arm(at)); + await this.host.serial(()=>this.host.ensure(unb64(provision.config.signing_key)));this.watchers=this.host.watchers; + } + return this.host; + } + /** #enterPut counts a put from its arrival, and turns away one more than the isolate can hold in memory. */ #enterPut() { if (this.flight.count >= this.limits.concurrentPuts) throw rateLimited(1); @@ -156,6 +200,7 @@ export class IdentityDO extends DurableObject { const erase=await this.host.serial(async()=>{ await this.host.ensure(Uint8Array.from(atob(provision.config.signing_key),c=>c.charCodeAt(0))); this.watchers=this.host.watchers;this.watchers.expire();tenant.stats.flush(); + await this.host.peers.deadlines(Date.now()); await this.host.maintenance(Date.now()); const at=this.host.retireAt; if(at && at<=Date.now())return true; diff --git a/relay/hosted/src/raft/README.md b/relay/hosted/src/raft/README.md index fefbcaa64b..e009a7b8b4 100644 --- a/relay/hosted/src/raft/README.md +++ b/relay/hosted/src/raft/README.md @@ -63,7 +63,7 @@ dependent effects for an internal router to deliver rather than silently dropping them. Run the bounded local proof with `sh relay/hosted/test/raft/actors-run.sh -/tmp/codeaf-p08`. It exercises real workerd actors and native SQLite peers, +/tmp/codeaf-actors`. It exercises real workerd actors and native SQLite peers, signed messages, ReadIndex/minority refusal, both snapshot directions, process death, full joint configuration restart and co-resident identities. Unit tests cover durability waits, queue bounds, all snapshot namespaces, @@ -109,16 +109,16 @@ reports whether a signed settlement may be proposed. It never synthesizes that settlement signature or declares a join successful. `joinStatus(barrier)` additionally checks current authorization and the learner's own applied index. -These are internal membership methods. Client enrollment, registry publication, -quiet-mode guards, voter swaps and delegated worker certificates each need their -own host door before a client can use them. -Do not advertise a public leader/worker command from these internal methods. +The identity coordinator exposes personal learner admission, committed registry +routing and status through public authenticated doors. Voter swaps, personal +leadership and delegated learner activation are not available here yet. -Provisioned anchor keys receive a restricted committed authority that can only -settle an existing intent or close their request session. They cannot enroll, -remove, read directory content, mutate chats, pair or rotate. Thus a crashed -requester need not return for an already authorized admission to finish. The -settlement still requires the exact applied configuration and pending revision. +Provisioned anchor keys receive restricted committed authority for membership +maintenance and request-session closure. Personal enrollment or retirement still +requires the original device admission proof, checked again by the replicated +evaluator. Anchors cannot authorize directory reads, chat mutations, pairing or +rotation. A crashed requester need not return for an already authorized admission +to finish; settlement requires the exact applied configuration and pending revision. ## Committed holds, timers and wake @@ -168,7 +168,31 @@ normalized before original-proof comparison, preserving replacement semantics. Public source serialization spans coordination, while external blob I/O leaves that queue and passes a new authority barrier afterward. Public watch sockets carry presence without the legacy socket-vouch promise; v1 clients keep ordinary -beats. The hosted relay does not offer personal-member learners or leadership -requests yet; its committed holds and idle maintenance follow the shared Go -policy. Explicit retirement has its own retained +beats. The hosted relay admits personal learners and serves their status, exchange +and hibernating sockets. Leadership requests remain unavailable; its committed +holds and idle maintenance follow the shared Go policy. Explicit retirement has its own retained all-replica erasure journal and never serves as pressure cleanup. + + +## Public personal member routing + +`POST /v2/cluster/enroll` retains the original personal certificate and signed +store token. Shared Go admission allocates from committed state; configuration +is a separate resumable stage. Status uses a fresh quorum read and Go projection. +An unsupported leadership request creates no intent. The application forwarding +upgrade is authenticated and passive until hosted leadership is available. + +The router's durable generation advances before its volatile queues reopen. +Peer and application sockets use the hibernation API with a separate tag and +serialized attachment containing their signed connection proof. Coordinator +eviction fences those old connections; the unchanged client refreshes the same +store's grant and reconnects. Watch presence and broadcasts filter these sockets. +Protocol queues use their own admission and reservations, outside public rates. + +ABI 36 supplies admission plans, enrollment bytes and control status from an +immutable member view. ABI 37 checks native request/envelope signatures, cursors' +scope, registered keys and permitted inner message kinds; ABI 38 supplies the +transport limits and response encoding. Routing owns per-lane delivery cursors, +acks and chunk credits. One outstanding chunk per target bounds catch-up, and +only its signed durable applied evidence completes snapshot delivery. Public +coordination never waits for a learner; peer frames bypass its public queue. diff --git a/relay/hosted/src/raft/hosted.js b/relay/hosted/src/raft/hosted.js index 1d35ee6299..c09ef827cf 100644 --- a/relay/hosted/src/raft/hosted.js +++ b/relay/hosted/src/raft/hosted.js @@ -1,3 +1,4 @@ +import { PersonalPeers } from './peers.js'; import { coordinationPolicy,CommittedHolds } from './liveness.js'; import { clusterIngress } from './ingress.js'; import { frame, response, HostCodec } from './abi.js'; @@ -100,7 +101,7 @@ export class HostedIdentity { if(!owned)await this.source.own(receipts); this.source.route(()=>true); this.watchers=new CommittedWatchers(this.ctx,this.tenant.limits.maxWatchers,this.tenant.watchers.ttlMs,at=>this.ctx.waitUntil(this.ctx.storage.setAlarm(at))); - this.tenant.watchers=this.watchers;this.ready=true; + this.tenant.watchers=this.watchers; this.effects=new RaftEffectsFacade(this,this.manifest.config.cluster); this.holds=new CommittedHolds(this,this.policy); this.holds.opened(Date.now(),this.watchers.holdEvidence(Date.now()).some(socket=>socket.holds.length)); @@ -111,23 +112,55 @@ export class HostedIdentity { } const root=await this.barrierRecord({namespace:'state',key:'identity'}); this.retireAt=this.project('identity',root).rotation?.retire_at; - await this.wakeMaintenance(); + const generation=await this.ctx.storage.get('raft-router-generation')??'0'; + if(typeof generation!=='string'||!/^(0|[1-9][0-9]*)$/.test(generation)||BigInt(generation)>=(1n<<64n)-1n)throw unavailable(); + const next=String(BigInt(generation)+1n); + await this.ctx.storage.put('raft-router-generation',next);await this.ctx.storage.sync(); + this.peers=new PersonalPeers(this,next); + for(const ws of this.peers.sockets())this.peers.close(ws); + await this.peers.refresh(); + await this.peers.recover(); + this.membershipPending=!!(header.raw&&decodeCanonical(dec.decode(header.raw)).pending); + await this.reconcileMembership(); + await this.wakeMaintenance();this.ready=true; } accept(id,e){if(!e)return;if(e.status)this.states.set(String(id),e.status);this.queue.push(...(e.send??[]).map(m=>({...m,from:String(id),to:String(m.to)})));this.reads.push(...(e.reads??[]).map(r=>({...r,node:String(id)})));} async input(id,tag,value){const result=await this.members.get(String(id)).hostInput(tag,value);this.accept(id,result);return result;} + // Reserve durable sequence ranges before emission. Concurrent public and + // peer turns share one allocator, and recovery skips every unused sequence. + async nextSequence(){ + for(;;){ + if(this.sequence!==undefined&&this.sequence{ + const range=await this.ctx.storage.transaction(async tx=>{ + const saved=await tx.get('raft-transport-sequence')??'0',count=BigInt(this.policy.sequence_reservation); + if(typeof saved!=='string'||!/^(0|[1-9][0-9]*)$/.test(saved)||BigInt(saved)>(1n<<64n)-1n-count)throw unavailable(); + const ceiling=BigInt(saved)+count;await tx.put('raft-transport-sequence',String(ceiling));return {start:saved,ceiling:String(ceiling)}; + });await this.ctx.storage.sync();this.sequence=BigInt(range.start);this.sequenceCeiling=BigInt(range.ceiling); + })().finally(()=>{this.sequencePending=null;}); + await this.sequencePending; + } + } async envelope(from,to,payload,chunk=null){ - const next=BigInt(await this.ctx.storage.get('raft-transport-sequence')??'0')+1n; - await this.ctx.storage.put('raft-transport-sequence',String(next));await this.ctx.storage.sync(); - const e={scope:this.config(from).scope,from,to,incarnation:'1',destination:'1',sequence:String(next),chunk,digest:Array.from(this.codec.hash(payload).digest,b=>b.toString(16).padStart(2,'0')).join(''),payload:b64(payload),signature:null}; + const next=await this.nextSequence(); + const e={scope:this.config(from).scope,from,to,incarnation:'1',destination:this.peers?.view[to]?.incarnation??'1',sequence:String(next),chunk,digest:Array.from(this.codec.hash(payload).digest,b=>b.toString(16).padStart(2,'0')).join(''),payload:b64(payload),signature:null}; const rank=this.manifest.config.anchors.indexOf(from); const key=await crypto.subtle.importKey('pkcs8',unb64(this.provisioning.anchors[rank]),'Ed25519',false,['sign']); e.signature=b64(new Uint8Array(await crypto.subtle.sign('Ed25519',key,enc.encode('codeaf/raft/envelope/1\n'+JSON.stringify(e))))); return enc.encode(JSON.stringify(e)); } + hash(raw){return Array.from(this.codec.hash(raw).digest,b=>b.toString(16).padStart(2,'0')).join('');} async pump(){ + if(this.pumping){await this.pumping;if(this.queue.length)return this.pump();return;} + if(this.peers)this.peers.blocked=new Set(); + this.pumping=this.pumpQueue(); + try{await this.pumping;}finally{this.pumping=null;} + if(this.peers)await this.peers.push(); + } + async pumpQueue(){ for(let n=0;this.queue.length;n++){ if(n>=2048){this.queue=[];throw unavailable();} - const m=this.queue.shift(),to=this.members.get(m.to);if(!to)continue; + const m=this.queue.shift(),to=this.members.get(m.to);if(!to){if(this.peers)await this.peers.send(m);continue;} const snapshot=this.codec.inspect(4,m.bytes).snapshot; try{ if(snapshot.length){ @@ -144,7 +177,8 @@ export class HostedIdentity { await to.hostSnapshot('complete',id); } } - const result=await to.peer(await this.envelope(m.from,m.to,m.bytes));this.accept(m.to,result); + const hash=this.hash(m.bytes),chunk=m.bytes.length>this.policy.max_message_bytes?{generation:hash,offset:'0',hash}:null; + const result=await to.peer(await this.envelope(m.from,m.to,m.bytes,chunk));this.accept(m.to,result); if(snapshot.length)await this.input(m.from,12,{peer:m.to,snapshot:true,success:result.installed===true}); }catch(error){ if(snapshot.length)await this.input(m.from,12,{peer:m.to,snapshot:true,success:false}); @@ -168,6 +202,7 @@ export class HostedIdentity { async commit(raw){ await this.elect();const id=this.leader(),proposal=await this.input(id,3,raw);await this.pump(); const result=await this.members.get(id)?.hostResult(raw,proposal.proposalAfter);if(!result)throw unavailable(); + if(this.peers)await this.peers.refresh(); return {raw:result.raw,value:decodeCanonical(dec.decode(result.raw)),index:result.index}; } async commitEffect(change,now=Date.now()){ @@ -210,6 +245,7 @@ export class HostedIdentity { async wakeMaintenance(){this.asleep=false;await this.armAlarm(Date.now()+this.policy.tick_every);} async maintenance(now=Date.now(),elapsed=Math.floor(performance.now())){ if(this.asleep)return; + if(this.membershipPending)await this.reconcileMembership(); this.elapsed=Math.max(this.elapsed,elapsed-this.started); // Alarm/recovery effects are retained until their exact batch is consumed. for(const [id,member]of this.members){ @@ -259,11 +295,54 @@ export class HostedIdentity { let sleeping=true; for(const [node,member]of this.members){const effects=await member.schedule(this.elapsed,now,false,acks);this.accept(node,effects);sleeping&&=effects.plan.sleeping;} await this.pump();this.asleep=sleeping; + if(sleeping&&this.peers){await this.peers.quiesce();await this.pump();} }catch{this.asleep=false;} } if(!this.asleep)await this.armAlarm(now+this.policy.tick_every); if(this.holds.next!==null)await this.armAlarm(this.holds.next); } + async personal(action,{proof,actor='',certificate=''}={}){ + const b=await this.barrier(); + return this.members.get(b.node).hostPersonal(b.index,{action,identity:this.manifest.config.identity,epoch:this.manifest.config.generation,generation:this.peers.generation,leader:this.leader(),actor,certificate, + proof:proof??{method:'',uri:'',identity_key:null,certificate:'',at:'',signature:null,body:null},progress:[...this.peers.progress].map(([node,applied])=>({node,applied}))}); + } + + async reconcileMembership(){ + if(!this.membershipPending)return; + for(let stage=0;stage<3;stage++){ + const raw=await this.barrierRecord({namespace:'membership',key:'registry'}),h=decodeCanonical(dec.decode(raw)); + if(!h.pending){this.membershipPending=false;await this.peers.refresh();return;} + const id=this.leader(),plan=await this.members.get(id).membership();this.accept(id,plan);await this.pump(); + if(plan.settle){requireOK((await this.anchorCommand({settle_membership:{revision:h.revision,node:h.pending.node}},Date.now())).value.code);} + } + const h=decodeCanonical(dec.decode(await this.barrierRecord({namespace:'membership',key:'registry'})));this.membershipPending=!!h.pending; + await this.peers.refresh(); + } + async enroll(proof){ + await this.reconcileMembership(); + for(let stage=0;stage<3;stage++){ + const out=await this.personal('admission',{proof}); + if(out.code!=='ok')return this.personalRefusal(out); + if(!out.plan.change){const grant=await this.personal('grant',{proof});return grant.code==='ok'?this.jsonBytes(unb64(grant.wire)):this.personalRefusal(grant);} + const result=await this.anchorCommand(out.plan.change,Date.now()); + if(result.value.code!=='ok'){ + return this.personalRefusal({code:result.value.code,message:undefined}); + } + this.membershipPending=true; + // Registry admission and configuration are separate durable stages. A + // subsequent activation or maintenance owns completion after a lost reply. + if(out.plan.change.enroll_member){const grant=await this.personal('grant',{proof});return this.jsonBytes(unb64(grant.wire));} + await this.reconcileMembership(); + } + throw unavailable(); + } + personalRefusal(out){const r=response(this.runtime.call(frame(38).u32(2).str(out.code).finish())),message=r.str();r.end();return new Response(out.message||message||'this computer is not allowed to join this cluster',{status:out.code==='capacity'?429:out.code==='update_required'?409:403,headers:{'cache-control':'no-store'}});} + async control(method,actor,certificate){ + await this.reconcileMembership();const out=await this.personal('status',{actor,certificate}); + if(out.code!=='ok')return new Response('this computer is not allowed to coordinate this cluster',{status:403}); + if(method==='POST')return new Response('this relay does not offer cluster leadership yet; your saved chats are still here',{status:404}); + return this.jsonBytes(unb64(out.wire)); + } async api(raw,q,actor) { await this.ensure(unb64(q.identity_key)); await this.wakeMaintenance(); diff --git a/relay/hosted/src/raft/inbox.js b/relay/hosted/src/raft/inbox.js index 8836706767..4e849e883a 100644 --- a/relay/hosted/src/raft/inbox.js +++ b/relay/hosted/src/raft/inbox.js @@ -8,6 +8,7 @@ const check = v => { if (!v) throw new Error('raft inbox: peer authorization ref export const INBOX_BYTES = 1024 * 1024; const budgets = [INBOX_BYTES - 128 * 1024 - 384 * 1024, 128 * 1024, 384 * 1024]; const slots = [4, 2, 2]; +const NORMAL_BYTES=64*1024; const shared = new Map(); // Authentication is repeated at dispatch, after every async signature wait. @@ -34,8 +35,9 @@ export class MemberInbox { const canonical = { ...unsigned, signature: e.signature }; check(JSON.stringify(canonical) === new TextDecoder().decode(raw)); if (e.chunk !== null) { check(Object.keys(e.chunk).join(',') === 'generation,offset,hash' && hash(e.chunk.generation) && uint(e.chunk.offset, false) && e.chunk.hash === e.digest && payload.length > 0 && payload.length <= 256 * 1024); + if(e.chunk.generation===e.chunk.hash&&e.chunk.offset==='0'){const m=this.codec.inspect(4,payload);check(payload.length>NORMAL_BYTES&&m.type===3&&String(m.from)===e.from&&String(m.to)===e.to);} } else { - check(payload.length > 0 && payload.length <= 64 * 1024); + check(payload.length > 0 && payload.length <= NORMAL_BYTES); const m = this.codec.inspect(4, payload); check(String(m.from) === e.from && String(m.to) === e.to); } const key = await crypto.subtle.importKey('raw', unb64(from.key), 'Ed25519', false, ['verify']); diff --git a/relay/hosted/src/raft/member.js b/relay/hosted/src/raft/member.js index 0bb8dc3756..b1785d60fc 100644 --- a/relay/hosted/src/raft/member.js +++ b/relay/hosted/src/raft/member.js @@ -5,7 +5,7 @@ import { MemberActor, sharedRuntime } from './actor.js'; import { MemberInbox, INBOX_BYTES } from './inbox.js'; import { directoryEvaluator } from './directory.js'; import { DEFAULT_ROWS_BYTES, MemberStorage, b64 } from './storage.js'; -import { HostCodec } from './abi.js'; +import { HostCodec,frame,response } from './abi.js'; import { encodeCanonical, decodeCanonical } from './codec.js'; import { dropTables } from '../erasure.js'; import { committedRegistry, provisionedRegistry } from './membership.js'; @@ -110,6 +110,20 @@ export class RaftMemberDO extends DurableObject { return value&&value.index>BigInt(after)&&value.index<=this.actor.status.applied?value:null; }); } + async hostPersonal(index,input){ + await this.load();return this.actor.serial(()=>{ + if(this.actor.stopped||this.actor.status.applied { const effects = this.recoveredEffects; this.recoveredEffects = null; return effects; }); } + // Only durable application can change protocol authority. Reuse its exact + // view within that index, and invalidate it after every committed change. + committedPeers(){ + const index=this.actor.store.application().applied; + if(this.registryIndex!==index){this.registryCache=this.actor.store.read('membership','registry')?committedRegistry(sharedRuntime(),this.actor.store):provisionedRegistry(sharedRuntime(),this.config);this.registryIndex=index;} + return this.registryCache; + } async peer(raw) { await this.load(); const config = this.config; if (!this.actor.store.read('membership', 'registry') && !config.anchors) throw new Error('raft member: enrollment is absent'); - this.inbox ??= new MemberInbox(config.scope, config.id, this.actor.codec, () => this.actor.store.read('membership', 'registry') ? committedRegistry(sharedRuntime(), this.actor.store) : provisionedRegistry(sharedRuntime(), config), fn => this.actor.serial(fn)); + this.inbox ??= new MemberInbox(config.scope, config.id, this.actor.codec, () => this.committedPeers(), fn => this.actor.serial(fn)); return this.inbox.receive(raw, async (envelope, payload) => { if (envelope.chunk) { + if(envelope.chunk.offset==='0'&&envelope.chunk.generation===envelope.chunk.hash)return this.actor.drive(2,payload); if (!this.actor.store.one('SELECT id FROM snapshots WHERE id=?', envelope.chunk.generation)) return { staged: false }; await this.actor.store.putChunk(envelope.chunk.generation, Number(BigInt(envelope.chunk.offset)), payload, envelope.chunk.hash); return { staged: true }; @@ -282,8 +304,8 @@ export class RaftMemberDO extends DurableObject { async refreshEnrollment() { await this.load(); return this.actor.serial(async () => { - const view = committedRegistry(sharedRuntime(), this.actor.store); - await this.ctx.storage.put('raft-registry', view); await this.ctx.storage.sync(); this.registry = view; + const view = this.committedPeers(); + if(JSON.stringify(view)!==JSON.stringify(this.registry)){await this.ctx.storage.put('raft-registry',view);await this.ctx.storage.sync();this.registry=view;} return view; }); } diff --git a/relay/hosted/src/raft/peers.js b/relay/hosted/src/raft/peers.js new file mode 100644 index 0000000000..4e84a2dbce --- /dev/null +++ b/relay/hosted/src/raft/peers.js @@ -0,0 +1,188 @@ +import {acceptHibernatingSocket} from '../watch.js'; +import {frame,response} from './abi.js'; +import {encodeCanonical,decodeCanonical} from './codec.js'; +import {b64,unb64} from './storage.js'; + +const enc=new TextEncoder(),dec=new TextDecoder('utf-8',{fatal:true}); +const data=v=>enc.encode(encodeCanonical(v)); +const refused=code=>Object.assign(new Error('member authorization refused'),{peerCode:code}); +export const peerPath=path=>['/v2/cluster/exchange','/v2/cluster/socket','/v2/cluster/forward'].includes(path); +export function peerLimits(runtime){const r=response(runtime.call(frame(38).u32(0).finish()));const limits=JSON.parse(dec.decode(r.blob()));r.end();return limits;} +export const peerSocket=ws=>ws.deserializeAttachment()?.clusterPeer===true; + +// Routing memory is expendable. A durable generation fences all old cursors +// before a recovered coordinator can serve another exchange or socket frame. +export class PersonalPeers { + constructor(host,generation){ + this.host=host;this.generation=generation;this.scope=host.config('1').scope;this.view={};this.boxes=new Map();this.used=[0,0,0];this.progress=new Map();this.snapshots=new Map();this.acks=new Map();this.tail=Promise.resolve();this.pending=0;this.blocked=new Set();this.checkpoints=new Map(); + this.limits=peerLimits(host.runtime); + } + serial(fn){if(this.pending>=64)return Promise.reject(refused('busy'));this.pending++;const p=this.tail.then(fn);this.tail=p.catch(()=>{});return p.finally(()=>this.pending--);} + members(){return Object.entries(this.view).map(([node,p])=>({node,...p}));} + async refresh(){ + const id=this.host.leader();if(!id)return; + const applied=String(this.host.states.get(id).applied);if(this.viewApplied===id+'/'+applied)return; + const view=await this.host.members.get(id).refreshEnrollment(); + this.viewApplied=id+'/'+applied; + if(JSON.stringify(view)===JSON.stringify(this.view))return; + this.view=view;this.used=[0,0,0]; + for(const [node,p]of Object.entries(view)){ + const old=this.boxes.get(node); + this.boxes.set(node,{queues:[[],[],[]],next:old?.next??[0n,0n,0n],issued:old?.issued??[0n,0n,0n],acked:old?.acked??[0n,0n,0n],credits:old?.credits??new Map()}); + if(p.revoked)this.progress.delete(node); + } + for(const [to,s]of this.snapshots){this.snapshots.delete(to);await this.host.input(s.from,12,{peer:to,snapshot:true,success:false});} + for(const ws of this.sockets())if(this.view[ws.deserializeAttachment().node]?.revoked)this.close(ws); + } + trusted(){const s=this.host.states.get(this.host.leader());return {scope:this.scope,generation:this.generation,members:this.members(),applied:String(s?.applied??0),term:String(s?.term??0),sleeping:this.host.asleep};} + validate(raw){ + const r=response(this.host.runtime.call(frame(37).blob(raw).blob(data(this.trusted())).finish()));const result=JSON.parse(dec.decode(r.blob()));r.end(); + if(result.code!=='ok')throw refused(result.code); + return {request:JSON.parse(dec.decode(raw)),obsolete:new Set(result.obsolete)}; + } + sockets(){return this.host.ctx.getWebSockets('cluster-peer');} + close(ws){try{ws.close(1012,'reconnecting');}catch{}} + expire(){const now=Date.now();for(const box of this.boxes.values())for(let lane=0;lane<3;lane++)box.queues[lane]=box.queues[lane].filter(q=>{if(q.expires>now)return true;this.used[lane]-=q.size;return false;});} + ack(req){ + const box=this.boxes.get(req.node);if(!box)throw refused('unauthorized'); + for(let lane=0;lane<3;lane++)if(BigInt(req.ack[lane])>box.issued[lane])throw refused('cursor'); + for(let lane=0;lane<3;lane++){ + const ack=BigInt(req.ack[lane]);if(ack<=box.acked[lane])continue; + box.queues[lane]=box.queues[lane].filter(q=>{ + if(BigInt(q.cursor)>ack)return true;this.used[lane]-=q.size; + if(q.envelope.chunk){const source=this.boxes.get(q.envelope.from);if(source)source.credits.set(req.node,q.envelope.sequence);} + return false; + });box.acked[lane]=ack; + } + } + lane(e){return e.chunk?2:this.view[e.from]?.voter&&this.view[e.to]?.voter?0:1;} + enqueue(envelopes){ + this.expire();const staged=[],added=[0,0,0],counts=new Map(); + for(const e of envelopes){ + const box=this.boxes.get(e.to);if(!box||this.view[e.to]?.revoked)throw refused('unauthorized'); + const duplicate=[...box.queues.flat(),...staged].find(q=>q.envelope.to===e.to&&q.envelope.from===e.from&&q.envelope.sequence===e.sequence); + if(duplicate){if(JSON.stringify(duplicate.envelope)!==JSON.stringify(e))throw refused('unauthorized');continue;} + const lane=this.lane(e),size=enc.encode(JSON.stringify(e)).length,k=e.to+'/'+lane; + counts.set(k,(counts.get(k)??0)+1);added[lane]+=size; + if(box.queues[lane].length+counts.get(k)>this.limits.Slots[lane]||this.used[lane]+added[lane]>this.limits.Budget[lane]||box.next[lane]+BigInt(counts.get(k))>=(1n<<64n))throw refused('busy'); + staged.push({envelope:e,lane,size,expires:Date.now()+this.limits.Retention}); + } + for(const q of staged){const box=this.boxes.get(q.envelope.to);q.cursor=String(++box.next[q.lane]);box.queues[q.lane].push(q);this.used[q.lane]+=q.size;} + } + deliveries(node){ + this.expire();const box=this.boxes.get(node),messages=[];let normal=0,snapshot=false,size=0; + if(!box)return messages; + for(let lane=0;lane<3;lane++)for(const q of box.queues[lane]){ + const bytes=unb64(q.envelope.payload).length; + if(q.envelope.chunk){if(snapshot||normal)break;snapshot=true;}else{if(snapshot||normal+bytes>this.limits.Batch)break;normal+=bytes;} + const d={lane,cursor:q.cursor,envelope:q.envelope},cost=enc.encode(JSON.stringify(d)).length; + if(size+cost>this.limits.Exchange-1024)break;size+=cost;messages.push(d);box.issued[lane]=BigInt(q.cursor)>box.issued[lane]?BigInt(q.cursor):box.issued[lane]; + } + return messages; + } + wire(node,extra={}){ + const box=this.boxes.get(node),chunks=[...(box?.credits??[])].sort(([a],[b])=>BigInt(a)({node,sequence})); + const r=response(this.host.runtime.call(frame(38).u32(1).blob(enc.encode(JSON.stringify({...extra,chunks,generation:this.generation,messages:this.deliveries(node)}))).finish()));const raw=r.blob();r.end();return raw; + } + async recover(){ + const from=this.host.leader(); + for(const [to,member]of Object.entries(this.view))if(!member.revoked&&!this.host.members.has(to)){ + await this.host.input(from,12,{peer:to,snapshot:true,success:false}); + await this.host.input(from,12,{peer:to,snapshot:false,success:false}); + } + await this.host.pump(); + } + async send(m){ + await this.refresh(); + if(this.host.asleep||this.blocked.has(m.to)||!this.view[m.to]||this.view[m.to].revoked)return; + const inspected=this.host.codec.inspect(4,m.bytes),snapshot=inspected.snapshot; + if(m.bytes.length>this.limits.Batch){ + this.blocked.add(m.to);const applied=String(this.host.states.get(m.from).applied); + if(this.checkpoints.get(m.from)!==applied){await this.host.members.get(m.from).hostCheckpoint();this.checkpoints.set(m.from,applied);} + await this.host.input(m.from,12,{peer:m.to,snapshot:false,success:false});return; + } + const lane=this.view[m.from]?.voter&&this.view[m.to]?.voter?0:1; + if(this.boxes.get(m.to).queues[lane].length>=this.limits.Slots[lane]){this.blocked.add(m.to);await this.host.input(m.from,12,{peer:m.to,snapshot:snapshot.length>0,success:false});return;} + try{ + this.enqueue([JSON.parse(dec.decode(await this.host.envelope(m.from,m.to,m.bytes)))]); + if(snapshot.length){const meta=this.host.codec.inspect(3,snapshot),manifest=JSON.parse(dec.decode(meta.data));this.snapshots.set(m.to,{from:m.from,manifest,generation:this.host.hash(meta.data),offset:0,waiting:null,started:Date.now()});} + }catch(e){if(e.peerCode!=='busy')throw e;this.blocked.add(m.to);await this.host.input(m.from,12,{peer:m.to,snapshot:snapshot.length>0,success:false});} + } + async chunks(){ + if(this.host.asleep)return; + for(const [to,s]of this.snapshots){ + if(BigInt(this.progress.get(to)??0)>=BigInt(s.manifest.index)){ + this.snapshots.delete(to);await this.host.input(s.from,12,{peer:to,snapshot:true,success:true});continue; + } + if(Date.now()-s.started>=this.limits.Retention){this.snapshots.delete(to);await this.host.input(s.from,12,{peer:to,snapshot:true,success:false});continue;} + if(s.waiting&&this.boxes.get(s.from)?.credits.get(to)!==s.waiting)continue; + s.waiting=null;if(s.offset>=Number(s.manifest.chunks))continue; + const chunk=await this.host.members.get(s.from).hostSnapshot('read',s.generation,s.offset); + const e=JSON.parse(dec.decode(await this.host.envelope(s.from,to,chunk,{generation:s.generation,offset:String(s.offset),hash:this.host.hash(chunk)}))); + try{this.enqueue([e]);s.offset++;s.waiting=e.sequence;}catch(err){if(err.peerCode!=='busy')throw err;} + } + } + async anchors(){ + for(let round=0;round{ + await this.refresh();let {request:req,obsolete}=this.validate(raw);this.ack(req); + let rejected=false; + try{this.enqueue((req.messages??[]).filter(e=>!obsolete.has(e.sequence)));}catch(err){if(err.peerCode!=='busy')throw err;rejected=true;} + if(req.applied)this.progress.set(req.node,String(BigInt(req.applied)>BigInt(this.progress.get(req.node)??0)?req.applied:this.progress.get(req.node))); + for(const e of req.messages??[])if(!obsolete.has(e.sequence))await this.host.wakeMaintenance(); + await this.anchors();await this.chunks();await this.host.pump(); + await this.push(req.node); + return {request:req,raw:this.wire(req.node,{[rejected?'rejected':'accepted']:(req.messages??[]).map(e=>e.sequence)}),rejected}; + });} + async push(except){ + await this.refresh(); + for(const ws of this.sockets()){ + const a=ws.deserializeAttachment(); + if(a.kind!=='peer'||a.node===except)continue; + try{ + this.validate(unb64(a.auth));const raw=this.wire(a.node),text=this.host.hash(raw); + if(text===a.last)continue; + const body=JSON.parse(dec.decode(raw));if(!body.messages.length&&!body.chunks?.length)continue; + ws.send(raw);const deadline=a.deadline??(body.messages.length?Date.now()+this.limits.Retention:null);ws.serializeAttachment({...a,last:text,deadline});if(deadline)await this.host.armAlarm(deadline); + }catch{this.close(ws);} + } + } + async accept(request,raw,kind){ + await this.refresh();const checked=this.validate(raw);if(checked.request.messages?.length)throw refused('unauthorized'); + const {request:req}=await this.exchange(raw); + if(this.sockets().filter(ws=>ws.deserializeAttachment().node===req.node).length>=this.limits.Inflight)throw refused('busy'); + if(request.headers.get('upgrade')?.toLowerCase()!=='websocket')return new Response('upgrade required',{status:426}); + const initial=kind==='peer'?this.wire(req.node):null; + const deadline=initial&&JSON.parse(dec.decode(initial)).messages.length?Date.now()+this.limits.Retention:null; + const {client,server}=acceptHibernatingSocket(this.host.ctx,['cluster-peer'],{clusterPeer:true,kind,node:req.node,generation:this.generation,auth:b64(raw),last:initial?this.host.hash(initial):null,deadline}); + if(deadline)await this.host.armAlarm(deadline); + if(initial)server.send(initial); + return new Response(null,{status:101,webSocket:client}); + } + async frame(ws,raw){ + const att=ws.deserializeAttachment(); + try{ + this.validate(unb64(att.auth));if(att.kind!=='peer'||!(raw instanceof ArrayBuffer))throw refused('invalid'); + const bytes=new Uint8Array(raw),next=JSON.parse(dec.decode(bytes));if(next.node!==att.node||next.generation!==att.generation)throw refused('unauthorized'); + const result=await this.exchange(bytes);ws.send(result.raw);const deadline=JSON.parse(dec.decode(result.raw)).messages.length?Date.now()+this.limits.Retention:null;ws.serializeAttachment({...att,last:this.host.hash(result.raw),deadline});if(deadline)await this.host.armAlarm(deadline); + }catch{this.close(ws);} + } + async deadlines(now){for(const ws of this.sockets()){const at=ws.deserializeAttachment().deadline;if(at&&at<=now)this.close(ws);else if(at)await this.host.armAlarm(at);}} + async quiesce(){ + for(const [node,box]of this.boxes)if(!this.view[node]?.voter)for(let lane=0;lane<3;lane++){for(const q of box.queues[lane])this.used[lane]-=q.size;box.queues[lane]=[];} + for(const [to,s]of this.snapshots){this.snapshots.delete(to);await this.host.input(s.from,12,{peer:to,snapshot:true,success:false});} + } +} +export function peerRefusal(e){const code=e.peerCode??'invalid',status={unauthorized:403,generation:409,cursor:409,busy:429}[code]??400;return new Response(JSON.stringify({error:code}),{status,headers:{'content-type':'application/json','cache-control':'no-store',...(code==='busy'?{'retry-after':'1'}:{})}});} diff --git a/relay/hosted/src/watch.js b/relay/hosted/src/watch.js index 0eb087bf6d..03d75bdca2 100644 --- a/relay/hosted/src/watch.js +++ b/relay/hosted/src/watch.js @@ -1,4 +1,4 @@ -// The directory watch socket of one identity (contract 21). It is the server half of a push that costs +// The directory watch socket of one identity. It is the server half of a push that costs // nothing while idle: the object accepts sockets with the hibernation API, so a socket that is open and // quiet holds no memory and bills no time, and it sets no timer and opens no connection for any socket. // A socket carries one number, the directory's version, and never a record. The list of sockets is the @@ -11,14 +11,15 @@ import { Wire } from './wire.js'; /** The close codes a client acts on: a revoked device stops for good; a rotation stops the watching. */ export const CLOSE_REVOKED = 4401; export const CLOSE_ROTATED = 4410; -/** A socket that stopped answering is closed with this code; a client that reads it redials at once (contract 21.12.4). */ +/** A socket that stopped answering is closed with this code; a client that reads it redials at once. */ export const CLOSE_SILENT = 4408; -/** The answer headers that tell a client this relay counts its socket as proof of life, and judges presence by it (contract 21.11.1, 21.12.3). */ +/** The answer headers that tell a client this relay counts its socket as proof of life, and judges presence by it. */ const ANSWER_HEADERS = { 'Codeaf-Vouch': '1', 'Codeaf-Presence': '1' }; /** What a socket carries through hibernation; an attachment from before a field existed reads as its default. */ const FRESH = { receipts: false, at: 0, holds: [], events: false, beat: PRESENCE.defaultBeat, lapsed: false }; +const watchSockets=ctx=>ctx.getWebSockets().filter(ws=>ws.deserializeAttachment?.()?.clusterPeer!==true); const attachmentOf = (ws) => ({ ...FRESH, ...ws.deserializeAttachment() }); /** @@ -58,6 +59,15 @@ const frame = (version) => `{"v":${version}}`; /** NOBODY hears nothing: what a directory uses when no socket can ever be open (tests, the pure rules). */ export const NOBODY = { broadcast() {}, closeDevice() {}, closeAll() {}, vouchedUntil: () => 0, announce() {} }; +// The hibernation API keeps accepted watch and protocol sockets passive between +// frames. Attachments and tags are installed together before either can send. +export function acceptHibernatingSocket(ctx, tags, attachment) { + const { 0: client, 1: server } = new WebSocketPair(); + ctx.acceptWebSocket(server, tags); + server.serializeAttachment(attachment); + return { client, server }; +} + export class Watchers { constructor(ctx, cap, ttlMs, arm = () => {}, seen = () => {}) { this.ctx = ctx; @@ -77,16 +87,15 @@ export class Watchers { /** * accept opens a socket for `device`, tells it `version` at once, and answers the 101 that completes the upgrade. * The holds the socket names and the time it was accepted travel in its attachment, which the platform keeps - * with the socket through hibernation, so vouching for a lease costs no storage write (contract 21.11). + * with the socket through hibernation, so vouching for a lease costs no storage write. * A socket opened with `events` also hears the event frames (presence, joined, revoked) and, at once, who is online; - * it is also the viewer that makes the presence sweep worth arming (contract 21.12.4). `beat` is the ping period the + * it is also the viewer that makes the presence sweep worth arming. `beat` is the ping period the * client declared, from which the socket's window follows. */ accept(device, version, now, { holds, beat, events = false, receipts = false, certificate = undefined }) { - if (this.ctx.getWebSockets().length >= this.cap) throw new Wire('too_many_watchers', 429); - const { 0: client, 1: server } = new WebSocketPair(); - this.ctx.acceptWebSocket(server, [device]); - server.serializeAttachment({ ...FRESH, at: now, holds, events, beat, receipts, ...(certificate?{certificate}:{}) }); + if (watchSockets(this.ctx).length >= this.cap) throw new Wire('too_many_watchers', 429); + const { client, server } = acceptHibernatingSocket(this.ctx, [device], + { ...FRESH, at: now, holds, events, beat, receipts, ...(certificate?{certificate}:{}) }); server.send(frame(version)); if (events) for (const text of this.liveness.snapshot(device)) server.send(text); this.liveness.opened(device); @@ -107,7 +116,7 @@ export class Watchers { // Only the committed-renewal bridge consumes this snapshot. The platform's // hibernating ping timestamp remains advisory until its exact holds commit. holdEvidence(now) { - return this.ctx.getWebSockets().map(ws => { + return watchSockets(this.ctx).map(ws => { const att = attachmentOf(ws); // Presence may forgive a closing socket, but renewal evidence must // fail closed when the platform cannot supply its timestamp. @@ -122,7 +131,7 @@ export class Watchers { /** broadcast tells every socket of the identity the new version. */ broadcast(version) { const text = frame(version); - for (const ws of this.ctx.getWebSockets()) attempt(() => ws.send(text)); + for (const ws of watchSockets(this.ctx)) attempt(() => ws.send(text)); } /** #evidence is what a socket names and when it last showed life, which is all a lease needs of it. */ @@ -148,7 +157,7 @@ export class Watchers { } #peers() { - return this.ctx.getWebSockets().map((ws) => this.#peer(ws)); + return watchSockets(this.ctx).map((ws) => this.#peer(ws)); } /** left runs when a socket has closed. It answers the device that now holds no live socket, or undefined: a socket that was already silent leaves nothing to tell. */ @@ -162,12 +171,12 @@ export class Watchers { this.liveness.expire(); } - /** online is the set of devices that hold a live watch socket now: one inside its window (contract 21.12.2). */ + /** online is the set of devices that hold a live watch socket now: one inside its window. */ online() { return this.liveness.online(); } - /** announce sends an event frame (contract 5) to every event socket except the sockets of `except`. */ + /** announce sends an event frame to every event socket except the sockets of `except`. */ announce(frame, except) { this.liveness.tell(frame, except); } @@ -177,7 +186,7 @@ export class Watchers { } closeAll(code, reason) { - for (const ws of this.ctx.getWebSockets()) attempt(() => ws.close(code, reason)); + for (const ws of watchSockets(this.ctx)) attempt(() => ws.close(code, reason)); } } diff --git a/relay/hosted/test/raft/hosted-control.test.js b/relay/hosted/test/raft/hosted-control.test.js index 8fa85736e4..d1541e0ff3 100644 --- a/relay/hosted/test/raft/hosted-control.test.js +++ b/relay/hosted/test/raft/hosted-control.test.js @@ -4,26 +4,26 @@ import { stat } from 'node:fs/promises'; import { join } from 'node:path'; import { hostedFixture } from './hosted-fixture.js'; -// R6/R9/R23/R30 keep hosted member and leadership doors absent until their -// runtime can serve an external member. Refusing avoids an intent that -// cannot finish. Restart runs the same bundle over the same persistent state. -test('R6 R9 R23 R30 hosted member doors stay absent across a runtime restart',{timeout:30000},async()=>{ +// R9/R12/R23/R30 expose authenticated learner doors while leadership remains unavailable. +test('R9 R12 R23 R30 hosted member doors and same-bundle restart',{timeout:30000},async()=>{ const h=await hostedFixture({base:'/fabric'}),a=h.client(); try{ await h.start(); - for(const path of ['/v2/cluster/control','/v2/cluster/enroll','/v2/cluster/exchange','/v2/cluster/socket']){ - for(const method of ['GET','POST']){ - const r=await h.request(a,method,path); - assert.equal(r.status,404,path);await r.text(); - } + let r=await h.request(a,'GET','/v2/cluster/control'); + assert.equal(r.status,200,await r.clone().text()); + let status=await r.json();assert.equal(status.self,'0');assert.equal(status.leadership_unavailable,true); + assert.deepEqual(status.configuration.voters,['1','2','3']); + const enroll=async()=>{const r=await h.request(a,'POST','/v2/cluster/enroll',{store:'a'.repeat(64),protocol:1});assert.equal(r.status,200,await r.clone().text());return r.json();}; + const first=await enroll(),second=await enroll();assert.equal(first.node,second.node);assert.notEqual(second.barrier,'0'); + r=await h.request(a,'GET','/v2/cluster/control');status=await r.json();assert.equal(status.self,first.node);assert(status.configuration.learners.includes(first.node));assert(!status.ready?.includes(first.node)); + r=await h.request(a,'POST','/v2/cluster/control');assert.equal(r.status,404);assert.equal((await r.text()).trim(),'this relay does not offer cluster leadership yet; your saved chats are still here'); + for(const path of ['control','enroll','exchange','socket','forward']){ + const r=await h.mf.dispatchFetch('http://local/fabric/v2/cluster/'+path,{method:path==='enroll'||path==='exchange'?'POST':'GET'});assert([400,403].includes(r.status),path+' '+r.status);await r.text(); } const bundle=await stat(join(h.dir,'worker.js')); await h.start(); const restarted=await stat(join(h.dir,'worker.js')); assert.equal(restarted.mtimeNs??restarted.mtimeMs,bundle.mtimeNs??bundle.mtimeMs,'restart rebuilt the program'); - for(const path of ['/v2/cluster/control','/v2/cluster/enroll','/v2/cluster/exchange','/v2/cluster/socket']){ - const r=await h.request(a,'GET',path); - assert.equal(r.status,404,path);await r.text(); - } + const returned=await enroll();assert.equal(returned.node,first.node);assert(BigInt(returned.generation)>BigInt(first.generation)); }finally{await h.close();} }); diff --git a/relay/hosted/test/raft/hosted-learner-go.test.js b/relay/hosted/test/raft/hosted-learner-go.test.js new file mode 100644 index 0000000000..7f4d04a7bd --- /dev/null +++ b/relay/hosted/test/raft/hosted-learner-go.test.js @@ -0,0 +1,26 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {spawn} from 'node:child_process'; +import {join} from 'node:path'; +import {hostedFixture} from './hosted-fixture.js'; +const root=new URL('../../../../',import.meta.url),footprint=new URL('../../../../scripts/raft-proof-footprint.sh',import.meta.url).pathname; +async function run(args,env={}){ + const child=spawn('sh',[footprint,...args],{cwd:root,env:{...process.env,...env},stdio:['ignore','pipe','pipe']});let output=''; + child.stdout.on('data',b=>output+=b);child.stderr.on('data',b=>output+=b); + const code=await new Promise((resolve,reject)=>{child.once('error',reject);child.once('exit',resolve)});return {code,output}; +} +const binary=join(process.env.CODEAF_RAFT_PROOF_DIR,'hosted-client.test'); +const compiled=await run(['go','test','-c','-p=1','-timeout','15m','-o',binary,'./internal/syncsetup']);console.log(compiled.output);assert.equal(compiled.code,0,compiled.output); +// R9/R23 use the production Worker and the terminal's real state owner. +// Independent identities use independent runtime lifetimes so completed cases +// do not consume the next case's active-member memory reservation. +for(const name of ['TestR9R23HostedLearnerRunStateReopenAndWipe','TestR23HostedQuietHomesMakeNoClusterRequests','TestR9HostedLearnerSnapshotChunksAndLog','TestR9R27HostedLearnerMaximumCommand']){ + test('R9 R23 hosted real client '+name,{timeout:300000},async()=>{ + const h=await hostedFixture({base:'/fabric'}); + try{ + await h.start();const url=await h.mf.ready; + const {code,output}=await run([binary,'-test.run','^'+name+'$','-test.count=1','-test.timeout=15m','-test.parallel=1','-test.v'],{CODEAF_HOSTED_PROOF_URL:url.origin+'/fabric'}); + console.log(output);assert.equal(code,0,output);assert.doesNotMatch(output,/--- SKIP:/); + }finally{await h.close();} + }); +} diff --git a/relay/hosted/test/raft/hosted-personal.test.js b/relay/hosted/test/raft/hosted-personal.test.js new file mode 100644 index 0000000000..16ed7cefcc --- /dev/null +++ b/relay/hosted/test/raft/hosted-personal.test.js @@ -0,0 +1,133 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {sign,createHash} from 'node:crypto'; +import {hostedFixture} from './hosted-fixture.js'; +const enc=new TextEncoder(),sha=b=>createHash('sha256').update(b).digest('hex'); +const token=n=>n.toString(16).padStart(64,'0'); +async function enroll(h,c,store=token(1),protocol=1){const r=await h.request(c,'POST','/v2/cluster/enroll',{store,protocol});assert.equal(r.status,200,await r.clone().text());return r.json();} +async function joined(h,c,store=token(1)){await enroll(h,c,store);return enroll(h,c,store);} +function signed(c,g,{messages=null,applied,ack=['0','0','0'],generation=g.generation}={}){ + const r={...(applied?{applied:String(applied)}:{}),scope:g.scope,node:g.node,incarnation:g.incarnation,generation,messages,signature:null,ack}; + r.signature=sign(null,Buffer.from('codeaf/raft/exchange/1\n'+JSON.stringify(r)),c.key.privateKey).toString('base64');return JSON.stringify(r); +} +async function exchange(h,raw){return h.mf.dispatchFetch('http://local/v2/cluster/exchange',{method:'POST',body:raw});} +function envelope(c,g,payload,sequence='1'){ + const e={scope:g.scope,from:g.node,to:'1',incarnation:g.incarnation,destination:'1',sequence,chunk:null,digest:sha(payload),payload:Buffer.from(payload).toString('base64'),signature:null}; + e.signature=sign(null,Buffer.from('codeaf/raft/envelope/1\n'+JSON.stringify(e)),c.key.privateKey).toString('base64');return e; +} +const nextFrame=ws=>new Promise((resolve,reject)=>{const timer=setTimeout(()=>reject(Error('no peer response')),5000);ws.addEventListener('message',e=>{clearTimeout(timer);resolve(JSON.parse(new TextDecoder().decode(e.data)));},{once:true});}); + +// R9/R12/R16 preserve admission proof, retry identity and permanent retirement. +test('R9 R12 R16 hosted admission retries, wipes, protocol and revoked keys',{timeout:60000},async()=>{ + const h=await hostedFixture(),a=h.client(),b=h.client(); + try{ + await h.start();const first=await joined(h,a),same=await joined(h,a);assert.equal(same.node,first.node);assert.equal(same.barrier,first.barrier); + const replacement=await joined(h,a,token(2));assert(BigInt(replacement.node)>BigInt(first.node)); + let r=await h.request(a,'POST','/v2/cluster/enroll',{store:token(1),protocol:1});assert.equal(r.status,403);await r.text(); + r=await exchange(h,signed(a,first));assert.equal(r.status,403);await r.text(); + r=await h.request(a,'POST','/v2/cluster/enroll',{store:token(3),protocol:2});assert.equal(r.status,409);assert.equal((await r.text()).trim(),'this codeaf build must be updated before it can join this cluster; your saved chats are still here'); + await h.call(b,'PUT','/v1/dir/devices/'+b.device,{V:1,name:'sealed'}); + const other=await joined(h,b,token(4));assert.notEqual(other.node,replacement.node); + assert.equal((await h.call(a,'POST','/v1/dir/devices/'+b.device+'/revoke')).status,204); + r=await h.request(b,'POST','/v2/cluster/enroll',{store:token(5),protocol:1});assert.equal(r.status,403);assert.equal((await r.text()).trim(),'this computer is not allowed to join this cluster'); + r=await exchange(h,signed(b,other));assert.equal(r.status,403);await r.text(); + const worker=h.client();worker.cert=h.delegate({cluster:replacement.scope.cluster,key:worker.key.publicKey.export({format:'der',type:'spki'}).subarray(-32).toString('base64'),made:Date.now()-1000,expires:Date.now()+60000,cells:['chat'],learner:true});r=await h.request(worker,'POST','/v2/cluster/enroll',{store:token(6),protocol:1});assert.equal(r.status,403);await r.text(); + }finally{await h.close();} +}); + +// R9/R27 count actual live admissions; no record is deleted to admit the 33rd. +test('R9 R27 hosted learner capacity is 32',{timeout:120000},async()=>{ + const h=await hostedFixture(); + try{ + await h.start();let last; + for(let n=1;n<=32;n++)last=await joined(h,h.client(),token(n)); + const r=await h.request(h.client(),'POST','/v2/cluster/enroll',{store:token(33),protocol:1});assert.equal(r.status,429);assert.equal((await r.text()).trim(),'this cluster has reached its enrollment limit; remove a joined computer or ask the relay operator for capacity'); + assert.equal(last.node,'35'); + }finally{await h.close();} +}); + +// R9/R12 recover configuration from committed admission without another request. +test('R9 R12 hosted activation repairs the enrollment/configuration cut',{timeout:30000},async()=>{ + const h=await hostedFixture(),a=h.client(); + try{ + await h.start({needle:'this.membershipPending=true;',after:"throw new Error('interrupted admission');"}); + const r=await h.request(a,'POST','/v2/cluster/enroll',{store:token(1),protocol:1});assert.equal(r.status,500);await r.text(); + await h.start();const status=await h.call(a,'GET','/v2/cluster/control');assert.equal(status.status,200,JSON.stringify(status));assert.equal(status.body.self,'4');assert.deepEqual(status.body.configuration.learners,['4']);assert.equal(status.body.membership.next_node,'5');assert(!status.body.membership.pending); + const grant=await enroll(h,a);assert.equal(grant.node,'4');assert.notEqual(grant.barrier,'0'); + }finally{await h.close();} +}); + +// R23 validates the native wire, cursor bounds, authentication and durable fences. +test('R23 hosted exchange and hibernating sockets reject foreign, local and stale traffic',{timeout:60000},async()=>{ + const h=await hostedFixture(),a=h.client(); + try{ + await h.start();const g=await joined(h,a); + let r=await exchange(h,signed(a,g,{ack:['999','0','0']}));assert.equal(r.status,409);assert.deepEqual(await r.json(),{error:'cursor'}); + for(const type of [0,1,2,11,15,16,19,20,21,22]){ + const payload=Uint8Array.of(8,type,16,1,24,Number(g.node),32,1),e=envelope(a,g,payload,String(type+1)); + r=await exchange(h,signed(a,g,{messages:[e]}));assert.equal(r.status,400,'local/proposal/storage type '+type);await r.text(); + } + r=await exchange(h,signed(a,g,{applied:(1n<<64n)-1n}));assert.equal(r.status,400);await r.text(); + r=await exchange(h,signed(h.client(),g));assert.equal(r.status,403);await r.text(); + const foreign={...g,scope:{...g.scope,identity:'id_'+'e'.repeat(32)}}; + r=await exchange(h,signed(a,foreign));assert.equal(r.status,403);await r.text(); + r=await exchange(h,signed(a,g)+' ');assert.equal(r.status,400);await r.text(); + const raw=signed(a,g),headers={Upgrade:'websocket','Codeaf-Cluster-Auth':Buffer.from(raw).toString('base64url')}; + const peer=await h.mf.dispatchFetch('http://local/v2/cluster/socket',{headers});assert.equal(peer.status,101);const ws=peer.webSocket;ws.accept();const initial=await nextFrame(ws);assert.equal(initial.generation,g.generation); + const appResponse=Uint8Array.of(8,4,16,1,24,Number(g.node),32,1); + const busy=[1,2,3].map(n=>envelope(a,g,appResponse,String(n))); + const rejected=nextFrame(ws);ws.send(enc.encode(signed(a,g,{messages:busy})));assert.deepEqual((await rejected).rejected,['1','2','3']); + const live=nextFrame(ws);ws.send(enc.encode(signed(a,g)));assert.equal((await live).generation,g.generation,'backpressure closed the peer socket'); + + const forward=await h.mf.dispatchFetch('http://local/v2/cluster/forward',{headers});assert.equal(forward.status,101);forward.webSocket.accept();forward.webSocket.close();ws.close(); + await h.start();r=await exchange(h,signed(a,g));assert.equal(r.status,409);assert.deepEqual(await r.json(),{error:'generation'}); + const refreshed=await enroll(h,a);assert(BigInt(refreshed.generation)>BigInt(g.generation));assert.equal(refreshed.node,g.node); + }finally{await h.close();} +}); + +// R9/R27 a vanished learner cannot gate public commitment, compaction or sleep. +test('R9 R26 R27 stalled hosted learner cannot keep voters awake or delay a write',{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();const grant=await joined(h,a); + assert.equal((await h.call(a,'POST','/v1/dir/cells/while-gone',{head:'saved',class:'work',size:0,keys:{}})).status,200); + assert.equal((await h.call(a,'POST','/v1/dir/cells/while-gone/release',{fence:1})).status,204); + const read=await h.call(a,'GET','/v1/dir/cells/while-gone');assert.equal(read.body.cell.head,'saved'); + const socket=await h.mf.dispatchFetch('http://local/v2/cluster/socket',{headers:{Upgrade:'websocket','Codeaf-Cluster-Auth':Buffer.from(signed(a,grant)).toString('base64url')}});assert.equal(socket.status,101); + const dying=socket.webSocket;dying.accept();dying.send(enc.encode(signed(a,grant,{messages:[envelope(a,grant,Uint8Array.of(8,9,16,1,24,Number(grant.node),32,1))]})));dying.close(); + assert.equal((await h.call(a,'POST','/v1/dir/vault',{old:'',new:'after-interrupted-peer'})).status,204); + + const idle=await h.control({op:'idle-schedule'});assert.equal(idle.sleeping,true);assert.equal(idle.recurringMemberCalls,0);assert.equal(idle.appliedBefore,idle.appliedAfter);assert.equal(idle.membershipCalls,0,'settled accounts ran recurring membership reconciliation'); + const quiet=await exchange(h,signed(a,grant));assert.equal(quiet.status,200);assert.deepEqual((await quiet.json()).messages,[]); + const asleep=await h.control({op:'peer-idle'});assert.equal(asleep.sleeping,true);assert.equal(asleep.snapshots,0);assert.deepEqual(asleep.bytes,[0,0,0]); + assert.equal((await h.call(a,'POST','/v1/dir/vault',{old:'after-interrupted-peer',new:'after-sleep'})).status,204); + const list=await h.call(a,'GET','/v1/dir/list');assert.equal(list.body.identity.vault,'after-sleep'); + }finally{await h.close();} +}); + +// R23 refuses unknown identities without activating storage, and peer delivery +// remains independent of a public call waiting on its own coordination queue. +test('R23 hosted unknown peer creates no storage and public waits do not block peers',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();const scope={identity:h.identity,cluster:'unknown',epoch:'1',protocol:1},g={scope,node:'4',incarnation:'1',generation:'1'}; + const before=await h.control({op:'transport-state'}); + let r=await exchange(h,signed(a,g));assert.equal(r.status,403);await r.text(); + assert.deepEqual(await h.control({op:'transport-state'}),before);assert.deepEqual(before.keys,[]);assert.deepEqual(before.tables,[]); + const grant=await joined(h,a);await h.control({op:'pause-public'}); + try{const heartbeat=envelope(a,grant,Uint8Array.of(8,9,16,1,24,Number(grant.node),32,1));r=await exchange(h,signed(a,grant,{messages:[heartbeat]}));assert.equal(r.status,200,await r.clone().text());assert.deepEqual((await r.json()).accepted,['1']);}finally{await h.control({op:'resume-public'});} + const compacted=await h.control({op:'checkpoint-state'});assert.equal(compacted.retained,0);assert.equal(compacted.snapshot,compacted.applied); + }finally{await h.close();} +}); + +// R23 concurrent senders and coordinator recovery cannot reuse a signed sequence. +test('R23 hosted outbound sequences stay unique across concurrency and recovery',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + const before=await h.control({op:'sequence-proof'});assert.equal(new Set(before.sequences).size,before.sequences.length); + await h.start();assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + const after=await h.control({op:'sequence-proof'});assert.equal(new Set(after.sequences).size,after.sequences.length);assert(BigInt(after.sequences[0])>BigInt(before.sequences.at(-1))); + console.log('R23 durable sequence ceiling',before.ceiling,after.ceiling); + }finally{await h.close();} +}); diff --git a/relay/hosted/test/raft/hosted-worker.js b/relay/hosted/test/raft/hosted-worker.js index de80434936..896ea94931 100644 --- a/relay/hosted/test/raft/hosted-worker.js +++ b/relay/hosted/test/raft/hosted-worker.js @@ -7,7 +7,7 @@ import { RaftMemberDO as Member } from '../../src/raft/member.js'; import { Directory } from '../../src/directory.js'; import { unb64 } from '../../src/raft/storage.js'; import { runtimeMetrics } from '../../src/raft/actor.js'; -const measurements={turns:0,wall:[]}; +const measurements={turns:0,wall:[],membership:0}; async function measured(fn){const start=performance.now();measurements.turns++;try{return await fn();}finally{measurements.wall.push(performance.now()-start);}} import { policyOf } from '../../src/rules.js'; @@ -19,6 +19,23 @@ export class IdentityDO extends Source { } return result; } + async sequenceProof(){return this.host.serial(async()=>{ + const from=this.host.manifest.config.anchors[0],to=this.host.manifest.config.anchors[1],term=Number(this.host.states.get(this.host.leader()).term); + const payload=Uint8Array.of(8,8,16,Number(to),24,Number(from),32,term); + const messages=await Promise.all(Array.from({length:16},()=>this.host.envelope(from,to,payload))); + return {sequences:messages.map(raw=>JSON.parse(new TextDecoder().decode(raw)).sequence),ceiling:await this.ctx.storage.get('raft-transport-sequence')}; + });} + peerIdle(){return {sleeping:this.host.asleep,bytes:this.host.peers.used,snapshots:this.host.peers.snapshots.size};} + transportState(){return {keys:[...this.ctx.storage.kv.list()].map(([key])=>key),tables:this.ctx.storage.sql.exec("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '_cf_%'").toArray().map(r=>r.name)};} + async pausePublic(){ + let began;const started=new Promise(resolve=>{began=resolve;}); + this.ctx.waitUntil(this.host.serial(async()=>{began();await new Promise(resolve=>{this.releasePublic=resolve;});}));await started; + } + resumePublic(){this.releasePublic?.();} + async checkpointState(){ + const id=this.host.leader();await this.host.members.get(id).hostCheckpoint(); + return this.host.members.get(id).transportCheckpoint(); + } holdUpload(){ const put=this.host.tenant.store.put.bind(this.host.tenant.store); this.pauseUpload=new Promise(resolve=>{this.releaseUpload=resolve;}); @@ -51,17 +68,17 @@ export class IdentityDO extends Source { }); } async idleSchedule(){ - // R26 allows eviction between the HTTP request and this fixture RPC. + // The object may hibernate between the HTTP request and this fixture RPC. // Restore through the production alarm before driving its virtual clock. if(!this.host)await super.alarm(); return this.host.serial(async()=>{ - const base=Date.now(); + const base=Date.now(),membershipBefore=measurements.membership; for(let n=1;n<=20;n++)await this.host.maintenance(base+n*this.host.policy.tick_every,this.host.started+n*this.host.policy.tick_every); const id=this.host.leader(),before=this.host.states.get(id).applied; const sleeping=this.host.asleep; const turns=measurements.turns; for(let n=21;n<=140;n++)await this.host.maintenance(base+n*this.host.policy.tick_every,this.host.started+n*this.host.policy.tick_every); - return {sleeping,appliedBefore:String(before),appliedAfter:String(this.host.states.get(id).applied),recurringMemberCalls:measurements.turns-turns,idleMilliseconds:120*this.host.policy.tick_every}; + return {sleeping,membershipCalls:measurements.membership-membershipBefore,appliedBefore:String(before),appliedAfter:String(this.host.states.get(id).applied),recurringMemberCalls:measurements.turns-turns,idleMilliseconds:120*this.host.policy.tick_every}; });} inspect() { return {gate:this.ctx.storage.sql.exec('SELECT doc FROM legacy_gate WHERE one=1').toArray()[0]?.doc, @@ -69,10 +86,12 @@ export class IdentityDO extends Source { } } export class RaftMemberDO extends Member { + membership(...args){measurements.membership++;return measured(()=>super.membership(...args));} hostInput(...args){return measured(()=>super.hostInput(...args));} peer(...args){return measured(()=>super.peer(...args));} hostView(...args){return measured(()=>super.hostView(...args));} hostResult(...args){return measured(()=>super.hostResult(...args));} + async transportCheckpoint(){await this.load();return this.actor.serial(()=>({applied:this.actor.store.application().applied,retained:this.actor.store.one('SELECT count(*) AS n FROM journal').n,snapshot:this.actor.codec.inspect(3,unb64(this.actor.store.get('snapshot'))).index.toString()}));} async seedTransportTail(){await this.load();await this.ctx.storage.put('raft-scheduled-part/interrupted/0',new Uint8Array([1,2,3]));await this.ctx.storage.sync();} async retainedControl(){return [...(await this.ctx.storage.list({prefix:'raft-'})).keys()].filter(key=>key!=='raft-erased');} async oldSnapshotAndCheckpoint(){ @@ -94,8 +113,14 @@ export default { const url=new URL(request.url); if(url.pathname==='/fixture'){ const {op,identity,records,node,cluster}=await request.json(); + if(op==='peer-idle')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).peerIdle()); + if(op==='sequence-proof')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).sequenceProof()); + if(op==='transport-state')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).transportState()); + if(op==='pause-public'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).pausePublic();return Response.json(true);} + if(op==='resume-public'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).resumePublic();return Response.json(true);} + if(op==='checkpoint-state')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).checkpointState()); if(op==='metrics')return Response.json({...runtimeMetrics(),...measurements}); - if(op==='reset-metrics'){measurements.turns=0;measurements.wall=[];return Response.json(true);} + if(op==='reset-metrics'){measurements.turns=0;measurements.wall=[];measurements.membership=0;return Response.json(true);} if(op==='seed'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).seed(identity,records);return Response.json(true);} if(op==='hold-upload'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).holdUpload();return Response.json(true);} if(op==='release-upload'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).releaseHeldUpload();return Response.json(true);} From d60214f6917d1f842b973ae32a73b2206661cddb Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 05:09:19 -0400 Subject: [PATCH 067/103] Keep hosted presence independent of sleeping coordination. Persist outstanding maintenance separately from advisory presence so idle connections survive coordinator reconstruction without waking voters. Preserve racing peer wakes and require fresh committed authority for retirement. Document quiet home traffic and the presence sweep cost. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../manual/chat/coordination-and-workers.md | 10 +++- internal/manual/hosted_quiet_test.go | 31 ++++++++++++ relay/hosted/src/identity.js | 31 +++++++++--- relay/hosted/src/raft/README.md | 21 ++++++-- relay/hosted/src/raft/hosted.js | 49 +++++++++++++++---- relay/hosted/src/raft/peers.js | 14 +++++- 6 files changed, 135 insertions(+), 21 deletions(-) create mode 100644 internal/manual/hosted_quiet_test.go diff --git a/internal/manual/chat/coordination-and-workers.md b/internal/manual/chat/coordination-and-workers.md index 15cb076406..afb2bc1a33 100644 --- a/internal/manual/chat/coordination-and-workers.md +++ b/internal/manual/chat/coordination-and-workers.md @@ -46,7 +46,15 @@ The sync connection carries presence hints and committed lease acknowledgments. ## Does syncing keep sending requests when nothing is happening? -A home that has never paired sends no sync traffic. `CODEAF_SYNC_URL=off` also blocks new sync requests from clients that were opened before the setting changed. Presence pings on an open view are hints; they do not write chat ownership. On the native relay, an open idle home uses its learner’s applied vault pointer to notice shared memory and preference changes. Local edits are detected locally and pushed on the next state round. A caught-up idle home sends no recurring state polls or peer keepalives. Home also follows its local applied directory for display changes; its roster and backstop read that same local hint. Quiet neighbours’ presence can age out without forcing a network ping. Continuing a chat and all ownership decisions still use fresh majority reads. A committed change wakes peer delivery and the next state round pulls the changed vault. A paired open home on the hosted relay also joins as a learner and keeps its connection open. If an older hosted relay has no learner door, the home keeps its existing five-second state polling. On the consensus directory, a group with no work or held chats can stop its periodic coordination timer after an idle barrier has reached every voter. The first operation wakes coordination and must obtain a fresh majority response. A saved leader address cannot answer that operation by itself. Losing the majority leaves syncing unavailable and keeps local work intact. +A home that has never paired sends no sync traffic. `CODEAF_SYNC_URL=off` also blocks new sync requests from clients that were opened before the setting changed. Presence pings on an open view are hints; they do not write chat ownership. On the hosted relay and the native relay, an open idle home uses its learner’s applied vault pointer to notice shared memory and preference changes. Local edits are detected locally and pushed on the next state round. Over ten idle minutes, two caught-up open homes make zero HTTP requests, zero peer or forwarding frames and zero client attempts. A caught-up idle home sends no recurring state polls or peer keepalives. Home also follows its local applied directory for display changes; its roster and backstop read that same local hint. Quiet neighbours’ presence can age out without forcing a network ping. Continuing a chat and all ownership decisions still use fresh majority reads. A committed change wakes peer delivery and the next state round pulls the changed vault. If an older hosted relay has no learner door, the home keeps its existing five-second state polling. On the consensus directory, a group with no work or held chats can stop its periodic coordination timer after an idle barrier has reached every voter. The first operation wakes coordination and must obtain a fresh majority response. A saved leader address cannot answer that operation by itself. Losing the majority leaves syncing unavailable and keeps local work intact. + +## Hosted idle presence sweeps and sleeping connections + +A hosted group with no work or held chats stops its maintenance alarm after every voter applies the idle barrier. Its three anchors then schedule no recurring alarms. Reconstructing the sleeping coordinator leaves its learner and forwarding sockets open; a presence sweep neither wakes the anchors nor reconnects those computers. The first real request obtains a fresh majority response before answering. + +A compatibility events viewer pings every 10 seconds. The platform answers without running coordination. While somebody watches presence, the relay checks at the next 25-second silence deadline so a stopped computer appears offline within 30 seconds. With two viewers pinging together, the simulated ten-minute measurement uses 29 short sweeps; those sweeps make no member calls or data writes. With no events viewer, no presence sweep recurs. Home follows the local learner once joined, so its ordinary display needs no remote presence viewer. + +A connection naming held chats still renews through committed changes every 30 seconds while fresh hold evidence is available. Presence alone never renews a chat. ## Can a worker access all my chats or approve pairing? diff --git a/internal/manual/hosted_quiet_test.go b/internal/manual/hosted_quiet_test.go new file mode 100644 index 0000000000..278f58cf9d --- /dev/null +++ b/internal/manual/hosted_quiet_test.go @@ -0,0 +1,31 @@ +package manual + +import ( + "fmt" + "strings" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +// R26/R27/C30 keep hosted idle costs and the fresh-read wake reachable. +func TestR26R27C30HostedQuietManual(t *testing.T) { + for _, query := range []string{"hosted idle presence sweeps sleeping connections", "hosted open home idle memory preferences"} { + found := false + for _, section := range Chat().Search(query, DefaultResults) { + found = found || section.Page == "coordination-and-workers" + } + if !found { + t.Errorf("R27 quiet answer unreachable: %q", query) + } + } + page, ok := Chat().Page("coordination-and-workers") + if !ok { + t.Fatal("missing coordination page") + } + for _, claim := range []string{"zero HTTP requests", "zero peer or forwarding frames", "zero client attempts", "no recurring alarms", "neither wakes the anchors nor reconnects", "first real request obtains a fresh majority", fmt.Sprintf("every %d seconds", cs.PresenceEveryMillis/1000), fmt.Sprintf("every %d seconds", cs.RenewEveryMillis/1000)} { + if !strings.Contains(page, claim) { + t.Errorf("R27 missing hosted idle behavior %q", claim) + } + } +} diff --git a/relay/hosted/src/identity.js b/relay/hosted/src/identity.js index 14288d2ae1..178c43873f 100644 --- a/relay/hosted/src/identity.js +++ b/relay/hosted/src/identity.js @@ -3,7 +3,7 @@ // none of its legacy directory methods are a production writer. import { DurableObject } from 'cloudflare:workers'; import { LegacyAdmission } from './raft/legacy.js'; -import { peerPath,peerSocket,peerRefusal } from './raft/peers.js'; +import { peerPath,peerSocket,peerRefusal,expirePeerSockets } from './raft/peers.js'; import { HostedIdentity } from './raft/hosted.js'; import { clusterIngress,clusterRequestLimit } from './raft/ingress.js'; import { sharedRuntime } from './raft/actor.js'; @@ -75,7 +75,7 @@ export class IdentityDO extends DurableObject { async #noteLeft(ws) { const device = this.watchers.left(ws); const identity = device && (await this.ctx.storage.get('identity')); - if (identity && !(await this.#isGone()) && this.host?.ready) await this.host.serial(() => this.host.commitEffect({device_seen:{device}},Date.now())); + if (identity && !(await this.#isGone()) && this.host?.ready) await this.host.serial(async()=>{await this.host.wakeMaintenance();await this.host.commitEffect({device_seen:{device}},Date.now());}); } async #serve(request) { @@ -196,14 +196,29 @@ export class IdentityDO extends DurableObject { // request supplies the public root; an alarm cannot invent that key. this.watchers.closeAll(1012,'reconnecting');return; } + // Presence is advisory and survives hibernation in socket attachments. A + // sweep cannot acquire authority or reopen a sleeping replication group. + this.watchers.expire(); + const now=Date.now(); + await expirePeerSockets(this.ctx,now,at=>this.#arm(at)); + const maintenance=await this.ctx.storage.get('raft-maintenance'); + const retirement=await this.ctx.storage.get('raft-retire-at'); + const retireDue=retirement&&retirement<=now; + if(retirement&&!retireDue)await this.#arm(retirement); + if(!maintenance&&!retireDue)return; const tenant=this.#tenantOf(identity);this.host ??= new HostedIdentity(this.ctx,this.env,tenant,this.legacyAdmission,at=>this.#arm(at)); const erase=await this.host.serial(async()=>{ await this.host.ensure(Uint8Array.from(atob(provision.config.signing_key),c=>c.charCodeAt(0))); - this.watchers=this.host.watchers;this.watchers.expire();tenant.stats.flush(); - await this.host.peers.deadlines(Date.now()); - await this.host.maintenance(Date.now()); + this.watchers=this.host.watchers;tenant.stats.flush(); + // A retained deadline is only a wake hint. Retirement still needs the + // current committed identity before any irreversible erasure starts. + if(retireDue||!this.host.retirementKnown){ + const root=await this.host.barrierRecord({namespace:'state',key:'identity'}); + await this.host.rememberRetirement(this.host.project('identity',root)); + } + await this.host.maintenance(now); const at=this.host.retireAt; - if(at && at<=Date.now())return true; + if(at && at<=now)return true; if(at)await this.#arm(at);return false; }); // Uploads must finish their post-I/O authority check before leaving @@ -212,7 +227,8 @@ export class IdentityDO extends DurableObject { } /** - * #arm wakes the object at `at` unless it is already set to wake sooner, so no caller can push another's alarm later. + * #arm wakes the object at `at` unless it is already set to wake sooner. Final + * all-voter sleep passes null before rearming independent presence deadlines. * Calls run one after another: two that overlapped would both read the same alarm and the later write would win. */ #arm(at) { @@ -222,6 +238,7 @@ export class IdentityDO extends DurableObject { } async #armSooner(at) { + if(at===null){await this.ctx.storage.deleteAlarm();return;} const set = await this.ctx.storage.getAlarm(); if (set === null || at < set) await this.ctx.storage.setAlarm(at); } diff --git a/relay/hosted/src/raft/README.md b/relay/hosted/src/raft/README.md index e009a7b8b4..356f1f8560 100644 --- a/relay/hosted/src/raft/README.md +++ b/relay/hosted/src/raft/README.md @@ -184,9 +184,10 @@ upgrade is authenticated and passive until hosted leadership is available. The router's durable generation advances before its volatile queues reopen. Peer and application sockets use the hibernation API with a separate tag and -serialized attachment containing their signed connection proof. Coordinator -eviction fences those old connections; the unchanged client refreshes the same -store's grant and reconnects. Watch presence and broadcasts filter these sockets. +serialized attachment containing their signed connection proof. A cold presence +sweep preserves those connections without opening coordination or advancing the +generation. A real request reactivates the coordinator with fresh quorum reads, +fences the lost queues, and lets the client refresh the same store's grant. Watch presence and broadcasts filter these sockets. Protocol queues use their own admission and reservations, outside public rates. ABI 36 supplies admission plans, enrollment bytes and control status from an @@ -196,3 +197,17 @@ transport limits and response encoding. Routing owns per-lane delivery cursors, acks and chunk credits. One outstanding chunk per target bounds catch-up, and only its signed durable applied evidence completes snapshot delivery. Public coordination never waits for a learner; peer frames bypass its public queue. + + +## Sleeping accounts and advisory presence + +The coordinator persists an outstanding-maintenance hint only while consensus +work is scheduled. All-voter sleep removes it and clears stale peer retention +deadlines before cancelling the maintenance alarm. Presence and explicit +retirement deadlines are then rearmed independently. An alarm on a cold sleeping +object reads socket attachments and that hint without constructing a tenant, +loading member handles, acquiring a read barrier or reopening peer queues. +Platform auto-response timestamps supply presence without data writes. Real +application input still activates coordination and takes a fresh quorum barrier; +a remembered leader never authorizes a reply. Watch hold renewals retain their +committed path and shared renewal period. diff --git a/relay/hosted/src/raft/hosted.js b/relay/hosted/src/raft/hosted.js index c09ef827cf..f8f56c8a84 100644 --- a/relay/hosted/src/raft/hosted.js +++ b/relay/hosted/src/raft/hosted.js @@ -34,8 +34,8 @@ async function keypair(){ // replies below depend on the members' actual durable application and quorum. export class HostedIdentity { constructor(ctx,env,tenant,admission,arm){ - this.armAlarm=arm??(async at=>{const old=await ctx.storage.getAlarm();if(old===null||at{if(at===null){await ctx.storage.deleteAlarm();return;}const old=await ctx.storage.getAlarm();if(old===null||attrue); - this.watchers=new CommittedWatchers(this.ctx,this.tenant.limits.maxWatchers,this.tenant.watchers.ttlMs,at=>this.ctx.waitUntil(this.ctx.storage.setAlarm(at))); + this.watchers=new CommittedWatchers(this.ctx,this.tenant.limits.maxWatchers,this.tenant.watchers.ttlMs,at=>this.ctx.waitUntil(this.armAlarm(at))); this.tenant.watchers=this.watchers; this.effects=new RaftEffectsFacade(this,this.manifest.config.cluster); this.holds=new CommittedHolds(this,this.policy); @@ -111,7 +111,7 @@ export class HostedIdentity { requireOK((await this.anchorCommand({init_membership:{anchors,limit:this.policy.max_member_records}},Date.now())).value.code); } const root=await this.barrierRecord({namespace:'state',key:'identity'}); - this.retireAt=this.project('identity',root).rotation?.retire_at; + await this.rememberRetirement(this.project('identity',root)); const generation=await this.ctx.storage.get('raft-router-generation')??'0'; if(typeof generation!=='string'||!/^(0|[1-9][0-9]*)$/.test(generation)||BigInt(generation)>=(1n<<64n)-1n)throw unavailable(); const next=String(BigInt(generation)+1n); @@ -214,6 +214,7 @@ export class HostedIdentity { projectJSON(kind,raw){const r=response(this.runtime.call(frame(31).str(kind).blob(raw).finish())),value=dec.decode(r.blob());r.end();return value;} project(kind,raw){return JSON.parse(this.projectJSON(kind,raw));} async authorize(device,certificate,write=false){ + await this.wakeMaintenance(); requireOK((await this.commitEffect({device_seen:{device}},Date.now())).code); const b=await this.barrier(),member=this.members.get(b.node),root=await member.hostView(b.index,'state','identity'),actor=await member.hostView(b.index,'state','device/'+segment(device)); const r=response(this.runtime.call(frame(32).blob(root.raw).blob(actor.raw??new Uint8Array()).str(device).str(certificate).u32(write?1:0).finish()));requireOK(r.str());r.end();return {barrier:b,app:root.app,identity:this.project('identity',root.raw)}; @@ -242,9 +243,39 @@ export class HostedIdentity { const r=response(this.runtime.call(frame(34).str(this.manifest.config.cluster).u64(id).u64(now).blob(pkcs8.slice(-32)).blob(saved??new Uint8Array()).blob(data(change)).finish())); const command=r.blob();r.end();return this.commit(command); } - async wakeMaintenance(){this.asleep=false;await this.armAlarm(Date.now()+this.policy.tick_every);} + async rememberRetirement(identity){ + const at=identity.rotation?.retire_at; + if(this.retirementKnown&&at===this.retireAt)return; + // Cache only confirmed storage so an interrupted write remains repairable. + this.retirementKnown=false; + if(at)await this.ctx.storage.put('raft-retire-at',at); + else await this.ctx.storage.delete('raft-retire-at'); + this.retireAt=at;this.retirementKnown=true; + } + wakeMaintenance(){ + this.asleep=false;this.wakeVersion++; + // Peer ingress has its own queue. Serialize only timer transitions so a + // wake racing the final sleep cannot lose its durable maintenance hint. + const task=this.alarmTail.then(async()=>{ + if(!this.maintaining){await this.ctx.storage.put('raft-maintenance',true);await this.ctx.storage.sync();this.maintaining=true;} + await this.armAlarm(Date.now()+this.policy.tick_every); + }); + this.alarmTail=task.catch(()=>{});return task; + } + sleepMaintenance(version){ + const task=this.alarmTail.then(async()=>{ + if(version!==this.wakeVersion){this.asleep=false;return;} + await this.ctx.storage.delete('raft-maintenance');await this.ctx.storage.sync();this.maintaining=false; + await this.armAlarm(null); + this.watchers.liveness.rearm(); + if(this.retireAt)await this.armAlarm(this.retireAt); + if(this.holds.next!==null)await this.armAlarm(this.holds.next); + }); + this.alarmTail=task.catch(()=>{});return task; + } async maintenance(now=Date.now(),elapsed=Math.floor(performance.now())){ if(this.asleep)return; + const wakeVersion=this.wakeVersion; if(this.membershipPending)await this.reconcileMembership(); this.elapsed=Math.max(this.elapsed,elapsed-this.started); // Alarm/recovery effects are retained until their exact batch is consumed. @@ -294,8 +325,8 @@ export class HostedIdentity { const acks=[];for(const voter of marker.voters)acks.push(await this.members.get(voter).sleepAck(marker.index,marker.term)); let sleeping=true; for(const [node,member]of this.members){const effects=await member.schedule(this.elapsed,now,false,acks);this.accept(node,effects);sleeping&&=effects.plan.sleeping;} - await this.pump();this.asleep=sleeping; - if(sleeping&&this.peers){await this.peers.quiesce();await this.pump();} + await this.pump();this.asleep=sleeping&&wakeVersion===this.wakeVersion; + if(this.asleep&&this.peers){await this.peers.quiesce();await this.pump();await this.sleepMaintenance(wakeVersion);} }catch{this.asleep=false;} } if(!this.asleep)await this.armAlarm(now+this.policy.tick_every); @@ -355,7 +386,7 @@ export class HostedIdentity { const result=await this.commit(data(q.command)); if(result.value.code==='ok'){ const b=await this.barrier(),row=await this.members.get(b.node).hostView(b.index,'state','identity'); - const identity=this.project('identity',row.raw);this.retireAt=identity.rotation?.retire_at; + const identity=this.project('identity',row.raw);await this.rememberRetirement(identity); this.watchers.broadcast(row.app.version); if(identity.rotation)this.watchers.closeAll(CLOSE_ROTATED,'rotated'); if(this.retireAt)await this.armAlarm(this.retireAt); @@ -401,7 +432,7 @@ export class HostedIdentity { let raw;try{raw=v1Command(this.runtime,this.manifest.config.cluster,Date.now(),this.proof(request,body));}catch{throw new Wire('bad_request',400);} const result=await this.commit(raw);requireOK(result.value.code); const b=await this.barrier(),snapshot=await this.members.get(b.node).hostView(b.index,'state','identity'),identity=this.project('identity',snapshot.raw),version=snapshot.app.version; - this.retireAt=identity.rotation?.retire_at; + await this.rememberRetirement(identity); this.watchers.broadcast(version); if(join){const code=decodeURIComponent(join[1]).trim().toLowerCase();await this.effects.deliver(this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(code)),code,Date.now());unwrap(await this.env.LINK_GATE.get(this.env.LINK_GATE.idFromName('gate')).decided(code));} if(request.method==='GET'){ diff --git a/relay/hosted/src/raft/peers.js b/relay/hosted/src/raft/peers.js index 4e84a2dbce..197f366f63 100644 --- a/relay/hosted/src/raft/peers.js +++ b/relay/hosted/src/raft/peers.js @@ -10,6 +10,16 @@ export const peerPath=path=>['/v2/cluster/exchange','/v2/cluster/socket','/v2/cl export function peerLimits(runtime){const r=response(runtime.call(frame(38).u32(0).finish()));const limits=JSON.parse(dec.decode(r.blob()));r.end();return limits;} export const peerSocket=ws=>ws.deserializeAttachment()?.clusterPeer===true; +// Retention belongs to the socket attachment, so an alarm can expire lost +// delivery without activating coordination or replacing an idle generation. +export async function expirePeerSockets(ctx,now,arm){ + for(const ws of ctx.getWebSockets('cluster-peer')){ + const at=ws.deserializeAttachment()?.deadline; + if(at&&at<=now){try{ws.close(1012,'reconnecting');}catch{}} + else if(at)await arm(at); + } +} + // Routing memory is expendable. A durable generation fences all old cursors // before a recovered coordinator can serve another exchange or socket frame. export class PersonalPeers { @@ -179,8 +189,10 @@ export class PersonalPeers { const result=await this.exchange(bytes);ws.send(result.raw);const deadline=JSON.parse(dec.decode(result.raw)).messages.length?Date.now()+this.limits.Retention:null;ws.serializeAttachment({...att,last:this.host.hash(result.raw),deadline});if(deadline)await this.host.armAlarm(deadline); }catch{this.close(ws);} } - async deadlines(now){for(const ws of this.sockets()){const at=ws.deserializeAttachment().deadline;if(at&&at<=now)this.close(ws);else if(at)await this.host.armAlarm(at);}} async quiesce(){ + // Final voter acknowledgments discard learner delivery. Its old retention + // deadline must not wake a cold router merely to reconnect idle sockets. + for(const ws of this.sockets()){const a=ws.deserializeAttachment();if(a.deadline)ws.serializeAttachment({...a,deadline:null});} for(const [node,box]of this.boxes)if(!this.view[node]?.voter)for(let lane=0;lane<3;lane++){for(const q of box.queues[lane])this.used[lane]-=q.size;box.queues[lane]=[];} for(const [to,s]of this.snapshots){this.snapshots.delete(to);await this.host.input(s.from,12,{peer:to,snapshot:true,success:false});} } From e01a08fd54dab59c6624ee6ccd3eedf708acb9f3 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 05:09:19 -0400 Subject: [PATCH 068/103] Exercise quiet hosted homes and retain useful local failure diagnostics. Cover real home lifetimes, observers, compatibility pings, sleeping alarms, cold wake and prompt state delivery. Retain bounded redacted runtime output and exact failing proxy routes. Fresh upstream connections on the counting hop prevent a local idle close from masquerading as a directory refusal, while real clients keep their ordinary transport. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/raftdirectory/v1_journey_test.go | 12 + .../syncsetup/member_hosted_quiet_test.go | 361 ++++++++++++++++++ relay/hosted/test/raft/fixture-log.js | 13 + relay/hosted/test/raft/hosted-fixture.js | 6 +- .../hosted/test/raft/hosted-quiet-go.test.js | 24 ++ .../test/raft/hosted-quiet-unit.test.js | 45 +++ relay/hosted/test/raft/hosted-quiet.test.js | 70 ++++ relay/hosted/test/raft/hosted-worker.js | 93 ++++- 8 files changed, 610 insertions(+), 14 deletions(-) create mode 100644 internal/syncsetup/member_hosted_quiet_test.go create mode 100644 relay/hosted/test/raft/fixture-log.js create mode 100644 relay/hosted/test/raft/hosted-quiet-go.test.js create mode 100644 relay/hosted/test/raft/hosted-quiet-unit.test.js create mode 100644 relay/hosted/test/raft/hosted-quiet.test.js diff --git a/internal/raftdirectory/v1_journey_test.go b/internal/raftdirectory/v1_journey_test.go index 0026a36f75..879d05941f 100644 --- a/internal/raftdirectory/v1_journey_test.go +++ b/internal/raftdirectory/v1_journey_test.go @@ -100,7 +100,19 @@ func TestR1R2R23R25R29HostedTwoHomes(t *testing.T) { } celltest.Available(t) proxy := httputil.NewSingleHostReverseProxy(target) + // The counting hop must not reuse a connection as the local runtime closes + // its idle pipeline. Real clients retain their own ordinary pooled transport. + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DisableKeepAlives = true + proxy.Transport = transport + defer transport.CloseIdleConnections() var hits atomic.Int64 + // Preserve the failing route and request count because a helper's caller + // line alone cannot distinguish bootstrap from a later connection failure. + proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { + t.Logf("R27 hosted proxy failure: request=%d method=%s path=%s error=%v", hits.Load(), r.Method, r.URL.Path, err) + http.Error(w, http.StatusText(http.StatusBadGateway), http.StatusBadGateway) + } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1); proxy.ServeHTTP(w, r) })) defer server.Close() runV1TwoHomeJourney(t, server.URL, &hits) diff --git a/internal/syncsetup/member_hosted_quiet_test.go b/internal/syncsetup/member_hosted_quiet_test.go new file mode 100644 index 0000000000..700a5d1fae --- /dev/null +++ b/internal/syncsetup/member_hosted_quiet_test.go @@ -0,0 +1,361 @@ +package syncsetup + +import ( + "bytes" + "context" + "encoding/json" + "maps" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/dirwatch" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/store" +) + +func hostedQuietControl(t *testing.T, endpoint, identity, op string) map[string]json.RawMessage { + t.Helper() + raw, _ := json.Marshal(map[string]string{"identity": identity, "op": op}) + req, e := http.NewRequestWithContext(t.Context(), http.MethodPost, endpoint+"/fixture", bytes.NewReader(raw)) + mustNil(t, e) + resp, e := http.DefaultClient.Do(req) + mustNil(t, e) + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("R27 fixture %s: %s", op, resp.Status) + } + var result map[string]json.RawMessage + if op == "reset-metrics" || op == "presence-turn" { + return nil + } + mustNil(t, json.NewDecoder(resp.Body).Decode(&result)) + return result +} +func hostedSocketCounts(t *testing.T, result map[string]json.RawMessage) map[string]int { + t.Helper() + var sockets []struct{ Kind string } + mustNil(t, json.Unmarshal(result["sockets"], &sockets)) + counts := map[string]int{} + for _, socket := range sockets { + kind := socket.Kind + if kind == "" { + kind = "watch" + } + counts[kind]++ + } + return counts +} + +func hostedNumber(t *testing.T, result map[string]json.RawMessage, key string) uint64 { + t.Helper() + var n uint64 + mustNil(t, json.Unmarshal(result[key], &n)) + return n +} +func hostedTicks(t *testing.T, s *Sync) func() { + t.Helper() + ctx, cancel := context.WithCancel(t.Context()) + ticks := make(chan time.Time) + done := make(chan struct{}) + go func() { s.runState(ctx, ticks); close(done) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(requestTimeout): + t.Error("R23 hosted state lifetime did not stop") + } + }) + step := func() { + select { + case ticks <- time.Now(): + case <-time.After(requestTimeout): + t.Fatal("R27 hosted state round stalled") + } + } + step() + step() + return func() { step(); step() } +} +func hostedRefreshed(t *testing.T, s *Sync, tick func(), generation uint64) { + t.Helper() + deadline := time.NewTimer(requestTimeout) + defer deadline.Stop() + retry := time.NewTicker(100 * time.Millisecond) + defer retry.Stop() + for { + tick() + s.memberMu.Lock() + m := s.member + ready := false + if m != nil { + m.mu.Lock() + ready = m.failure == nil && m.conn != nil && m.forwardConn != nil && m.record.Grant.Generation > generation + m.mu.Unlock() + } + s.memberMu.Unlock() + if ready { + return + } + select { + case <-retry.C: + case <-deadline.C: + t.Fatal("R26 real input did not refresh the retained learner connection") + } + } +} + +func hostedSettled(t *testing.T, s *Sync) { + t.Helper() + waitMember(t, s, func(m *personalMember) bool { return m.idle && len(m.sent) == 0 && len(m.pending) == 0 }) +} + +// R23/R26/R27/C30 measure both real terminal lifetimes, local Home observers +// and platform watch pings, then require quorum wake and shared state delivery. +func TestR23R26R27C30HostedOpenHomesIdleEvictionAndWake(t *testing.T) { + endpoint := hostedLearnerEndpoint(t) + a, ok, e := Open(machine(t, endpoint)) + mustNil(t, e) + if !ok { + t.Fatal("open") + } + homeB := machine(t, endpoint) + _, e = identity.Adopt(homeB, a.Identity, true) + mustNil(t, e) + b, ok, e := Open(homeB) + mustNil(t, e) + if !ok { + t.Fatal("second open") + } + tickA, tickB := hostedTicks(t, a), hostedTicks(t, b) + hostedReady(t, a) + hostedReady(t, b) + for n := 0; n < 3; n++ { + tickA() + tickB() + hostedSettled(t, a) + hostedSettled(t, b) + } + screens := []*Sync{} + follows := []dirwatch.Follower{} + streams := []dirwatch.Stream{} + watches := []dirwatch.Stream{} + for _, s := range []*Sync{a, b} { + screen, ok, e := Open(s.Home) + mustNil(t, e) + if !ok { + t.Fatal("screen open") + } + screens = append(screens, screen) + follow := screen.Follow() + defer follow.Close() + // The local observer and the compatibility watch both remain open. Only + // the latter uses platform auto-response for advisory presence. + stream, e := screen.followStream(t.Context(), screen.Dir.(directory.Watcher)) + mustNil(t, e) + defer stream.Close() + streams = append(streams, stream) + watch, e := screen.Dir.(directory.Watcher).Watch(t.Context()) + mustNil(t, e) + defer watch.Close() + watches = append(watches, watch) + follows = append(follows, follow) + } + idle := hostedQuietControl(t, endpoint, a.Identity.ID(), "idle-schedule") + if string(idle["sleeping"]) != "true" || hostedNumber(t, idle, "recurringMemberCalls") != 0 { + t.Fatalf("R26 idle schedule: %s", idle) + } + for _, s := range []*Sync{a, b} { + hostedSettled(t, s) + } + ping := func(stream dirwatch.Stream) { + mustNil(t, stream.Ping(t.Context())) + for { + frame, e := stream.Next(t.Context()) + mustNil(t, e) + if frame.Pong { + return + } + } + } + count := func() uint64 { + var n uint64 + for _, s := range append([]*Sync{a, b}, screens...) { + n += s.relayAttempts.Load() + s.clusterFrames.Load() + } + return n + } + before := hostedQuietControl(t, endpoint, a.Identity.ID(), "quiet-state") + var alarms map[string]json.RawMessage + mustNil(t, json.Unmarshal(before["memberAlarms"], &alarms)) + for node, alarm := range alarms { + if string(alarm) != "null" { + t.Fatalf("R26 sleeping voter %s retained alarm: %s", node, alarm) + } + } + if len(alarms) != 3 || string(before["maintenance"]) != "false" { + t.Fatalf("R26 sleeping group retained maintenance: %s", before) + } + if !maps.Equal(hostedSocketCounts(t, before), map[string]int{"peer": 2, "forward": 2, "watch": 2}) { + t.Fatalf("R26 missing retained sockets: %s", before) + } + generation := recordOf(t, a).Grant.Generation + cold := hostedQuietControl(t, endpoint, a.Identity.ID(), "evict-idle") + if string(cold["ready"]) != "false" { + t.Fatal("R26 fixture did not reconstruct coordinator") + } + hostedQuietControl(t, endpoint, a.Identity.ID(), "reset-metrics") + attempts := count() + pongs := 0 + for n := 0; n < 120; n++ { + tickA() + tickB() + for _, follow := range follows { + follow.Probe() + } + for i, screen := range screens { + ping(streams[i]) + ping(watches[i]) + pongs++ + _, _, e := screen.RowsAt(t.Context()) + mustNil(t, e) + _, e = screen.Roster(t.Context()) + mustNil(t, e) + _, e = screen.FleetSize(t.Context()) + mustNil(t, e) + } + } + schedule := hostedQuietControl(t, endpoint, a.Identity.ID(), "presence-schedule") + if hostedNumber(t, schedule, "milliseconds") != 600000 { + t.Fatal("R27 missing ten-minute presence measurement") + } + metrics := hostedQuietControl(t, endpoint, a.Identity.ID(), "metrics") + httpCount := hostedNumber(t, metrics, "http") + frames := hostedNumber(t, metrics, "peerIn") + hostedNumber(t, metrics, "peerOut") + clientAttempts := count() - attempts + t.Logf("R23 R27 hosted ten-minute idle: Worker HTTP=%d peer/forward frames=%d client attempts=%d platform presence ping/pongs=%d sweeps=%d", httpCount, frames, clientAttempts, pongs, hostedNumber(t, schedule, "sweeps")) + if httpCount != 0 || frames != 0 || clientAttempts != 0 || hostedNumber(t, metrics, "writes") != 0 || hostedNumber(t, metrics, "turns") != 0 { + t.Fatalf("R27 idle activity: %s", metrics) + } + // R26/R27 retain real hibernating sockets across two complete presence + // windows. Auto-response timestamps, rather than a mocked renewal, age them. + hostedQuietControl(t, endpoint, a.Identity.ID(), "reset-metrics") + started := time.Now() + nextPing := time.NewTicker(dirwatch.KeepEvery) + defer nextPing.Stop() + for time.Since(started) < 2*dirwatch.DeadAfter+dirwatch.KeepEvery { + select { + case <-nextPing.C: + for _, watch := range watches { + ping(watch) + pongs++ + } + case <-t.Context().Done(): + t.Fatal(t.Context().Err()) + } + } + real := hostedQuietControl(t, endpoint, a.Identity.ID(), "metrics") + after := hostedQuietControl(t, endpoint, a.Identity.ID(), "quiet-state") + if hostedNumber(t, real, "sweeps") < 2 || hostedNumber(t, real, "turns") != 0 || hostedNumber(t, real, "writes") != 0 || hostedNumber(t, real, "peerIn")+hostedNumber(t, real, "peerOut") != 0 || hostedNumber(t, real, "http") != 0 { + t.Fatalf("R26 cold real-time presence: %s", real) + } + if string(after["ready"]) != "false" || string(after["generation"]) != string(before["generation"]) { + t.Fatal("R26 sweep activated coordinator or changed generation") + } + if !maps.Equal(hostedSocketCounts(t, before), hostedSocketCounts(t, after)) { + t.Fatal("R26 cold presence lost a retained socket") + } + t.Logf("R26 R27 real-time presence: duration=%s sweeps=%d member calls=0 writes=0 peer frames=0", time.Since(started), hostedNumber(t, real, "sweeps")) + for _, watch := range watches { + watch.Close() + } + // A first actual read must obtain authority from the voters. The following + // preference and memory write must reach the other learner after eviction. + _, e = b.Dir.List(t.Context()) + mustNil(t, e) + hostedRefreshed(t, a, tickA, generation) + hostedRefreshed(t, b, tickB, generation) + mustNil(t, config.WriteChatModel(b.Home, "stub/hosted-wake")) + brain, e := store.Open(filepath.Join(b.Home, "graph.db")) + mustNil(t, e) + memory, e := brain.AddMemory(store.Memory{Owner: store.OwnerUser, Text: "A hosted idle wake carries memory.", Origin: store.OriginPerson}) + mustNil(t, e) + mustNil(t, brain.Close()) + mustNil(t, b.SyncState(t.Context())) + want, e := b.Dir.List(t.Context()) + mustNil(t, e) + // Generation fencing may refresh the same durable stores on real input. + tickA() + tickB() + waitMember(t, a, func(m *personalMember) bool { return appliedVault(m) == want.Identity.Vault && m.idle }) + mustNil(t, a.SyncState(t.Context())) + raw, e := os.ReadFile(filepath.Join(a.Home, "config.json")) + mustNil(t, e) + if !strings.Contains(string(raw), "stub/hosted-wake") { + t.Fatal("C30 preference missed the first state round") + } + received, e := store.Open(filepath.Join(a.Home, "graph.db")) + mustNil(t, e) + got, found, e := received.MemoryRecord(memory.ID) + mustNil(t, e) + mustNil(t, received.Close()) + if !found || got.Text != memory.Text { + t.Fatal("C30 memory missed the first state round") + } + mustNil(t, config.WriteChatModel(a.Home, "stub/hosted-local")) + mustNil(t, a.SyncState(t.Context())) + latest, e := a.Dir.List(t.Context()) + mustNil(t, e) + if latest.Identity.Vault == want.Identity.Vault { + t.Fatal("C30 local change was not pushed promptly") + } +} + +// R23 every use after off or unpair must stop the joined member without an +// enrollment retry, HTTP request or protocol frame reaching the Worker. +func TestR23HostedJoinedLearnerQuietGuards(t *testing.T) { + endpoint := hostedLearnerEndpoint(t) + for _, mode := range []string{"off", "unpair"} { + t.Run(mode, func(t *testing.T) { + s, ok, e := Open(machine(t, endpoint)) + mustNil(t, e) + if !ok { + t.Fatal("open") + } + hostedOwner(t, s) + hostedReady(t, s) + hostedSettled(t, s) + hostedQuietControl(t, endpoint, s.Identity.ID(), "idle-schedule") + hostedSettled(t, s) + hostedQuietControl(t, endpoint, s.Identity.ID(), "reset-metrics") + before := s.relayAttempts.Load() + s.clusterFrames.Load() + if mode == "off" { + t.Setenv(URLVar, "off") + } else { + mustNil(t, os.Remove(filepath.Join(s.Home, identity.File))) + _, e = identity.EnsureSolo(s.Home) + mustNil(t, e) + } + mustNil(t, s.memberRound(t.Context())) + for n := 0; n < 3; n++ { + mustNil(t, s.SyncState(t.Context())) + _, _, e = s.RowsAt(t.Context()) + mustNil(t, e) + } + if s.member != nil || s.relayAttempts.Load()+s.clusterFrames.Load() != before { + t.Fatal("R23 joined quiet home kept sending") + } + metrics := hostedQuietControl(t, endpoint, s.Identity.ID(), "metrics") + if hostedNumber(t, metrics, "http") != 0 { + t.Fatalf("R23 quiet Worker requests: %s", metrics) + } + t.Logf("R23 %s: further client requests and frames=0", mode) + }) + } +} diff --git a/relay/hosted/test/raft/fixture-log.js b/relay/hosted/test/raft/fixture-log.js new file mode 100644 index 0000000000..15707560df --- /dev/null +++ b/relay/hosted/test/raft/fixture-log.js @@ -0,0 +1,13 @@ +import {Log,LogLevel} from 'miniflare'; + +// Runtime failures need bounded diagnostics without signed headers or payloads. +export class FixtureLog extends Log { + constructor(write=message=>console.error(message)){super(LogLevel.WARN);this.write=write;this.lines=[];this.bytes=0;} + capture(level,message){ + if(!/warn|error|fatal|^log$/i.test(String(level)))return; + if(this.lines.length>=64||this.bytes>=16384)return; + const text=String(message).replace(/((?:codeaf-[\w-]+|authorization|cookie|body|payload|signature|seed|private[_-]?key)["']?\s*[=:]\s*)[\s\S]*/i,'$1[redacted]').replace(/https?:\/\/[^\s)]+/g,'[url]').replace(/((?:codeaf-[\w-]+|authorization|cookie|body|payload|signature|seed|private[_-]?key)["']?\s*[=:]\s*)[^\n,]+/gi,'$1[redacted]').replace(/[A-Za-z0-9+/_=-]{48,}/g,'[redacted]').replace(/\x1b\[[0-9;]*m/g,'').slice(0,Math.min(2048,16384-this.bytes)); + const line='hosted runtime '+level+': '+text;this.lines.push(line);this.bytes+=text.length;this.write(line); + } + logWithLevel(level,message){if(level<=LogLevel.WARN)this.capture(level===LogLevel.ERROR?'error':'warn',message);} +} diff --git a/relay/hosted/test/raft/hosted-fixture.js b/relay/hosted/test/raft/hosted-fixture.js index 2e3b506e09..b93051a74b 100644 --- a/relay/hosted/test/raft/hosted-fixture.js +++ b/relay/hosted/test/raft/hosted-fixture.js @@ -1,4 +1,5 @@ import assert from 'node:assert/strict'; +import {FixtureLog} from './fixture-log.js'; import { prefixOf } from '../../src/base.js'; import { encodeCanonical } from '../../src/raft/codec.js'; import { Miniflare, convertV4MiniflareOptions } from 'miniflare'; @@ -23,6 +24,7 @@ export async function hostedFixture({fixture=false,base='',go=false}={}) { }; await copyFile(join(out,'core.wasm'),join(dir,'core.wasm')); let mf, compiledCut; + const diagnostics=new FixtureLog(); const bundles=new Map(); const compile=async(cut)=>{ // A restart must retain the same program and storage. Cache each injected @@ -45,7 +47,7 @@ export async function hostedFixture({fixture=false,base='',go=false}={}) { const start=async(cut)=>{ if(mf)await mf.dispose();await compile(cut); const worker={name:'hosted-proof',modulesRoot:dir,modules:[{type:'ESModule',path:join(dir,'worker.js')},{type:'CompiledWasm',path:join(dir,'core.wasm')}],compatibilityDate:'2026-09-01',durableObjects:Object.fromEntries(Object.entries({IDENTITY:'IdentityDO',RAFT_MEMBERS:'RaftMemberDO',NEWCOMERS:'NewcomerGate',PAIR_GATE:'PairGate',MAILBOX:'Mailbox',LINK_GATE:'LinkGate',LINK_REQUEST:'LinkRequest'}).map(([binding,className])=>[binding,{className,useSQLite:true}])),r2Buckets:['FRAMES'],bindings:{BASE_PATH:base,LEASE_POLICY:'amended',CAF_LIMITS:JSON.stringify({requestsPerMinute:100000,requestsPerMinutePerDevice:100000,minGraceMs:2000})},outboundService:()=>{throw Error('H4 outbound network forbidden');}}; - mf=new Miniflare(convertV4MiniflareOptions({host:'127.0.0.1',inspectorPort:0,resourcePersistencePath:join(dir,'persist'),telemetry:{enabled:false},cf:false,workers:[worker]})); + mf=new Miniflare(convertV4MiniflareOptions({log:diagnostics,handleStructuredLogs:entry=>diagnostics.capture(entry.level,entry.message),host:'127.0.0.1',inspectorPort:0,resourcePersistencePath:join(dir,'persist'),telemetry:{enabled:false},cf:false,workers:[worker]})); await mf.ready;return mf; }; const request=async(c,method,path,body)=>{ @@ -66,5 +68,5 @@ export async function hostedFixture({fixture=false,base='',go=false}={}) { certificate={v:1,identity,...certificate,signature:null}; certificate.signature=sign(null,Buffer.from('codeaf/worker-cert/v1\0'+encodeCanonical(certificate)),root.privateKey).toString('base64');return certificate; }; - return {identity,client,dir,start,request,call,control,api,delegate,get mf(){return mf;},async close(){if(mf)await mf.dispose();await rm(dir,{recursive:true,force:true});}}; + return {identity,client,dir,start,request,call,control,api,delegate,diagnostics,get mf(){return mf;},async close(){if(mf)await mf.dispose();await rm(dir,{recursive:true,force:true});}}; } diff --git a/relay/hosted/test/raft/hosted-quiet-go.test.js b/relay/hosted/test/raft/hosted-quiet-go.test.js new file mode 100644 index 0000000000..21d8f86cd3 --- /dev/null +++ b/relay/hosted/test/raft/hosted-quiet-go.test.js @@ -0,0 +1,24 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {spawn} from 'node:child_process'; +import {join} from 'node:path'; +import {hostedFixture} from './hosted-fixture.js'; +const root=new URL('../../../../',import.meta.url),footprint=new URL('../../../../scripts/raft-proof-footprint.sh',import.meta.url).pathname; +async function run(args,env={}){ + const child=spawn('sh',[footprint,...args],{cwd:root,env:{...process.env,...env},stdio:['ignore','pipe','pipe']});let output=''; + child.stdout.on('data',b=>{output+=b;process.stdout.write(b);});child.stderr.on('data',b=>{output+=b;process.stderr.write(b);}); + const code=await new Promise((resolve,reject)=>{child.once('error',reject);child.once('exit',resolve)});return {code,output}; +} +const binary=join(process.env.CODEAF_RAFT_PROOF_DIR,'hosted-quiet.test'); +const compiled=await run(['go','test','-c','-p=1','-timeout','15m','-o',binary,'./internal/syncsetup']);assert.equal(compiled.code,0,compiled.output); +for(const name of ['TestR23R26R27C30HostedOpenHomesIdleEvictionAndWake','TestR23HostedJoinedLearnerQuietGuards','TestR23HostedQuietHomesMakeNoClusterRequests']){ + test('R23 R26 R27 C30 hosted real idle client '+name,{timeout:300000},async()=>{ + const h=await hostedFixture({fixture:true}); + try{ + await h.start();const url=await h.mf.ready; + const {code,output}=await run([binary,'-test.run','^'+name+'$','-test.count=1','-test.timeout=15m','-test.parallel=1','-test.v'],{CODEAF_HOSTED_PROOF_URL:url.origin}); + assert.equal(code,0,output);assert.doesNotMatch(output,/--- SKIP:/); + if(name==='TestR23HostedQuietHomesMakeNoClusterRequests')assert.equal((await h.control({op:'metrics'})).http,0); + }finally{await h.close();} + }); +} diff --git a/relay/hosted/test/raft/hosted-quiet-unit.test.js b/relay/hosted/test/raft/hosted-quiet-unit.test.js new file mode 100644 index 0000000000..978aa18c41 --- /dev/null +++ b/relay/hosted/test/raft/hosted-quiet-unit.test.js @@ -0,0 +1,45 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {HostedIdentity} from '../../src/raft/hosted.js'; + +// R26 a real input racing the final timer cancellation retains active work. +test('R26 wake during durable sleep cancellation retains a maintenance alarm',async()=>{ + let began,release; + const started=new Promise(resolve=>{began=resolve;}),blocked=new Promise(resolve=>{release=resolve;}); + const values=new Map([['raft-maintenance',true]]);let alarm=5000; + const host=Object.assign(Object.create(HostedIdentity.prototype),{ + asleep:true,wakeVersion:0,alarmTail:Promise.resolve(),maintaining:true, + policy:{tick_every:5000},holds:{next:null},watchers:{liveness:{rearm(){}}}, + ctx:{storage:{put:async(key,value)=>values.set(key,value),delete:async key=>{began();await blocked;values.delete(key);},sync:async()=>{}}}, + armAlarm:async at=>{alarm=at;}, + }); + const sleep=host.sleepMaintenance(0);await started; + const wake=host.wakeMaintenance();release();await Promise.all([sleep,wake]); + assert.equal(host.asleep,false);assert.equal(values.get('raft-maintenance'),true);assert(alarm>Date.now()); +}); + +// R26 an already admitted wake must invalidate an older sleep completion. +test('R26 stale sleep cannot cancel a newer input',async()=>{ + let alarm=5000,writes=0; + const host=Object.assign(Object.create(HostedIdentity.prototype),{ + asleep:true,wakeVersion:2,alarmTail:Promise.resolve(),ctx:{storage:{delete:async()=>{writes++;}}},armAlarm:async at=>{alarm=at;}, + }); + await host.sleepMaintenance(1); + assert.equal(host.asleep,false);assert.equal(alarm,5000);assert.equal(writes,0); +}); + +// R26 a failed durable hint must remain repairable on the next activation. +test('R26 retirement hints retry after an interrupted durable write',async()=>{ + for(const removal of [false,true]){ + const values=new Map(removal?[['raft-retire-at',100]]:[]);let interrupted=true; + const write=async fn=>{if(interrupted){interrupted=false;throw Error('storage unavailable');}fn();}; + const host=Object.assign(Object.create(HostedIdentity.prototype),{ + retirementKnown:true,retireAt:removal?100:undefined, + ctx:{storage:{put:(key,value)=>write(()=>values.set(key,value)),delete:key=>write(()=>values.delete(key))}}, + }); + const root=removal?{}:{rotation:{retire_at:100}}; + await assert.rejects(host.rememberRetirement(root),/storage unavailable/); + await host.rememberRetirement(root); + assert.equal(values.get('raft-retire-at'),removal?undefined:100); + } +}); diff --git a/relay/hosted/test/raft/hosted-quiet.test.js b/relay/hosted/test/raft/hosted-quiet.test.js new file mode 100644 index 0000000000..97504ad6f3 --- /dev/null +++ b/relay/hosted/test/raft/hosted-quiet.test.js @@ -0,0 +1,70 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {hostedFixture} from './hosted-fixture.js'; + +// R26/R27 advisory sweeps must stay independent of quorum and router recovery. +test('R26 R27 cold presence sweeps leave a sleeping learner group untouched',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();const store='a'.repeat(64); + for(let n=0;n<2;n++){const r=await h.call(a,'POST','/v2/cluster/enroll',{store,protocol:1});assert.equal(r.status,200);} + const idle=await h.control({op:'idle-schedule'});assert.equal(idle.sleeping,true);assert.equal(idle.recurringMemberCalls,0); + const before=await h.control({op:'quiet-state'});assert.equal(before.maintenance,false);assert.equal(before.alarm,null);assert.deepEqual(before.memberAlarms,{'1':null,'2':null,'3':null}); + const cold=await h.control({op:'evict-idle'});assert.equal(cold.ready,false); + const schedule=await h.control({op:'presence-schedule'});assert.equal(schedule.sweeps,0,'no events viewer scheduled presence work'); + await h.control({op:'reset-metrics'}); + for(let n=0;n<24;n++)await h.control({op:'presence-turn'}); + const metrics=await h.control({op:'metrics'}),after=await h.control({op:'quiet-state'}); + assert.equal(after.ready,false);assert.equal(after.generation,before.generation);assert.equal(after.alarm,null); + assert.equal(metrics.turns,0);assert.equal(metrics.writes,0);assert.equal(metrics.peerOut,0); + assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + assert.equal((await h.control({op:'quiet-state'})).ready,true); + console.log('R26 R27 cold sweeps',JSON.stringify({sweeps:metrics.sweeps,memberCalls:metrics.turns,writes:metrics.writes,peerFrames:metrics.peerOut})); + }finally{await h.close();} +}); + +// R27 runtime warnings remain observable, bounded and free of signed material. +test('R27 local runtime diagnostics preserve warnings and redact credentials',async()=>{ + const {FixtureLog}=await import('./fixture-log.js'),lines=[],log=new FixtureLog(line=>lines.push(line)); + log.warn('connection reset by peer; codeaf-cert: private certificate'); + log.error(new Error('workerd failed; signature='+ 'a'.repeat(80))); + log.capture('log','workerd raw connection reset'); + log.capture('error','{"authorization":"tiny-secret"}'); + log.capture('error','{"body":"private body"}'); + for(let n=0;n<100;n++)log.capture('error','bounded failure'); + assert.equal(lines.length,64);assert(log.bytes<=16384); + assert.match(lines[0],/connection reset/);assert.match(lines[2],/raw connection reset/);assert.doesNotMatch(lines.join('\n'),/private certificate|a{80}|tiny-secret|private body/); +}); + +// R4/R26 a sleeping coordinator's remembered leader has no read authority. +test('R4 R26 first read after cold idle requires a surviving majority',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + assert.equal((await h.control({op:'idle-schedule'})).sleeping,true); + await h.control({op:'evict-minority'}); + const r=await h.request(a,'GET','/v1/dir/list');assert.equal(r.status,503);await r.text(); + }finally{await h.close();} +}); + +// R27 capture the actual local runtime stream, not only an injected logger. +test('R27 workerd warnings and errors reach bounded local test diagnostics',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}); + try{ + await h.start();await h.control({op:'diagnostic'}); + for(let n=0;n<100&&!h.diagnostics.lines.some(line=>line.includes('local runtime error capture'));n++)await new Promise(resolve=>setTimeout(resolve,10)); + assert(h.diagnostics.lines.some(line=>line.includes('local runtime warning capture'))); + assert(h.diagnostics.lines.some(line=>line.includes('local runtime error capture'))); + }finally{await h.close();} +}); + +// R16/R26 timer metadata cannot authorize irreversible identity erasure. +test('R16 R26 stale retirement deadline rechecks committed authority',{timeout:30000},async()=>{ + const h=await hostedFixture({fixture:true}),a=h.client(); + try{ + await h.start();assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + assert.equal((await h.control({op:'idle-schedule'})).sleeping,true); + await h.control({op:'evict-idle'});await h.control({op:'stale-retirement'}); + assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200,'a timer hint erased an unretired identity'); + }finally{await h.close();} +}); diff --git a/relay/hosted/test/raft/hosted-worker.js b/relay/hosted/test/raft/hosted-worker.js index 896ea94931..674b1f11f5 100644 --- a/relay/hosted/test/raft/hosted-worker.js +++ b/relay/hosted/test/raft/hosted-worker.js @@ -6,14 +6,61 @@ import { IdentityDO as Source } from '../../src/identity.js'; import { RaftMemberDO as Member } from '../../src/raft/member.js'; import { Directory } from '../../src/directory.js'; import { unb64 } from '../../src/raft/storage.js'; -import { runtimeMetrics } from '../../src/raft/actor.js'; -const measurements={turns:0,wall:[],membership:0}; +import { runtimeMetrics,sharedRuntime } from '../../src/raft/actor.js'; +import {coordinationPolicy} from '../../src/raft/liveness.js'; +const measurements={turns:0,wall:[],membership:0,http:0,peerIn:0,peerOut:0,sweeps:0,writes:0}; async function measured(fn){const start=performance.now();measurements.turns++;try{return await fn();}finally{measurements.wall.push(performance.now()-start);}} import { policyOf } from '../../src/rules.js'; export class IdentityDO extends Source { + constructor(ctx,env){ + super(ctx,env); + const accept=ctx.acceptWebSocket.bind(ctx); + ctx.acceptWebSocket=(ws,tags)=>{accept(ws,tags);const send=ws.send.bind(ws);ws.send=raw=>{if(ws.deserializeAttachment()?.clusterPeer)measurements.peerOut++;return send(raw);};}; + for(const key of ['put','delete']){const call=ctx.storage[key].bind(ctx.storage);ctx.storage[key]=(...args)=>{measurements.writes++;return call(...args);};} + for(const key of ['put','delete']){const call=ctx.storage.kv[key].bind(ctx.storage.kv);ctx.storage.kv[key]=(...args)=>{measurements.writes++;return call(...args);};} + const exec=ctx.storage.sql.exec.bind(ctx.storage.sql); + ctx.storage.sql.exec=(query,...args)=>{if(!/^\s*(SELECT|PRAGMA)/i.test(query))measurements.writes++;return exec(query,...args);}; + } + get current(){return this.rebuilt??this;} + async evictIdle(){ + const old=this.current.host; + if(old){old.ready=false;old.members.clear();old.states.clear();old.queue=[];old.reads=[];old.peers=null;old.holds=null;} + this.current.host=null;this.host=null;this.capturing=false; + this.rebuilt=new Source(this.ctx,this.env);return this.quietState(); + } + async quietState(){ + const config=await this.ctx.storage.get('raft-provision'),alarms={}; + if(config)for(const node of config.config.anchors)alarms[node]=await this.env.RAFT_MEMBERS.get(this.env.RAFT_MEMBERS.idFromName(config.config.cluster+'/'+node)).alarmState(); + return {ready:!!this.current.host?.ready,generation:await this.ctx.storage.get('raft-router-generation'),maintenance:await this.ctx.storage.get('raft-maintenance')??false,alarm:await this.ctx.storage.getAlarm(),memberAlarms:alarms,sockets:this.ctx.getWebSockets().map(ws=>({peer:!!ws.deserializeAttachment()?.clusterPeer,kind:ws.deserializeAttachment()?.kind,deadline:ws.deserializeAttachment()?.deadline}))}; + } + async staleRetirement(){await this.ctx.storage.put('raft-retire-at',Date.now()-1);try{await this.presenceTurn();}catch{}return true;} + async evictMinority(){ + const config=await this.ctx.storage.get('raft-provision'),namespace=this.env.RAFT_MEMBERS; + const missing=new Set(config.config.anchors.slice(1).map(node=>namespace.idFromName(config.config.cluster+'/'+node).toString())); + const unavailable=new Proxy({},{get:()=>async()=>{throw new Error('anchor unavailable');}}); + this.rebuilt=new Source(this.ctx,{...this.env,RAFT_MEMBERS:{idFromName:name=>namespace.idFromName(name),get:id=>missing.has(id.toString())?unavailable:namespace.get(id)}}); + return true; + } + async presenceSchedule(){ + const watchers=this.current.watchers,clock=watchers.liveness.clock,timestamp=this.ctx.getWebSocketAutoResponseTimestamp.bind(this.ctx),base=Date.now();let now=base,sweeps=0; + const period=coordinationPolicy(sharedRuntime()).presence_every; + // Only the loopback clock advances. Socket metadata and the production + // alarm path still choose every sweep; no consensus timer is substituted. + watchers.liveness.clock=()=>now; + this.ctx.getWebSocketAutoResponseTimestamp=ws=>ws.deserializeAttachment()?.clusterPeer?timestamp(ws):new Date(base+Math.floor((now-base)/period)*period); + try{ + for(;;){const at=watchers.liveness.nextSweep();if(at===undefined||at>base+600000)break;now=at;await this.presenceTurn();sweeps++;} + return {milliseconds:600000,sweeps}; + }finally{watchers.liveness.clock=clock;this.ctx.getWebSocketAutoResponseTimestamp=timestamp;await this.ctx.storage.deleteAlarm();watchers.liveness.rearm();} + } + async presenceTurn(){measurements.sweeps++;return this.rebuilt?this.rebuilt.alarm():super.alarm();} + alarm(){return this.presenceTurn();} + webSocketMessage(ws,raw){if(ws.deserializeAttachment()?.clusterPeer)measurements.peerIn++;return this.rebuilt?this.rebuilt.webSocketMessage(ws,raw):super.webSocketMessage(ws,raw);} + webSocketClose(ws,code){return this.rebuilt?this.rebuilt.webSocketClose(ws,code):super.webSocketClose(ws,code);} + webSocketError(ws){return this.rebuilt?this.rebuilt.webSocketError(ws):super.webSocketError(ws);} async fetch(request){ - const result=await measured(()=>super.fetch(request)); + const result=await measured(()=>this.rebuilt?this.rebuilt.fetch(request):super.fetch(request)); if(this.host&&!this.capturing){ const commit=this.host.commit.bind(this.host);this.host.commit=async raw=>{this.lastCommand=raw.slice();return commit(raw);};this.capturing=true; } @@ -70,15 +117,16 @@ export class IdentityDO extends Source { async idleSchedule(){ // The object may hibernate between the HTTP request and this fixture RPC. // Restore through the production alarm before driving its virtual clock. - if(!this.host)await super.alarm(); - return this.host.serial(async()=>{ + if(!this.current.host)await this.current.alarm(); + const host=this.current.host; + return host.serial(async()=>{ const base=Date.now(),membershipBefore=measurements.membership; - for(let n=1;n<=20;n++)await this.host.maintenance(base+n*this.host.policy.tick_every,this.host.started+n*this.host.policy.tick_every); - const id=this.host.leader(),before=this.host.states.get(id).applied; - const sleeping=this.host.asleep; + for(let n=1;n<=20;n++)await host.maintenance(base+n*host.policy.tick_every,host.started+n*host.policy.tick_every); + const id=host.leader(),before=host.states.get(id).applied; + const sleeping=host.asleep; const turns=measurements.turns; - for(let n=21;n<=140;n++)await this.host.maintenance(base+n*this.host.policy.tick_every,this.host.started+n*this.host.policy.tick_every); - return {sleeping,membershipCalls:measurements.membership-membershipBefore,appliedBefore:String(before),appliedAfter:String(this.host.states.get(id).applied),recurringMemberCalls:measurements.turns-turns,idleMilliseconds:120*this.host.policy.tick_every}; + for(let n=21;n<=140;n++)await host.maintenance(base+n*host.policy.tick_every,host.started+n*host.policy.tick_every); + return {sleeping,membershipCalls:measurements.membership-membershipBefore,appliedBefore:String(before),appliedAfter:String(host.states.get(id).applied),recurringMemberCalls:measurements.turns-turns,idleMilliseconds:120*host.policy.tick_every}; });} inspect() { return {gate:this.ctx.storage.sql.exec('SELECT doc FROM legacy_gate WHERE one=1').toArray()[0]?.doc, @@ -86,6 +134,19 @@ export class IdentityDO extends Source { } } export class RaftMemberDO extends Member { + alarmState(){return this.ctx.storage.getAlarm();} + alarm(...args){return measured(()=>super.alarm(...args));} + importReceipt(...args){return measured(()=>super.importReceipt(...args));} + hostSnapshot(...args){return measured(()=>super.hostSnapshot(...args));} + hostCheckpoint(...args){return measured(()=>super.hostCheckpoint(...args));} + acknowledgeScheduled(...args){return measured(()=>super.acknowledgeScheduled(...args));} + schedule(...args){return measured(()=>super.schedule(...args));} + initialize(...args){return measured(()=>super.initialize(...args));} + drainRecovery(...args){return measured(()=>super.drainRecovery(...args));} + scheduled(...args){return measured(()=>super.scheduled(...args));} + sleepAck(...args){return measured(()=>super.sleepAck(...args));} + refreshEnrollment(...args){return measured(()=>super.refreshEnrollment(...args));} + hostPersonal(...args){return measured(()=>super.hostPersonal(...args));} membership(...args){measurements.membership++;return measured(()=>super.membership(...args));} hostInput(...args){return measured(()=>super.hostInput(...args));} peer(...args){return measured(()=>super.peer(...args));} @@ -113,6 +174,14 @@ export default { const url=new URL(request.url); if(url.pathname==='/fixture'){ const {op,identity,records,node,cluster}=await request.json(); + if(op==='stale-retirement')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).staleRetirement()); + if(op==='diagnostic'){console.warn('local runtime warning capture');console.error('local runtime error capture');return Response.json(true);} + if(op==='evict-minority')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).evictMinority()); + if(op==='presence-schedule')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).presenceSchedule()); + if(op==='quiet-state')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).quietState()); + if(op==='evict-idle')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).evictIdle()); + if(op==='presence-turn')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).presenceTurn()??true); + if(op==='member-alarm')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).alarmState()); if(op==='peer-idle')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).peerIdle()); if(op==='sequence-proof')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).sequenceProof()); if(op==='transport-state')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).transportState()); @@ -120,7 +189,7 @@ export default { if(op==='resume-public'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).resumePublic();return Response.json(true);} if(op==='checkpoint-state')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).checkpointState()); if(op==='metrics')return Response.json({...runtimeMetrics(),...measurements}); - if(op==='reset-metrics'){measurements.turns=0;measurements.wall=[];measurements.membership=0;return Response.json(true);} + if(op==='reset-metrics'){for(const key of Object.keys(measurements))measurements[key]=key==='wall'?[]:0;return Response.json(true);} if(op==='seed'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).seed(identity,records);return Response.json(true);} if(op==='hold-upload'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).holdUpload();return Response.json(true);} if(op==='release-upload'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).releaseHeldUpload();return Response.json(true);} @@ -135,6 +204,6 @@ export default { if(op==='member')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).inspect()); return new Response(null,{status:404}); } - return production.fetch(request,env); + measurements.http++;return production.fetch(request,env); } }; From 268f632f2b1e7f01a2d8fe9c7db0a9231365126b Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 08:46:10 -0400 Subject: [PATCH 069/103] Allow joined computers to lead the hosted relay safely. Keep public coordination available through signed forwarding and fresh quorum reads while a personal computer leads. Reuse the native membership planner and application wire, and preserve quiet learner homes. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterctl/forward.go | 59 +- internal/clusterstate/personal_host.go | 1 + internal/clusterwire/forward.go | 71 +++ .../manual/chat/coordination-and-workers.md | 12 +- internal/manual/hosted_personal_test.go | 16 +- internal/raftabi/README.md | 13 +- internal/raftabi/abi.go | 2 + internal/raftabi/forward.go | 86 +++ internal/raftabi/forward_test.go | 58 ++ internal/raftabi/peer.go | 9 +- internal/raftabi/personal.go | 20 +- internal/rafttransport/README.md | 12 +- internal/rafttransport/envelope.go | 15 +- internal/rafttransport/exchange.go | 2 +- internal/rafttransport/router.go | 2 +- internal/syncsetup/member_hosted_lead_test.go | 553 ++++++++++++++++++ .../syncsetup/member_hosted_learner_test.go | 10 +- relay/hosted/src/identity.js | 30 +- relay/hosted/src/raft/README.md | 21 +- relay/hosted/src/raft/forward.js | 41 ++ relay/hosted/src/raft/hosted.js | 176 ++++-- relay/hosted/src/raft/peers.js | 79 ++- relay/hosted/test/raft/build.mjs | 2 +- relay/hosted/test/raft/hosted-control.test.js | 6 +- relay/hosted/test/raft/hosted-forward.test.js | 189 ++++++ relay/hosted/test/raft/hosted-lead-go.test.js | 37 ++ relay/hosted/test/raft/hosted-worker.js | 10 +- 27 files changed, 1358 insertions(+), 174 deletions(-) create mode 100644 internal/clusterwire/forward.go create mode 100644 internal/raftabi/forward.go create mode 100644 internal/raftabi/forward_test.go create mode 100644 internal/syncsetup/member_hosted_lead_test.go create mode 100644 relay/hosted/src/raft/forward.js create mode 100644 relay/hosted/test/raft/hosted-forward.test.js create mode 100644 relay/hosted/test/raft/hosted-lead-go.test.js diff --git a/internal/clusterctl/forward.go b/internal/clusterctl/forward.go index a4957b1ef0..bbf4d6282f 100644 --- a/internal/clusterctl/forward.go +++ b/internal/clusterctl/forward.go @@ -1,61 +1,14 @@ package clusterctl import ( - "crypto/ed25519" - "time" - - cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/clusterwire" "github.com/Agent-Field/codeaf/internal/raftcore" - "github.com/Agent-Field/codeaf/internal/rafttransport" ) -const ForwardPath = "/v2/cluster/forward" -const ForwardTimeout = 30 * time.Second - -// Forward is application ingress on a SEPARATE outbound connection. Original -// commands never become peer envelopes or untrusted Raft proposal messages. -// Each direction signs its complete scope, request ID and body independently. -type Forward struct { - Scope rafttransport.Scope `json:"scope"` - Generation uint64 `json:"generation,string"` - From uint64 `json:"from,string"` - To uint64 `json:"to,string"` - ID string `json:"id"` - Action string `json:"action"` - Command []byte `json:"command,omitempty"` - Revision uint64 `json:"revision,omitempty"` - Barrier uint64 `json:"barrier,omitempty"` - Schedule *raftcore.ScheduleInput `json:"schedule,omitempty"` - Now int64 `json:"now,omitempty"` - Result *cs.Result `json:"result,omitempty"` - Evidence *raftcore.LeadershipEvidence `json:"evidence,omitempty"` - Receipt *cs.LeadershipReceipt `json:"receipt,omitempty"` - Plan *raftcore.SchedulePlan `json:"plan,omitempty"` - Membership *raftcore.MembershipPlan `json:"membership,omitempty"` - SleepAck *raftcore.SleepAck `json:"sleep_ack,omitempty"` - Status ForwardStatus `json:"status"` - Applied uint64 `json:"applied,omitempty"` - Error string `json:"error,omitempty"` - Signature []byte `json:"signature"` -} - -func (f Forward) bytes() []byte { - f.Signature = nil - b, _ := rafttransport.Encode(f) - return append([]byte("codeaf/cluster/forward/1\n"), b...) -} -func (f *Forward) Sign(sign func([]byte) []byte) { f.Signature = sign(f.bytes()) } -func (f Forward) Verify(key ed25519.PublicKey) bool { - return len(key) == ed25519.PublicKeySize && f.ID != "" && len(f.ID) <= raftcore.MaxRequestIDBytes && ed25519.Verify(key, f.bytes(), f.Signature) -} +const ForwardPath = clusterwire.ForwardPath +const ForwardTimeout = clusterwire.ForwardTimeout -// ForwardStatus contains only routing hints; quorum replies carry their own barrier. -type ForwardStatus struct { - Leader uint64 - Term uint64 - Applied uint64 -} +type Forward = clusterwire.Forward +type ForwardStatus = clusterwire.ForwardStatus -func PeerStatus(s raftcore.Status) ForwardStatus { - return ForwardStatus{Leader: s.Leader, Term: s.Term, Applied: s.Applied} -} +func PeerStatus(s raftcore.Status) ForwardStatus { return clusterwire.PeerStatus(s) } diff --git a/internal/clusterstate/personal_host.go b/internal/clusterstate/personal_host.go index 49aa587f7e..16b8cf7aff 100644 --- a/internal/clusterstate/personal_host.go +++ b/internal/clusterstate/personal_host.go @@ -205,6 +205,7 @@ func PersonalControlView(cluster, actor, certificate string, conf Configuration, } if !m.Retired { out.Self = m.Node + out.LeadershipUnavailable = false } return out, OK, nil } diff --git a/internal/clusterwire/forward.go b/internal/clusterwire/forward.go new file mode 100644 index 0000000000..7bf2a5b572 --- /dev/null +++ b/internal/clusterwire/forward.go @@ -0,0 +1,71 @@ +// Package clusterwire preserves signed application routing without network dependencies. +package clusterwire + +import ( + "crypto/ed25519" + "encoding/json" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" +) + +const ForwardPath = "/v2/cluster/forward" +const ForwardTimeout = 30 * time.Second + +// Scope binds every signature to an identity, cluster epoch and protocol. All +// uint64 fields cross JavaScript as canonical unsigned decimal strings. +type Scope struct { + Identity string `json:"identity"` + Cluster string `json:"cluster"` + Epoch uint64 `json:"epoch,string"` + Protocol uint16 `json:"protocol"` +} + +// Forward is application ingress on a separate outbound connection. Original +// commands never become peer envelopes or untrusted Raft proposal messages. +// Each direction signs its complete scope, request ID and body independently. +type Forward struct { + Scope Scope `json:"scope"` + Generation uint64 `json:"generation,string"` + From uint64 `json:"from,string"` + To uint64 `json:"to,string"` + ID string `json:"id"` + Action string `json:"action"` + Command []byte `json:"command,omitempty"` + Revision uint64 `json:"revision,omitempty"` + Barrier uint64 `json:"barrier,omitempty"` + Schedule *raftcore.ScheduleInput `json:"schedule,omitempty"` + Now int64 `json:"now,omitempty"` + Result *cs.Result `json:"result,omitempty"` + Evidence *raftcore.LeadershipEvidence `json:"evidence,omitempty"` + Receipt *cs.LeadershipReceipt `json:"receipt,omitempty"` + Plan *raftcore.SchedulePlan `json:"plan,omitempty"` + Membership *raftcore.MembershipPlan `json:"membership,omitempty"` + SleepAck *raftcore.SleepAck `json:"sleep_ack,omitempty"` + Status ForwardStatus `json:"status"` + Applied uint64 `json:"applied,omitempty"` + Error string `json:"error,omitempty"` + Signature []byte `json:"signature"` +} + +func (f Forward) bytes() []byte { + f.Signature = nil + b, _ := json.Marshal(f) + return append([]byte("codeaf/cluster/forward/1\n"), b...) +} +func (f *Forward) Sign(sign func([]byte) []byte) { f.Signature = sign(f.bytes()) } +func (f Forward) Verify(key ed25519.PublicKey) bool { + return len(key) == ed25519.PublicKeySize && f.ID != "" && len(f.ID) <= raftcore.MaxRequestIDBytes && ed25519.Verify(key, f.bytes(), f.Signature) +} + +// ForwardStatus contains only routing hints; quorum replies carry their own barrier. +type ForwardStatus struct { + Leader uint64 + Term uint64 + Applied uint64 +} + +func PeerStatus(s raftcore.Status) ForwardStatus { + return ForwardStatus{Leader: s.Leader, Term: s.Term, Applied: s.Applied} +} diff --git a/internal/manual/chat/coordination-and-workers.md b/internal/manual/chat/coordination-and-workers.md index afb2bc1a33..9666f5680e 100644 --- a/internal/manual/chat/coordination-and-workers.md +++ b/internal/manual/chat/coordination-and-workers.md @@ -17,14 +17,16 @@ The membership protocol admits at most 32 live learners and retains retired reco `codeaf cluster status` on the hosted relay, or a self-hosted `codeaf relay` with storage, reports the committed voters and current leader after a fresh majority read. A paired computer joins with persistent storage as a learner before status shows its node. Readiness requires its own durable application through the advertised join position. Status names `this computer`, `learning computers`, `ready computers` and `voting computers` when known. `codeaf cluster lead` on an unjoined computer says `this computer must join the cluster with persistent storage before it can lead`. -On a self-hosted `codeaf relay` with storage, run `codeaf cluster lead` on the joined computer that should coordinate. It first catches up, replaces the third anchor’s vote through a joint change, durably finishes that change with the third anchor still learning, and transfers leadership. The voters are then two anchors and this computer. Keep codeaf open to keep its member running; a command used with no open terminal runs its member only for that command. Leadership moves coordination metadata, never chat content or chat ownership. - -The hosted relay admits paired computers as learners and shows them in `codeaf cluster status`. Leadership is not offered there yet: `codeaf cluster lead` says `this relay does not offer cluster leadership yet; your saved chats are still here`. Ordinary hosted chat sync remains available. +On the hosted relay or a self-hosted `codeaf relay` with storage, run `codeaf cluster lead` on the joined computer that should coordinate. It first catches up, replaces the third anchor’s vote through a joint change, durably finishes that change with the third anchor still learning, and transfers leadership. The voters are then two anchors and this computer. Keep codeaf open to keep its member running; a command used with no open terminal runs its member only for that command. Leadership moves coordination metadata, never chat content or chat ownership. Do not treat pairing or a saved leadership request as a successful transfer. Success requires the selected computer to be elected, commit its current term and pass a fresh majority read; its receipt names the computer, term, revision and read barrier. Retrying `codeaf cluster lead` preserves an already saved signed request. Continue chats through the ordinary continue door. Three voters tolerate one unavailable member. Losing a majority pauses coordination; saved chats remain. The refusal is `this cluster cannot coordinate right now; reconnect enough of its voting computers and try again; your saved chats are still here`. +## Hosted relay while this computer leads + +The hosted relay keeps serving directory requests, watches, pairing decisions and worker assignments while this computer leads. It routes signed coordination requests through the computer’s outbound connection and waits for its own durable copy before replying. `codeaf cluster status` on either joined home reports the elected computer and the same three voters. + ## Pending leadership request, retry and refusal A lost reply or restarted home keeps the original signed leadership request. Retry `codeaf cluster lead` to resume it; a pending request cannot choose a different computer. A completed refusal says `the leadership request could not complete; refresh cluster status and try again` and is acknowledged before another request is saved. Only one configuration change can run at a time. A different personal coordinator cannot directly replace an active personal coordinator; let the anchors resume, then request leadership again. @@ -46,11 +48,11 @@ The sync connection carries presence hints and committed lease acknowledgments. ## Does syncing keep sending requests when nothing is happening? -A home that has never paired sends no sync traffic. `CODEAF_SYNC_URL=off` also blocks new sync requests from clients that were opened before the setting changed. Presence pings on an open view are hints; they do not write chat ownership. On the hosted relay and the native relay, an open idle home uses its learner’s applied vault pointer to notice shared memory and preference changes. Local edits are detected locally and pushed on the next state round. Over ten idle minutes, two caught-up open homes make zero HTTP requests, zero peer or forwarding frames and zero client attempts. A caught-up idle home sends no recurring state polls or peer keepalives. Home also follows its local applied directory for display changes; its roster and backstop read that same local hint. Quiet neighbours’ presence can age out without forcing a network ping. Continuing a chat and all ownership decisions still use fresh majority reads. A committed change wakes peer delivery and the next state round pulls the changed vault. If an older hosted relay has no learner door, the home keeps its existing five-second state polling. On the consensus directory, a group with no work or held chats can stop its periodic coordination timer after an idle barrier has reached every voter. The first operation wakes coordination and must obtain a fresh majority response. A saved leader address cannot answer that operation by itself. Losing the majority leaves syncing unavailable and keeps local work intact. +A home that has never paired sends no sync traffic. `CODEAF_SYNC_URL=off` also blocks new sync requests from clients that were opened before the setting changed. Presence pings on an open view are hints; they do not write chat ownership. On the hosted relay and the native relay, an open idle home uses its learner’s applied vault pointer to notice shared memory and preference changes. Local edits are detected locally and pushed on the next state round. Over ten idle minutes, two caught-up open homes joined as learners make zero HTTP requests, zero peer or forwarding frames and zero client attempts. A caught-up idle home sends no recurring state polls or peer keepalives. Home also follows its local applied directory for display changes; its roster and backstop read that same local hint. Quiet neighbours’ presence can age out without forcing a network ping. Continuing a chat and all ownership decisions still use fresh majority reads. A committed change wakes peer delivery and the next state round pulls the changed vault. If an older hosted relay has no learner door, the home keeps its existing five-second state polling. On the consensus directory, a group with no work or held chats can stop its periodic coordination timer after an idle barrier has reached every voter. The first operation wakes coordination and must obtain a fresh majority response. A saved leader address cannot answer that operation by itself. Losing the majority leaves syncing unavailable and keeps local work intact. ## Hosted idle presence sweeps and sleeping connections -A hosted group with no work or held chats stops its maintenance alarm after every voter applies the idle barrier. Its three anchors then schedule no recurring alarms. Reconstructing the sleeping coordinator leaves its learner and forwarding sockets open; a presence sweep neither wakes the anchors nor reconnects those computers. The first real request obtains a fresh majority response before answering. +With its three anchors voting, a hosted group with no work or held chats stops its maintenance alarm after every voter applies the idle barrier. Its three anchors then schedule no recurring alarms. Reconstructing the sleeping coordinator leaves its learner and forwarding sockets open; a presence sweep neither wakes the anchors nor reconnects those computers. The first real request obtains a fresh majority response before answering. A compatibility events viewer pings every 10 seconds. The platform answers without running coordination. While somebody watches presence, the relay checks at the next 25-second silence deadline so a stopped computer appears offline within 30 seconds. With two viewers pinging together, the simulated ten-minute measurement uses 29 short sweeps; those sweeps make no member calls or data writes. With no events viewer, no presence sweep recurs. Home follows the local learner once joined, so its ordinary display needs no remote presence viewer. diff --git a/internal/manual/hosted_personal_test.go b/internal/manual/hosted_personal_test.go index 639542ea8c..4cd7b26d94 100644 --- a/internal/manual/hosted_personal_test.go +++ b/internal/manual/hosted_personal_test.go @@ -5,10 +5,9 @@ import ( "testing" ) -// R9/R23 keep current hosted enrollment and status distinct from the older -// relay fallback, while leaving leadership availability explicit. -func TestR9R23HostedLearnerManualTruth(t *testing.T) { - for _, query := range []string{"hosted relay learner cluster status", "paired open home hosted learner leadership"} { +// R6/R9/R23 keep hosted enrollment, personal leadership and forwarding reachable. +func TestR6R9R23HostedPersonalManualTruth(t *testing.T) { + for _, query := range []string{"hosted relay learner cluster status", "paired open home hosted learner leadership", "hosted relay personal leader forwarding"} { found := false for _, section := range Chat().Search(query, DefaultResults) { found = found || section.Page == "coordination-and-workers" @@ -23,15 +22,18 @@ func TestR9R23HostedLearnerManualTruth(t *testing.T) { } for _, claim := range []string{ "joins the hosted relay", - "hosted relay admits paired computers as learners", + "as a durable learner", + "`codeaf cluster lead`", + "The voters are then two anchors and this computer", + "routes signed coordination requests", + "waits for its own durable copy before replying", "`codeaf cluster status`", - "this relay does not offer cluster leadership yet; your saved chats are still here", } { if !strings.Contains(page, claim) { t.Errorf("R9/R23 current hosted capability missing: %q", claim) } } - for _, obsolete := range []string{"The hosted relay has no learner door", "The hosted relay has no cluster control door"} { + for _, obsolete := range []string{"The hosted relay has no learner door", "The hosted relay has no cluster control door", "this relay does not offer cluster leadership yet"} { if strings.Contains(page, obsolete) { t.Errorf("R9 obsolete current-hosted denial: %q", obsolete) } diff --git a/internal/raftabi/README.md b/internal/raftabi/README.md index 12ad1ff51c..e8f0f15069 100644 --- a/internal/raftabi/README.md +++ b/internal/raftabi/README.md @@ -9,7 +9,7 @@ JavaScript lease or consensus rules are implemented here. Build/run the bounded local proof from the repository root: ```sh -sh relay/hosted/test/raft/run.sh /tmp/codeaf-p03 +sh relay/hosted/test/raft/run.sh /tmp/codeaf-raft-proof ``` Install the hosted relay's pinned npm lockfile first if needed. It uses the installed @@ -200,4 +200,13 @@ operation 27, it runs under the member queue after current ingress time commits and a fresh quorum barrier passes; the ABI never supplies a local success path. Assignments, per-cell active slots and incarnation-bound holds use the existing state namespace and survive ordinary native/hosted snapshots. No storage format -or old ABI tag changes. \ No newline at end of file +or old ABI tag changes. +## Signed application forwarding + +Operation 39 signs a canonical typed host request or authenticates an exact +native application reply. The signed frame retains the established transport +encoding; a separate canonical projection keeps reply indices lossless in +JavaScript. Scope, router generation, connection and request token are checked +before dispatching completion. Hosts use only provisioned anchor keys to route +original signed commands, reads and member coordination. The reactor imports no +network client, and application frames never enter the Raft peer message path. diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index 7b12ebf035..3e0d5dabf5 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -164,6 +164,8 @@ func (r *Runtime) Call(b []byte) (out []byte) { func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { e := encoder{} switch op { + case 39: + return hostedForward(d) case 36: return hostedPersonal(d) case 37: diff --git a/internal/raftabi/forward.go b/internal/raftabi/forward.go new file mode 100644 index 0000000000..028474cd5a --- /dev/null +++ b/internal/raftabi/forward.go @@ -0,0 +1,86 @@ +package raftabi + +import ( + "crypto/ed25519" + "encoding/json" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/clusterwire" + "github.com/Agent-Field/codeaf/internal/raftcore" +) + +// Application signatures use the same native encoding, including omitted +// fields and numeric routing hints, without importing a network client. +func hostedForward(d *decoder) ([]byte, error) { + op, raw, key := d.u32(), d.blob(), d.blob() + if d.end() != nil || len(raw) > peerMaxExchangeBytes { + return nil, errFrame + } + var work clusterwire.Forward + if op == 0 { + var in forwardInput + if cs.Decode(raw, &in) != nil { + return nil, errFrame + } + work = clusterwire.Forward{Scope: clusterwire.Scope{Identity: in.Scope.Identity, Cluster: in.Scope.Cluster, Epoch: in.Scope.Epoch, Protocol: in.Scope.Protocol}, Generation: in.Generation, From: in.From, To: in.To, ID: in.ID, Action: in.Action, Command: in.Command, Revision: in.Revision, Barrier: in.Barrier, Schedule: in.Schedule, Now: in.Now} + } else if peerDecode(raw, &work) != nil { + return nil, errFrame + } + e := encoder{} + switch op { + case 0: + if len(key) != ed25519.SeedSize { + return nil, errFrame + } + private := ed25519.NewKeyFromSeed(key) + work.Sign(func(b []byte) []byte { return ed25519.Sign(private, b) }) + b, err := json.Marshal(work) + if err != nil { + return nil, err + } + e.blob(b) + case 1: + if work.Verify(key) { + e.u32(1) + b, err := cs.Encode(forwardReply{work.ID, work.Error, work.Barrier, work.Applied, work.Result, work.Evidence, work.Receipt, work.Plan, work.Membership, work.SleepAck, work.Status}) + if err != nil { + return nil, err + } + e.blob(b) + } else { + e.u32(0) + } + default: + return nil, errFrame + } + return e.b, nil +} + +// The canonical host projection keeps every index lossless in JavaScript; +// the signed peer frame retains the established native wire exactly. +type forwardInput struct { + Scope peerScopeState `json:"scope"` + Generation uint64 `json:"generation"` + From uint64 `json:"from"` + To uint64 `json:"to"` + ID string `json:"id"` + Action string `json:"action"` + Command []byte `json:"command"` + Revision uint64 `json:"revision"` + Barrier uint64 `json:"barrier"` + Schedule *raftcore.ScheduleInput `json:"schedule"` + Now int64 `json:"now"` +} +type forwardReply struct { + ID string `json:"id"` + Error string `json:"error"` + Barrier uint64 `json:"barrier"` + Applied uint64 `json:"applied"` + Result *cs.Result `json:"result"` + Evidence *raftcore.LeadershipEvidence `json:"evidence"` + Receipt *cs.LeadershipReceipt `json:"receipt"` + Plan *raftcore.SchedulePlan `json:"plan"` + Membership *raftcore.MembershipPlan `json:"membership"` + SleepAck *raftcore.SleepAck `json:"sleep_ack"` + Status clusterwire.ForwardStatus `json:"status"` +} diff --git a/internal/raftabi/forward_test.go b/internal/raftabi/forward_test.go new file mode 100644 index 0000000000..2554285a34 --- /dev/null +++ b/internal/raftabi/forward_test.go @@ -0,0 +1,58 @@ +package raftabi + +import ( + "bytes" + "crypto/ed25519" + "encoding/json" + "testing" + + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + rt "github.com/Agent-Field/codeaf/internal/rafttransport" +) + +// R6 preserves the signed application wire across native and hosted routing. +func TestR6HostedForwardUsesOriginalSignedWire(t *testing.T) { + key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{31}, 32)) + work := clusterctl.Forward{Scope: rt.Scope{Identity: "identity", Cluster: "cluster", Epoch: 1, Protocol: 1}, Generation: 9, From: 1, To: 4, ID: "request-one", Action: "submit", Command: []byte("original signed command")} + raw, err := cs.Encode(forwardInput{Scope: peerScopeState{"identity", "cluster", 1, 1}, Generation: 9, From: 1, To: 4, ID: work.ID, Action: work.Action, Command: work.Command}) + if err != nil { + t.Fatal(err) + } + f := request(39) + f.u32(0) + f.blob(raw) + f.blob(key.Seed()) + d := answer(t, New().Call(f.b)) + signed := d.blob() + const unsigned = `{"scope":{"identity":"identity","cluster":"cluster","epoch":"1","protocol":1},"generation":"9","from":"1","to":"4","id":"request-one","action":"submit","command":"b3JpZ2luYWwgc2lnbmVkIGNvbW1hbmQ=","status":{"Leader":0,"Term":0,"Applied":0},"signature":null}` + var signature struct { + Signature []byte `json:"signature"` + } + if json.Unmarshal(signed, &signature) != nil || !ed25519.Verify(key.Public().(ed25519.PublicKey), append([]byte("codeaf/cluster/forward/1\n"), []byte(unsigned)...), signature.Signature) { + t.Fatal("R6 hosted frame changed the established native signed encoding") + } + + var got clusterctl.Forward + if rt.Decode(signed, &got) != nil || !got.Verify(key.Public().(ed25519.PublicKey)) || !bytes.Equal(got.Command, work.Command) || got.ID != work.ID { + t.Fatal("R6 hosted signing changed the original request") + } + f = request(39) + f.u32(1) + f.blob(signed) + f.blob(key.Public().(ed25519.PublicKey)) + d = answer(t, New().Call(f.b)) + if d.u32() != 1 { + t.Fatal("R6 signed reply was refused") + } + got.Barrier++ + raw, _ = rt.Encode(got) + f = request(39) + f.u32(1) + f.blob(raw) + f.blob(key.Public().(ed25519.PublicKey)) + d = answer(t, New().Call(f.b)) + if d.u32() != 0 { + t.Fatal("R6 altered barrier was accepted") + } +} diff --git a/internal/raftabi/peer.go b/internal/raftabi/peer.go index baf1835577..4f367271e0 100644 --- a/internal/raftabi/peer.go +++ b/internal/raftabi/peer.go @@ -6,6 +6,7 @@ import ( "strconv" cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/clusterwire" pb "go.etcd.io/raft/v3/raftpb" "google.golang.org/protobuf/proto" ) @@ -124,10 +125,10 @@ func hostedPeerWire(d *decoder) ([]byte, error) { return nil, errFrame } raw, err := json.Marshal(struct { - Batch, Chunk, Exchange, Messages, Inflight, Retention int - Slots [3]int - Budget [3]int - }{peerMaxBatchBytes, peerMaxChunkBytes, peerMaxExchangeBytes, peerMaxMessages, peerMaxInflight, int(cs.PeerRetentionMillis), [3]int{peerMaxInflight / 2, peerMaxInflight / 4, peerMaxInflight / 4}, [3]int{3 * peerMaxQueuedBytes / 8, peerMaxQueuedBytes / 8, peerMaxQueuedBytes / 2}}) + Batch, Chunk, Exchange, Messages, Inflight, Retention, ForwardTimeout int + Slots [3]int + Budget [3]int + }{peerMaxBatchBytes, peerMaxChunkBytes, peerMaxExchangeBytes, peerMaxMessages, peerMaxInflight, int(cs.PeerRetentionMillis), int(clusterwire.ForwardTimeout.Milliseconds()), [3]int{peerMaxInflight / 2, peerMaxInflight / 4, peerMaxInflight / 4}, [3]int{3 * peerMaxQueuedBytes / 8, peerMaxQueuedBytes / 8, peerMaxQueuedBytes / 2}}) if err != nil { return nil, err } diff --git a/internal/raftabi/personal.go b/internal/raftabi/personal.go index 302545e71a..4d28bf4703 100644 --- a/internal/raftabi/personal.go +++ b/internal/raftabi/personal.go @@ -83,7 +83,7 @@ func hostedPersonal(d *decoder) ([]byte, error) { want(cs.IdentityKey()) want(cs.MembershipKey()) actor := in.Actor - if in.Action != "status" { + if in.Action != "status" && in.Action != "lead" { var identity cs.IdentityRec if cs.Decode(rows[cs.IdentityKey()], &identity) == nil { request, _, who, e := cs.VerifyPersonalAdmission(in.Proof, identity, in.Now) @@ -101,7 +101,7 @@ func hostedPersonal(d *decoder) ([]byte, error) { if cs.Decode(rows[cs.MemberDeviceKey(actor)], &mapping) == nil && mapping.Node != 0 { want(cs.MemberKey(mapping.Node)) } - if in.Action != "status" { + if in.Action != "status" && in.Action != "lead" { var identity cs.IdentityRec if cs.Decode(rows[cs.IdentityKey()], &identity) == nil { request, _, _, e := cs.VerifyPersonalAdmission(in.Proof, identity, in.Now) @@ -111,7 +111,7 @@ func hostedPersonal(d *decoder) ([]byte, error) { } } } - if in.Action == "status" || in.Action == "grant" { + if in.Action == "status" || in.Action == "lead" || in.Action == "grant" { var h cs.Membership if cs.Decode(rows[cs.MembershipKey()], &h) == nil { for _, node := range h.Nodes { @@ -176,7 +176,7 @@ func hostedPersonal(d *decoder) ([]byte, error) { } } } - case "status": + case "status", "lead": progress := map[uint64]uint64{} for _, p := range in.Progress { progress[p.Node] = p.Applied @@ -185,6 +185,18 @@ func hostedPersonal(d *decoder) ([]byte, error) { status, out.Code, err = cs.PersonalControlView(in.Cluster, in.Actor, in.Certificate, raftcore.Configuration(conf), in.Applied, in.Leader, in.CommittedNow, progress, read) if err == nil && out.Code == cs.OK { out.Wire, err = cs.Encode(status) + if in.Action == "lead" { + c, e := cs.DecodeCommand(in.Proof.Body) + if e != nil || cs.ClientCommand(c) != nil || c.Actor != in.Actor || c.Change.RequestLeadership == nil { + out.Code = cs.BadRequest + out.Wire = nil + } else if status.Self == 0 { + out.Code = cs.Unauthorized + out.Wire = nil + } else { + out.Wire = in.Proof.Body + } + } } default: return nil, errFrame diff --git a/internal/rafttransport/README.md b/internal/rafttransport/README.md index 7cf36b7e5e..c1ef0eade6 100644 --- a/internal/rafttransport/README.md +++ b/internal/rafttransport/README.md @@ -228,7 +228,7 @@ wakes the ordinary quorum path. The codeaf client starts its personal learner only inside the existing terminal state-sync lifetime, or a scoped status activation, after current quiet guards. Open remains local. Hosted relays offer enrollment and control status. Worker -setup remains unavailable. Native personal leadership uses the separate application ingress below; hosted leadership remains unavailable. +setup remains unavailable. Personal leadership uses the separate application ingress below on native and hosted relays. The native host advances its checksummed `transport-generation` file with atomic rename and directory fsync before opening its router; it never mutates the source @@ -242,7 +242,7 @@ and successful voter operations intact. ## Native personal leaders and application forwarding -A native personal member also opens `/v2/cluster/forward` outward. This is a +A personal member also opens `/v2/cluster/forward` outward. This is a separate application connection, never a Raft peer envelope. The upgrade uses the same enrolled, signed scope/node/incarnation/router-generation proof. Every application frame signs its scope, generation, source/destination, request ID, @@ -301,7 +301,7 @@ leader; actual work, read requests and higher-term messages still do. The hosted coordinator publishes routing only from committed anchor records. Its generation is durably advanced before restoring expendable inboxes, so old cursors and socket proofs cannot survive coordinator memory loss. Hibernating -peer and passive application sockets retain separate tags and attachments; +peer and application sockets retain separate tags and attachments; watch presence, broadcasts and watcher admission never count them. A stale connection closes and the same durable home refreshes its enrollment grant. The relay never opens an outbound connection to a personal member. @@ -311,8 +311,10 @@ admission survives a lost reply before configuration; activation or maintenance reconciles only pending intent. The original signed device request remains in the enrollment command. A fresh quorum status reports readiness only after the enrolled key signs durable application through its advertised join barrier. -Leadership remains unavailable on the hosted relay, and application sockets -stay passive. Peer ingress progresses independently of the public request queue. +Personal leadership uses the same Go planner for the joint swap, durable leave, +settlement and actual transfer. Application sockets carry original signed commands +and fresh reads; completion waits for an anchor to apply through each returned +index. Peer ingress progresses independently of the public operation queue. Snapshots send a manifest followed by chunks on their reserved transport lane. Each target has one outstanding chunk and advances only on signed delivery diff --git a/internal/rafttransport/envelope.go b/internal/rafttransport/envelope.go index a37daa8e3d..f5d9508acd 100644 --- a/internal/rafttransport/envelope.go +++ b/internal/rafttransport/envelope.go @@ -13,6 +13,7 @@ import ( "strings" "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/clusterwire" "github.com/Agent-Field/codeaf/internal/raftcore" "github.com/Agent-Field/codeaf/internal/raftsnapshot" "go.etcd.io/raft/v3" @@ -40,14 +41,8 @@ var ( ErrCursor = errors.New("rafttransport: invalid delivery cursor") ) -// Scope binds every signature to an identity, cluster epoch and protocol. All -// uint64 fields cross JavaScript as canonical unsigned decimal strings. -type Scope struct { - Identity string `json:"identity"` - Cluster string `json:"cluster"` - Epoch uint64 `json:"epoch,string"` - Protocol uint16 `json:"protocol"` -} +// Scope binds peer and application signatures to the same cluster. +type Scope = clusterwire.Scope func token(s string) bool { if len(s) == 0 || len(s) > 128 { @@ -60,7 +55,7 @@ func token(s string) bool { } return true } -func (s Scope) valid() bool { +func validScope(s Scope) bool { return token(s.Identity) && token(s.Cluster) && s.Epoch != 0 && s.Protocol == Protocol } func node(id uint64) bool { return id != 0 && !raft.IsLocalMsgTarget(id) } @@ -139,7 +134,7 @@ func (e Envelope) AppendChunk() bool { } func (e Envelope) validate() error { - if !e.Scope.valid() || !node(e.From) || !node(e.To) || e.From == e.To || e.Incarnation == 0 || e.Destination == 0 || e.Sequence == 0 || e.Digest != digest(e.Payload) { + if !validScope(e.Scope) || !node(e.From) || !node(e.To) || e.From == e.To || e.Incarnation == 0 || e.Destination == 0 || e.Sequence == 0 || e.Digest != digest(e.Payload) { return ErrWire } if e.Chunk != nil { diff --git a/internal/rafttransport/exchange.go b/internal/rafttransport/exchange.go index 715128e480..3aca7e06a1 100644 --- a/internal/rafttransport/exchange.go +++ b/internal/rafttransport/exchange.go @@ -111,7 +111,7 @@ func (r Request) Verify(key ed25519.PublicKey) error { return nil } func (r Request) validate() error { - if !r.Scope.valid() || !node(r.Node) || r.Incarnation == 0 || r.Generation == 0 || len(r.Messages) > MaxMessages { + if !validScope(r.Scope) || !node(r.Node) || r.Incarnation == 0 || r.Generation == 0 || len(r.Messages) > MaxMessages { return ErrWire } normal, chunks := 0, 0 diff --git a/internal/rafttransport/router.go b/internal/rafttransport/router.go index 0cb3e19c9d..0b032f264c 100644 --- a/internal/rafttransport/router.go +++ b/internal/rafttransport/router.go @@ -94,7 +94,7 @@ func NewRouter(limits Limits) (*Router, error) { // the exchange lock fences requests authenticated before a committed revoke. // Hosts must reload durable views before accepting traffic after a restart. func (r *Router) Install(v View) error { - if !v.Scope.valid() || v.Revision == 0 || v.Generation == 0 || len(v.Members) > r.limits.Records { + if !validScope(v.Scope) || v.Revision == 0 || v.Generation == 0 || len(v.Members) > r.limits.Records { return ErrWire } members := make(map[uint64]Member, len(v.Members)) diff --git a/internal/syncsetup/member_hosted_lead_test.go b/internal/syncsetup/member_hosted_lead_test.go new file mode 100644 index 0000000000..ae41cdf8bd --- /dev/null +++ b/internal/syncsetup/member_hosted_lead_test.go @@ -0,0 +1,553 @@ +package syncsetup + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/blobstore" + "github.com/Agent-Field/codeaf/internal/clusterapi" + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/reqsign" + "go.etcd.io/raft/v3" +) + +type hostedLeadRig struct { + t *testing.T + server *httptest.Server +} + +func newHostedLeadRig(t *testing.T) *hostedLeadRig { + t.Helper() + endpoint := hostedLearnerEndpoint(t) + target, e := url.Parse(endpoint) + mustNil(t, e) + proxy := httputil.NewSingleHostReverseProxy(target) + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DisableKeepAlives = true + proxy.Transport = transport + server := httptest.NewServer(proxy) + t.Cleanup(server.Close) + t.Cleanup(transport.CloseIdleConnections) + return &hostedLeadRig{t, server} +} +func (r *hostedLeadRig) home(id *identity.Identity) *Sync { + r.t.Helper() + home := machine(r.t, r.server.URL) + if id != nil { + _, e := identity.Adopt(home, *id, true) + mustNil(r.t, e) + } + s, ok, e := Open(home) + mustNil(r.t, e) + if !ok { + r.t.Fatal("open") + } + hostedOwner(r.t, s) + return s +} +func hostedPersonalLeader(t *testing.T, p *Sync) cs.LeadershipReceipt { + t.Helper() + receipt, e := p.LeadCluster(t.Context(), 0) + mustNil(t, e) + waitMember(t, p, func(m *personalMember) bool { + return m.status.Role == raft.StateLeader && m.status.Term == receipt.Term && m.status.Applied >= receipt.Barrier + }) + status, e := p.ClusterStatus(t.Context()) + mustNil(t, e) + if status.Leader != receipt.Node || status.LeadershipUnavailable || status.Configuration.Joint() || !slices.Equal(status.Configuration.Voters, []uint64{1, 2, receipt.Node}) || !slices.Contains(status.Configuration.Learners, 3) || status.Membership.Leadership == nil || !status.Membership.Leadership.Completed || receipt.Revision != status.Membership.Revision { + t.Fatalf("R6 incomplete leadership receipt/status: %+v %+v", receipt, status) + } + return receipt +} + +// R6/R15/R18 keep every public ingress usable while the elected personal core leads. +func TestR6R15R18HostedLeaderOriginalWorkerIngressAndThinStore(t *testing.T) { + r := newHostedLeadRig(t) + p := r.home(nil) + var blobRequests atomic.Uint64 + signed, _ := http.NewRequest(http.MethodGet, r.server.URL, nil) + reqsign.Sign(signed, nil, deviceSigner{p.Identity, p.Device}, time.Now()) + certP := signed.Header.Get(reqsign.HeaderCert) + originalHandler := r.server.Config.Handler + r.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, q *http.Request) { + if strings.HasPrefix(q.URL.Path, "/v1/store/") && q.Header.Get(reqsign.HeaderCert) == certP { + blobRequests.Add(1) + } + originalHandler.ServeHTTP(w, q) + }) + hostedPersonalLeader(t, p) + id := p.Identity + b := r.home(&id) + const cell = "01J0000000000000000000000A" + head := strings.Repeat("1", 64) + _, err := b.Dir.Create(t.Context(), cell, directory.CellInit{Head: head}) + mustNil(t, err) + mustNil(t, b.Dir.Release(t.Context(), cell, 1)) + status, err := p.clusterClient().Status(t.Context()) + mustNil(t, err) + owner := b.ClusterAPI(status.Cluster) + owner.Now = time.Now + sequence := uint64(0) + signOwner := func(change cs.Mutation) cs.Command { + sequence++ + c, err := b.SignClusterCommand(cs.Command{Schema: cs.SchemaVersion, Cluster: status.Cluster, Session: "external-ingress", Sequence: sequence, Ack: sequence - 1, Now: time.Now().UnixMilli(), Change: change}) + mustNil(t, err) + return c + } + submit := func(client clusterapi.Client, c cs.Command, want cs.Code) cs.Result { + t.Helper() + result, err := client.Submit(t.Context(), c) + mustNil(t, err) + if result.Code != want { + t.Fatalf("R6 original proposal: want %s got %+v", want, result) + } + return result + } + _, key, err := ed25519.GenerateKey(rand.Reader) + mustNil(t, err) + cert, err := identity.IssueWorkerCert(id, key.Public().(ed25519.PublicKey), status.Cluster, []string{cell}, time.Now(), time.Now().Add(time.Hour), false) + mustNil(t, err) + worker := cert.DeviceID() + incarnation := uint64(0) + wire := clusterapi.Client{Base: r.server.URL, HTTP: r.server.Client(), Now: time.Now, Allowed: func() bool { return true }, Sign: func(b []byte) []byte { return ed25519.Sign(key, b) }, Request: func() cs.ClusterRequest { + return cs.ClusterRequest{IdentityKey: id.PublicKey(), Cluster: status.Cluster, Worker: &cert, Incarnation: incarnation} + }} + n := uint64(0) + signWorker := func(change cs.Mutation) cs.Command { + n++ + c, err := cs.Sign(cs.Command{Schema: cs.SchemaVersion, Cluster: status.Cluster, Actor: worker, Session: "external-worker", Sequence: n, Ack: n - 1, WorkerIncarnation: incarnation, Now: time.Now().UnixMilli(), Change: change}, key) + mustNil(t, err) + return c + } + original := signOwner(cs.Mutation{EnrollWorker: &cs.EnrollWorker{Certificate: cert, Grants: map[string][]byte{cell: {0x81, 0x82}}}}) + raw, err := cs.EncodeCommand(original) + mustNil(t, err) + submit(owner, original, cs.OK) + submit(wire, signWorker(cs.Mutation{RegisterWorker: &cs.RegisterWorker{Incarnation: 1}}), cs.OK) + incarnation = 1 + view, grant, err := wire.Cell(t.Context(), cell) + mustNil(t, err) + if view.Cell.Head != head || !bytes.Equal(grant, []byte{0x81, 0x82}) { + t.Fatal("R18 scoped read lost grant", view, grant) + } + if _, _, err = wire.Cell(t.Context(), "outside"); err == nil { + t.Fatal("R18 worker escaped its grant") + } + hb := cs.WorkerHeartbeat{Worker: worker, Incarnation: 1, Sequence: 1, SeenAt: time.Now().UnixMilli()} + hraw, err := cs.WorkerHeartbeatBytes(status.Cluster, hb) + mustNil(t, err) + hb.Signature = ed25519.Sign(key, hraw) + submit(wire, signWorker(cs.Mutation{WorkerBeatBatch: &cs.WorkerBeatBatch{Items: []cs.WorkerHeartbeat{hb}}}), cs.OK) + assignment := cs.Assign{ID: "external-run", Revision: 1, Cell: cell, Expect: cs.LeaseExpectation{Device: b.Device.ID(), Fence: 1}, Target: worker, Incarnation: 1, Expires: time.Now().Add(time.Minute).UnixMilli(), SealedPayload: []byte{0x91}} + submit(owner, signOwner(cs.Mutation{Assign: &assignment}), cs.OK) + ref := (cs.Assignment{Spec: assignment}).Ref() + current, _, err := wire.Current(t.Context(), ref) + mustNil(t, err) + if current.Status != "pending" { + t.Fatal(current) + } + claim := signWorker(cs.Mutation{ClaimAssignment: &cs.ClaimAssignment{Ref: ref, Expect: assignment.Expect}}) + result := submit(wire, claim, cs.OK) + if result.View == nil || result.View.Cell.Lease.Fence != 2 { + t.Fatal(result) + } + submit(wire, claim, cs.OK) + _, err = b.Dir.Acquire(t.Context(), cell, directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: worker, Fence: 2}}) + mustNil(t, err) + current, view, err = wire.Current(t.Context(), ref) + mustNil(t, err) + if current.Fence != 2 || view.Cell.Lease.Fence != 3 { + t.Fatal("R18 cached claim became current authority", current, view) + } + watch, err := b.Dir.(directory.HoldWatcher).WatchHolding(t.Context(), []directory.Hold{{Cell: cell, Fence: 3}}) + mustNil(t, err) + defer watch.Close() + ctx, cancel := context.WithTimeout(t.Context(), requestTimeout) + defer cancel() + _, err = watch.Next(ctx) + mustNil(t, err) + cancel() + renewAfter := time.Now().Add(time.Duration(cs.RenewEveryMillis-cs.PresenceEveryMillis) * time.Millisecond).UnixMilli() + for elapsed := int64(0); elapsed < cs.RenewEveryMillis; elapsed += cs.PresenceEveryMillis { + time.Sleep(time.Duration(cs.PresenceEveryMillis) * time.Millisecond) + ping, stop := context.WithTimeout(t.Context(), requestTimeout) + mustNil(t, watch.Ping(ping)) + stop() + } + ctx, cancel = context.WithTimeout(t.Context(), requestTimeout) + defer cancel() + receipt := false + for !receipt { + frame, err := watch.Next(ctx) + mustNil(t, err) + for _, hold := range frame.Holds { + if hold.Cell == cell && hold.Fence == 3 && hold.Index > 0 && hold.At >= renewAfter { + receipt = true + } + } + } + // C1/C3 decisions use the same original public write/read path while P + // leads. Pairing itself never adds a vote or a peer on the new device. + requests := directory.NewLinkHTTP(r.server.URL, r.server.Client()) + for _, approve := range []bool{true, false} { + seed, e := identity.NewSeed() + mustNil(t, e) + opened, e := requests.CreateRequest(t.Context(), directory.NewRequest{Pubkey: base64.RawURLEncoding.EncodeToString(seed.Public()), X25519: base64.RawURLEncoding.EncodeToString(bytes.Repeat([]byte{0x37}, 32)), NameSealed: base64.RawURLEncoding.EncodeToString(bytes.Repeat([]byte{0x81}, 48)), Platform: "linux"}) + mustNil(t, e) + if approve { + certRaw, e := json.Marshal(identity.IssueCert(id, seed.Public(), time.Now())) + mustNil(t, e) + approval := directory.Approval{Device: directory.Device{V: 1, Name: "c2VhbGVk", Platform: "linux"}, Cert: base64.RawURLEncoding.EncodeToString(certRaw), Grant: base64.RawURLEncoding.EncodeToString(bytes.Repeat([]byte{0x87}, 64))} + mustNil(t, b.Dir.ApproveRequest(t.Context(), opened.Code, approval)) + mustNil(t, b.Dir.ApproveRequest(t.Context(), opened.Code, approval)) + } else { + mustNil(t, b.Dir.DenyRequest(t.Context(), opened.Code)) + } + decision, e := requests.GetRequest(t.Context(), opened.Code, 0) + mustNil(t, e) + if approve && (decision.State != directory.RequestApproved || decision.Grant == nil) || !approve && decision.State != directory.RequestDenied { + t.Fatal("R6 pairing decision did not commit", decision) + } + } + // A content object exists at the relay; only its uploader may request it. + + sentinel := []byte("P-MUST-NOT-STORE-THIS-CONTENT-5f99") + frame, err := blobstore.Encode(strings.Repeat("0", 32), []blobstore.Object{{RID: strings.Repeat("a", 64), Bytes: append([]byte("AGEO\x01"), sentinel...)}}) + mustNil(t, err) + _, err = b.Store.PutFrame(t.Context(), frame) + mustNil(t, err) + _, err = b.Dir.List(t.Context()) + mustNil(t, err) + waitMember(t, p, func(m *personalMember) bool { return m.status.Applied > 0 }) + p.member.mu.Lock() + recovered, err := p.member.store.Recover(t.Context()) + p.member.mu.Unlock() + mustNil(t, err) + found := false + for _, entry := range recovered.Core.Entries { + found = found || bytes.Equal(entry.GetData(), raw) + } + if !found { + t.Fatal("R6 relay replaced the original signed proposal") + } + err = filepath.WalkDir(p.memberPath(), func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + raw, err := os.ReadFile(path) + if err != nil { + return err + } + if bytes.Contains(raw, sentinel) { + t.Error("R15 P stored content", path) + } + } + return nil + }) + mustNil(t, err) + if blobRequests.Load() != 0 { + t.Fatal("R15 P fetched content") + } +} + +// R6 requires both homes to observe the elected personal voter after fresh reads. +func TestR6HostedPersonalLeadershipBothHomesStatus(t *testing.T) { + r := newHostedLeadRig(t) + p := r.home(nil) + id := p.Identity + b := r.home(&id) + receipt := hostedPersonalLeader(t, p) + status, e := b.ClusterStatus(t.Context()) + mustNil(t, e) + if status.Leader != receipt.Node || status.LeadershipUnavailable || !slices.Equal(status.Configuration.Voters, receipt.Voters) || status.Self == receipt.Node { + t.Fatalf("R6 second home status: %+v", status) + } +} + +// R6 retains one saved signed request across a lost reply and client restart. +func TestR6HostedPersonalLeadershipLostReplyResumes(t *testing.T) { + r := newHostedLeadRig(t) + s := r.home(nil) + original := r.server.Config.Handler + var mu sync.Mutex + var requests [][]byte + lost := false + r.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, q *http.Request) { + if q.URL.Path != clusterctl.Path || q.Method != http.MethodPost { + original.ServeHTTP(w, q) + return + } + raw, e := io.ReadAll(q.Body) + if e != nil { + panic(e) + } + q.Body = io.NopCloser(bytes.NewReader(raw)) + record := httptest.NewRecorder() + original.ServeHTTP(record, q) + mu.Lock() + defer mu.Unlock() + requests = append(requests, raw) + if record.Code == 200 && !lost { + lost = true + http.Error(w, "reply lost", 503) + return + } + for key, values := range record.Header() { + w.Header()[key] = values + } + w.WriteHeader(record.Code) + w.Write(record.Body.Bytes()) + }) + if _, e := s.LeadCluster(t.Context(), 0); e == nil { + t.Fatal("R6 lost reply claimed success") + } + path := filepath.Join(s.Home, "cluster-leadership-"+s.Device.ID()+".json") + raw, e := os.ReadFile(path) + mustNil(t, e) + var saved leadershipSession + mustNil(t, json.Unmarshal(raw, &saved)) + if saved.Pending == nil { + t.Fatal("R6 unknown outcome discarded request") + } + again := buildAt(s.Home, s.Relay, s.Identity, s.Device, s.Interval, time.Now) + if _, e = again.LeadCluster(t.Context(), saved.Pending.Change.RequestLeadership.Node+1); e == nil { + t.Fatal("R6 pending target changed") + } + receipt, e := again.LeadCluster(t.Context(), 0) + mustNil(t, e) + mu.Lock() + same := lost && len(requests) == 2 && bytes.Equal(requests[0], requests[1]) + mu.Unlock() + if !same || receipt.Node != saved.Pending.Change.RequestLeadership.Node { + t.Fatal("R6 retry allocated a different request") + } + raw, e = os.ReadFile(path) + mustNil(t, e) + var acknowledged leadershipSession + mustNil(t, json.Unmarshal(raw, &acknowledged)) + if acknowledged.Pending != nil || acknowledged.Sequence != saved.Pending.Sequence { + t.Fatal("R6 completion was not acknowledged") + } +} + +// R6 acknowledges a committed refusal before allocating the next signed request. +func TestR6HostedPersonalLeadershipCommittedRefusal(t *testing.T) { + r := newHostedLeadRig(t) + s := r.home(nil) + status, e := s.clusterClient().Status(t.Context()) + mustNil(t, e) + c, e := s.SignClusterCommand(cs.Command{Schema: cs.SchemaVersion, Cluster: status.Cluster, Session: "leadership", Sequence: 1, Now: time.Now().UnixMilli(), Change: cs.Mutation{RequestLeadership: &cs.RequestLeadership{Revision: status.Membership.Revision + 1, Node: status.Self, Device: s.Device.ID()}}}) + mustNil(t, e) + path := filepath.Join(s.Home, "cluster-leadership-"+s.Device.ID()+".json") + mustNil(t, saveLeadership(path, leadershipSession{Pending: &c})) + _, e = s.LeadCluster(t.Context(), 0) + var refusal clusterctl.CommittedRefusal + if !errors.As(e, &refusal) || refusal.Sequence != 1 { + t.Fatal("R6 committed refusal not returned", e) + } + raw, e := os.ReadFile(path) + mustNil(t, e) + var saved leadershipSession + mustNil(t, json.Unmarshal(raw, &saved)) + if saved.Pending != nil || saved.Sequence != 1 { + t.Fatal("R6 refusal not acknowledged") + } + receipt, e := s.LeadCluster(context.Background(), 0) + mustNil(t, e) + if receipt.Node != status.Self { + t.Fatal("R6 next request did not complete") + } +} + +// R6 resumes the saved operation when the member home closes before admission. +func TestR6HostedPersonalLeadershipHomeRestartBeforeAdmission(t *testing.T) { + r := newHostedLeadRig(t) + home := machine(t, r.server.URL) + s, ok, e := Open(home) + mustNil(t, e) + if !ok { + t.Fatal("open") + } + stop := hostedOwner(t, s) + original := r.server.Config.Handler + var requests [][]byte + lost := false + var mu sync.Mutex + r.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, q *http.Request) { + if q.Method == http.MethodPost && q.URL.Path == clusterctl.Path { + raw, e := io.ReadAll(q.Body) + if e != nil { + panic(e) + } + q.Body = io.NopCloser(bytes.NewReader(raw)) + mu.Lock() + requests = append(requests, raw) + first := !lost + lost = true + mu.Unlock() + if first { + http.Error(w, "request interrupted before admission", 503) + return + } + } + original.ServeHTTP(w, q) + }) + if _, e = s.LeadCluster(t.Context(), 0); e == nil { + t.Fatal("R6 interrupted request claimed success") + } + path := filepath.Join(home, "cluster-leadership-"+s.Device.ID()+".json") + raw, e := os.ReadFile(path) + mustNil(t, e) + var pending leadershipSession + mustNil(t, json.Unmarshal(raw, &pending)) + if pending.Pending == nil { + t.Fatal("R6 interrupted request was discarded") + } + stop() + again, ok, e := Open(home) + mustNil(t, e) + if !ok { + t.Fatal("reopen") + } + receipt, e := again.LeadCluster(t.Context(), 0) + mustNil(t, e) + mu.Lock() + same := len(requests) == 2 && bytes.Equal(requests[0], requests[1]) + mu.Unlock() + if !same || receipt.Node != pending.Pending.Change.RequestLeadership.Node { + t.Fatal("R6 restarted home replaced its saved operation") + } +} + +// R6 keeps ordinary directory mutations, reads and rotation on the forwarded path. +func TestR6HostedLeaderPublicDirectoryVerbs(t *testing.T) { + r := newHostedLeadRig(t) + p := r.home(nil) + hostedPersonalLeader(t, p) + id := p.Identity + b := r.home(&id) + ctx := t.Context() + mustNil(t, b.Dir.PutDevice(ctx, b.Device.ID(), directory.Device{V: 1, Name: "c2VhbGVk", Platform: "linux"})) + const cell = "01J0000000000000000000000B" + head := strings.Repeat("2", 64) + view, e := b.Dir.Create(ctx, cell, directory.CellInit{Head: head}) + mustNil(t, e) + view, e = b.Dir.Heartbeat(ctx, cell, directory.Beat{Fence: view.Cell.Lease.Fence, Pending: 1}) + mustNil(t, e) + view, e = b.Dir.Publish(ctx, cell, directory.Publish{Fence: view.Cell.Lease.Fence, OldHead: head, Head: head, Size: 1}) + mustNil(t, e) + mustNil(t, b.Dir.Release(ctx, cell, view.Cell.Lease.Fence)) + view, e = b.Dir.Acquire(ctx, cell, directory.AcquireOpts{}) + mustNil(t, e) + mustNil(t, b.Dir.Release(ctx, cell, view.Cell.Lease.Fence)) + mustNil(t, b.Dir.Archive(ctx, cell)) + got, e := b.Dir.Cell(ctx, cell) + mustNil(t, e) + if !got.Cell.Archived { + t.Fatal("R6 forwarded archive was not applied") + } + listing, e := b.Dir.List(ctx) + mustNil(t, e) + vault := strings.Repeat("3", 64) + mustNil(t, b.Dir.SetVault(ctx, listing.Identity.Vault, vault)) + listing, e = b.Dir.List(ctx) + mustNil(t, e) + if listing.Identity.Vault != vault { + t.Fatal("R6 forwarded vault CAS was not applied") + } + _, e = b.Dir.Presence(ctx) + mustNil(t, e) + _, e = b.Dir.Rotation(ctx) + mustNil(t, e) + _, e = b.Dir.Rotate(ctx, directory.RotationReq{Op: directory.OpFreeze}) + mustNil(t, e) + _, e = b.Dir.Rotate(ctx, directory.RotationReq{Op: directory.OpThaw}) + mustNil(t, e) + seed, e := identity.NewSeed() + mustNil(t, e) + cert := identity.IssueCert(id, seed.Public(), time.Now()) + revoked := cert.DeviceID() + dev, e := seed.Join(cert) + mustNil(t, e) + caller := directory.NewHTTP(r.server.URL, reqsign.SignFor(deviceSigner{id, dev}, time.Now), r.server.Client()) + mustNil(t, caller.PutDevice(ctx, revoked, directory.Device{V: 1, Name: "c2VhbGVk", Platform: "linux"})) + mustNil(t, b.Dir.Revoke(ctx, revoked)) + if _, e = caller.List(ctx); e == nil { + t.Fatal("R6 revoked forwarded caller retained authority") + } +} + +// R6/R13/R27 retain the largest legal command through the existing chunk lane. +func TestR6R13R27HostedLeaderKeepsMaximumPublicCommand(t *testing.T) { + r := newHostedLeadRig(t) + p := r.home(nil) + hostedPersonalLeader(t, p) + status, err := p.clusterClient().Status(t.Context()) + mustNil(t, err) + encodedSize := func(n int) int { + t.Helper() + body, _ := json.Marshal(directory.Device{V: 1, Name: strings.Repeat("x", n)}) + request, err := http.NewRequest(http.MethodPut, r.server.URL+"/v1/dir/devices/"+p.Device.ID(), bytes.NewReader(body)) + mustNil(t, err) + reqsign.Sign(request, body, deviceSigner{p.Identity, p.Device}, time.Now()) + proof, err := reqsign.Capture(request, body) + mustNil(t, err) + command, err := cs.V1Command(status.Cluster, time.Now().UnixMilli(), proof) + mustNil(t, err) + raw, err := cs.EncodeCommand(command) + mustNil(t, err) + return len(raw) + } + low, high := 0, cs.MaxCommandBytes + for low < high { + middle := (low + high + 1) / 2 + if encodedSize(middle) <= cs.MaxCommandBytes { + low = middle + } else { + high = middle - 1 + } + } + if encodedSize(low) > cs.MaxCommandBytes || encodedSize(low) < cs.MaxCommandBytes-4 { + t.Fatal("R27 fixture missed the legal command boundary") + } + mustNil(t, p.Dir.PutDevice(t.Context(), p.Device.ID(), directory.Device{V: 1, Name: strings.Repeat("x", low)})) + _, err = p.Dir.List(t.Context()) + mustNil(t, err) + p.member.mu.Lock() + app, _, err := p.member.store.ApplicationView(t.Context()) + key := raftstore.Key{Namespace: raftstore.State, Key: cs.DeviceKey(p.Device.ID()).Key} + var rows map[raftstore.Key][]byte + if err == nil { + rows, _, err = p.member.store.Read(t.Context(), app.Applied, []raftstore.Key{key}) + } + p.member.mu.Unlock() + mustNil(t, err) + var saved cs.Device + mustNil(t, cs.Decode(rows[key], &saved)) + if saved.Name != strings.Repeat("x", low) { + t.Fatal("R6 maximum public command was not applied on P") + } +} diff --git a/internal/syncsetup/member_hosted_learner_test.go b/internal/syncsetup/member_hosted_learner_test.go index ae64e1035d..ff5129866a 100644 --- a/internal/syncsetup/member_hosted_learner_test.go +++ b/internal/syncsetup/member_hosted_learner_test.go @@ -15,7 +15,6 @@ import ( "testing" "time" - "github.com/Agent-Field/codeaf/internal/clusterctl" cs "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/directory" "github.com/Agent-Field/codeaf/internal/identity" @@ -80,7 +79,7 @@ func hostedReady(t *testing.T, s *Sync) uint64 { for time.Now().Before(deadline) { status, e := s.clusterClient().Status(t.Context()) mustNil(t, e) - if len(status.Configuration.Voters) != 3 || !status.LeadershipUnavailable { + if len(status.Configuration.Voters) != 3 || status.LeadershipUnavailable { t.Fatalf("R9 hosted voter/leadership policy: %+v", status) } if status.Self != 0 && slices.Contains(status.Ready, status.Self) && slices.Contains(status.Configuration.Learners, status.Self) { @@ -123,12 +122,11 @@ func TestR9R23HostedLearnerRunStateReopenAndWipe(t *testing.T) { if !slices.Contains(status.Configuration.Learners, node) || !slices.Contains(status.Configuration.Learners, otherNode) { t.Fatal("R9 both homes must remain learners") } - _, e = a.LeadCluster(t.Context(), 0) - if !errors.Is(e, clusterctl.ErrUnsupported) { - t.Fatalf("R9 leadership refusal: %v", e) + if status.LeadershipUnavailable { + t.Fatal("R6 joined hosted home cannot request leadership") } if _, e = os.Stat(filepath.Join(home, "cluster-leadership-"+a.Device.ID()+".json")); !errors.Is(e, os.ErrNotExist) { - t.Fatal("R9 unsupported lead wrote an intent", e) + t.Fatal("R9 joining allocated a leadership request", e) } stop() reopened, _, e := Open(home) diff --git a/relay/hosted/src/identity.js b/relay/hosted/src/identity.js index 178c43873f..f56c8e9019 100644 --- a/relay/hosted/src/identity.js +++ b/relay/hosted/src/identity.js @@ -64,18 +64,28 @@ export class IdentityDO extends DurableObject { /** webSocketClose completes the close handshake the client started, and notes when a device's last socket went. */ async webSocketClose(ws, code) { finishClose(ws, code); - if(!peerSocket(ws))await this.#noteLeft(ws); + if(peerSocket(ws))await this.#peerDeparted(ws);else await this.#noteLeft(ws); } webSocketError(ws) { - if(!peerSocket(ws))return this.#noteLeft(ws); + if(peerSocket(ws))return this.#peerDeparted(ws);return this.#noteLeft(ws); + } + + async #peerDeparted(ws){ + if(this.host?.peers)return this.host.peers.departed(ws); + // A retained voter list is only a wake hint. Disconnection grants no + // leadership or membership authority, including after hibernation. + const voters=await this.ctx.storage.get('raft-router-voters'); + if(voters?.includes(ws.deserializeAttachment()?.node)){ + await this.ctx.storage.put('raft-maintenance',true);await this.ctx.storage.sync();await this.#arm(Date.now()); + } } /** #noteLeft records last_seen of a device that holds no socket now; the offline frame follows 15 s later, by alarm. */ async #noteLeft(ws) { const device = this.watchers.left(ws); const identity = device && (await this.ctx.storage.get('identity')); - if (identity && !(await this.#isGone()) && this.host?.ready) await this.host.serial(async()=>{await this.host.wakeMaintenance();await this.host.commitEffect({device_seen:{device}},Date.now());}); + if (identity && !(await this.#isGone()) && this.host?.ready) await this.host.operation(async()=>{await this.host.wakeMaintenance();await this.host.commitEffect({device_seen:{device}},Date.now());}); } async #serve(request) { @@ -89,7 +99,7 @@ export class IdentityDO extends DurableObject { const raw=socket?fromBase64Url(request.headers.get('Codeaf-Cluster-Auth')):await readBody(request,host.peers.limits.Exchange,'too_large'); if(!raw)return peerRefusal({peerCode:'invalid'}); if(socket)return await host.peers.accept(request,raw,url.pathname.endsWith('/forward')?'forward':'peer'); - const result=await host.peers.exchange(raw);if(result.rejected)return peerRefusal({peerCode:'busy'}); + const result=await host.peers.httpExchange(raw);if(result.rejected)return peerRefusal({peerCode:'busy'}); return new Response(result.raw,{headers:{'content-type':'application/json','cache-control':'no-store'}}); }catch(error){return peerRefusal(error);} } @@ -103,10 +113,10 @@ export class IdentityDO extends DurableObject { if(request.method==='GET'&&body.length)return new Response('unexpected request body',{status:400}); const tenant=this.#tenantOf(who.identity);this.#admit(tenant,who.device); this.host??=new HostedIdentity(this.ctx,this.env,tenant,this.legacyAdmission,at=>this.#arm(at)); - const reply=await this.host.serial(async()=>{ + const reply=await this.host.operation(async()=>{ await this.host.ensure(fromBase64Url(request.headers.get('codeaf-identity'))); await this.host.wakeMaintenance(); - return enrolling?this.host.enroll(this.host.proof(request,body)):this.host.control(request.method,who.device,request.headers.get('codeaf-cert')); + return enrolling?this.host.enroll(this.host.proof(request,body)):this.host.control(request.method,who.device,request.headers.get('codeaf-cert'),this.host.proof(request,body)); });this.watchers=this.host.watchers;reply.headers.set('cache-control','no-store');return reply; } if(url.pathname==='/v2/cluster/api'){ @@ -121,7 +131,7 @@ export class IdentityDO extends DurableObject { const identity=await identityOf(request.headers.get('codeaf-identity')); const tenant=this.#tenantOf(identity);this.#admit(tenant,actor); this.host??=new HostedIdentity(this.ctx,this.env,tenant,this.legacyAdmission,at=>this.#arm(at)); - const reply=await this.host.serial(()=>this.host.api(raw,proof,actor)); + const reply=await this.host.operation(()=>this.host.api(raw,proof,actor)); this.watchers=this.host.watchers;return reply; } const route = matchRoute(request.method, url.pathname); @@ -148,9 +158,9 @@ export class IdentityDO extends DurableObject { await this.#refuseGone(); const provision=await this.ctx.storage.get('raft-provision'),identity=await this.ctx.storage.get('identity'); if(!provision||!identity)throw Object.assign(new Error('member authorization refused'),{peerCode:'unauthorized'}); - if(!this.host?.ready){ + if(!this.host?.ready&&!this.host?.peers){ const tenant=this.#tenantOf(identity);this.host??=new HostedIdentity(this.ctx,this.env,tenant,this.legacyAdmission,at=>this.#arm(at)); - await this.host.serial(()=>this.host.ensure(unb64(provision.config.signing_key)));this.watchers=this.host.watchers; + await this.host.operation(()=>this.host.ensure(unb64(provision.config.signing_key)));this.watchers=this.host.watchers; } return this.host; } @@ -207,7 +217,7 @@ export class IdentityDO extends DurableObject { if(retirement&&!retireDue)await this.#arm(retirement); if(!maintenance&&!retireDue)return; const tenant=this.#tenantOf(identity);this.host ??= new HostedIdentity(this.ctx,this.env,tenant,this.legacyAdmission,at=>this.#arm(at)); - const erase=await this.host.serial(async()=>{ + const erase=await this.host.operation(async()=>{ await this.host.ensure(Uint8Array.from(atob(provision.config.signing_key),c=>c.charCodeAt(0))); this.watchers=this.host.watchers;tenant.stats.flush(); // A retained deadline is only a wake hint. Retirement still needs the diff --git a/relay/hosted/src/raft/README.md b/relay/hosted/src/raft/README.md index 356f1f8560..4d129f5362 100644 --- a/relay/hosted/src/raft/README.md +++ b/relay/hosted/src/raft/README.md @@ -110,8 +110,8 @@ settlement signature or declares a join successful. `joinStatus(barrier)` additi checks current authorization and the learner's own applied index. The identity coordinator exposes personal learner admission, committed registry -routing and status through public authenticated doors. Voter swaps, personal -leadership and delegated learner activation are not available here yet. +routing, status and personal leadership through public authenticated doors. +Delegated learner activation is not available here yet. Provisioned anchor keys receive restricted committed authority for membership maintenance and request-session closure. Personal enrollment or retirement still @@ -169,8 +169,8 @@ Public source serialization spans coordination, while external blob I/O leaves that queue and passes a new authority barrier afterward. Public watch sockets carry presence without the legacy socket-vouch promise; v1 clients keep ordinary beats. The hosted relay admits personal learners and serves their status, exchange -and hibernating sockets. Leadership requests remain unavailable; its committed -holds and idle maintenance follow the shared Go policy. Explicit retirement has its own retained +and hibernating sockets. Joined computers can explicitly request leadership; +committed holds and idle maintenance follow the shared Go policy. Explicit retirement has its own retained all-replica erasure journal and never serves as pressure cleanup. @@ -179,8 +179,15 @@ all-replica erasure journal and never serves as pressure cleanup. `POST /v2/cluster/enroll` retains the original personal certificate and signed store token. Shared Go admission allocates from committed state; configuration is a separate resumable stage. Status uses a fresh quorum read and Go projection. -An unsupported leadership request creates no intent. The application forwarding -upgrade is authenticated and passive until hosted leadership is available. +Leadership sequences the shared Go planner through catch-up, joint configuration, +durable leave and settlement before transfer. A receipt requires the elected +personal core’s current-term commit, a fresh read, durable completion and another +fresh receipt. The application forwarding connection authenticates every frame +with provisioned anchor keys and the current personal member key. Bounded pending +requests release the coordinator turn while waiting, and peer traffic continues +on its independent queue. Original signed commands remain unchanged. Fresh read +and admission indices must reach an anchor’s durable application before replies. +ABI 39 preserves the native signed application wire without network imports. The router's durable generation advances before its volatile queues reopen. Peer and application sockets use the hibernation API with a separate tag and @@ -202,7 +209,7 @@ coordination never waits for a learner; peer frames bypass its public queue. ## Sleeping accounts and advisory presence The coordinator persists an outstanding-maintenance hint only while consensus -work is scheduled. All-voter sleep removes it and clears stale peer retention +work is scheduled. With the anchor topology, all-voter sleep removes it and clears stale peer retention deadlines before cancelling the maintenance alarm. Presence and explicit retirement deadlines are then rearmed independently. An alarm on a cold sleeping object reads socket attachments and that hint without constructing a tenant, diff --git a/relay/hosted/src/raft/forward.js b/relay/hosted/src/raft/forward.js new file mode 100644 index 0000000000..1212b9da0c --- /dev/null +++ b/relay/hosted/src/raft/forward.js @@ -0,0 +1,41 @@ +import {Flight} from '../flight.js'; +import {frame,response} from './abi.js'; +import {encodeCanonical} from './codec.js'; +import {unb64} from './storage.js'; + +const enc=new TextEncoder(),dec=new TextDecoder('utf-8',{fatal:true}); +const unavailable=()=>new Error('personal member unavailable'); + +// Public waits retain only their operation queue. Replies resolve bounded +// tokens independently, allowing the peer stream to finish durable replication. +export class PersonalForward { + constructor(host){this.host=host;this.pending=new Map();} + socket(node){return this.host.peers.sockets().find(ws=>{const a=ws.deserializeAttachment();return a.kind==='forward'&&a.node===node&&a.generation===this.host.peers.generation;});} + async begin(node,work){ + const host=this.host,ws=this.socket(node); + if(!ws||this.pending.size>=host.peers.limits.Inflight)throw unavailable(); + host.peers.validate(unb64(ws.deserializeAttachment().auth)); + const id=work.id??crypto.randomUUID(); + if(this.pending.has(id))throw unavailable(); + const value={command:null,revision:'0',barrier:'0',schedule:null,now:0,...work,scope:host.peers.scope,generation:host.peers.generation,from:host.manifest.config.anchors[0],to:node,id}; + const r=response(host.runtime.call(frame(39).u32(0).blob(enc.encode(encodeCanonical(value))).blob(unb64(host.provisioning.anchors[0]).slice(-32)).finish())),raw=r.blob();r.end(); + const flight=new Flight(),pending={ws,node,release:flight.enter(),reply:null}; + const promise=flight.idle(host.peers.limits.ForwardTimeout).then(completed=>{if(!completed)this.finish(id,null);return pending.reply;}); + this.pending.set(id,pending); + try{ws.send(raw);}catch{this.finish(id,null);} + return {id,promise}; + } + finish(id,reply){const p=this.pending.get(id);if(!p)return;this.pending.delete(id);p.reply=reply;p.release();this.host.signal();} + frame(ws,raw){ + const host=this.host,a=ws.deserializeAttachment(); + host.peers.validate(unb64(a.auth)); + if(!(raw instanceof ArrayBuffer))throw unavailable(); + const peer=host.peers.view[a.node],r=response(host.runtime.call(frame(39).u32(1).blob(new Uint8Array(raw)).blob(unb64(peer.key)).finish())); + if(!r.u32())throw unavailable(); + const f=JSON.parse(dec.decode(r.blob()));r.end(); + const wire=JSON.parse(dec.decode(new Uint8Array(raw))); + if(wire.scope.identity!==host.peers.scope.identity||wire.scope.cluster!==host.peers.scope.cluster||wire.scope.epoch!==host.peers.scope.epoch||wire.scope.protocol!==host.peers.scope.protocol||wire.generation!==host.peers.generation||wire.from!==a.node||wire.to!==host.manifest.config.anchors[0])throw unavailable(); + const p=this.pending.get(f.id);if(p?.ws===ws&&p.node===wire.from)this.finish(f.id,f); + } + departed(ws){for(const [id,p]of this.pending)if(p.ws===ws)this.finish(id,null);} +} diff --git a/relay/hosted/src/raft/hosted.js b/relay/hosted/src/raft/hosted.js index f8f56c8a84..2d1a97fe5f 100644 --- a/relay/hosted/src/raft/hosted.js +++ b/relay/hosted/src/raft/hosted.js @@ -1,3 +1,5 @@ +import { Flight } from '../flight.js'; +import { PersonalForward } from './forward.js'; import { PersonalPeers } from './peers.js'; import { coordinationPolicy,CommittedHolds } from './liveness.js'; import { clusterIngress } from './ingress.js'; @@ -19,7 +21,7 @@ const enc=new TextEncoder(),dec=new TextDecoder('utf-8',{fatal:true}); const segment=s=>Array.from(enc.encode(s),b=>b.toString(16).padStart(2,'0')).join(''); const unsegment=s=>dec.decode(Uint8Array.from(s.match(/../g)??[],h=>parseInt(h,16))); const data=v=>enc.encode(encodeCanonical(v)); -const MIGRATION_PAGE=64; +const MIGRATION_PAGE=64,COORDINATOR_QUEUE=64; const unavailable=()=>new Wire('unreachable',503,1); const status={not_found:404,unauthorized:401,revoked:401,rotated:410,bad_request:400,bad_grace:400,self_revoke:400,capacity:507,full:507}; function requireOK(code){if(code!=='ok')throw new Wire(code,status[code]??409);} @@ -37,12 +39,51 @@ export class HostedIdentity { this.armAlarm=arm??(async at=>{if(at===null){await ctx.storage.deleteAlarm();return;}const old=await ctx.storage.getAlarm();if(old===null||at=64)return Promise.reject(unavailable());this.pending++; + if(this.pending>=COORDINATOR_QUEUE)return Promise.reject(unavailable());this.pending++; const task=this.tail.then(fn);this.tail=task.catch(()=>{});return task.finally(()=>this.pending--); } + operation(fn){ + if(this.operations>=COORDINATOR_QUEUE)return Promise.reject(unavailable());this.operations++; + const task=this.operationTail.then(fn);this.operationTail=task.catch(()=>{});return task.finally(()=>this.operations--); + } + signal(){this.progressVersion++;for(const resolve of this.waiters)resolve();this.waiters.clear();} + async progress(deadline,version=this.progressVersion){ + if(Date.now()>=deadline)throw unavailable(); + if(version!==this.progressVersion)return; + const flight=new Flight(),release=flight.enter();this.waiters.add(release); + try{if(!await flight.idle(Math.max(0,deadline-Date.now()))||Date.now()>=deadline)throw unavailable();} + finally{release();this.waiters.delete(release);} + } + replica(){return [...this.states].filter(([id])=>this.members.has(id)).sort(([,a],[,b])=>a.applied===b.applied?Number(b.term-a.term):a.applied>b.applied?-1:1)[0]?.[0];} + leaderID(){const hints=[...this.states].filter(([id])=>this.members.has(id)).sort(([,a],[,b])=>a.term===b.term?(a.applied>b.applied?-1:a.appliedb.term?-1:1);return this.leader()??String(hints[0]?.[1].leader??0);} + async forward(node,work){ + const token=await this.serial(()=>this.forwarding.begin(node,work)); + const reply=await token.promise; + if(!reply||reply.error)throw Object.assign(unavailable(),{forwardID:token.id}); + return reply; + } + async refreshRoute(){ + for(const id of this.members.keys())await this.input(id,0);await this.pump(); + } + async remote(work){ + work={...work,id:work.id??crypto.randomUUID()}; + const target=this.leaderID(); + try{return await this.forward(target,work);}catch(error){ + await this.refreshRoute(); + const next=this.leaderID(); + if(!this.leader()&&next!=='0'&&next!==target)return this.forward(next,{...work,id:error.forwardID??work.id}); + throw error; + } + } + async appliedReplica(index,deadline=Date.now()+this.peers.limits.ForwardTimeout){ + for(;;){const id=this.replica();if(!id||!index||BigInt(index)===0n)throw unavailable(); + if(this.states.get(id).applied>=BigInt(index))return {node:id,index:BigInt(index)}; + await this.progress(deadline); + } + } member(id){return this.env.RAFT_MEMBERS.get(this.env.RAFT_MEMBERS.idFromName(this.manifest.config.cluster+'/'+id));} config(id){ const c=this.manifest.config; @@ -97,7 +138,7 @@ export class HostedIdentity { }catch(error){this.members.delete(id);this.states.delete(id);if(!owned)throw error;} } if(this.members.size<2)throw unavailable(); - await this.pump();await this.elect();await this.barrier(); + await this.pump();if(owned)await this.restorePeers();await this.elect();await this.barrier(); if(!owned)await this.source.own(receipts); this.source.route(()=>true); this.watchers=new CommittedWatchers(this.ctx,this.tenant.limits.maxWatchers,this.tenant.watchers.ttlMs,at=>this.ctx.waitUntil(this.armAlarm(at))); @@ -112,6 +153,12 @@ export class HostedIdentity { } const root=await this.barrierRecord({namespace:'state',key:'identity'}); await this.rememberRetirement(this.project('identity',root)); + if(!this.peers)await this.restorePeers();else await this.peers.recover(); + this.membershipPending=!!(header.raw&&decodeCanonical(dec.decode(header.raw)).pending); + await this.reconcileMembership(); + await this.wakeMaintenance();this.ready=true; + } + async restorePeers(){ const generation=await this.ctx.storage.get('raft-router-generation')??'0'; if(typeof generation!=='string'||!/^(0|[1-9][0-9]*)$/.test(generation)||BigInt(generation)>=(1n<<64n)-1n)throw unavailable(); const next=String(BigInt(generation)+1n); @@ -120,11 +167,8 @@ export class HostedIdentity { for(const ws of this.peers.sockets())this.peers.close(ws); await this.peers.refresh(); await this.peers.recover(); - this.membershipPending=!!(header.raw&&decodeCanonical(dec.decode(header.raw)).pending); - await this.reconcileMembership(); - await this.wakeMaintenance();this.ready=true; } - accept(id,e){if(!e)return;if(e.status)this.states.set(String(id),e.status);this.queue.push(...(e.send??[]).map(m=>({...m,from:String(id),to:String(m.to)})));this.reads.push(...(e.reads??[]).map(r=>({...r,node:String(id)})));} + accept(id,e){if(!e)return;if(e.status)this.states.set(String(id),e.status);this.queue.push(...(e.send??[]).map(m=>({...m,from:String(id),to:String(m.to)})));this.reads.push(...(e.reads??[]).map(r=>({...r,node:String(id)})));this.signal();} async input(id,tag,value){const result=await this.members.get(String(id)).hostInput(tag,value);this.accept(id,result);return result;} // Reserve durable sequence ranges before emission. Concurrent public and // peer turns share one allocator, and recovery skips every unused sequence. @@ -186,27 +230,48 @@ export class HostedIdentity { } } } - leader(){return [...this.states].find(([id,s])=>this.members.has(id)&&s.role===2)?.[0];} + leader(){const term=[...this.states.values()].reduce((max,s)=>s.term>max?s.term:max,0n);return [...this.states].find(([id,s])=>this.members.has(id)&&s.term===term&&s.role===2)?.[0];} + externalVoter(){return Object.entries(this.peers?.view??{}).some(([node,peer])=>peer.voter&&!peer.revoked&&!this.manifest.config.anchors.includes(node));} async elect(){ - if(this.leader())return; + if(this.leader()||(this.leaderID()!=='0'&&!this.members.has(this.leaderID())))return; + if(this.transferTarget){const deadline=Date.now()+this.peers.limits.ForwardTimeout;while(!this.leader()&&(this.leaderID()==='0'||this.members.has(this.leaderID())))await this.progress(deadline);return;} for(const id of this.members.keys()){await this.input(id,16);await this.pump();if(this.leader())return;} throw unavailable(); } - async barrier(){ - await this.elect();const leader=this.leader(),id=crypto.randomUUID();this.reads=[]; - await this.input(leader,4,id);await this.pump(); - const read=this.reads.find(r=>r.node===leader&&r.id===id);this.reads=[]; - if(!read) {await this.input(leader,10,id);throw unavailable();} - return {node:leader,index:read.index}; + async barrier(id=crypto.randomUUID(),deadline=Date.now()+(this.peers?.limits.ForwardTimeout??this.policy.tick_every)){ + await this.elect();const leader=this.leader(); + if(!leader){try{const reply=await this.remote({action:'read',id});return this.appliedReplica(reply.barrier,deadline);}catch(error){if(this.leader())return this.barrier(id,deadline);throw error;}} + try{await this.input(leader,4,id);await this.pump();} + catch(error){await this.refreshRoute();if(this.leader()!==leader)return this.barrier(id,deadline);throw error;} + try{for(;;){ + if(this.leader()!==leader){await this.refreshRoute();if(this.leader()!==leader)return this.barrier(id,deadline);} + const at=this.reads.findIndex(r=>r.node===leader&&r.id===id);if(at>=0)return this.reads.splice(at,1)[0]; + if(!this.externalVoter())throw unavailable();await this.progress(deadline); + }} + finally{await this.input(leader,10,id);this.reads=this.reads.filter(r=>r.id!==id);} } - async commit(raw){ - await this.elect();const id=this.leader(),proposal=await this.input(id,3,raw);await this.pump(); - const result=await this.members.get(id)?.hostResult(raw,proposal.proposalAfter);if(!result)throw unavailable(); - if(this.peers)await this.peers.refresh(); - return {raw:result.raw,value:decodeCanonical(dec.decode(result.raw)),index:result.index}; + async commit(raw,deadline=Date.now()+(this.peers?.limits.ForwardTimeout??this.policy.tick_every)){ + await this.elect();const id=this.leader(); + if(!id){ + let reply;try{reply=await this.remote({action:'submit',command:b64(raw)});}catch(error){if(this.leader())return this.commit(raw,deadline);throw error;} + if(!reply.result)throw unavailable();await this.appliedReplica(reply.applied,deadline); + await this.peers.refresh();return {raw:data(reply.result),value:reply.result,index:BigInt(reply.applied)}; + } + let proposal; + try{proposal=await this.input(id,3,raw);await this.pump();} + catch(error){await this.refreshRoute();if(this.leader()!==id)return this.commit(raw,deadline);throw error;} + for(;;){ + if(this.leader()!==id){await this.refreshRoute();if(this.leader()!==id)return this.commit(raw,deadline);} + const version=this.progressVersion,result=await this.members.get(id)?.hostResult(raw,proposal.proposalAfter); + if(result){if(this.peers)await this.peers.refresh();return {raw:result.raw,value:decodeCanonical(dec.decode(result.raw)),index:result.index};} + // All hosted persistence and replies have drained. Only an external + // voter can still make progress without another input to this operation. + if(!this.externalVoter())throw unavailable(); + await this.progress(deadline,version); + } } async commitEffect(change,now=Date.now()){ - const r=response(this.runtime.call(frame(30).str(this.manifest.config.cluster).u64(now).blob(unb64(this.provisioning.effects)).blob(enc.encode(JSON.stringify(change))).finish())); + const r=response(this.runtime.call(frame(30).str(this.manifest.config.cluster).u64(now).blob(unb64(this.provisioning.effects)).blob(data(change)).finish())); const raw=r.blob();r.end();return (await this.commit(raw)).value; } async barrierRecord({namespace,key}){const b=await this.barrier();return (await this.members.get(b.node).hostView(b.index,namespace,key)).raw;} @@ -288,10 +353,18 @@ export class HostedIdentity { // Scanning is an internal activity hint only. Delivery obtains its own // fresh barrier and authority check before touching a public link request. const leader=this.leader(); + if(!leader&&this.leaderID()!=='0'&&!this.members.has(this.leaderID())){ + for(const [id,member]of this.members){this.accept(id,await member.schedule(this.elapsed,now,this.forwarding.pending.size>0));} + await this.pump(); + // External leadership keeps its ordinary clock until every voting + // member can participate in a completed sleep transition. + if(this.forwarding.socket(this.leaderID()))await this.forward(this.leaderID(),{action:'schedule',now,schedule:{now:this.elapsed,busy:true,marker:{index:'0',term:'0',barrier:'0',voters:null},acks:null}}); + await this.armAlarm(now+this.policy.tick_every);return; + } if(!leader){ // A missing leader still needs ordinary follower election ticks. An // unavailable authority read cannot be a reason to cancel that clock. - for(const [id,member]of this.members){const effects=await member.schedule(this.elapsed,now,false);this.accept(id,effects);} + for(const [id,member]of this.members){const effects=await member.schedule(this.elapsed,now,this.forwarding.pending.size>0);this.accept(id,effects);} await this.pump();await this.armAlarm(now+this.policy.tick_every);return; } let scan=this.outboxScan??''; @@ -301,7 +374,7 @@ export class HostedIdentity { await this.effects.replay(code=>this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(code)),scan,now); } let plan; - for(const [id,member]of this.members){const effects=await member.schedule(this.elapsed,now,false);this.accept(id,effects);if(id===this.leader())plan=effects.plan;} + for(const [id,member]of this.members){const effects=await member.schedule(this.elapsed,now,this.forwarding.pending.size>0);this.accept(id,effects);if(id===this.leader())plan=effects.plan;} await this.pump(); if(plan?.barrier){ const id=crypto.randomUUID();this.reads=[];await this.input(this.leader(),15,id);await this.pump(); @@ -325,7 +398,7 @@ export class HostedIdentity { const acks=[];for(const voter of marker.voters)acks.push(await this.members.get(voter).sleepAck(marker.index,marker.term)); let sleeping=true; for(const [node,member]of this.members){const effects=await member.schedule(this.elapsed,now,false,acks);this.accept(node,effects);sleeping&&=effects.plan.sleeping;} - await this.pump();this.asleep=sleeping&&wakeVersion===this.wakeVersion; + await this.pump();this.asleep=sleeping&&this.forwarding.pending.size===0&&wakeVersion===this.wakeVersion; if(this.asleep&&this.peers){await this.peers.quiesce();await this.pump();await this.sleepMaintenance(wakeVersion);} }catch{this.asleep=false;} } @@ -334,7 +407,7 @@ export class HostedIdentity { } async personal(action,{proof,actor='',certificate=''}={}){ const b=await this.barrier(); - return this.members.get(b.node).hostPersonal(b.index,{action,identity:this.manifest.config.identity,epoch:this.manifest.config.generation,generation:this.peers.generation,leader:this.leader(),actor,certificate, + return this.members.get(b.node).hostPersonal(b.index,{action,identity:this.manifest.config.identity,epoch:this.manifest.config.generation,generation:this.peers.generation,leader:this.leaderID(),actor,certificate, proof:proof??{method:'',uri:'',identity_key:null,certificate:'',at:'',signature:null,body:null},progress:[...this.peers.progress].map(([node,applied])=>({node,applied}))}); } @@ -343,7 +416,9 @@ export class HostedIdentity { for(let stage=0;stage<3;stage++){ const raw=await this.barrierRecord({namespace:'membership',key:'registry'}),h=decodeCanonical(dec.decode(raw)); if(!h.pending){this.membershipPending=false;await this.peers.refresh();return;} - const id=this.leader(),plan=await this.members.get(id).membership();this.accept(id,plan);await this.pump(); + const id=this.leader();let plan; + if(id){plan=await this.members.get(id).membership();this.accept(id,plan);await this.pump();} + else {const reply=await this.remote({action:'membership'});if(!reply.membership)throw unavailable();plan={change:reply.membership.Change,settle:reply.membership.Settle};if(plan.change)continue;} if(plan.settle){requireOK((await this.anchorCommand({settle_membership:{revision:h.revision,node:h.pending.node}},Date.now())).value.code);} } const h=decodeCanonical(dec.decode(await this.barrierRecord({namespace:'membership',key:'registry'})));this.membershipPending=!!h.pending; @@ -368,11 +443,42 @@ export class HostedIdentity { throw unavailable(); } personalRefusal(out){const r=response(this.runtime.call(frame(38).u32(2).str(out.code).finish())),message=r.str();r.end();return new Response(out.message||message||'this computer is not allowed to join this cluster',{status:out.code==='capacity'?429:out.code==='update_required'?409:403,headers:{'cache-control':'no-store'}});} - async control(method,actor,certificate){ - await this.reconcileMembership();const out=await this.personal('status',{actor,certificate}); - if(out.code!=='ok')return new Response('this computer is not allowed to coordinate this cluster',{status:403}); - if(method==='POST')return new Response('this relay does not offer cluster leadership yet; your saved chats are still here',{status:404}); - return this.jsonBytes(unb64(out.wire)); + async reconcileLeadership(){ + const raw=await this.barrierRecord({namespace:'membership',key:'registry'}),h=decodeCanonical(dec.decode(raw)),l=h.leadership,id=this.leader(); + if(!l||!id)return false; + const target=l.restore?l.standby:l.node;let evidence={node:'0',revision:'0',applied:'0',protocol:0,durable:false}; + try{ + if(this.members.has(target))evidence=await this.members.get(target).leadershipEvidence(h.revision,l.barrier); + else {try{evidence=(await this.forward(target,{action:'evidence',revision:h.revision,barrier:l.barrier})).evidence??evidence;}catch{const request=crypto.randomUUID();await this.input(id,15,request);await this.input(id,10,request);await this.pump();return false;}} + if(this.leader()!==id)return true; + const result=await this.members.get(id).leadership(evidence);if(result.plan.transfer!=='0')this.transferTarget=result.plan.transfer;this.accept(id,result);await this.pump();const plan=result.plan; + if(plan.configure){await this.peers.refresh();return true;} + if(plan.settle){requireOK((await this.anchorCommand({settle_leadership:{revision:h.revision}},Date.now())).value.code);return true;} + return false; + }catch(error){ + if(/requires the elected leader/.test(error.message)){await this.input(id,0);await this.pump();if(this.leader()!==id)return true;} + if(/target|transfer|busy|barrier not applied/.test(error.message))return false;throw error; + } + } + async control(method,actor,certificate,proof){ + await this.reconcileMembership();const out=await this.personal(method==='POST'?'lead':'status',{actor,certificate,proof}); + if(out.code!=='ok')return new Response('this computer is not allowed to coordinate this cluster',{status:out.code==='bad_request'?400:403}); + if(method!=='POST')return this.jsonBytes(unb64(out.wire)); + const command=decodeCanonical(dec.decode(unb64(out.wire))),result=await this.commit(unb64(out.wire)); + const refusal=code=>new Response(encodeCanonical({code,sequence:command.sequence}),{status:409,headers:{'content-type':'application/json'}}); + if(result.value.code!=='ok')return refusal(result.value.code); + const deadline=Date.now()+this.peers.limits.ForwardTimeout; + while(Date.now()this.serve(c,route)); + if(!new URL(c.request.url).pathname.startsWith('/v1/store/'))return this.operation(()=>this.serve(c,route)); const cert=c.request.headers.get('codeaf-cert'); - await this.serial(async()=>{ + await this.operation(async()=>{ await this.ensure(fromBase64Url(c.request.headers.get('codeaf-identity'))); await this.authorize(c.device,cert,route.write); }); // Blob waits cannot hold the coordination queue: revoke/rotation must be // able to commit during I/O, and Has must not block queued frame puts. const result=await route.handler(c,route.args);await this.ctx.storage.sync(); - await this.serial(()=>this.authorize(c.device,cert,route.write));return result; + await this.operation(()=>this.authorize(c.device,cert,route.write));return result; } async serve(c,route){ const {request,device,body}=c,url=new URL(request.url),path=url.pathname,cert=request.headers.get('codeaf-cert'); diff --git a/relay/hosted/src/raft/peers.js b/relay/hosted/src/raft/peers.js index 197f366f63..150811ace0 100644 --- a/relay/hosted/src/raft/peers.js +++ b/relay/hosted/src/raft/peers.js @@ -24,27 +24,46 @@ export async function expirePeerSockets(ctx,now,arm){ // before a recovered coordinator can serve another exchange or socket frame. export class PersonalPeers { constructor(host,generation){ - this.host=host;this.generation=generation;this.scope=host.config('1').scope;this.view={};this.boxes=new Map();this.used=[0,0,0];this.progress=new Map();this.snapshots=new Map();this.acks=new Map();this.tail=Promise.resolve();this.pending=0;this.blocked=new Set();this.checkpoints=new Map(); + this.host=host;this.generation=generation;this.scope=host.config('1').scope;this.view={};this.boxes=new Map();this.used=[0,0,0];this.progress=new Map();this.snapshots=new Map();this.acks=new Map();this.tail=Promise.resolve();this.pending=0;this.blocked=new Set();this.retryTargets=new Set();this.checkpoints=new Map(); this.limits=peerLimits(host.runtime); } serial(fn){if(this.pending>=64)return Promise.reject(refused('busy'));this.pending++;const p=this.tail.then(fn);this.tail=p.catch(()=>{});return p.finally(()=>this.pending--);} members(){return Object.entries(this.view).map(([node,p])=>({node,...p}));} + // Public and peer turns publish in order, so a delayed member read cannot + // restore an older transport authority after another turn has advanced it. async refresh(){ - const id=this.host.leader();if(!id)return; + if(this.refreshing){await this.refreshing;return this.refresh();} + this.refreshing=this.refreshView(); + try{await this.refreshing;}finally{this.refreshing=null;} + } + async refreshView(){ + const id=this.host.leader()??this.host.replica();if(!id)return; const applied=String(this.host.states.get(id).applied);if(this.viewApplied===id+'/'+applied)return; const view=await this.host.members.get(id).refreshEnrollment(); - this.viewApplied=id+'/'+applied; - if(JSON.stringify(view)===JSON.stringify(this.view))return; - this.view=view;this.used=[0,0,0]; + if(JSON.stringify(view)===JSON.stringify(this.view)){this.viewApplied=id+'/'+applied;return;} + const previous=this.view; + const unchanged=node=>previous[node]&&view[node]&&!view[node].revoked&&previous[node].incarnation===view[node].incarnation&&previous[node].key===view[node].key&&previous[node].voter===view[node].voter; + const voters=Object.keys(view).filter(id=>view[id].voter&&!view[id].revoked); + if(JSON.stringify(voters)!==JSON.stringify(this.host.voterHint)){ + await this.host.ctx.storage.put('raft-router-voters',voters);await this.host.ctx.storage.sync();this.host.voterHint=voters; + } + this.view=view;this.viewApplied=id+'/'+applied;this.used=[0,0,0]; for(const [node,p]of Object.entries(view)){ const old=this.boxes.get(node); - this.boxes.set(node,{queues:[[],[],[]],next:old?.next??[0n,0n,0n],issued:old?.issued??[0n,0n,0n],acked:old?.acked??[0n,0n,0n],credits:old?.credits??new Map()}); + // A new learner cannot discard an admitted quorum reply between existing + // members. Changed authority still fences all of its old deliveries. + const queues=old&&unchanged(node)?old.queues.map(lane=>lane.filter(q=>unchanged(q.envelope.from))):[[],[],[]]; + for(let lane=0;lane<3;lane++)for(const q of queues[lane])this.used[lane]+=q.size; + this.boxes.set(node,{queues,next:old?.next??[0n,0n,0n],issued:old?.issued??[0n,0n,0n],acked:old?.acked??[0n,0n,0n],credits:old?.credits??new Map()}); if(p.revoked)this.progress.delete(node); } - for(const [to,s]of this.snapshots){this.snapshots.delete(to);await this.host.input(s.from,12,{peer:to,snapshot:true,success:false});} + for(const [to,s]of this.snapshots)if(!unchanged(to)||!unchanged(s.from)){this.snapshots.delete(to);await this.host.input(s.from,12,{peer:to,snapshot:true,success:false});} for(const ws of this.sockets())if(this.view[ws.deserializeAttachment().node]?.revoked)this.close(ws); + const leader=this.host.leader(); + if(leader){const request=crypto.randomUUID();await this.host.input(leader,15,request);await this.host.input(leader,10,request);} + else {const target=this.host.leaderID();if(this.host.forwarding.socket(target)){const token=await this.host.forwarding.begin(target,{action:'poke'});token.promise.catch(()=>{});}} } - trusted(){const s=this.host.states.get(this.host.leader());return {scope:this.scope,generation:this.generation,members:this.members(),applied:String(s?.applied??0),term:String(s?.term??0),sleeping:this.host.asleep};} + trusted(){const s=this.host.states.get(this.host.replica());return {scope:this.scope,generation:this.generation,members:this.members(),applied:String(s?.applied??0),term:String(s?.term??0),sleeping:this.host.asleep};} validate(raw){ const r=response(this.host.runtime.call(frame(37).blob(raw).blob(data(this.trusted())).finish()));const result=JSON.parse(dec.decode(r.blob()));r.end(); if(result.code!=='ok')throw refused(result.code); @@ -95,7 +114,7 @@ export class PersonalPeers { const r=response(this.host.runtime.call(frame(38).u32(1).blob(enc.encode(JSON.stringify({...extra,chunks,generation:this.generation,messages:this.deliveries(node)}))).finish()));const raw=r.blob();r.end();return raw; } async recover(){ - const from=this.host.leader(); + const from=this.host.leader();if(!from)return; for(const [to,member]of Object.entries(this.view))if(!member.revoked&&!this.host.members.has(to)){ await this.host.input(from,12,{peer:to,snapshot:true,success:false}); await this.host.input(from,12,{peer:to,snapshot:false,success:false}); @@ -104,19 +123,31 @@ export class PersonalPeers { } async send(m){ await this.refresh(); - if(this.host.asleep||this.blocked.has(m.to)||!this.view[m.to]||this.view[m.to].revoked)return; + if(this.host.asleep||!this.view[m.to]||this.view[m.to].revoked)return; + if(this.blocked.has(m.to)){this.retryTargets.add(m.to);return;} const inspected=this.host.codec.inspect(4,m.bytes),snapshot=inspected.snapshot; if(m.bytes.length>this.limits.Batch){ - this.blocked.add(m.to);const applied=String(this.host.states.get(m.from).applied); + this.blocked.add(m.to);this.retryTargets.add(m.to);const applied=String(this.host.states.get(m.from).applied); if(this.checkpoints.get(m.from)!==applied){await this.host.members.get(m.from).hostCheckpoint();this.checkpoints.set(m.from,applied);} await this.host.input(m.from,12,{peer:m.to,snapshot:false,success:false});return; } const lane=this.view[m.from]?.voter&&this.view[m.to]?.voter?0:1; - if(this.boxes.get(m.to).queues[lane].length>=this.limits.Slots[lane]){this.blocked.add(m.to);await this.host.input(m.from,12,{peer:m.to,snapshot:snapshot.length>0,success:false});return;} + if(this.boxes.get(m.to).queues[lane].length>=this.limits.Slots[lane]){this.blocked.add(m.to);this.retryTargets.add(m.to);await this.host.input(m.from,12,{peer:m.to,snapshot:snapshot.length>0,success:false});return;} try{ this.enqueue([JSON.parse(dec.decode(await this.host.envelope(m.from,m.to,m.bytes)))]); if(snapshot.length){const meta=this.host.codec.inspect(3,snapshot),manifest=JSON.parse(dec.decode(meta.data));this.snapshots.set(m.to,{from:m.from,manifest,generation:this.host.hash(meta.data),offset:0,waiting:null,started:Date.now()});} - }catch(e){if(e.peerCode!=='busy')throw e;this.blocked.add(m.to);await this.host.input(m.from,12,{peer:m.to,snapshot:snapshot.length>0,success:false});} + }catch(e){if(e.peerCode!=='busy')throw e;this.blocked.add(m.to);this.retryTargets.add(m.to);await this.host.input(m.from,12,{peer:m.to,snapshot:snapshot.length>0,success:false});} + } + // A full voter queue can drop every response to one fresh read. Retry only + // after its acknowledgments free the reservation, using the elected member's + // ordinary IdleRead; the original public read still needs its own completion. + async retry(node){ + if(!this.retryTargets.has(node))return; + if(!this.view[node]?.voter||this.view[node].revoked||this.host.members.has(node)){this.retryTargets.delete(node);return;} + if(this.boxes.get(node).queues[0].length||this.host.leaderID()!==node||!this.host.forwarding.socket(node))return; + this.retryTargets.delete(node); + try{const token=await this.host.serial(()=>this.host.forwarding.begin(node,{action:'poke'}));token.promise.catch(()=>{});} + catch{this.retryTargets.add(node);} } async chunks(){ if(this.host.asleep)return; @@ -153,9 +184,23 @@ export class PersonalPeers { if(req.applied)this.progress.set(req.node,String(BigInt(req.applied)>BigInt(this.progress.get(req.node)??0)?req.applied:this.progress.get(req.node))); for(const e of req.messages??[])if(!obsolete.has(e.sequence))await this.host.wakeMaintenance(); await this.anchors();await this.chunks();await this.host.pump(); - await this.push(req.node); + await this.retry(req.node);await this.push(req.node); return {request:req,raw:this.wire(req.node,{[rejected?'rejected':'accepted']:(req.messages??[]).map(e=>e.sequence)}),rejected}; });} + async httpExchange(raw){ + const result=await this.exchange(raw),req=result.request,leader=this.host.leaderID(); + const behind=BigInt(req.applied??0)<(this.host.states.get(this.host.replica())?.applied??0n); + if(this.host.asleep||this.host.leader()||this.view[req.node]?.voter||!behind||JSON.parse(dec.decode(result.raw)).messages.length||!this.host.forwarding.socket(leader))return result; + // The personal sender handles this learner's acknowledgment asynchronously. + // Release the peer queue before asking it to retry, so a completed snapshot + // can receive its following append before the home abandons HTTP activation. + try{await this.host.forward(leader,{action:'poke'});}catch{return result;} + return this.serial(async()=>{ + if(this.host.peers!==this)throw refused('generation'); + await this.refresh();this.validate(raw); + return {...result,raw:this.wire(req.node,{[result.rejected?'rejected':'accepted']:(req.messages??[]).map(e=>e.sequence)})}; + }); + } async push(except){ await this.refresh(); for(const ws of this.sockets()){ @@ -184,11 +229,15 @@ export class PersonalPeers { async frame(ws,raw){ const att=ws.deserializeAttachment(); try{ - this.validate(unb64(att.auth));if(att.kind!=='peer'||!(raw instanceof ArrayBuffer))throw refused('invalid'); + this.validate(unb64(att.auth));if(att.kind==='forward'){this.host.forwarding.frame(ws,raw);return;}if(att.kind!=='peer'||!(raw instanceof ArrayBuffer))throw refused('invalid'); const bytes=new Uint8Array(raw),next=JSON.parse(dec.decode(bytes));if(next.node!==att.node||next.generation!==att.generation)throw refused('unauthorized'); const result=await this.exchange(bytes);ws.send(result.raw);const deadline=JSON.parse(dec.decode(result.raw)).messages.length?Date.now()+this.limits.Retention:null;ws.serializeAttachment({...att,last:this.host.hash(result.raw),deadline});if(deadline)await this.host.armAlarm(deadline); }catch{this.close(ws);} } + async departed(ws){ + this.host.forwarding.departed(ws); + if(this.view[ws.deserializeAttachment()?.node]?.voter){await this.host.wakeMaintenance();this.host.signal();} + } async quiesce(){ // Final voter acknowledgments discard learner delivery. Its old retention // deadline must not wake a cold router merely to reconnect idle sockets. diff --git a/relay/hosted/test/raft/build.mjs b/relay/hosted/test/raft/build.mjs index 6cdcbc4254..e7b3d18103 100644 --- a/relay/hosted/test/raft/build.mjs +++ b/relay/hosted/test/raft/build.mjs @@ -35,7 +35,7 @@ async function walk(path) { else if ((/\.(go|js|json|mod|sum|patch)$/.test(p) && !p.endsWith('_test.go')) || entry.name === 'LICENSE') source.push([relative(resolve('../..'), p), sha(await readFile(p))]); } } -for (const p of ['../../internal/raftsnapshot', '../../internal/raftcore', '../../internal/raftabi', '../../internal/clusterstate', '../../cmd/codeaf-raft-wasm', '../../third_party/raft', '../../third_party/raft-patch', 'src/raft']) await walk(resolve(p)); +for (const p of ['../../internal/raftsnapshot', '../../internal/raftcore', '../../internal/raftabi', '../../internal/clusterstate', '../../internal/clusterwire', '../../cmd/codeaf-raft-wasm', '../../third_party/raft', '../../third_party/raft-patch', 'src/raft']) await walk(resolve(p)); for (const p of ['../../go.mod','../../go.sum']) source.push([relative(resolve('../..'),resolve(p)),sha(await readFile(p))]); source.sort((a,b)=>a[0].localeCompare(b[0])); const manifest = { abi: 1, coreProtocol: 1, electionTiming: 2, rowSchema: 1, snapshotSchema: 1, snapshotFormatSHA256: sha(await readFile('../../internal/raftsnapshot/format.go')), go: 'go1.26.5', sourceHash: sha(JSON.stringify(source)), wasmSHA256: sha(wasm), sources: source }; diff --git a/relay/hosted/test/raft/hosted-control.test.js b/relay/hosted/test/raft/hosted-control.test.js index d1541e0ff3..694b872e81 100644 --- a/relay/hosted/test/raft/hosted-control.test.js +++ b/relay/hosted/test/raft/hosted-control.test.js @@ -4,7 +4,7 @@ import { stat } from 'node:fs/promises'; import { join } from 'node:path'; import { hostedFixture } from './hosted-fixture.js'; -// R9/R12/R23/R30 expose authenticated learner doors while leadership remains unavailable. +// R9/R12/R23/R30 expose authenticated learner doors with explicit personal leadership. test('R9 R12 R23 R30 hosted member doors and same-bundle restart',{timeout:30000},async()=>{ const h=await hostedFixture({base:'/fabric'}),a=h.client(); try{ @@ -15,8 +15,8 @@ test('R9 R12 R23 R30 hosted member doors and same-bundle restart',{timeout:30000 assert.deepEqual(status.configuration.voters,['1','2','3']); const enroll=async()=>{const r=await h.request(a,'POST','/v2/cluster/enroll',{store:'a'.repeat(64),protocol:1});assert.equal(r.status,200,await r.clone().text());return r.json();}; const first=await enroll(),second=await enroll();assert.equal(first.node,second.node);assert.notEqual(second.barrier,'0'); - r=await h.request(a,'GET','/v2/cluster/control');status=await r.json();assert.equal(status.self,first.node);assert(status.configuration.learners.includes(first.node));assert(!status.ready?.includes(first.node)); - r=await h.request(a,'POST','/v2/cluster/control');assert.equal(r.status,404);assert.equal((await r.text()).trim(),'this relay does not offer cluster leadership yet; your saved chats are still here'); + r=await h.request(a,'GET','/v2/cluster/control');status=await r.json();assert.equal(status.self,first.node);assert(status.configuration.learners.includes(first.node));assert(!status.ready?.includes(first.node));assert(!status.leadership_unavailable); + r=await h.request(a,'POST','/v2/cluster/control');assert.equal(r.status,400);await r.text(); for(const path of ['control','enroll','exchange','socket','forward']){ const r=await h.mf.dispatchFetch('http://local/fabric/v2/cluster/'+path,{method:path==='enroll'||path==='exchange'?'POST':'GET'});assert([400,403].includes(r.status),path+' '+r.status);await r.text(); } diff --git a/relay/hosted/test/raft/hosted-forward.test.js b/relay/hosted/test/raft/hosted-forward.test.js new file mode 100644 index 0000000000..455665a2b9 --- /dev/null +++ b/relay/hosted/test/raft/hosted-forward.test.js @@ -0,0 +1,189 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {generateKeyPairSync,sign,verify} from 'node:crypto'; +import {HostCodec,Reactor} from '../../src/raft/abi.js'; +import {PersonalForward} from '../../src/raft/forward.js'; +import {encodeCanonical} from '../../src/raft/codec.js'; +import {HostedIdentity} from '../../src/raft/hosted.js'; +import {PersonalPeers,peerLimits} from '../../src/raft/peers.js'; +const runtime=new Reactor(new WebAssembly.Module(readFileSync(process.env.CODEAF_RAFT_PROOF_DIR+'/core.wasm'))); +const keys=()=>{const pair=generateKeyPairSync('ed25519');return {...pair,raw:pair.publicKey.export({type:'spki',format:'der'}).subarray(-32).toString('base64'),private:pair.privateKey.export({type:'pkcs8',format:'der'}).toString('base64')};}; +function rig(){ + const anchor=keys(),personal=keys(),scope={identity:'identity',cluster:'cluster',epoch:'1',protocol:1},sent=[]; + const ws={deserializeAttachment:()=>({kind:'forward',node:'4',generation:'2',auth:''}),send:raw=>sent.push(raw)}; + const host=Object.assign(Object.create(HostedIdentity.prototype),{runtime,tail:Promise.resolve(),pending:0,operationTail:Promise.resolve(),operations:0,waiters:new Set(),provisioning:{anchors:[anchor.private]},manifest:{config:{anchors:['1','2','3']}},peers:{scope,generation:'2',limits:peerLimits(runtime),view:{'4':{key:personal.raw}},sockets:()=>[ws],validate(){}}}); + host.forwarding=new PersonalForward(host); + const reply=(id,barrier=10)=>{ + const value={scope,generation:'2',from:'4',to:'1',id,action:'read',barrier,status:{Leader:4,Term:2,Applied:barrier},applied:barrier,signature:null}; + value.signature=sign(null,Buffer.from('codeaf/cluster/forward/1\n'+JSON.stringify(value)),personal.privateKey).toString('base64'); + const raw=Buffer.from(JSON.stringify(value));return raw.buffer.slice(raw.byteOffset,raw.byteOffset+raw.byteLength); + }; + return {host,ws,sent,anchor,personal,reply}; +} + +// R6 public completion releases the member turn, preserving peer progress. +test('R6 a forwarded public wait leaves coordinator and peer replies live',async()=>{ + const r=rig();let completed=false; + const waiting=r.host.operation(()=>r.host.forward('4',{action:'read',id:'one'})).then(reply=>{completed=true;return reply;}); + await r.host.serial(()=>{});await r.host.serial(()=>{}); + assert.equal(completed,false);assert.equal(r.host.forwarding.pending.size,1); + const frame=JSON.parse(new TextDecoder().decode(r.sent[0]));const signature=Buffer.from(frame.signature,'base64');frame.signature=null; + assert(verify(null,Buffer.from('codeaf/cluster/forward/1\n'+JSON.stringify(frame)),r.anchor.publicKey,signature)); + r.host.forwarding.frame(r.ws,r.reply('one')); + const result=await waiting;assert.equal(result.barrier,'10');assert.equal(r.host.forwarding.pending.size,0); +}); + +// R6 signatures bind the exact reply, connection, scope and request token. +test('R6 altered or cross-connection replies cannot complete a public request',async()=>{ + const r=rig(),token=await r.host.serial(()=>r.host.forwarding.begin('4',{action:'read',id:'two'})); + const raw=r.reply('two'),value=JSON.parse(new TextDecoder().decode(raw));value.barrier++; + assert.throws(()=>r.host.forwarding.frame(r.ws,new TextEncoder().encode(JSON.stringify(value)).buffer)); + r.host.forwarding.frame({deserializeAttachment:r.ws.deserializeAttachment},raw);assert.equal(r.host.forwarding.pending.size,1); + r.host.forwarding.frame(r.ws,raw);assert.equal((await token.promise).barrier,'10'); +}); + +// R6/R27 application admission has the existing transport inflight bound. +test('R6 R27 application inflight admission stays bounded and disconnection is retryable',async()=>{ + const r=rig(),tokens=[]; + for(let n=0;nt.promise))).every(v=>v===null));assert.equal(r.host.forwarding.pending.size,0); +}); + +// R6 a stale address cannot allocate a new request or supply cached success. +test('R6 stale external hints retry the original request ID',async()=>{ + const calls=[];let target='4'; + const host=Object.assign(Object.create(HostedIdentity.prototype),{members:new Map([['1',{}]]),leader:()=>null,leaderID:()=>target,input:async()=>{target='5';},pump:async()=>{},forward:async(node,work)=>{calls.push({node,...work});if(node==='4')throw Object.assign(Error('retry'),{forwardID:work.id});return {id:work.id,barrier:'12'};}}); + const result=await host.remote({action:'read',id:'original'});assert.equal(result.barrier,'12');assert.deepEqual(calls.map(c=>[c.node,c.id]),[['4','original'],['5','original']]); +}); + +// R6/R22 committed renewal preparation retains exact unsigned ownership fences. +test('R6 R22 hosted effects preserve canonical renewal fences',async()=>{ + const r=rig(),fence='9007199254740993'; + r.host.provisioning.effects=Buffer.from(r.anchor.private,'base64').subarray(-32).toString('base64'); + r.host.manifest.config.cluster='cluster'; + r.host.commit=async raw=>({value:JSON.parse(new TextDecoder().decode(raw))}); + const command=await r.host.commitEffect({renew_holds:{device:'personal',seen_at:1000,items:[{cell:'cell',fence}]}},1000); + assert.equal(command.change.renew_holds.items[0].fence,fence); + assert.equal(command.change.renew_holds.seen_at,1000); +}); + +// R6 a changed local leader must perform the original request's fresh read. +test('R6 stale external hint retries the same ReadIndex context on a hosted leader',async()=>{ + let local=false;const inputs=[]; + const host=Object.assign(Object.create(HostedIdentity.prototype),{reads:[],peers:{limits:peerLimits(runtime)},elect:async()=>{},leader:()=>local?'1':undefined,remote:async work=>{assert.equal(work.id,'same-read');local=true;throw Error('stale address');},input:async(node,tag,id)=>{inputs.push({node,tag,id});if(tag===4)host.reads.push({node,id,index:9n});},pump:async()=>{}}); + const read=await host.barrier('same-read');assert.equal(read.index,9n); + assert.deepEqual(inputs.filter(i=>i.tag===4),[{node:'1',tag:4,id:'same-read'}]); +}); + +// R6 a missing old connection still reserves one ID before refreshing its hint. +test('R6 a stale disconnected hint cannot allocate a second forwarding ID',async()=>{ + const calls=[];let target='4'; + const host=Object.assign(Object.create(HostedIdentity.prototype),{members:new Map([['1',{}]]),leader:()=>null,leaderID:()=>target,input:async()=>{target='5';},pump:async()=>{},forward:async(node,work)=>{calls.push({node,...work});if(node==='4')throw Error('connection unavailable');return {id:work.id,barrier:'12'};}}); + await host.remote({action:'read'});assert.equal(typeof calls[0].id,'string');assert.equal(calls[0].id,calls[1].id); +}); + +// R6/R9 a joining home must receive the append after its snapshot without +// blocking delivery of the personal leader's response on the peer queue. +test('R6 R9 HTTP catch-up lets peer progress finish before returning an empty response',{timeout:peerLimits(runtime).ForwardTimeout},async()=>{ + let delivered=false; + const peer=Object.assign(Object.create((await import('../../src/raft/peers.js')).PersonalPeers.prototype),{tail:Promise.resolve(),pending:0,view:{'5':{voter:false}},refresh:async()=>{},validate:()=>{},wire:()=>new TextEncoder().encode(JSON.stringify({messages:delivered?['append']:[]})),exchange:async()=>({request:{node:'5',applied:'14',messages:[]},raw:new TextEncoder().encode('{"messages":[]}'),rejected:false})}); + const host={peers:peer,asleep:false,leader:()=>undefined,leaderID:()=> '4',replica:()=> '1',states:new Map([['1',{applied:15n}]]),forwarding:{socket:()=>({})},forward:async(node,work)=>{assert.equal(node,'4');assert.equal(work.action,'poke');await peer.serial(()=>{delivered=true;});return {barrier:'15'};}}; + peer.host=host; + const result=await peer.httpExchange(new Uint8Array());assert.deepEqual(JSON.parse(new TextDecoder().decode(result.raw)).messages,['append']); +}); + +// R6/R9 admitting another home cannot discard the elected computer's in-flight +// quorum reply while publishing the new home’s transport authority. +test('R6 R9 enrollment retains admitted traffic between unchanged members',async()=>{ + const r=rig(),members={'1':{incarnation:'1',key:r.anchor.raw,voter:true,revoked:false},'4':{incarnation:'1',key:r.personal.raw,voter:true,revoked:false}}; + let view=members; + const host={runtime,config:()=>({scope:r.host.peers.scope}),states:new Map([['1',{applied:13n}]]),leader:()=>undefined,replica:()=> '1',leaderID:()=> '4',members:new Map([['1',{refreshEnrollment:async()=>view}]]),ctx:{storage:{put:async()=>{},sync:async()=>{}}},voterHint:['1','4'],forwarding:{socket:()=>null},input:async()=>{},hash:()=>'',asleep:false}; + const peers=new PersonalPeers(host,'2');host.peers=peers;peers.sockets=()=>[];await peers.refresh(); + peers.enqueue([{from:'4',to:'1',sequence:'7',payload:'AQ=='}]); + const before=structuredClone(peers.deliveries('1')),used=[...peers.used]; + view={...members,'5':{incarnation:'1',key:r.personal.raw,voter:false,revoked:false}};host.states.get('1').applied=14n; + await peers.refresh(); + assert.deepEqual(peers.deliveries('1'),before);assert.deepEqual(peers.used,used);assert(peers.members().some(p=>p.node==='5')); + view={...view,'4':{...members['4'],revoked:true}};host.states.get('1').applied=15n;await peers.refresh(); + assert.deepEqual(peers.deliveries('1'),[]);assert.deepEqual(peers.used,[0,0,0]); +}); + +// R6/R9 concurrent publication cannot restore an older transport authority. +test('R6 R9 overlapping membership refreshes leave the latest committed view',async()=>{ + const first=Promise.withResolvers(),second=Promise.withResolvers();let calls=0; + const old={'1':{incarnation:'1',key:'anchor',voter:true,revoked:false}},newer={...old,'5':{incarnation:'1',key:'personal',voter:false,revoked:false}}; + const host={runtime,config:()=>({scope:rig().host.peers.scope}),states:new Map([['1',{applied:13n}]]),leader:()=>undefined,replica:()=> '1',leaderID:()=> '4',members:new Map([['1',{refreshEnrollment:()=>++calls===1?first.promise:second.promise}]]),ctx:{storage:{put:async()=>{},sync:async()=>{}}},voterHint:['1'],forwarding:{socket:()=>null},input:async()=>{},asleep:false}; + const peers=new PersonalPeers(host,'2');host.peers=peers;peers.sockets=()=>[]; + const earlier=peers.refresh();host.states.get('1').applied=14n;const later=peers.refresh(); + second.resolve(newer);await Promise.resolve();await Promise.resolve();first.resolve(old); + await Promise.all([earlier,later]);assert(peers.members().some(p=>p.node==='5')); +}); + +// R6/R27 a saturated transport may drop a read round, but draining its bounded +// queue must obtain a new quorum round without holding the peer queue. +test('R6 R27 a drained voter queue retries a dropped quorum round',async()=>{ + const r=rig(),view={'1':{incarnation:'1',key:r.anchor.raw,voter:true,revoked:false},'4':{incarnation:'1',key:r.personal.raw,voter:true,revoked:false}},reply=Promise.withResolvers(),actions=[]; + const host={runtime,codec:new HostCodec(runtime),config:()=>({scope:r.host.peers.scope}),states:new Map([['1',{applied:13n}]]),leader:()=>undefined,replica:()=> '1',leaderID:()=> '4',members:new Map([['1',{refreshEnrollment:async()=>view}]]),ctx:{storage:{put:async()=>{},sync:async()=>{}}},voterHint:['1','4'],forwarding:{socket:()=>({}),begin:async(node,work)=>{actions.push([node,work.action]);return {id:'retry',promise:reply.promise};}},serial:fn=>fn(),input:async()=>{},pump:async()=>{},wakeMaintenance:async()=>{},asleep:false}; + const peers=new PersonalPeers(host,'2');host.peers=peers;peers.sockets=()=>[];await peers.refresh();actions.length=0; + peers.anchors=async()=>{};peers.chunks=async()=>{};peers.push=async()=>{};peers.validate=raw=>({request:JSON.parse(new TextDecoder().decode(raw)),obsolete:new Set()}); + for(let n=0;n{});assert.deepEqual(actions,[['4','poke']]); + await peers.exchange(request);assert.deepEqual(actions,[['4','poke']]);reply.resolve(null); +}); + +// R6 a leadership change between evidence and dispatch resumes through a fresh +// barrier instead of turning the successful election into a refusal. +test('R6 leadership planning retries a route that changed during its member RPC',async()=>{ + let leader='1',polled=false; + const host=Object.assign(Object.create(HostedIdentity.prototype),{ + barrierRecord:async()=>new TextEncoder().encode(encodeCanonical({revision:'3',leadership:{node:'4',standby:'3',restore:false,barrier:'10'}})), + leader:()=>leader,members:new Map([['1',{leadership:async()=>{throw Error('raftcore: operation requires the elected leader');}}]]), + forward:async()=>({evidence:{node:'4'}}),input:async(id,tag)=>{assert.equal(id,'1');assert.equal(tag,0);polled=true;leader=undefined;},pump:async()=>{}, + }); + assert.equal(await host.reconcileLeadership(),true);assert(polled); +}); + +// R6 a hosted address that becomes stale must preserve the same public read +// context when the elected computer takes over during the short dispatch. +test('R6 stale hosted hint retries the original ReadIndex context on the personal leader',async()=>{ + let local=true;const ids=[]; + const host=Object.assign(Object.create(HostedIdentity.prototype),{members:new Map([['1',{}]]),peers:{limits:peerLimits(runtime)},elect:async()=>{},leader:()=>local?'1':undefined,leaderID:()=>local?'1':'4', + input:async(node,tag,id)=>{if(tag===4){ids.push(id);throw Error('raftcore: operation requires the elected leader');}assert.equal(tag,0);local=false;},pump:async()=>{}, + remote:async work=>{ids.push(work.id);return {barrier:'17'};},appliedReplica:async index=>{assert.equal(index,'17');return {node:'1',index:17n};}, + }); + assert.equal((await host.barrier('original-local-read')).index,17n);assert.deepEqual(ids,['original-local-read','original-local-read']); +}); + +// R6 a changed local address cannot replace an already signed proposal. +test('R6 stale hosted hint forwards the original signed proposal unchanged',async()=>{ + const command=Uint8Array.of(1,2,3);let local=true;const proposals=[]; + const host=Object.assign(Object.create(HostedIdentity.prototype),{members:new Map([['1',{}]]),peers:{refresh:async()=>{},limits:peerLimits(runtime)},elect:async()=>{},leader:()=>local?'1':undefined,leaderID:()=>local?'1':'4', + input:async(node,tag,raw)=>{if(tag===3){proposals.push([...raw]);throw Error('raftcore: operation requires the elected leader');}assert.equal(tag,0);local=false;},pump:async()=>{}, + remote:async work=>{proposals.push([...Buffer.from(work.command,'base64')]);return {applied:'18',result:{code:'ok'}};},appliedReplica:async index=>assert.equal(index,'18'), + }); + assert.equal((await host.commit(command)).value.code,'ok');assert.deepEqual(proposals,[[1,2,3],[1,2,3]]); +}); + +// R6 losing the hosted leader after dispatch also resumes the same operation. +test('R6 an unfinished hosted read follows the new leader with its original context',async()=>{ + let local=true,waits=0;const ids=[]; + const host=Object.assign(Object.create(HostedIdentity.prototype),{members:new Map([['1',{}]]),reads:[],peers:{limits:peerLimits(runtime)},elect:async()=>{},externalVoter:()=>true,leader:()=>local?'1':undefined,leaderID:()=>local?'1':'4', + input:async(node,tag,id)=>{if(tag===4)ids.push(id);else assert([0,10].includes(tag));},pump:async()=>{},progress:async()=>{if(waits++)throw Error('read did not follow the new leader');local=false;}, + remote:async work=>{ids.push(work.id);return {barrier:'17'};},appliedReplica:async()=>({node:'1',index:17n}), + }); + assert.equal((await host.barrier('pending-local-read')).index,17n);assert.deepEqual(ids,['pending-local-read','pending-local-read']); +}); + +// R6 an admitted proposal keeps its signature and session through a leader change. +test('R6 an unfinished hosted proposal follows the new leader without rewriting it',async()=>{ + const command=Uint8Array.of(1,2,3);let local=true,waits=0;const proposals=[]; + const host=Object.assign(Object.create(HostedIdentity.prototype),{members:new Map([['1',{hostResult:async()=>null}]]),peers:{refresh:async()=>{},limits:peerLimits(runtime)},elect:async()=>{},externalVoter:()=>true,leader:()=>local?'1':undefined,leaderID:()=>local?'1':'4', + input:async(node,tag,raw)=>{if(tag===3){proposals.push([...raw]);return {proposalAfter:1n};}assert.equal(tag,0);},pump:async()=>{},progress:async()=>{if(waits++)throw Error('proposal did not follow the new leader');local=false;}, + remote:async work=>{proposals.push([...Buffer.from(work.command,'base64')]);return {applied:'18',result:{code:'ok'}};},appliedReplica:async()=>{}, + }); + assert.equal((await host.commit(command)).value.code,'ok');assert.deepEqual(proposals,[[1,2,3],[1,2,3]]); +}); diff --git a/relay/hosted/test/raft/hosted-lead-go.test.js b/relay/hosted/test/raft/hosted-lead-go.test.js new file mode 100644 index 0000000000..4f5ef2b87e --- /dev/null +++ b/relay/hosted/test/raft/hosted-lead-go.test.js @@ -0,0 +1,37 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {spawn} from 'node:child_process'; +import {mkdtemp,rm} from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; +import {hostedFixture} from './hosted-fixture.js'; +const root=new URL('../../../../',import.meta.url); +const footprint=new URL('../../../../scripts/raft-proof-footprint.sh',import.meta.url).pathname; +async function run(args,env={},signal){ + const child=spawn('sh',[footprint,...args],{cwd:root,env:{...process.env,...env},stdio:['ignore','pipe','pipe'],signal}); + let output='';child.stdout.on('data',b=>{output+=b;process.stdout.write(b)});child.stderr.on('data',b=>{output+=b;process.stdout.write(b)}); + const code=await new Promise((resolve,reject)=>{child.once('error',reject);child.once('exit',resolve)});assert.equal(code,0,output);return output; +} +const dir=await mkdtemp(join(tmpdir(),'codeaf-lead-proof-')),binary=join(dir,'client.test'); +try{ + await run(['go','test','-p=1','./internal/syncsetup','-c','-o',binary]); + // Each identity proof gets a fresh isolate, retaining the shared admission cap. + for(const name of [ + 'TestR6R15R18HostedLeaderOriginalWorkerIngressAndThinStore', + 'TestR6HostedPersonalLeadershipBothHomesStatus', + 'TestR6HostedPersonalLeadershipLostReplyResumes', + 'TestR6HostedPersonalLeadershipCommittedRefusal', + 'TestR6HostedPersonalLeadershipHomeRestartBeforeAdmission', + 'TestR6HostedLeaderPublicDirectoryVerbs', + 'TestR6R13R27HostedLeaderKeepsMaximumPublicCommand', + ]){ + test('R6 R15 R18 '+name,{timeout:300000},async t=>{ + const h=await hostedFixture();try{ + await h.start();const url=await h.mf.ready; + const output=await run([binary,'-test.run=^'+name+'$','-test.count=1','-test.timeout=15m','-test.v'],{CODEAF_HOSTED_PROOF_URL:url.origin},t.signal); + assert.doesNotMatch(output,/--- SKIP:/);assert.match(output,new RegExp('--- PASS: '+name)); + }finally{await h.close();} + }); + } + test.after(()=>rm(dir,{recursive:true,force:true})); +}catch(error){await rm(dir,{recursive:true,force:true});throw error;} diff --git a/relay/hosted/test/raft/hosted-worker.js b/relay/hosted/test/raft/hosted-worker.js index 674b1f11f5..5fefb8afcc 100644 --- a/relay/hosted/test/raft/hosted-worker.js +++ b/relay/hosted/test/raft/hosted-worker.js @@ -66,7 +66,7 @@ export class IdentityDO extends Source { } return result; } - async sequenceProof(){return this.host.serial(async()=>{ + async sequenceProof(){return this.host.operation(async()=>{ const from=this.host.manifest.config.anchors[0],to=this.host.manifest.config.anchors[1],term=Number(this.host.states.get(this.host.leader()).term); const payload=Uint8Array.of(8,8,16,Number(to),24,Number(from),32,term); const messages=await Promise.all(Array.from({length:16},()=>this.host.envelope(from,to,payload))); @@ -76,7 +76,7 @@ export class IdentityDO extends Source { transportState(){return {keys:[...this.ctx.storage.kv.list()].map(([key])=>key),tables:this.ctx.storage.sql.exec("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '_cf_%'").toArray().map(r=>r.name)};} async pausePublic(){ let began;const started=new Promise(resolve=>{began=resolve;}); - this.ctx.waitUntil(this.host.serial(async()=>{began();await new Promise(resolve=>{this.releasePublic=resolve;});}));await started; + this.ctx.waitUntil(this.host.operation(async()=>{began();await new Promise(resolve=>{this.releasePublic=resolve;});}));await started; } resumePublic(){this.releasePublic?.();} async checkpointState(){ @@ -90,7 +90,7 @@ export class IdentityDO extends Source { } releaseHeldUpload(){this.releaseUpload?.();} async retryWithoutQuorum(){ - return this.host.serial(async()=>{ + return this.host.operation(async()=>{ const leader=this.host.leader();for(const id of [...this.host.members.keys()])if(id!==leader)this.host.members.delete(id); this.host.queue=[]; try{await this.host.commit(this.lastCommand);return 'acknowledged';}catch{return 'unavailable';} @@ -105,7 +105,7 @@ export class IdentityDO extends Source { await this.ctx.storage.put({identity,admitted:true});await this.ctx.storage.sync(); } async replayStale() { - return this.host.serial(async()=>{ + return this.host.operation(async()=>{ const from=this.host.leader(),to=this.host.manifest.config.anchors.find(id=>id!==from); const member=this.host.member(to),snapshot=await member.oldSnapshotAndCheckpoint(); const uint=v=>{let n=BigInt(v),b=[];do{const x=Number(n&127n);n>>=7n;b.push(x|(n?128:0));}while(n);return b;}; @@ -119,7 +119,7 @@ export class IdentityDO extends Source { // Restore through the production alarm before driving its virtual clock. if(!this.current.host)await this.current.alarm(); const host=this.current.host; - return host.serial(async()=>{ + return host.operation(async()=>{ const base=Date.now(),membershipBefore=measurements.membership; for(let n=1;n<=20;n++)await host.maintenance(base+n*host.policy.tick_every,host.started+n*host.policy.tick_every); const id=host.leader(),before=host.states.get(id).applied; From 960294cd8fabb9bc4ca4e4ac547ea445cf2ab682 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 10:33:54 -0400 Subject: [PATCH 070/103] Complete hosted fallback and idle personal leadership. Advance surviving voters before authority reads and use the shared leadership planner to restore the standby after election. Preserve quiet scheduling with pending work accounted for, durable sleep cursors and a continuous monotonic clock across coordinator activation. Cover process death, sync-off, cold wake, minority refusal and returning learners through the real client and local Worker. Document the fallback windows and quiet behavior on both relays. Keep background-job and self-hosting questions reaching their explanatory pages. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../manual/chat/coordination-and-workers.md | 6 +- .../chat/use-this-on-another-computer.md | 2 +- internal/manual/chat/what-i-can-do.md | 2 +- .../syncsetup/member_hosted_fallback_test.go | 311 ++++++++++++++++++ .../member_hosted_leader_quiet_test.go | 296 +++++++++++++++++ relay/hosted/src/identity.js | 8 +- relay/hosted/src/raft/README.md | 29 +- relay/hosted/src/raft/actor.js | 3 + relay/hosted/src/raft/hosted.js | 210 ++++++++---- relay/hosted/src/raft/member.js | 2 +- relay/hosted/src/raft/peers.js | 21 +- .../test/raft/hosted-fallback-go.test.js | 37 +++ .../hosted/test/raft/hosted-fallback.test.js | 55 ++++ relay/hosted/test/raft/hosted-worker.js | 37 ++- 14 files changed, 920 insertions(+), 99 deletions(-) create mode 100644 internal/syncsetup/member_hosted_fallback_test.go create mode 100644 internal/syncsetup/member_hosted_leader_quiet_test.go create mode 100644 relay/hosted/test/raft/hosted-fallback-go.test.js create mode 100644 relay/hosted/test/raft/hosted-fallback.test.js diff --git a/internal/manual/chat/coordination-and-workers.md b/internal/manual/chat/coordination-and-workers.md index 9666f5680e..b9744853bc 100644 --- a/internal/manual/chat/coordination-and-workers.md +++ b/internal/manual/chat/coordination-and-workers.md @@ -33,7 +33,7 @@ A lost reply or restarted home keeps the original signed leadership request. Ret ## Sleeping coordinator, fallback timing and no automatic regain -When a personal coordinator sleeps, exits or turns sync off, the two remaining anchor voters can elect a coordinator and restore the third anchor after it catches up. The missing computer need not approve its removal. A returning computer catches up as a learner and does not automatically regain leadership: request `codeaf cluster lead` again. +On the hosted relay and a self-hosted relay with storage, when a personal coordinator sleeps, exits or turns sync off, the two remaining anchor voters can elect a coordinator and restore the third anchor after it catches up. The missing computer need not approve its removal. A returning computer catches up as a learner and does not automatically regain leadership: request `codeaf cluster lead` again. Coordination uses five-second ticks. The provisioned anchors’ ranked election windows are six–seven, eight–nine and ten–eleven ticks (30–35, 40–45 and 50–55 seconds); other computers use twelve–thirteen ticks (60–65 seconds). Healthy fallback targets about 65 seconds. Scheduling delays or a partition can take longer. Fully idle groups stop periodic ticks only after every voter has applied the same idle barrier; a new operation or a leading computer’s disconnection wakes coordination and needs a fresh majority response. @@ -48,11 +48,11 @@ The sync connection carries presence hints and committed lease acknowledgments. ## Does syncing keep sending requests when nothing is happening? -A home that has never paired sends no sync traffic. `CODEAF_SYNC_URL=off` also blocks new sync requests from clients that were opened before the setting changed. Presence pings on an open view are hints; they do not write chat ownership. On the hosted relay and the native relay, an open idle home uses its learner’s applied vault pointer to notice shared memory and preference changes. Local edits are detected locally and pushed on the next state round. Over ten idle minutes, two caught-up open homes joined as learners make zero HTTP requests, zero peer or forwarding frames and zero client attempts. A caught-up idle home sends no recurring state polls or peer keepalives. Home also follows its local applied directory for display changes; its roster and backstop read that same local hint. Quiet neighbours’ presence can age out without forcing a network ping. Continuing a chat and all ownership decisions still use fresh majority reads. A committed change wakes peer delivery and the next state round pulls the changed vault. If an older hosted relay has no learner door, the home keeps its existing five-second state polling. On the consensus directory, a group with no work or held chats can stop its periodic coordination timer after an idle barrier has reached every voter. The first operation wakes coordination and must obtain a fresh majority response. A saved leader address cannot answer that operation by itself. Losing the majority leaves syncing unavailable and keeps local work intact. +A home that has never paired sends no sync traffic. `CODEAF_SYNC_URL=off` also blocks new sync requests from clients that were opened before the setting changed. Presence pings on an open view are hints; they do not write chat ownership. On the hosted relay and the native relay, an open idle home uses its learner’s applied vault pointer to notice shared memory and preference changes. Local edits are detected locally and pushed on the next state round. Over ten idle minutes, caught-up open homes make zero HTTP requests, zero peer or forwarding frames and zero client attempts, including while one home leads and after it sleeps and the anchors resume. A caught-up idle home sends no recurring state polls or peer keepalives. Home also follows its local applied directory for display changes; its roster and backstop read that same local hint. Quiet neighbours’ presence can age out without forcing a network ping. Continuing a chat and all ownership decisions still use fresh majority reads. A committed change wakes peer delivery and the next state round pulls the changed vault. If an older hosted relay has no learner door, the home keeps its existing five-second state polling. On the consensus directory, a group with no work or held chats can stop its periodic coordination timer after an idle barrier has reached every voter. The first operation wakes coordination and must obtain a fresh majority response. A saved leader address cannot answer that operation by itself. Losing the majority leaves syncing unavailable and keeps local work intact. ## Hosted idle presence sweeps and sleeping connections -With its three anchors voting, a hosted group with no work or held chats stops its maintenance alarm after every voter applies the idle barrier. Its three anchors then schedule no recurring alarms. Reconstructing the sleeping coordinator leaves its learner and forwarding sockets open; a presence sweep neither wakes the anchors nor reconnects those computers. The first real request obtains a fresh majority response before answering. +With its three anchors voting, or with a joined computer leading, a hosted group with no work or held chats stops its maintenance alarm after every voter applies the idle barrier. Its three anchors then schedule no recurring alarms. Reconstructing the sleeping coordinator leaves its learner and forwarding sockets open; a presence sweep neither wakes the anchors nor reconnects those computers. The first real request obtains a fresh majority response before answering. A compatibility events viewer pings every 10 seconds. The platform answers without running coordination. While somebody watches presence, the relay checks at the next 25-second silence deadline so a stopped computer appears offline within 30 seconds. With two viewers pinging together, the simulated ten-minute measurement uses 29 short sweeps; those sweeps make no member calls or data writes. With no events viewer, no presence sweep recurs. Home follows the local learner once joined, so its ordinary display needs no remote presence viewer. diff --git a/internal/manual/chat/use-this-on-another-computer.md b/internal/manual/chat/use-this-on-another-computer.md index 94f874ea8c..0bcd5bdb5b 100644 --- a/internal/manual/chat/use-this-on-another-computer.md +++ b/internal/manual/chat/use-this-on-another-computer.md @@ -282,7 +282,7 @@ If something reaches for the relay while quiet, it gets the word `sync is waitin The first-run line about the hosted relay is said once, when chats are first copied, after pairing or offering a code. `CODEAF_SYNC_URL=off` keeps even a paired machine from contacting a relay, including pairing with `--via`. -## Where your chats are stored — the hosted relay, turning it off, your own relay, is my code private +## Where your chats are stored — the hosted relay, turning it off, self-host your own relay, is my code private Moving chats and pairing are available by default on Linux, macOS and native Windows on every build with a compatible engine. Pair both computers first. diff --git a/internal/manual/chat/what-i-can-do.md b/internal/manual/chat/what-i-can-do.md index 44d42fd4e9..d209fc3b66 100644 --- a/internal/manual/chat/what-i-can-do.md +++ b/internal/manual/chat/what-i-can-do.md @@ -290,7 +290,7 @@ truly enormous amounts before it was kept as a job, the log begins where codeaf' rolling tail begins — the last few hundred kilobytes — rather than at the very first line. -## Does codeaf poll a background job, or does it get told — how does it know a job finished? +## Does codeaf poll a background job with repeated sleep and tail commands, or get told when it finishes? **It gets told, and it never has to poll.** Two things arrive without anybody asking for them. diff --git a/internal/syncsetup/member_hosted_fallback_test.go b/internal/syncsetup/member_hosted_fallback_test.go new file mode 100644 index 0000000000..5555b10478 --- /dev/null +++ b/internal/syncsetup/member_hosted_fallback_test.go @@ -0,0 +1,311 @@ +package syncsetup + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "slices" + "strings" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" +) + +type hostedFallbackState struct { + Leaders []string `json:"leaders"` + Errors []string `json:"errors"` + Leader uint64 `json:"leader"` + Cluster string `json:"cluster"` + Configuration cs.Configuration `json:"configuration"` + Membership cs.Membership `json:"membership"` + Campaigns int `json:"campaigns"` +} + +func hostedFallbackTicks(t *testing.T, s *Sync, ticks int, missing string) hostedFallbackState { + t.Helper() + raw, err := json.Marshal(map[string]any{"op": "leadership-ticks", "identity": s.Identity.ID(), "ticks": ticks, "missing": missing}) + mustNil(t, err) + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, hostedLearnerEndpoint(t)+"/fixture", bytes.NewReader(raw)) + mustNil(t, err) + resp, err := http.DefaultClient.Do(req) + mustNil(t, err) + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("R7 virtual maintenance: %s", resp.Status) + } + var out hostedFallbackState + body, err := io.ReadAll(resp.Body) + mustNil(t, err) + mustNil(t, cs.Decode(body, &out)) + return out +} +func hostedLeaderDeparted(t *testing.T, s *Sync) { + t.Helper() + deadline := time.NewTimer(requestTimeout) + defer deadline.Stop() + retry := time.NewTicker(10 * time.Millisecond) + defer retry.Stop() + for { + state := hostedQuietControl(t, hostedLearnerEndpoint(t), s.Identity.ID(), "quiet-state") + counts := hostedSocketCounts(t, state) + if counts["peer"] == 0 && counts["forward"] == 0 { + return + } + select { + case <-retry.C: + case <-deadline.C: + t.Fatal("R7 leader sockets did not close") + } + } +} + +// R7 requires surviving anchors to elect by their own timers and restore the +// standby without contacting the absent personal voter for permission. +func TestR7HostedPersonalLeaderFallbackAndNoAutomaticRegain(t *testing.T) { + r := newHostedLeadRig(t) + p, ok, err := Open(machine(t, r.server.URL)) + mustNil(t, err) + if !ok { + t.Fatal("open") + } + stop := hostedOwner(t, p) + receipt := hostedPersonalLeader(t, p) + saved := recordOf(t, p) + stop() + hostedLeaderDeparted(t, p) + before := hostedQuietControl(t, hostedLearnerEndpoint(t), p.Identity.ID(), "metrics") + state := hostedFallbackTicks(t, p, 13, "") + if state.Leader == 0 || state.Leader == receipt.Node || !slices.Equal(state.Configuration.Voters, []uint64{1, 2, 3}) || state.Configuration.Joint() || state.Membership.Leadership != nil || !slices.Contains(state.Configuration.Learners, receipt.Node) { + t.Fatalf("R7 fallback did not restore anchors within thirteen ticks: %+v", state) + } + if uint64(state.Campaigns) != hostedNumber(t, before, "campaigns") { + t.Fatal("R7 fallback used an explicit campaign") + } + const cell = "01J0000000000000000000FAL" + _, err = p.Dir.Create(t.Context(), cell, directory.CellInit{Head: strings.Repeat("1", 64)}) + mustNil(t, err) + _, err = p.Dir.Acquire(t.Context(), cell, directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: p.Device.ID(), Fence: 1}}) + mustNil(t, err) + hostedOwner(t, p) + hostedReady(t, p) + if nodeOf(p) != receipt.Node || recordOf(t, p).Store != saved.Store { + t.Fatal("R7 return replaced durable identity") + } + status, err := p.clusterClient().Status(t.Context()) + mustNil(t, err) + if status.Leader == receipt.Node || !slices.Contains(status.Configuration.Learners, receipt.Node) { + t.Fatal("R7 returning computer regained leadership", status) + } + again := hostedPersonalLeader(t, p) + if again.Node != receipt.Node { + t.Fatal("R7 explicit lead changed the returning node") + } +} + +// R8 keeps the existing cluster unavailable when two of its three voters are +// absent, even though a nonvoting anchor retains a complete learning copy. +func TestR8HostedPersonalLeaderAndAnchorLossNeverCreatesSecondCluster(t *testing.T) { + r := newHostedLeadRig(t) + p, ok, err := Open(machine(t, r.server.URL)) + mustNil(t, err) + if !ok { + t.Fatal("open") + } + stop := hostedOwner(t, p) + receipt := hostedPersonalLeader(t, p) + before, err := p.clusterClient().Status(t.Context()) + mustNil(t, err) + stop() + hostedLeaderDeparted(t, p) + state := hostedFallbackTicks(t, p, 13, "1") + if state.Cluster != before.Cluster || !slices.Equal(state.Configuration.Voters, []uint64{1, 2, receipt.Node}) || state.Membership.Leadership == nil || state.Membership.Leadership.Restore { + t.Fatal("R8 minority rewrote its cluster", state) + } + _, err = p.clusterClient().Status(t.Context()) + if err == nil || err.Error() != cs.LeadershipUnavailable { + t.Fatal("R8 minority status refusal", err) + } + _, err = p.Dir.Create(t.Context(), "01J0000000000000000000MIN", directory.CellInit{Head: strings.Repeat("1", 64)}) + if err == nil { + t.Fatal("R8 minority accepted a mutation") + } + after := hostedFallbackTicks(t, p, 1, "") + if after.Cluster != state.Cluster || !slices.Equal(after.Configuration.Voters, state.Configuration.Voters) { + t.Fatal("R8 minority manufactured a majority", after) + } +} + +// R7 kills a real personal member process without graceful shutdown. The +// surviving quorum must restore the standby before that process can return. +func TestR7HostedPersonalLeaderProcessDeathRestoresAnchors(t *testing.T) { + r := newHostedLeadRig(t) + p, ok, err := Open(machine(t, r.server.URL)) + mustNil(t, err) + if !ok { + t.Fatal("open") + } + cmd := exec.Command(os.Args[0], "-test.run=^TestR7HostedPersonalMemberProcessHelper$", "-test.v", "-test.timeout=15m") + cmd.Env = append(os.Environ(), "CODEAF_TEST_MEMBER_HOME="+p.Home) + stdout, err := cmd.StdoutPipe() + mustNil(t, err) + var stderr bytes.Buffer + cmd.Stderr = &stderr + mustNil(t, cmd.Start()) + defer func() { + if cmd.ProcessState == nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + } + }() + elected := make(chan cs.LeadershipReceipt, 1) + failure := make(chan error, 1) + go func() { + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + line := scanner.Text() + t.Logf("R7 child: %s", line) + if strings.HasPrefix(line, "R7 elected ") { + var receipt cs.LeadershipReceipt + if err := json.Unmarshal([]byte(strings.TrimPrefix(line, "R7 elected ")), &receipt); err != nil { + failure <- err + return + } + elected <- receipt + return + } + } + failure <- fmt.Errorf("R7 process exited: %v", scanner.Err()) + }() + var receipt cs.LeadershipReceipt + select { + case receipt = <-elected: + case err := <-failure: + t.Fatal(err, stderr.String()) + case <-time.After(requestTimeout): + t.Fatal("R7 process never led", stderr.String()) + } + status, err := p.clusterClient().Status(t.Context()) + mustNil(t, err) + if status.Leader != receipt.Node || receipt.Barrier == 0 { + t.Fatal("R7 process was not leader", status) + } + saved := recordOf(t, p) + mustNil(t, cmd.Process.Kill()) + if cmd.Wait() == nil { + t.Fatal("R7 process exited gracefully") + } + hostedLeaderDeparted(t, p) + before := hostedQuietControl(t, hostedLearnerEndpoint(t), p.Identity.ID(), "metrics") + state := hostedFallbackTicks(t, p, 13, "") + if state.Leader == 0 || state.Leader == receipt.Node || !slices.Equal(state.Configuration.Voters, []uint64{1, 2, 3}) || state.Configuration.Joint() || state.Membership.Leadership != nil || uint64(state.Campaigns) != hostedNumber(t, before, "campaigns") { + t.Fatal("R7 process death did not restore anchors by ordinary ticks", state) + } + const cell = "01J000000000000000000KILL" + _, err = p.Dir.Create(t.Context(), cell, directory.CellInit{Head: strings.Repeat("1", 64)}) + mustNil(t, err) + _, err = p.Dir.Acquire(t.Context(), cell, directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: p.Device.ID(), Fence: 1}}) + mustNil(t, err) + hostedOwner(t, p) + hostedReady(t, p) + if nodeOf(p) != receipt.Node || recordOf(t, p).Store != saved.Store { + t.Fatal("R7 process return changed durable identity") + } + status, err = p.clusterClient().Status(t.Context()) + mustNil(t, err) + if status.Leader == receipt.Node || !slices.Contains(status.Configuration.Learners, receipt.Node) { + t.Fatal("R7 killed process automatically regained leadership", status) + } + p.memberMu.Lock() + m := p.member + m.mu.Lock() + sequence := m.sequence + m.mu.Unlock() + p.memberMu.Unlock() + if sequence < saved.SequenceCeiling { + t.Fatal("R7 process reused a reserved transport sequence") + } +} + +// R7 keeps the child under the same RunState owner as an open terminal. Its +// parent kills this address space after observing the actual elected receipt. +func TestR7HostedPersonalMemberProcessHelper(t *testing.T) { + home := os.Getenv("CODEAF_TEST_MEMBER_HOME") + if home == "" { + return + } + p, ok, err := Open(home) + mustNil(t, err) + if !ok { + t.Fatal("open") + } + hostedOwner(t, p) + receipt := hostedPersonalLeader(t, p) + raw, err := json.Marshal(receipt) + mustNil(t, err) + fmt.Println("R7 elected " + string(raw)) + select {} +} + +// R26 requires a first cold read to use a real majority without interrupting +// the connected personal leader or discarding its durable routing progress. +func TestR26HostedPersonalLeaderColdReadKeepsQuorum(t *testing.T) { + r := newHostedLeadRig(t) + p := r.home(nil) + receipt := hostedPersonalLeader(t, p) + id := p.Identity + b := r.home(&id) + idle := hostedQuietControl(t, hostedLearnerEndpoint(t), id.ID(), "idle-schedule") + if string(idle["sleeping"]) != "true" { + t.Fatal("R26 group did not sleep", idle) + } + hostedQuietControl(t, hostedLearnerEndpoint(t), id.ID(), "evict-idle") + _, err := b.Dir.List(t.Context()) + if err != nil { + t.Logf("cold routing: %s", hostedQuietControl(t, hostedLearnerEndpoint(t), id.ID(), "quiet-state")) + } + mustNil(t, err) + status, err := b.clusterClient().Status(t.Context()) + mustNil(t, err) + if status.Leader != receipt.Node || !slices.Equal(status.Configuration.Voters, []uint64{1, 2, receipt.Node}) { + t.Fatal("R26 cold read interrupted personal leadership", status) + } +} + +// R7/R26 a sleeping leader's departure must wake a cold coordinator's real +// follower clocks even while initialization cannot obtain an authority read. +func TestR7R26HostedColdSleepingLeaderDepartureRestoresAnchors(t *testing.T) { + r := newHostedLeadRig(t) + p, ok, err := Open(machine(t, r.server.URL)) + mustNil(t, err) + if !ok { + t.Fatal("open") + } + tick := hostedTicks(t, p) + hostedReady(t, p) + receipt := hostedPersonalLeader(t, p) + tick() + hostedSettled(t, p) + idle := hostedQuietControl(t, hostedLearnerEndpoint(t), p.Identity.ID(), "idle-schedule") + if string(idle["sleeping"]) != "true" { + t.Fatal("R26 group did not sleep", idle) + } + hostedQuietControl(t, hostedLearnerEndpoint(t), p.Identity.ID(), "evict-idle") + p.stopMember() + hostedLeaderDeparted(t, p) + before := hostedQuietControl(t, hostedLearnerEndpoint(t), p.Identity.ID(), "metrics") + state := hostedFallbackTicks(t, p, 13, "") + if state.Leader == 0 || state.Leader == receipt.Node || state.Membership.Leadership != nil || state.Configuration.Joint() || !slices.Equal(state.Configuration.Voters, []uint64{1, 2, 3}) || uint64(state.Campaigns) != hostedNumber(t, before, "campaigns") { + t.Fatal("R7 R26 cold departure did not restore anchors through election timers", state) + } + _, err = p.Dir.Create(t.Context(), "01J00000000000000000COLD", directory.CellInit{Head: strings.Repeat("1", 64)}) + mustNil(t, err) + _, err = p.Dir.Acquire(t.Context(), "01J00000000000000000COLD", directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: p.Device.ID(), Fence: 1}}) + mustNil(t, err) +} diff --git a/internal/syncsetup/member_hosted_leader_quiet_test.go b/internal/syncsetup/member_hosted_leader_quiet_test.go new file mode 100644 index 0000000000..5f0aec4c71 --- /dev/null +++ b/internal/syncsetup/member_hosted_leader_quiet_test.go @@ -0,0 +1,296 @@ +package syncsetup + +import ( + "encoding/json" + "maps" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/dirwatch" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/store" +) + +// R7/R23/R26/R27 keep whole homes quiet with a personal leader and after its +// use-time sync-off departure, including cold presence and first-input wake. +func TestR23R26R27HostedLeadingHomesIdleEvictionAndWake(t *testing.T) { + hostedLeadingHomesQuiet(t, false) +} +func TestR7R23R26R27HostedSyncOffFallbackIdleEvictionAndWake(t *testing.T) { + hostedLeadingHomesQuiet(t, true) +} + +func hostedLeadingHomesQuiet(t *testing.T, fallback bool) { + endpoint := hostedLearnerEndpoint(t) + a, ok, e := Open(machine(t, endpoint)) + mustNil(t, e) + if !ok { + t.Fatal("open") + } + homeB := machine(t, endpoint) + _, e = identity.Adopt(homeB, a.Identity, true) + mustNil(t, e) + b, ok, e := Open(homeB) + mustNil(t, e) + if !ok { + t.Fatal("second open") + } + tickA, tickB := hostedTicks(t, a), hostedTicks(t, b) + hostedReady(t, a) + hostedReady(t, b) + receipt := hostedPersonalLeader(t, a) + live := []*Sync{a, b} + round := func() { tickA(); tickB() } + if fallback { + hostedQuietControl(t, endpoint, a.Identity.ID(), "idle-schedule") + hostedSettled(t, a) + hostedSettled(t, b) + waitMember(t, a, func(m *personalMember) bool { return m.sleeping }) + mustNil(t, SaveRelayURL(b.Home, endpoint)) + t.Setenv(URLVar, "") + attempts := a.relayAttempts.Load() + a.clusterFrames.Load() + mustNil(t, SaveRelayURL(a.Home, "off")) + mustNil(t, a.memberRound(t.Context())) + if a.member != nil || a.relayAttempts.Load()+a.clusterFrames.Load() != attempts { + t.Fatal("R7 R23 sync off sent work") + } + // The surviving learner remains connected, so only the leader's pair leaves. + deadline := time.Now().Add(requestTimeout) + for { + counts := hostedSocketCounts(t, hostedQuietControl(t, endpoint, a.Identity.ID(), "quiet-state")) + if counts["peer"] == 1 && counts["forward"] == 1 { + break + } + if time.Now().After(deadline) { + t.Fatal("R7 leading home did not sleep") + } + time.Sleep(time.Millisecond) + } + state := hostedFallbackTicks(t, b, 13, "") + if state.Leader == 0 || state.Leader == receipt.Node || state.Membership.Leadership != nil || !slices.Equal(state.Configuration.Voters, []uint64{1, 2, 3}) { + t.Fatal("R7 idle departure did not restore anchors", state) + } + live = []*Sync{b} + round = tickB + } + for n := 0; n < 3; n++ { + round() + for _, s := range live { + hostedSettled(t, s) + } + } + screens := []*Sync{} + follows := []dirwatch.Follower{} + streams := []dirwatch.Stream{} + watches := []dirwatch.Stream{} + for _, s := range live { + screen, ok, e := Open(s.Home) + mustNil(t, e) + if !ok { + t.Fatal("screen open") + } + screens = append(screens, screen) + follow := screen.Follow() + defer follow.Close() + // The local observer and the compatibility watch both remain open. Only + // the latter uses platform auto-response for advisory presence. + stream, e := screen.followStream(t.Context(), screen.Dir.(directory.Watcher)) + mustNil(t, e) + defer stream.Close() + streams = append(streams, stream) + watch, e := screen.Dir.(directory.Watcher).Watch(t.Context()) + mustNil(t, e) + defer watch.Close() + watches = append(watches, watch) + follows = append(follows, follow) + } + idle := hostedQuietControl(t, endpoint, a.Identity.ID(), "idle-schedule") + if string(idle["sleeping"]) != "true" || hostedNumber(t, idle, "recurringMemberCalls") != 0 { + t.Fatalf("R26 idle schedule: %s", idle) + } + for _, s := range live { + hostedSettled(t, s) + } + ping := func(stream dirwatch.Stream) { + mustNil(t, stream.Ping(t.Context())) + for { + frame, e := stream.Next(t.Context()) + mustNil(t, e) + if frame.Pong { + return + } + } + } + count := func() uint64 { + var n uint64 + for _, s := range append([]*Sync{a, b}, screens...) { + n += s.relayAttempts.Load() + s.clusterFrames.Load() + } + return n + } + before := hostedQuietControl(t, endpoint, a.Identity.ID(), "quiet-state") + var alarms map[string]json.RawMessage + mustNil(t, json.Unmarshal(before["memberAlarms"], &alarms)) + for node, alarm := range alarms { + if string(alarm) != "null" { + t.Fatalf("R26 sleeping voter %s retained alarm: %s", node, alarm) + } + } + if len(alarms) != 3 || string(before["maintenance"]) != "false" { + t.Fatalf("R26 sleeping group retained maintenance: %s", before) + } + if !maps.Equal(hostedSocketCounts(t, before), map[string]int{"peer": len(live), "forward": len(live), "watch": len(live)}) { + t.Fatalf("R26 missing retained sockets: %s", before) + } + generation := recordOf(t, a).Grant.Generation + cold := hostedQuietControl(t, endpoint, a.Identity.ID(), "evict-idle") + if string(cold["ready"]) != "false" { + t.Fatal("R26 fixture did not reconstruct coordinator") + } + hostedQuietControl(t, endpoint, a.Identity.ID(), "reset-metrics") + attempts := count() + pongs := 0 + for n := 0; n < 120; n++ { + round() + for _, follow := range follows { + follow.Probe() + } + for i, screen := range screens { + ping(streams[i]) + ping(watches[i]) + pongs++ + _, _, e := screen.RowsAt(t.Context()) + mustNil(t, e) + _, e = screen.Roster(t.Context()) + mustNil(t, e) + _, e = screen.FleetSize(t.Context()) + mustNil(t, e) + } + } + schedule := hostedQuietControl(t, endpoint, a.Identity.ID(), "presence-schedule") + if hostedNumber(t, schedule, "milliseconds") != 600000 { + t.Fatal("R27 missing ten-minute presence measurement") + } + metrics := hostedQuietControl(t, endpoint, a.Identity.ID(), "metrics") + httpCount := hostedNumber(t, metrics, "http") + frames := hostedNumber(t, metrics, "peerIn") + hostedNumber(t, metrics, "peerOut") + clientAttempts := count() - attempts + t.Logf("R27 hosted memory: peak core=%d reserved=%d budget=%d active anchors=%d", hostedNumber(t, metrics, "peakMemory"), hostedNumber(t, metrics, "reservedBytes"), hostedNumber(t, metrics, "coreBudget"), hostedNumber(t, metrics, "active")) + if hostedNumber(t, metrics, "peakMemory")+hostedNumber(t, metrics, "reservedBytes") > hostedNumber(t, metrics, "coreBudget") { + t.Fatal("R27 hosted core reservation exceeded its shared budget") + } + t.Logf("R23 R27 hosted personal-leader mode=%t ten-minute idle: Worker HTTP=%d peer/forward frames=%d client attempts=%d platform presence ping/pongs=%d sweeps=%d", fallback, httpCount, frames, clientAttempts, pongs, hostedNumber(t, schedule, "sweeps")) + if httpCount != 0 || frames != 0 || clientAttempts != 0 || hostedNumber(t, metrics, "writes") != 0 || hostedNumber(t, metrics, "turns") != 0 { + t.Fatalf("R27 idle activity: %s", metrics) + } + // R26/R27 retain real hibernating sockets across two complete presence + // windows. Auto-response timestamps, rather than a mocked renewal, age them. + hostedQuietControl(t, endpoint, a.Identity.ID(), "reset-metrics") + started := time.Now() + nextPing := time.NewTicker(dirwatch.KeepEvery) + defer nextPing.Stop() + for time.Since(started) < 2*dirwatch.DeadAfter+dirwatch.KeepEvery { + select { + case <-nextPing.C: + for _, watch := range watches { + ping(watch) + pongs++ + } + case <-t.Context().Done(): + t.Fatal(t.Context().Err()) + } + } + real := hostedQuietControl(t, endpoint, a.Identity.ID(), "metrics") + after := hostedQuietControl(t, endpoint, a.Identity.ID(), "quiet-state") + if hostedNumber(t, real, "sweeps") < 2 || hostedNumber(t, real, "turns") != 0 || hostedNumber(t, real, "writes") != 0 || hostedNumber(t, real, "peerIn")+hostedNumber(t, real, "peerOut") != 0 || hostedNumber(t, real, "http") != 0 { + t.Fatalf("R26 cold real-time presence: %s", real) + } + if string(after["ready"]) != "false" || string(after["generation"]) != string(before["generation"]) { + t.Fatal("R26 sweep activated coordinator or changed generation") + } + if !maps.Equal(hostedSocketCounts(t, before), hostedSocketCounts(t, after)) { + t.Fatal("R26 cold presence lost a retained socket") + } + t.Logf("R26 R27 real-time presence: duration=%s sweeps=%d member calls=0 writes=0 peer frames=0", time.Since(started), hostedNumber(t, real, "sweeps")) + for _, watch := range watches { + watch.Close() + } + deadline := time.Now().Add(requestTimeout) + for { + state := hostedQuietControl(t, endpoint, a.Identity.ID(), "quiet-state") + if hostedSocketCounts(t, state)["watch"] == 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("R27 closed viewers retained presence") + } + time.Sleep(time.Millisecond) + } + noViewers := hostedQuietControl(t, endpoint, a.Identity.ID(), "presence-schedule") + if hostedNumber(t, noViewers, "sweeps") != 0 { + t.Fatal("R27 no-viewer presence recurred", noViewers) + } + // A first actual read must obtain authority from the voters. The following + // preference and memory write must reach the other learner after eviction. + _, e = b.Dir.List(t.Context()) + mustNil(t, e) + if fallback { + mustNil(t, SaveRelayURL(a.Home, endpoint)) + tickA() + hostedReady(t, a) + } + wake := hostedQuietControl(t, endpoint, a.Identity.ID(), "quiet-state") + if string(wake["generation"]) != string(before["generation"]) { + hostedRefreshed(t, a, tickA, generation) + hostedRefreshed(t, b, tickB, generation) + } else { + for _, s := range []*Sync{a, b} { + waitMember(t, s, func(m *personalMember) bool { return m.failure == nil && m.conn != nil && m.forwardConn != nil }) + } + } + status, e := b.clusterClient().Status(t.Context()) + mustNil(t, e) + if !fallback && status.Leader != receipt.Node { + t.Fatal("R26 cold wake disrupted the connected personal leader", status) + } + mustNil(t, config.WriteChatModel(b.Home, "stub/hosted-wake")) + brain, e := store.Open(filepath.Join(b.Home, "graph.db")) + mustNil(t, e) + memory, e := brain.AddMemory(store.Memory{Owner: store.OwnerUser, Text: "A hosted idle wake carries memory.", Origin: store.OriginPerson}) + mustNil(t, e) + mustNil(t, brain.Close()) + mustNil(t, b.SyncState(t.Context())) + want, e := b.Dir.List(t.Context()) + mustNil(t, e) + // Generation fencing may refresh the same durable stores on real input. + tickA() + tickB() + waitMember(t, a, func(m *personalMember) bool { return appliedVault(m) == want.Identity.Vault && m.idle }) + mustNil(t, a.SyncState(t.Context())) + raw, e := os.ReadFile(filepath.Join(a.Home, "config.json")) + mustNil(t, e) + if !strings.Contains(string(raw), "stub/hosted-wake") { + t.Fatal("C30 preference missed the first state round") + } + received, e := store.Open(filepath.Join(a.Home, "graph.db")) + mustNil(t, e) + got, found, e := received.MemoryRecord(memory.ID) + mustNil(t, e) + mustNil(t, received.Close()) + if !found || got.Text != memory.Text { + t.Fatal("C30 memory missed the first state round") + } + mustNil(t, config.WriteChatModel(a.Home, "stub/hosted-local")) + mustNil(t, a.SyncState(t.Context())) + latest, e := a.Dir.List(t.Context()) + mustNil(t, e) + if latest.Identity.Vault == want.Identity.Vault { + t.Fatal("C30 local change was not pushed promptly") + } +} diff --git a/relay/hosted/src/identity.js b/relay/hosted/src/identity.js index f56c8e9019..50e02ca336 100644 --- a/relay/hosted/src/identity.js +++ b/relay/hosted/src/identity.js @@ -218,15 +218,17 @@ export class IdentityDO extends DurableObject { if(!maintenance&&!retireDue)return; const tenant=this.#tenantOf(identity);this.host ??= new HostedIdentity(this.ctx,this.env,tenant,this.legacyAdmission,at=>this.#arm(at)); const erase=await this.host.operation(async()=>{ - await this.host.ensure(Uint8Array.from(atob(provision.config.signing_key),c=>c.charCodeAt(0))); + try{await this.host.ensure(Uint8Array.from(atob(provision.config.signing_key),c=>c.charCodeAt(0)));} + catch(error){if(!this.host.initialized)throw error;await this.host.maintenance(now);return false;} this.watchers=this.host.watchers;tenant.stats.flush(); + if(retireDue||!this.host.retirementKnown)await this.host.wakeMaintenance(); + await this.host.maintenance(now); // A retained deadline is only a wake hint. Retirement still needs the // current committed identity before any irreversible erasure starts. if(retireDue||!this.host.retirementKnown){ const root=await this.host.barrierRecord({namespace:'state',key:'identity'}); await this.host.rememberRetirement(this.host.project('identity',root)); } - await this.host.maintenance(now); const at=this.host.retireAt; if(at && at<=now)return true; if(at)await this.#arm(at);return false; @@ -282,7 +284,7 @@ export class IdentityDO extends DurableObject { catch(error){await this.#arm(Date.now()+this.limits.statsFlushMs);throw error;} } dropTables(this.ctx.storage.sql); - await this.ctx.storage.delete(['identity', 'admitted','raft-provision']); + await this.ctx.storage.delete(['identity', 'admitted','raft-provision','raft-router-sleep']); await this.ctx.storage.sync();this.host=null; } diff --git a/relay/hosted/src/raft/README.md b/relay/hosted/src/raft/README.md index 4d129f5362..1439f65708 100644 --- a/relay/hosted/src/raft/README.md +++ b/relay/hosted/src/raft/README.md @@ -189,12 +189,16 @@ on its independent queue. Original signed commands remain unchanged. Fresh read and admission indices must reach an anchor’s durable application before replies. ABI 39 preserves the native signed application wire without network imports. -The router's durable generation advances before its volatile queues reopen. +The router's durable generation advances before lost volatile queues reopen. Peer and application sockets use the hibernation API with a separate tag and serialized attachment containing their signed connection proof. A cold presence sweep preserves those connections without opening coordination or advancing the -generation. A real request reactivates the coordinator with fresh quorum reads, -fences the lost queues, and lets the client refresh the same store's grant. Watch presence and broadcasts filter these sockets. +generation. Completed sleep retains the empty queues' cursor limits. With the same +committed membership and retained personal voter connections, a real request can +restore that progress and obtain a fresh quorum read without reconnecting the +leader. Otherwise it fences lost queues and lets the client refresh the same +store's grant. A saved leader is only a route for that new read. Watch presence +and broadcasts filter these sockets. Protocol queues use their own admission and reservations, outside public rates. ABI 36 supplies admission plans, enrollment bytes and control status from an @@ -209,7 +213,7 @@ coordination never waits for a learner; peer frames bypass its public queue. ## Sleeping accounts and advisory presence The coordinator persists an outstanding-maintenance hint only while consensus -work is scheduled. With the anchor topology, all-voter sleep removes it and clears stale peer retention +work is scheduled. With an anchor or a personal leader, all-voter sleep removes it and clears stale peer retention deadlines before cancelling the maintenance alarm. Presence and explicit retirement deadlines are then rearmed independently. An alarm on a cold sleeping object reads socket attachments and that hint without constructing a tenant, @@ -218,3 +222,20 @@ Platform auto-response timestamps supply presence without data writes. Real application input still activates coordination and takes a fresh quorum barrier; a remembered leader never authorizes a reply. Watch hold renewals retain their committed path and shared renewal period. + + +## Personal leader departure and anchor restoration + +Follower election clocks advance before membership and hold-renewal authority +reads. A personal voter's disconnection wakes ordinary ranked election timers; +it never triggers an explicit campaign or changes a voter list. A newly elected +anchor uses the shared leadership planner and its committed current-term evidence +to restore the standby through a reverse joint swap. The absent voter need not +approve its removal. A returning personal store remains a learner until another +explicit leadership request. Losing two voters leaves the existing cluster +unavailable without creating a replacement cluster. + +Remote idle scheduling uses the same safe idle read, durable marker and final +all-voter sleep acknowledgments as local leadership. Forwarding waits, voting +delivery and retry hints retain active work. A final sleep clears maintenance +only if no newer ingress has woken the coordinator. diff --git a/relay/hosted/src/raft/actor.js b/relay/hosted/src/raft/actor.js index f36331b01a..c3606f6a1e 100644 --- a/relay/hosted/src/raft/actor.js +++ b/relay/hosted/src/raft/actor.js @@ -156,6 +156,9 @@ export class MemberActor { const request = new TextEncoder().encode(encodeCanonical({now,busy,marker,acks})); const r = response(this.reactor.call(frame(23).u32(this.handle).blob(request).finish())); const plan = decodeCanonical(new TextDecoder().decode(r.blob())); r.end(); + // A replacement coordinator resumes this clock so retained cores do not + // wait for its new origin to catch up. + this.elapsed=Math.max(this.elapsed??0,now); const effects = plan.tick ? await this.drive(1) : {send:[],reads:[],status:this.status}; return { ...effects, plan }; } diff --git a/relay/hosted/src/raft/hosted.js b/relay/hosted/src/raft/hosted.js index 2d1a97fe5f..d23d827f0e 100644 --- a/relay/hosted/src/raft/hosted.js +++ b/relay/hosted/src/raft/hosted.js @@ -58,7 +58,7 @@ export class HostedIdentity { finally{release();this.waiters.delete(release);} } replica(){return [...this.states].filter(([id])=>this.members.has(id)).sort(([,a],[,b])=>a.applied===b.applied?Number(b.term-a.term):a.applied>b.applied?-1:1)[0]?.[0];} - leaderID(){const hints=[...this.states].filter(([id])=>this.members.has(id)).sort(([,a],[,b])=>a.term===b.term?(a.applied>b.applied?-1:a.appliedb.term?-1:1);return this.leader()??String(hints[0]?.[1].leader??0);} + leaderID(){const hints=[...this.states].filter(([id])=>this.members.has(id)).sort(([,a],[,b])=>a.term===b.term?(a.applied>b.applied?-1:a.appliedb.term?-1:1);const hint=String(hints[0]?.[1].leader??0);return this.leader()??(hint!=='0'?hint:this.routeHint??'0');} async forward(node,work){ const token=await this.serial(()=>this.forwarding.begin(node,work)); const reply=await token.promise; @@ -101,6 +101,7 @@ export class HostedIdentity { } async ensure(identityKey){ if(this.ready)return; + if(this.initialized)return this.completeEnsure(); this.provisioning=await this.provision(identityKey); if(!this.source.gate())await this.source.freeze(this.provisioning.config); this.manifest=await this.source.inventory(); @@ -133,19 +134,27 @@ export class HostedIdentity { try{ const receipt=await this.member(id).importReceipt(this.manifest);if(!receipt)continue; const member=this.member(id);this.members.set(id,member);receipts.push(receipt); - this.accept(id,await member.initialize(this.config(id))); + const initialized=await member.initialize(this.config(id));this.accept(id,initialized);this.elapsed=Math.max(this.elapsed,initialized.elapsed??0); this.accept(id,await member.drainRecovery()); }catch(error){this.members.delete(id);this.states.delete(id);if(!owned)throw error;} } if(this.members.size<2)throw unavailable(); - await this.pump();if(owned)await this.restorePeers();await this.elect();await this.barrier(); + if(owned){await this.pump();await this.restorePeers();} + else{await this.pump();await this.elect();await this.barrier();} if(!owned)await this.source.own(receipts); + this.started=Math.floor(performance.now())-this.elapsed; this.source.route(()=>true); this.watchers=new CommittedWatchers(this.ctx,this.tenant.limits.maxWatchers,this.tenant.watchers.ttlMs,at=>this.ctx.waitUntil(this.armAlarm(at))); this.tenant.watchers=this.watchers; this.effects=new RaftEffectsFacade(this,this.manifest.config.cluster); this.holds=new CommittedHolds(this,this.policy); this.holds.opened(Date.now(),this.watchers.holdEvidence(Date.now()).some(socket=>socket.holds.length)); + // Transport and follower clocks remain available while a cold cluster + // cannot yet answer authority reads. Publication still needs a quorum. + this.initialized=true;await this.wakeMaintenance(true); + return this.completeEnsure(); + } + async completeEnsure(){ const b=await this.barrier(),header=await this.members.get(b.node).hostView(b.index,'membership','registry'); if(!header.raw){ const anchors=this.config(this.manifest.config.anchors[0]).anchors; @@ -161,6 +170,21 @@ export class HostedIdentity { async restorePeers(){ const generation=await this.ctx.storage.get('raft-router-generation')??'0'; if(typeof generation!=='string'||!/^(0|[1-9][0-9]*)$/.test(generation)||BigInt(generation)>=(1n<<64n)-1n)throw unavailable(); + const saved=await this.ctx.storage.get('raft-router-sleep'),active=await this.ctx.storage.get('raft-maintenance'); + if(Number.isSafeInteger(saved?.elapsed)&&saved.elapsed>=0)this.elapsed=Math.max(this.elapsed,saved.elapsed); + if(saved?.generation===generation&&!active){ + const peers=new PersonalPeers(this,generation);peers.restoring=true;await peers.refresh(); + const live=Object.fromEntries(Object.entries(peers.view).filter(([,peer])=>!peer.revoked)); + const external=Object.keys(live).filter(node=>live[node].voter&&!this.members.has(node)); + if(external.length&&encodeCanonical(live)===encodeCanonical(saved.view)&&external.every(node=>['peer','forward'].every(kind=>peers.sockets().some(ws=>{const a=ws.deserializeAttachment();return a.node===node&&a.kind===kind&&a.generation===generation;})))){ + for(const [node,cursors]of Object.entries(saved.cursors)){ + const box=peers.boxes.get(node);if(!box)throw unavailable(); + for(const key of ['next','issued','acked']){if(cursors[key].length!==peers.limits.Slots.length)throw unavailable();box[key]=cursors[key].map(BigInt);} + peers.acks.set(node,box.acked.map(String)); + } + peers.progress=new Map(saved.progress);peers.restoring=false;this.peers=peers;this.routeHint=saved.leader;return; + } + } const next=String(BigInt(generation)+1n); await this.ctx.storage.put('raft-router-generation',next);await this.ctx.storage.sync(); this.peers=new PersonalPeers(this,next); @@ -234,7 +258,8 @@ export class HostedIdentity { externalVoter(){return Object.entries(this.peers?.view??{}).some(([node,peer])=>peer.voter&&!peer.revoked&&!this.manifest.config.anchors.includes(node));} async elect(){ if(this.leader()||(this.leaderID()!=='0'&&!this.members.has(this.leaderID())))return; - if(this.transferTarget){const deadline=Date.now()+this.peers.limits.ForwardTimeout;while(!this.leader()&&(this.leaderID()==='0'||this.members.has(this.leaderID())))await this.progress(deadline);return;} + if(this.transferTarget&&this.forwarding.socket(this.transferTarget.node)){const deadline=Date.now()+this.peers.limits.ForwardTimeout;while(!this.leader()&&(this.leaderID()==='0'||this.members.has(this.leaderID())))await this.progress(deadline);return;} + if(this.externalVoter())throw unavailable(); for(const id of this.members.keys()){await this.input(id,16);await this.pump();if(this.leader())return;} throw unavailable(); } @@ -317,8 +342,8 @@ export class HostedIdentity { else await this.ctx.storage.delete('raft-retire-at'); this.retireAt=at;this.retirementKnown=true; } - wakeMaintenance(){ - this.asleep=false;this.wakeVersion++; + wakeMaintenance(wake=false){ + this.wakePending=this.wakePending||this.asleep||wake;this.asleep=false;this.wakeVersion++; // Peer ingress has its own queue. Serialize only timer transitions so a // wake racing the final sleep cannot lose its durable maintenance hint. const task=this.alarmTail.then(async()=>{ @@ -338,72 +363,102 @@ export class HostedIdentity { }); this.alarmTail=task.catch(()=>{});return task; } + coordinationBusy(){ + if(this.forwarding.pending.size||this.queue.length||this.peers.pending)return true; + for(const [node,peer]of Object.entries(this.peers.view))if(peer.voter&&!peer.revoked){ + if(this.peers.retryTargets.has(node)||this.peers.boxes.get(node)?.queues.some(q=>q.length)||this.peers.snapshots.has(node))return true; + } + return false; + } + // Replies produced by this turn must drain before sleep without becoming + // another application activity sample that cancels the new marker. + async drainCoordination(){ + const deadline=Date.now()+this.peers.limits.ForwardTimeout; + while(this.coordinationBusy()){ + const version=this.progressVersion; + if([...this.members.keys()].some(node=>this.peers.boxes.get(node)?.queues.some(q=>q.length)))await this.peers.serial(()=>this.peers.anchors()); + await this.pump(); + if(!this.coordinationBusy())return; + await this.progress(deadline,version); + } + } async maintenance(now=Date.now(),elapsed=Math.floor(performance.now())){ if(this.asleep)return; - const wakeVersion=this.wakeVersion; - if(this.membershipPending)await this.reconcileMembership(); + const wakeVersion=this.wakeVersion,busy=this.coordinationBusy()||!!this.wakePending; + this.wakePending=false; this.elapsed=Math.max(this.elapsed,elapsed-this.started); - // Alarm/recovery effects are retained until their exact batch is consumed. + // Followers must advance their election clocks before any authority read: + // an absent leader cannot answer membership or hold-renewal requests. + const plans=new Map(); for(const [id,member]of this.members){ - this.accept(id,await member.drainRecovery()); - const batch=await member.scheduled();if(!batch)continue; - this.accept(id,{send:batch.send.map(m=>({to:m.to,bytes:unb64(m.bytes)}))});await this.pump();await member.acknowledgeScheduled(batch.id); - } - if(this.holds.next!==null)await this.holds.renew(now,this.watchers.holdEvidence(now)); - // Scanning is an internal activity hint only. Delivery obtains its own - // fresh barrier and authority check before touching a public link request. - const leader=this.leader(); - if(!leader&&this.leaderID()!=='0'&&!this.members.has(this.leaderID())){ - for(const [id,member]of this.members){this.accept(id,await member.schedule(this.elapsed,now,this.forwarding.pending.size>0));} - await this.pump(); - // External leadership keeps its ordinary clock until every voting - // member can participate in a completed sleep transition. - if(this.forwarding.socket(this.leaderID()))await this.forward(this.leaderID(),{action:'schedule',now,schedule:{now:this.elapsed,busy:true,marker:{index:'0',term:'0',barrier:'0',voters:null},acks:null}}); - await this.armAlarm(now+this.policy.tick_every);return; - } - if(!leader){ - // A missing leader still needs ordinary follower election ticks. An - // unavailable authority read cannot be a reason to cancel that clock. - for(const [id,member]of this.members){const effects=await member.schedule(this.elapsed,now,this.forwarding.pending.size>0);this.accept(id,effects);} - await this.pump();await this.armAlarm(now+this.policy.tick_every);return; - } - let scan=this.outboxScan??''; - const app=await this.members.get(leader).hostView(this.states.get(leader).applied,'outbox','',scan); - this.outboxScan=app.rows.at(-1)?.key??''; - if(app.rows.some(row=>{const value=decodeCanonical(dec.decode(row.data));return value.decision&&!value.outcome})){ - await this.effects.replay(code=>this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(code)),scan,now); + try{ + this.accept(id,await member.drainRecovery()); + const batch=await member.scheduled(); + if(batch){this.accept(id,{send:batch.send.map(m=>({to:m.to,bytes:unb64(m.bytes)}))});await this.pump();await member.acknowledgeScheduled(batch.id);} + const effects=await member.schedule(this.elapsed,now,busy);this.accept(id,effects);plans.set(id,effects.plan); + }catch{this.members.delete(id);this.states.delete(id);} } - let plan; - for(const [id,member]of this.members){const effects=await member.schedule(this.elapsed,now,this.forwarding.pending.size>0);this.accept(id,effects);if(id===this.leader())plan=effects.plan;} await this.pump(); - if(plan?.barrier){ - const id=crypto.randomUUID();this.reads=[];await this.input(this.leader(),15,id);await this.pump(); - if(!this.reads.some(r=>r.id===id))throw unavailable();this.reads=[]; - const effects=await this.members.get(this.leader()).schedule(this.elapsed,now,false);this.accept(this.leader(),effects);await this.pump();plan=effects.plan; - } - if(plan?.prepare){ - const id=this.leader(),s=this.states.get(id),view=await this.members.get(id).hostView(s.applied,'state','identity'); - // The IdleRead freshness is consumed at the SAME applied position. Read - // the anchor session locally so command construction does not wake it. - const saved=await this.members.get(id).hostView(s.applied,'sessions',segment('anchor/'+this.manifest.config.anchors[0])+'/'+segment('maintenance')); - const pkcs8=unb64(this.provisioning.anchors[0]); - const r=response(this.runtime.call(frame(34).str(this.manifest.config.cluster).u64(this.manifest.config.anchors[0]).u64(now).blob(pkcs8.slice(-32)).blob(saved.raw??new Uint8Array()).blob(data({prepare_quiescence:{barrier:String(s.applied),term:String(s.term),voters:this.configuration(view.app.conf).voters}})).finish())); - const raw=r.blob();r.end();requireOK((await this.commit(raw)).value.code); - await this.input(this.leader(),1);await this.pump(); - } - const id=this.leader(),s=this.states.get(id),row=await this.members.get(id).hostView(s.applied,'membership','quiescence'); - if(row.raw){ - const marker=decodeCanonical(dec.decode(row.raw)); - try{ - const acks=[];for(const voter of marker.voters)acks.push(await this.members.get(voter).sleepAck(marker.index,marker.term)); - let sleeping=true; - for(const [node,member]of this.members){const effects=await member.schedule(this.elapsed,now,false,acks);this.accept(node,effects);sleeping&&=effects.plan.sleeping;} - await this.pump();this.asleep=sleeping&&this.forwarding.pending.size===0&&wakeVersion===this.wakeVersion; - if(this.asleep&&this.peers){await this.peers.quiesce();await this.pump();await this.sleepMaintenance(wakeVersion);} - }catch{this.asleep=false;} + try{ + const leader=this.leader(),remote=leader?null:this.leaderID(); + if(!leader&&(remote==='0'||this.members.has(remote)||!this.forwarding.socket(remote)))return; + if(this.membershipPending)await this.reconcileMembership(); + // Local durable state is only a work hint. The shared planner obtains + // its own quorum read before proposing the reverse joint swap. + if(leader){ + const row=await this.members.get(leader).hostView(this.states.get(leader).applied,'membership','registry'); + if(row.raw&&decodeCanonical(dec.decode(row.raw)).leadership)await this.reconcileLeadership(); + } + if(this.holds.next!==null)await this.holds.renew(now,this.watchers.holdEvidence(now)); + const replica=this.leader()??this.replica(),member=this.members.get(replica); + let scan=this.outboxScan??''; + const app=await member.hostView(this.states.get(replica).applied,'outbox','',scan); + this.outboxScan=app.rows.at(-1)?.key??''; + if(app.rows.some(row=>{const value=decodeCanonical(dec.decode(row.data));return value.decision&&!value.outcome;}))await this.effects.replay(code=>this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(code)),scan,now); + let remoteAck,remoteStatus; + const schedule=async(acks=[])=>{ + if(remote){ + const reply=await this.forward(remote,{action:'schedule',now,schedule:{now:this.elapsed,busy,marker:{index:'0',term:'0',barrier:'0',voters:null},acks}}); + if(!reply.plan)throw unavailable();remoteAck=reply.sleep_ack;remoteStatus=reply.status;return reply.plan; + } + const id=this.leader(),effects=await this.members.get(id).schedule(this.elapsed,now,busy,acks);this.accept(id,effects);await this.pump();return effects.plan; + }; + let plan=remote?await schedule():plans.get(this.leader()); + if(plan?.barrier){ + const id=crypto.randomUUID(); + if(remote){const reply=await this.forward(remote,{action:'idle-read',id});await this.appliedReplica(reply.barrier);} + else{const leader=this.leader();await this.input(leader,15,id);await this.pump();if(!this.reads.some(r=>r.node===leader&&r.id===id))throw unavailable();this.reads=this.reads.filter(r=>r.id!==id);} + plan=await schedule(); + } + if(plan?.prepare){ + await this.drainCoordination(); + const id=this.leader()??this.replica(),s=this.states.get(id),view=await this.members.get(id).hostView(s.applied,'state','identity'); + // IdleRead freshness is consumed at the same applied position. Building + // this restricted command locally must not replace it with a user read. + const saved=await this.members.get(id).hostView(s.applied,'sessions',segment('anchor/'+this.manifest.config.anchors[0])+'/'+segment('maintenance')); + const pkcs8=unb64(this.provisioning.anchors[0]); + const r=response(this.runtime.call(frame(34).str(this.manifest.config.cluster).u64(this.manifest.config.anchors[0]).u64(now).blob(pkcs8.slice(-32)).blob(saved.raw??new Uint8Array()).blob(data({prepare_quiescence:{barrier:String(s.applied),term:String(remote?remoteStatus.Term:s.term),voters:this.configuration(view.app.conf).voters}})).finish())); + const raw=r.blob();r.end();requireOK((await this.commit(raw)).value.code); + if(!remote){await this.input(this.leader(),1);await this.pump();} + } + const id=this.leader()??this.replica(),s=this.states.get(id),row=await this.members.get(id).hostView(s.applied,'membership','quiescence'); + const marker=row.raw&&decodeCanonical(dec.decode(row.raw)); + if(marker&&marker.index!=='0'){ + try{ + const acks=[]; + for(const voter of marker.voters)if(this.members.has(voter))acks.push(await this.members.get(voter).sleepAck(marker.index,marker.term)); + await this.drainCoordination(); + if(remote){await schedule();if(remoteAck)acks.push(remoteAck);} + let sleeping=remote?(await schedule(acks)).sleeping:true; + for(const [node,member]of this.members){const effects=await member.schedule(this.elapsed,now,busy,acks);this.accept(node,effects);sleeping&&=effects.plan.sleeping;} + await this.pump();this.asleep=sleeping&&!this.coordinationBusy()&&wakeVersion===this.wakeVersion; + if(this.asleep){await this.peers.quiesce();await this.pump();await this.sleepMaintenance(wakeVersion);} + }catch{this.asleep=false;} + } + }finally{ + if(!this.asleep)await this.armAlarm(now+this.policy.tick_every); + if(this.holds.next!==null)await this.armAlarm(this.holds.next); } - if(!this.asleep)await this.armAlarm(now+this.policy.tick_every); - if(this.holds.next!==null)await this.armAlarm(this.holds.next); } async personal(action,{proof,actor='',certificate=''}={}){ const b=await this.barrier(); @@ -445,13 +500,24 @@ export class HostedIdentity { personalRefusal(out){const r=response(this.runtime.call(frame(38).u32(2).str(out.code).finish())),message=r.str();r.end();return new Response(out.message||message||'this computer is not allowed to join this cluster',{status:out.code==='capacity'?429:out.code==='update_required'?409:403,headers:{'cache-control':'no-store'}});} async reconcileLeadership(){ const raw=await this.barrierRecord({namespace:'membership',key:'registry'}),h=decodeCanonical(dec.decode(raw)),l=h.leadership,id=this.leader(); - if(!l||!id)return false; - const target=l.restore?l.standby:l.node;let evidence={node:'0',revision:'0',applied:'0',protocol:0,durable:false}; + if(!l||!id){if(!l)this.transferTarget=null;return false;} + if(l.restore||l.completed||this.transferTarget?.revision!==h.revision)this.transferTarget=null; + const empty={node:'0',revision:'0',applied:'0',protocol:0,durable:false}; try{ - if(this.members.has(target))evidence=await this.members.get(target).leadershipEvidence(h.revision,l.barrier); - else {try{evidence=(await this.forward(target,{action:'evidence',revision:h.revision,barrier:l.barrier})).evidence??evidence;}catch{const request=crypto.randomUUID();await this.input(id,15,request);await this.input(id,10,request);await this.pump();return false;}} - if(this.leader()!==id)return true; - const result=await this.members.get(id).leadership(evidence);if(result.plan.transfer!=='0')this.transferTarget=result.plan.transfer;this.accept(id,result);await this.pump();const plan=result.plan; + let result; + try{result=await this.members.get(id).leadership(empty);} + catch(error){ + if(!/target|transfer/.test(error.message))throw error; + const target=l.restore?l.standby:l.node;let evidence; + if(this.members.has(target))evidence=await this.members.get(target).leadershipEvidence(h.revision,l.barrier); + else {try{evidence=(await this.forward(target,{action:'evidence',revision:h.revision,barrier:l.barrier})).evidence??empty;}catch{const request=crypto.randomUUID();await this.input(id,15,request);await this.input(id,10,request);await this.pump();return false;}} + if(this.leader()!==id)return true; + result=await this.members.get(id).leadership(evidence); + } + const plan=result.plan; + if(plan.transfer!=='0')this.transferTarget={node:plan.transfer,revision:h.revision}; + this.accept(id,result);await this.pump(); + if(plan.fallback){this.transferTarget=null;requireOK((await this.anchorCommand({restore_anchors:{revision:h.revision}},Date.now())).value.code);return true;} if(plan.configure){await this.peers.refresh();return true;} if(plan.settle){requireOK((await this.anchorCommand({settle_leadership:{revision:h.revision}},Date.now())).value.code);return true;} return false; @@ -474,7 +540,7 @@ export class HostedIdentity { if(this.leaderID()===l.node&&!this.leader()&&l.settled){ if(!l.completed){const reply=await this.forward(l.node,{action:'receipt'});if(!reply.receipt)throw unavailable();await this.appliedReplica(reply.barrier,deadline);requireOK((await this.anchorCommand({complete_leadership:{revision:h.revision}},Date.now())).value.code);} const reply=await this.forward(l.node,{action:'receipt'});if(!reply.receipt||reply.receipt.node!==l.node||reply.receipt.revision!==h.revision)throw unavailable(); - await this.appliedReplica(reply.barrier,deadline);return this.jsonBytes(encodeCanonical(reply.receipt)); + await this.appliedReplica(reply.barrier,deadline);this.transferTarget=null;return this.jsonBytes(encodeCanonical(reply.receipt)); } const version=this.progressVersion;if(!await this.reconcileLeadership())await this.progress(deadline,version); } diff --git a/relay/hosted/src/raft/member.js b/relay/hosted/src/raft/member.js index b1785d60fc..2343042a6b 100644 --- a/relay/hosted/src/raft/member.js +++ b/relay/hosted/src/raft/member.js @@ -162,7 +162,7 @@ export class RaftMemberDO extends DurableObject { return this.loading; } this.loadingConfig = JSON.stringify(config); - this.loading = this.activate(config); + this.loading = this.activate(config).then(effects=>({...effects,elapsed:this.actor.elapsed??0})); try { return await this.loading; } finally { this.loading = null; this.loadingConfig = null; } } async activate(config) { diff --git a/relay/hosted/src/raft/peers.js b/relay/hosted/src/raft/peers.js index 150811ace0..fe4b54caa0 100644 --- a/relay/hosted/src/raft/peers.js +++ b/relay/hosted/src/raft/peers.js @@ -27,7 +27,7 @@ export class PersonalPeers { this.host=host;this.generation=generation;this.scope=host.config('1').scope;this.view={};this.boxes=new Map();this.used=[0,0,0];this.progress=new Map();this.snapshots=new Map();this.acks=new Map();this.tail=Promise.resolve();this.pending=0;this.blocked=new Set();this.retryTargets=new Set();this.checkpoints=new Map(); this.limits=peerLimits(host.runtime); } - serial(fn){if(this.pending>=64)return Promise.reject(refused('busy'));this.pending++;const p=this.tail.then(fn);this.tail=p.catch(()=>{});return p.finally(()=>this.pending--);} + serial(fn){if(this.pending>=64)return Promise.reject(refused('busy'));this.pending++;const p=this.tail.then(fn);this.tail=p.catch(()=>{});return p.finally(()=>{this.pending--;this.host.signal?.();});} members(){return Object.entries(this.view).map(([node,p])=>({node,...p}));} // Public and peer turns publish in order, so a delayed member read cannot // restore an older transport authority after another turn has advanced it. @@ -59,6 +59,7 @@ export class PersonalPeers { } for(const [to,s]of this.snapshots)if(!unchanged(to)||!unchanged(s.from)){this.snapshots.delete(to);await this.host.input(s.from,12,{peer:to,snapshot:true,success:false});} for(const ws of this.sockets())if(this.view[ws.deserializeAttachment().node]?.revoked)this.close(ws); + if(this.restoring)return; const leader=this.host.leader(); if(leader){const request=crypto.randomUUID();await this.host.input(leader,15,request);await this.host.input(leader,10,request);} else {const target=this.host.leaderID();if(this.host.forwarding.socket(target)){const token=await this.host.forwarding.begin(target,{action:'poke'});token.promise.catch(()=>{});}} @@ -124,19 +125,22 @@ export class PersonalPeers { async send(m){ await this.refresh(); if(this.host.asleep||!this.view[m.to]||this.view[m.to].revoked)return; - if(this.blocked.has(m.to)){this.retryTargets.add(m.to);return;} + // Only a dropped voting round needs a fresh quorum poke. Learner + // backpressure must not invalidate the elected member's idle marker. + const voterRound=this.view[m.from]?.voter&&this.view[m.to]?.voter; + if(this.blocked.has(m.to)){if(voterRound)this.retryTargets.add(m.to);return;} const inspected=this.host.codec.inspect(4,m.bytes),snapshot=inspected.snapshot; if(m.bytes.length>this.limits.Batch){ - this.blocked.add(m.to);this.retryTargets.add(m.to);const applied=String(this.host.states.get(m.from).applied); + this.blocked.add(m.to);if(voterRound)this.retryTargets.add(m.to);const applied=String(this.host.states.get(m.from).applied); if(this.checkpoints.get(m.from)!==applied){await this.host.members.get(m.from).hostCheckpoint();this.checkpoints.set(m.from,applied);} await this.host.input(m.from,12,{peer:m.to,snapshot:false,success:false});return; } const lane=this.view[m.from]?.voter&&this.view[m.to]?.voter?0:1; - if(this.boxes.get(m.to).queues[lane].length>=this.limits.Slots[lane]){this.blocked.add(m.to);this.retryTargets.add(m.to);await this.host.input(m.from,12,{peer:m.to,snapshot:snapshot.length>0,success:false});return;} + if(this.boxes.get(m.to).queues[lane].length>=this.limits.Slots[lane]){this.blocked.add(m.to);if(voterRound)this.retryTargets.add(m.to);await this.host.input(m.from,12,{peer:m.to,snapshot:snapshot.length>0,success:false});return;} try{ this.enqueue([JSON.parse(dec.decode(await this.host.envelope(m.from,m.to,m.bytes)))]); if(snapshot.length){const meta=this.host.codec.inspect(3,snapshot),manifest=JSON.parse(dec.decode(meta.data));this.snapshots.set(m.to,{from:m.from,manifest,generation:this.host.hash(meta.data),offset:0,waiting:null,started:Date.now()});} - }catch(e){if(e.peerCode!=='busy')throw e;this.blocked.add(m.to);this.retryTargets.add(m.to);await this.host.input(m.from,12,{peer:m.to,snapshot:snapshot.length>0,success:false});} + }catch(e){if(e.peerCode!=='busy')throw e;this.blocked.add(m.to);if(voterRound)this.retryTargets.add(m.to);await this.host.input(m.from,12,{peer:m.to,snapshot:snapshot.length>0,success:false});} } // A full voter queue can drop every response to one fresh read. Retry only // after its acknowledgments free the reservation, using the elected member's @@ -236,7 +240,7 @@ export class PersonalPeers { } async departed(ws){ this.host.forwarding.departed(ws); - if(this.view[ws.deserializeAttachment()?.node]?.voter){await this.host.wakeMaintenance();this.host.signal();} + if(this.view[ws.deserializeAttachment()?.node]?.voter){await this.host.wakeMaintenance(true);this.host.signal();} } async quiesce(){ // Final voter acknowledgments discard learner delivery. Its old retention @@ -244,6 +248,11 @@ export class PersonalPeers { for(const ws of this.sockets()){const a=ws.deserializeAttachment();if(a.deadline)ws.serializeAttachment({...a,deadline:null});} for(const [node,box]of this.boxes)if(!this.view[node]?.voter)for(let lane=0;lane<3;lane++){for(const q of box.queues[lane])this.used[lane]-=q.size;box.queues[lane]=[];} for(const [to,s]of this.snapshots){this.snapshots.delete(to);await this.host.input(s.from,12,{peer:to,snapshot:true,success:false});} + // Empty routing queues can resume after hibernation only when their exact + // cursor high-water marks survive with the completed all-voter sleep. + const view=Object.fromEntries(Object.entries(this.view).filter(([,peer])=>!peer.revoked)),cursors={}; + for(const node of Object.keys(view)){const box=this.boxes.get(node);cursors[node]=Object.fromEntries(['next','issued','acked'].map(key=>[key,box[key].map(String)]));} + await this.host.ctx.storage.put('raft-router-sleep',{generation:this.generation,leader:this.host.leaderID(),elapsed:this.host.elapsed,view,cursors,progress:[...this.progress].filter(([node])=>view[node])});await this.host.ctx.storage.sync(); } } export function peerRefusal(e){const code=e.peerCode??'invalid',status={unauthorized:403,generation:409,cursor:409,busy:429}[code]??400;return new Response(JSON.stringify({error:code}),{status,headers:{'content-type':'application/json','cache-control':'no-store',...(code==='busy'?{'retry-after':'1'}:{})}});} diff --git a/relay/hosted/test/raft/hosted-fallback-go.test.js b/relay/hosted/test/raft/hosted-fallback-go.test.js new file mode 100644 index 0000000000..f4cabeec31 --- /dev/null +++ b/relay/hosted/test/raft/hosted-fallback-go.test.js @@ -0,0 +1,37 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {spawn} from 'node:child_process'; +import {mkdtemp,rm} from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; +import {hostedFixture} from './hosted-fixture.js'; +const root=new URL('../../../../',import.meta.url); +const footprint=new URL('../../../../scripts/raft-proof-footprint.sh',import.meta.url).pathname; +async function run(args,env={},signal){ + const child=spawn('sh',[footprint,...args],{cwd:root,env:{...process.env,...env},stdio:['ignore','pipe','pipe'],signal}); + let output='';child.stdout.on('data',b=>{output+=b;process.stdout.write(b)});child.stderr.on('data',b=>{output+=b;process.stdout.write(b)}); + const code=await new Promise((resolve,reject)=>{child.once('error',reject);child.once('exit',resolve)});assert.equal(code,0,output);return output; +} +const dir=await mkdtemp(join(tmpdir(),'codeaf-lead-proof-')),binary=join(dir,'client.test'); +try{ + await run(['go','test','-p=1','-timeout=15m','./internal/syncsetup','-c','-o',binary]); + // Each identity proof gets a fresh isolate, retaining the shared admission cap. + for(const name of [ + 'TestR7HostedPersonalLeaderFallbackAndNoAutomaticRegain', + 'TestR7HostedPersonalLeaderProcessDeathRestoresAnchors', + 'TestR8HostedPersonalLeaderAndAnchorLossNeverCreatesSecondCluster', + 'TestR26HostedPersonalLeaderColdReadKeepsQuorum', + 'TestR7R26HostedColdSleepingLeaderDepartureRestoresAnchors', + 'TestR23R26R27HostedLeadingHomesIdleEvictionAndWake', + 'TestR7R23R26R27HostedSyncOffFallbackIdleEvictionAndWake', + ]){ + test('R7 R8 R23 R26 '+name,{timeout:300000},async t=>{ + const h=await hostedFixture({fixture:true});try{ + await h.start();const url=await h.mf.ready; + const output=await run([binary,'-test.run=^'+name+'$','-test.count=1','-test.timeout=15m','-test.v'],{CODEAF_HOSTED_PROOF_URL:url.origin},t.signal); + assert.doesNotMatch(output,/--- SKIP:/);assert.match(output,new RegExp('--- PASS: '+name)); + }finally{await h.close();} + }); + } + test.after(()=>rm(dir,{recursive:true,force:true})); +}catch(error){await rm(dir,{recursive:true,force:true});throw error;} diff --git a/relay/hosted/test/raft/hosted-fallback.test.js b/relay/hosted/test/raft/hosted-fallback.test.js new file mode 100644 index 0000000000..e17c3620c4 --- /dev/null +++ b/relay/hosted/test/raft/hosted-fallback.test.js @@ -0,0 +1,55 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {HostedIdentity} from '../../src/raft/hosted.js'; + +// R7 authority can be unavailable throughout an election. The two surviving +// voters still need their ordinary clocks and a retained maintenance alarm. +test('R7 pending membership cannot stop surviving follower election clocks',async()=>{ + const clocks=[],alarms=[],reads=[]; + const members=new Map(['1','2'].map(id=>[id,{ + drainRecovery:async()=>null,scheduled:async()=>null, + schedule:async elapsed=>{clocks.push([id,elapsed]);return {plan:{},status:{applied:1n,term:2n,leader:0n,role:0}};}, + }])); + members.set('3',{drainRecovery:async()=>{throw Error('standby unavailable');}}); + const host=Object.assign(Object.create(HostedIdentity.prototype),{ + asleep:false,wakeVersion:0,started:0,elapsed:0,operations:1,membershipPending:true, + manifest:{config:{anchors:['1','2','3']}},members,states:new Map(),queue:[],reads:[],waiters:new Set(),progressVersion:0, + holds:{next:null},policy:{tick_every:5000},forwarding:{pending:new Map(),socket:()=>null}, + peers:{view:{'4':{voter:true}},boxes:new Map(),snapshots:new Map(),retryTargets:new Set()}, + pump:async()=>{},reconcileMembership:async()=>{reads.push('read');throw Error('quorum unavailable');},armAlarm:async at=>alarms.push(at), + }); + await host.maintenance(5000,5000); + assert.deepEqual(clocks,[['1',5000],['2',5000]]); + assert.equal(reads.length,0);assert.deepEqual(alarms,[10000]); +}); + +// R23/R26 a forwarding request or undelivered voter work must finish before +// the group may sleep; a disconnected learner cannot keep the voters awake. +test('R23 R26 pending forwarding and voting delivery prevent idle suppression',()=>{ + const host=Object.assign(Object.create(HostedIdentity.prototype),{ + operations:1,queue:[],members:new Map([['1',{}]]),forwarding:{pending:new Map()}, + peers:{view:{'4':{voter:true},'5':{voter:false}},boxes:new Map([['4',{queues:[[],[],[]]}],['5',{queues:[[{}],[],[]]}]]),snapshots:new Map(),retryTargets:new Set()}, + }); + assert.equal(host.coordinationBusy(),false); + host.forwarding.pending.set('request',{});assert.equal(host.coordinationBusy(),true);host.forwarding.pending.clear(); + host.peers.boxes.get('4').queues[0].push({});assert.equal(host.coordinationBusy(),true);host.peers.boxes.get('4').queues[0]=[]; + host.peers.retryTargets.add('4');assert.equal(host.coordinationBusy(),true);host.peers.retryTargets.clear(); + host.peers.snapshots.set('4',{});assert.equal(host.coordinationBusy(),true);host.peers.snapshots.clear(); + host.peers.pending=1;assert.equal(host.coordinationBusy(),true);host.peers.pending=0; + host.peers.view['1']={voter:true};host.peers.boxes.set('1',{queues:[[{}],[],[]]});assert.equal(host.coordinationBusy(),true);host.peers.boxes.get('1').queues[0]=[]; + assert.equal(host.coordinationBusy(),false); +}); + +// R26/R27 dropped learner responses cannot cancel a voter sleep marker by +// starting an unrelated quorum read; a dropped voting round still retries. +test('R26 R27 learner backpressure cannot invalidate an idle voting barrier',async()=>{ + const {PersonalPeers}=await import('../../src/raft/peers.js'); + const peers=Object.assign(Object.create(PersonalPeers.prototype),{ + refresh:async()=>{},blocked:new Set(),retryTargets:new Set(), + view:{'1':{voter:true},'3':{voter:false},'4':{voter:true}}, + boxes:new Map([['4',{queues:[[],[{}],[]]}]]),limits:{Batch:1024,Slots:[2,1,1]}, + host:{asleep:false,codec:{inspect:()=>({snapshot:new Uint8Array()})},input:async()=>{}}, + }); + await peers.send({from:'3',to:'4',bytes:new Uint8Array()});assert.equal(peers.retryTargets.size,0); + await peers.send({from:'1',to:'4',bytes:new Uint8Array()});assert(peers.retryTargets.has('4')); +}); diff --git a/relay/hosted/test/raft/hosted-worker.js b/relay/hosted/test/raft/hosted-worker.js index 5fefb8afcc..e726d79719 100644 --- a/relay/hosted/test/raft/hosted-worker.js +++ b/relay/hosted/test/raft/hosted-worker.js @@ -6,9 +6,11 @@ import { IdentityDO as Source } from '../../src/identity.js'; import { RaftMemberDO as Member } from '../../src/raft/member.js'; import { Directory } from '../../src/directory.js'; import { unb64 } from '../../src/raft/storage.js'; +import {encodeCanonical} from '../../src/raft/codec.js'; +import {CORE_MEMORY_BUDGET} from '../../src/raft/abi.js'; import { runtimeMetrics,sharedRuntime } from '../../src/raft/actor.js'; import {coordinationPolicy} from '../../src/raft/liveness.js'; -const measurements={turns:0,wall:[],membership:0,http:0,peerIn:0,peerOut:0,sweeps:0,writes:0}; +const measurements={turns:0,wall:[],membership:0,campaigns:0,http:0,peerIn:0,peerOut:0,sweeps:0,writes:0}; async function measured(fn){const start=performance.now();measurements.turns++;try{return await fn();}finally{measurements.wall.push(performance.now()-start);}} import { policyOf } from '../../src/rules.js'; @@ -114,18 +116,36 @@ export class IdentityDO extends Source { this.host.queue.push({from,to,bytes:raw});await this.host.pump();return this.host.members.size; }); } + async leadershipTicks(count,missing){ + if(!this.current.host)await this.current.alarm(); + const host=this.current.host; + return host.operation(async()=>{ + if(missing){host.members.delete(missing);host.states.delete(missing);} + const errors=[],leaders=[],base=Date.now(); + for(let n=0;n{ - const base=Date.now(),membershipBefore=measurements.membership; - for(let n=1;n<=20;n++)await host.maintenance(base+n*host.policy.tick_every,host.started+n*host.policy.tick_every); - const id=host.leader(),before=host.states.get(id).applied; + const base=Date.now(),offset=host.elapsed,membershipBefore=measurements.membership; + const settle=()=>host.drainCoordination(); + await settle(); + for(let n=1;n<=20;n++){await host.maintenance(base+n*host.policy.tick_every,host.started+offset+n*host.policy.tick_every);await settle();} + const id=host.leader()??host.replica(),before=host.states.get(id).applied; const sleeping=host.asleep; const turns=measurements.turns; - for(let n=21;n<=140;n++)await host.maintenance(base+n*host.policy.tick_every,host.started+n*host.policy.tick_every); + for(let n=21;n<=140;n++)await host.maintenance(base+n*host.policy.tick_every,host.started+offset+n*host.policy.tick_every); return {sleeping,membershipCalls:measurements.membership-membershipBefore,appliedBefore:String(before),appliedAfter:String(host.states.get(id).applied),recurringMemberCalls:measurements.turns-turns,idleMilliseconds:120*host.policy.tick_every}; });} inspect() { @@ -148,7 +168,7 @@ export class RaftMemberDO extends Member { refreshEnrollment(...args){return measured(()=>super.refreshEnrollment(...args));} hostPersonal(...args){return measured(()=>super.hostPersonal(...args));} membership(...args){measurements.membership++;return measured(()=>super.membership(...args));} - hostInput(...args){return measured(()=>super.hostInput(...args));} + hostInput(...args){if(args[0]===16)measurements.campaigns++;return measured(()=>super.hostInput(...args));} peer(...args){return measured(()=>super.peer(...args));} hostView(...args){return measured(()=>super.hostView(...args));} hostResult(...args){return measured(()=>super.hostResult(...args));} @@ -173,7 +193,8 @@ export default { async fetch(request,env) { const url=new URL(request.url); if(url.pathname==='/fixture'){ - const {op,identity,records,node,cluster}=await request.json(); + const {op,identity,records,node,cluster,ticks,missing}=await request.json(); + if(op==='leadership-ticks')return new Response(encodeCanonical(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).leadershipTicks(ticks,missing))); if(op==='stale-retirement')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).staleRetirement()); if(op==='diagnostic'){console.warn('local runtime warning capture');console.error('local runtime error capture');return Response.json(true);} if(op==='evict-minority')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).evictMinority()); @@ -188,7 +209,7 @@ export default { if(op==='pause-public'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).pausePublic();return Response.json(true);} if(op==='resume-public'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).resumePublic();return Response.json(true);} if(op==='checkpoint-state')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).checkpointState()); - if(op==='metrics')return Response.json({...runtimeMetrics(),...measurements}); + if(op==='metrics')return Response.json({coreBudget:CORE_MEMORY_BUDGET,...runtimeMetrics(),...measurements}); if(op==='reset-metrics'){for(const key of Object.keys(measurements))measurements[key]=key==='wall'?[]:0;return Response.json(true);} if(op==='seed'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).seed(identity,records);return Response.json(true);} if(op==='hold-upload'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).holdUpload();return Response.json(true);} From 1bfc2cb29764aa9af2eef7edc6559da56656b99f Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 14:06:44 -0400 Subject: [PATCH 071/103] Keep large signed metadata within the existing command limits. Older clients retain their original body limit through bounded authenticated body pages and one atomic directory change. Lossless physical records and bounded projection preserve opaque fields without increasing the command or reactor envelopes. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterstate/apply.go | 144 +++++-- internal/clusterstate/codec.go | 264 ++++++++++++- internal/clusterstate/codec_large_test.go | 31 ++ internal/clusterstate/imports_test.go | 2 +- internal/clusterstate/legacy.go | 6 +- internal/clusterstate/record_decode.go | 134 +++++++ internal/clusterstate/record_encoding.go | 76 ++++ internal/clusterstate/types.go | 2 + internal/clusterstate/v1.go | 34 +- internal/clusterstate/v1_stage.go | 291 ++++++++++++++ internal/clusterstate/v1_stage_test.go | 365 ++++++++++++++++++ internal/directory/handler.go | 5 +- internal/raftabi/abi.go | 32 +- internal/raftabi/application_pages.go | 169 ++++++++ internal/raftabi/application_pages_test.go | 72 ++++ internal/raftabi/hosted.go | 59 ++- internal/raftabi/v1.go | 30 ++ .../testdata/v1-large-requests.json | 44 +++ 18 files changed, 1695 insertions(+), 65 deletions(-) create mode 100644 internal/clusterstate/codec_large_test.go create mode 100644 internal/clusterstate/record_decode.go create mode 100644 internal/clusterstate/record_encoding.go create mode 100644 internal/clusterstate/v1_stage.go create mode 100644 internal/clusterstate/v1_stage_test.go create mode 100644 internal/raftabi/application_pages.go create mode 100644 internal/raftabi/application_pages_test.go create mode 100644 internal/raftdirectory/testdata/v1-large-requests.json diff --git a/internal/clusterstate/apply.go b/internal/clusterstate/apply.go index ab6aadb7ce..31ca4f4b52 100644 --- a/internal/clusterstate/apply.go +++ b/internal/clusterstate/apply.go @@ -31,7 +31,8 @@ type RecordKey struct { } type RecordWrite struct { RecordKey - Data []byte `json:"data"` + Data []byte `json:"data"` + Delete bool `json:"delete,omitempty"` } type ReadRecord func(RecordKey) ([]byte, error) @@ -101,7 +102,8 @@ type Evaluation struct { // Digest excludes time, signature and acknowledgement. A retry may arrive at a // newer ingress time; its operation and durable sequence must remain identical. func Digest(c Command) (string, error) { - b, err := Encode(struct { + h := sha256.New() + err := encodeCanonicalTo(h, struct { WorkerIncarnation uint64 `json:"worker_incarnation,omitempty"` Cluster string `json:"cluster"` Actor string `json:"actor"` @@ -112,8 +114,7 @@ func Digest(c Command) (string, error) { if err != nil { return "", err } - h := sha256.Sum256(b) - return hex.EncodeToString(h[:]), nil + return hex.EncodeToString(h.Sum(nil)), nil } // SigningBytes binds all routing, sequence and time fields to this protocol. @@ -136,8 +137,9 @@ func Sign(c Command, key ed25519.PrivateKey) (Command, error) { return c, nil } -// Dependencies declares every possible row before evaluation. Hosts may page -// these immutable records, but must distinguish absent rows from unread rows. +// Dependencies declares ordinary operation rows before evaluation. Staging +// requests its fixed temporary slot separately. Hosts must distinguish absent +// immutable records from unread records. func Dependencies(c Command) []RecordKey { keys := []RecordKey{QuiescenceKey(), IdentityKey(), DeviceKey(c.Actor), AuthorityKey(c.Actor), SessionKey(c.Actor, c.Session), SessionsKey()} if c.Change.InitMembership != nil { @@ -242,6 +244,15 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er if err != nil { return out, err } + if c.Change.StageV1 != nil { + return evaluateV1Stage(c, meta, read) + } + return evaluateCommand(c, meta, read, false) +} + +func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged bool) (Evaluation, error) { + out := Evaluation{Now: meta.Now, Version: meta.Version, Writes: []RecordWrite{}, Result: Result{Schema: SchemaVersion, Code: Unauthorized, Now: meta.Now, Version: meta.Version}} + var err error rows := map[RecordKey][]byte{} n := 0 for _, k := range Dependencies(c) { @@ -250,12 +261,14 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er return out, e } n += len(k.Namespace) + len(k.Key) + len(b) - if n > MaxEvaluationBytes { + if !paged && n > MaxEvaluationBytes || paged && len(b) > MaxEvaluationBytes { return out, ErrDependency } rows[k] = bytes.Clone(b) } + cacheCellRecords := len(cellIDs(c.Change)) == 1 + var decodedCells map[RecordKey]Cell // Corrupt dependency bytes are storage evidence, never a not-found or policy // refusal. Validate every declared present row before interpreting any rule. for k, b := range rows { @@ -310,6 +323,12 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er if Decode(b, target) != nil { return out, ErrDependency } + if cell, ok := target.(*Cell); ok && cacheCellRecords && (len(b) >= MaxEvaluationBytes/2 || bytes.HasPrefix(b, compressedRecordPrefix)) { + if decodedCells == nil { + decodedCells = make(map[RecordKey]Cell) + } + decodedCells[k] = *cell + } } if meta.Now < 0 || meta.Now > int64(MaxSafeInteger)-LeaseTTLMillis { return out, ErrDependency @@ -343,9 +362,19 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er } } } - digest, _ := Digest(c) - signed, e := SigningBytes(c) - authenticated := e == nil && len(auth.Key) == ed25519.PublicKeySize && ed25519.Verify(ed25519.PublicKey(auth.Key), signed, c.Signature) + // A staged final operation has already rebuilt its digest from the complete + // original proof, after all immutable dependencies became available. + digest := c.RequestDigest + if !paged { + digest, _ = Digest(c) + } + var signed []byte + var e error + authenticated := false + if c.V1 == nil { + signed, e = SigningBytes(c) + authenticated = e == nil && len(auth.Key) == ed25519.PublicKeySize && ed25519.Verify(ed25519.PublicKey(auth.Key), signed, c.Signature) + } if c.Certificate != "" { certificateAuth, err := PersonalCommandAuthority(c, identity) if err != nil { @@ -415,13 +444,23 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er out.Now = max(meta.Now, c.Now) out.Result.Now = out.Now writes := map[RecordKey][]byte{} + var writtenCells map[RecordKey]Cell put := func(k RecordKey, v any) { - b, e := Encode(v) + b, e := encodeRecord(v) if e != nil { err = e return } writes[k] = b + if cell, ok := v.(Cell); ok { + if _, cached := decodedCells[k]; !cached { + return + } + if writtenCells == nil { + writtenCells = make(map[RecordKey]Cell) + } + writtenCells[k] = cell + } } code := OK if identity.Rotation != nil && c.Change.Rotate == nil && c.Change.AdvanceTime == nil && c.Change.DeviceSeen == nil && c.Change.CloseSession == nil && c.Change.CompleteJoin == nil { @@ -460,7 +499,7 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er } else if membershipMutation(c.Change) { code = applyMembership(c, meta, rows, put, &out.Result) } else { - code = applyMutation(c, out.Now, identity, rows, put, &out.Result) + code = applyMutation(c, out.Now, identity, rows, put, &out.Result, decodedCells) } } } @@ -468,9 +507,20 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er return out, err } changed := false + // Reuse the validated cell for mutation and visibility. Re-decoding the + // same retained frame plan creates several complete JSON trees per update. for k, b := range writes { - if k.Namespace == "state" && visibleChange(k, rows[k], b, out.Now) { - changed = true + if k.Namespace != "state" { + continue + } + if old, ok := decodedCells[k]; ok { + if next, present := writtenCells[k]; present { + changed = changed || visibleCellChange(old, next, out.Now) + } else { + changed = changed || visibleChange(k, rows[k], b, out.Now) + } + } else { + changed = changed || visibleChange(k, rows[k], b, out.Now) } } if c.Change.DeviceSeen != nil { @@ -508,7 +558,7 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er } save() - fits := func() bool { + fits := func() (bool, error) { size := meta.Bytes budget := 0 for k, b := range writes { @@ -517,7 +567,7 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er if old := rows[k]; old != nil { oldSize := RecordBytes(k, old) if oldSize > size { - return false + return false, nil } size -= oldSize } @@ -527,19 +577,39 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er if ceiling == 0 { ceiling = DefaultApplicationBytes } - return budget <= MaxEvaluationBytes && (size <= ceiling || size <= meta.Bytes) + if budget > MaxEvaluationBytes { + return false, nil + } + if size <= ceiling || size <= meta.Bytes { + return true, nil + } + if paged { + return false, nil + } + scratch, err := V1StagingBytes(read) + if err != nil { + return false, err + } + if scratch > meta.Bytes { + return false, ErrDependency + } + return size-scratch <= ceiling, nil } - if !fits() { + if ok, err := fits(); err != nil { + return out, err + } else if !ok { writes = map[RecordKey][]byte{} out.Version = meta.Version out.Result = Result{Schema: SchemaVersion, Code: Capacity, Now: out.Now, Version: meta.Version} save() - if !fits() { + if ok, err := fits(); err != nil { + return out, err + } else if !ok { return out, ErrDependency } } for k, b := range writes { - out.Writes = append(out.Writes, RecordWrite{k, b}) + out.Writes = append(out.Writes, RecordWrite{RecordKey: k, Data: b}) } sort.Slice(out.Writes, func(i, j int) bool { a, b := out.Writes[i].RecordKey, out.Writes[j].RecordKey @@ -551,7 +621,7 @@ func Evaluate(raw []byte, meta ApplicationMeta, read ReadRecord) (Evaluation, er return out, err } -func applyMutation(c Command, now int64, identity IdentityRec, rows map[RecordKey][]byte, put func(RecordKey, any), result *Result) Code { +func applyMutation(c Command, now int64, identity IdentityRec, rows map[RecordKey][]byte, put func(RecordKey, any), result *Result, decodedCells map[RecordKey]Cell) Code { m := c.Change device := func(id string) (Device, bool) { var d Device @@ -701,7 +771,7 @@ func applyMutation(c Command, now int64, identity IdentityRec, rows map[RecordKe sub := c sub.Change = Mutation{Beat: &b} r := Result{} - code := applyMutation(sub, now, identity, batchRows, func(k RecordKey, v any) { put(k, v); data, _ := Encode(v); batchRows[k] = data }, &r) + code := applyMutation(sub, now, identity, batchRows, func(k RecordKey, v any) { put(k, v); data, _ := Encode(v); batchRows[k] = data }, &r, nil) result.Items = append(result.Items, ItemResult{Cell: b.Cell, Code: code, View: r.View}) } return OK @@ -713,7 +783,9 @@ func applyMutation(c Command, now int64, identity IdentityRec, rows map[RecordKe id := ids[0] var cell Cell b := rows[CellKey(id)] - if b != nil && Decode(b, &cell) != nil { + if cached, ok := decodedCells[CellKey(id)]; ok { + cell = cached + } else if b != nil && Decode(b, &cell) != nil { return BadRequest } if m.Create != nil { @@ -780,6 +852,20 @@ func applyMutation(c Command, now int64, identity IdentityRec, rows map[RecordKe // Visible versions follow the existing projection: renewed expiry within the // same held state and advisory last-seen timestamps do not repaint Home. +func visibleCellChange(a, b Cell, now int64) bool { + if a.Lease.Expires > now { + a.Lease.Expires = 1 + } else { + a.Lease.Expires = 0 + } + if b.Lease.Expires > now { + b.Lease.Expires = 1 + } else { + b.Lease.Expires = 0 + } + return !reflect.DeepEqual(a, b) +} + func visibleChange(k RecordKey, old, next []byte, now int64) bool { if old == nil { return true @@ -793,17 +879,7 @@ func visibleChange(k RecordKey, old, next []byte, now int64) bool { if Decode(old, &a) != nil || Decode(next, &b) != nil { return true } - if a.Lease.Expires > now { - a.Lease.Expires = 1 - } else { - a.Lease.Expires = 0 - } - if b.Lease.Expires > now { - b.Lease.Expires = 1 - } else { - b.Lease.Expires = 0 - } - return !reflect.DeepEqual(a, b) + return visibleCellChange(a, b, now) case len(k.Key) > 7 && k.Key[:7] == "device/": var a, b Device if Decode(old, &a) != nil || Decode(next, &b) != nil { diff --git a/internal/clusterstate/codec.go b/internal/clusterstate/codec.go index 41ac404016..feeb827f57 100644 --- a/internal/clusterstate/codec.go +++ b/internal/clusterstate/codec.go @@ -5,7 +5,9 @@ import ( "encoding/json" "errors" "fmt" + "io" "reflect" + "sort" "strconv" "strings" "unicode/utf8" @@ -22,24 +24,244 @@ var ( // opaque, and byte slices use Go's padded standard base64. HTML and U+2028/2029 // escaping follows encoding/json. Hosts must pass these bytes without rewriting. func Encode(v any) ([]byte, error) { - b, err := json.Marshal(v) + tree, _, err := canonicalTree(v) if err != nil { - return nil, ErrCodec + return nil, err } - if !validStrings(reflect.ValueOf(v)) { - return nil, ErrCodec + if hasLargeString(tree) { + var out bytes.Buffer + if err := writeCanonicalTree(&out, tree, true); err != nil { + return nil, err + } + return out.Bytes(), nil + } + return json.Marshal(tree) +} + +// ProjectionJSON retains typed decimal counters in JSON consumed by a host. +// Compact HTML spelling avoids an expanded intermediate string; it is not a +// replacement for canonical protocol bytes. +func ProjectionJSON(v any) ([]byte, error) { + tree, _, err := canonicalTree(v) + if err != nil { + return nil, err + } + var out bytes.Buffer + if err = encodeCanonicalTreeTo(&out, tree, false); err != nil { + return nil, err + } + return out.Bytes(), nil +} + +func canonicalTree(v any) (any, int, error) { + b, err := marshalIntermediate(v) + if err != nil || !validStrings(reflect.ValueOf(v)) { + return nil, 0, ErrCodec } var tree any d := json.NewDecoder(bytes.NewReader(b)) d.UseNumber() if err = d.Decode(&tree); err != nil { - return nil, ErrCodec + return nil, 0, ErrCodec } tree, err = integers(tree, reflect.TypeOf(v), true) + return tree, len(b), err +} + +// Canonical sinks hash, compare or compress the final bytes without retaining +// another expanded copy. The encoder's sole trailing newline is not canonical. +func encodeCanonicalTo(w io.Writer, v any) error { + return encodeCanonicalValueTo(w, v, true) +} +func encodeCanonicalValueTo(w io.Writer, v any, escapeHTML bool) error { + tree, _, err := canonicalTree(v) if err != nil { + return err + } + return encodeCanonicalTreeTo(w, tree, escapeHTML) +} +func encodeCanonicalTreeTo(w io.Writer, tree any, escapeHTML bool) error { + if hasLargeString(tree) { + return writeCanonicalTree(w, tree, escapeHTML) + } + sink := &canonicalSink{writer: w} + encoder := json.NewEncoder(sink) + encoder.SetEscapeHTML(escapeHTML) + if err := encoder.Encode(tree); err != nil { + return err + } + if !sink.pending || sink.last != '\n' { + return ErrCodec + } + return nil +} + +// Large opaque strings are escaped in bounded pieces. Small subtrees retain +// the standard JSON encoder, including its established Unicode spelling. +func hasLargeString(tree any) bool { + switch v := tree.(type) { + case string: + return len(v) > MaxCommandBytes + case map[string]any: + for key, value := range v { + if len(key) > MaxCommandBytes || hasLargeString(value) { + return true + } + } + case []any: + for _, value := range v { + if hasLargeString(value) { + return true + } + } + } + return false +} +func writeCanonicalBytes(w io.Writer, b []byte) error { + n, err := w.Write(b) + if err == nil && n != len(b) { + return io.ErrShortWrite + } + return err +} +func writeCanonicalTree(w io.Writer, tree any, escapeHTML bool) error { + if !hasLargeString(tree) { + var b []byte + var err error + if escapeHTML { + b, err = json.Marshal(tree) + } else { + b, err = marshalIntermediate(tree) + } + if err != nil { + return err + } + return writeCanonicalBytes(w, b) + } + switch v := tree.(type) { + case string: + if err := writeCanonicalBytes(w, []byte{'"'}); err != nil { + return err + } + for start := 0; start < len(v); { + end := min(start+V1PageBytes/8, len(v)) + for end < len(v) && !utf8.RuneStart(v[end]) { + end-- + } + var b []byte + var err error + if escapeHTML { + b, err = json.Marshal(v[start:end]) + } else { + b, err = marshalIntermediate(v[start:end]) + } + if err != nil { + return err + } + if err = writeCanonicalBytes(w, b[1:len(b)-1]); err != nil { + return err + } + start = end + } + return writeCanonicalBytes(w, []byte{'"'}) + case map[string]any: + keys := make([]string, 0, len(v)) + for key := range v { + keys = append(keys, key) + } + sort.Strings(keys) + if err := writeCanonicalBytes(w, []byte{'{'}); err != nil { + return err + } + for i, key := range keys { + if i > 0 { + if err := writeCanonicalBytes(w, []byte{','}); err != nil { + return err + } + } + if err := writeCanonicalTree(w, key, escapeHTML); err != nil { + return err + } + if err := writeCanonicalBytes(w, []byte{':'}); err != nil { + return err + } + if err := writeCanonicalTree(w, v[key], escapeHTML); err != nil { + return err + } + } + return writeCanonicalBytes(w, []byte{'}'}) + case []any: + if err := writeCanonicalBytes(w, []byte{'['}); err != nil { + return err + } + for i, value := range v { + if i > 0 { + if err := writeCanonicalBytes(w, []byte{','}); err != nil { + return err + } + } + if err := writeCanonicalTree(w, value, escapeHTML); err != nil { + return err + } + } + return writeCanonicalBytes(w, []byte{']'}) + } + return ErrCodec +} + +type canonicalSink struct { + writer io.Writer + pending bool + last byte +} + +func (s *canonicalSink) Write(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + if s.pending { + if n, err := s.writer.Write([]byte{s.last}); err != nil { + return 0, err + } else if n != 1 { + return 0, io.ErrShortWrite + } + } + n, err := s.writer.Write(p[:len(p)-1]) + if err != nil { + return n, err + } + if n != len(p)-1 { + return n, io.ErrShortWrite + } + s.pending = true + s.last = p[len(p)-1] + return len(p), nil +} + +type canonicalComparison struct { + want []byte + offset int +} + +func (c *canonicalComparison) Write(p []byte) (int, error) { + if len(p) > len(c.want)-c.offset || !bytes.Equal(p, c.want[c.offset:c.offset+len(p)]) { + return 0, ErrCodec + } + c.offset += len(p) + return len(p), nil +} + +// Intermediate JSON only feeds the typed codec. Avoid expanding opaque HTML +// characters until the final canonical write, so legal legacy rows do not +// retain several escaped copies inside the bounded reactor. +func marshalIntermediate(v any) ([]byte, error) { + var out bytes.Buffer + encoder := json.NewEncoder(&out) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(v); err != nil { return nil, err } - return json.Marshal(tree) + return bytes.TrimSuffix(out.Bytes(), []byte("\n")), nil } // JSON's replacement of invalid UTF-8 would alter an opaque field. Reject it @@ -77,23 +299,36 @@ func validStrings(v reflect.Value) bool { // Decode accepts only canonical bytes into a typed destination. Re-encoding // rejects duplicate/unknown/missing fields, alternate integer spellings, lossy -// Unicode, and trailing data. It updates the destination only after validation. +// Unicode, and trailing data. Internal compressed records use compact HTML +// escaping; public canonical bytes retain their established escaping. The +// destination changes only after validation. func Decode(b []byte, into any) error { + compact := bytes.HasPrefix(b, compressedRecordPrefix) + var err error + b, err = recordBytes(b) + if err != nil { + return err + } v := reflect.ValueOf(into) if v.Kind() != reflect.Pointer || v.IsNil() { return ErrCodec } + if compact { + if handled, err := decodeRetainedRecord(b, v); handled { + return err + } + } d := json.NewDecoder(bytes.NewReader(b)) d.UseNumber() var tree any if err := d.Decode(&tree); err != nil { return ErrCodec } - tree, err := integers(tree, v.Type().Elem(), false) + tree, err = integers(tree, v.Type().Elem(), false) if err != nil { return err } - plain, err := json.Marshal(tree) + plain, err := marshalIntermediate(tree) if err != nil { return ErrCodec } @@ -103,8 +338,9 @@ func Decode(b []byte, into any) error { if err = d.Decode(candidate.Interface()); err != nil { return ErrCodec } - canonical, err := Encode(candidate.Elem().Interface()) - if err != nil || !bytes.Equal(b, canonical) { + comparison := canonicalComparison{want: b} + err = encodeCanonicalValueTo(&comparison, candidate.Elem().Interface(), !compact) + if err != nil || comparison.offset != len(b) { return ErrCodec } v.Elem().Set(candidate.Elem()) @@ -228,6 +464,9 @@ func EncodeCommand(c Command) ([]byte, error) { } func DecodeCommand(b []byte) (Command, error) { + if bytes.HasPrefix(b, compressedRecordPrefix) { + return Command{}, ErrCodec + } var c Command if err := Decode(b, &c); err != nil { return Command{}, err @@ -319,6 +558,9 @@ func DecodeResult(b []byte) (Result, error) { } func (r Result) Validate() error { + if r.ViewIndex != 0 && r.View != nil { + return ErrCodec + } if r.Schema != SchemaVersion { return ErrSchema } diff --git a/internal/clusterstate/codec_large_test.go b/internal/clusterstate/codec_large_test.go new file mode 100644 index 0000000000..ef4f25e408 --- /dev/null +++ b/internal/clusterstate/codec_large_test.go @@ -0,0 +1,31 @@ +package clusterstate_test + +import ( + "bytes" + "encoding/json" + "reflect" + "strings" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +// R14/R25 retain the established JSON spelling for large opaque Unicode, +// including escapes and map keys that cross bounded serialization pieces. +func TestR14LargeOpaqueJSONMatchesLegacyCanonicalBytes(t *testing.T) { + text := strings.Repeat("<>&\\\"\n\t\u2028🙂界", 10000) + for _, value := range []map[string]string{{"title": text, "opaque": "saved"}, {text: "key", "other": text}} { + want, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + got, err := cs.Encode(value) + if err != nil || !bytes.Equal(got, want) { + t.Fatal("large canonical JSON spelling changed", err) + } + var decoded map[string]string + if err = cs.Decode(want, &decoded); err != nil || !reflect.DeepEqual(decoded, value) { + t.Fatal("large Unicode round trip changed fields", err) + } + } +} diff --git a/internal/clusterstate/imports_test.go b/internal/clusterstate/imports_test.go index 40d885830f..6e7b0ee776 100644 --- a/internal/clusterstate/imports_test.go +++ b/internal/clusterstate/imports_test.go @@ -14,7 +14,7 @@ import ( // enter the pure graph when records or rules change. func TestR14PureImportBoundary(t *testing.T) { allowed := map[string]bool{ - "io": true, "bytes": true, "encoding/json": true, "errors": true, "fmt": true, + "compress/gzip": true, "io": true, "bytes": true, "encoding/json": true, "errors": true, "fmt": true, // Signature verification, hashing, key encoding and ordering are pure. "encoding/base64": true, "crypto/ed25519": true, "crypto/sha256": true, "encoding/hex": true, "sort": true, "reflect": true, "strconv": true, "strings": true, "unicode/utf8": true, diff --git a/internal/clusterstate/legacy.go b/internal/clusterstate/legacy.go index 30c5bb0319..4cf78aa8b7 100644 --- a/internal/clusterstate/legacy.go +++ b/internal/clusterstate/legacy.go @@ -149,7 +149,7 @@ func LegacyRow(c LegacyConfig, kind, id string, raw []byte, holdUntil int64) (Re if e != nil || len(data)+len(key.Key) > MaxEvaluationBytes { return RecordWrite{}, ErrMigration } - return RecordWrite{key, data}, nil + return RecordWrite{RecordKey: key, Data: data}, nil } func legacyDecode(raw []byte, v any) error { d := json.NewDecoder(bytes.NewReader(raw)) @@ -166,7 +166,7 @@ func legacyDecode(raw []byte, v any) error { } func LegacyEffects(c LegacyConfig) RecordWrite { b, _ := Encode(Authority{Cluster: c.Cluster, Key: c.EffectsKey, Effects: true}) - return RecordWrite{AuthorityKey(DeviceIDOfKey(c.EffectsKey)), b} + return RecordWrite{RecordKey: AuthorityKey(DeviceIDOfKey(c.EffectsKey)), Data: b} } // SealLegacyManifest counts its own retained provenance in the imported state. @@ -203,7 +203,7 @@ func LegacyAuthorities(c LegacyConfig) ([]RecordWrite, error) { if e != nil { return nil, e } - out = append(out, RecordWrite{AuthorityKey(DeviceIDOfKey(public)), b}) + out = append(out, RecordWrite{RecordKey: AuthorityKey(DeviceIDOfKey(public)), Data: b}) } sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) return out, nil diff --git a/internal/clusterstate/record_decode.go b/internal/clusterstate/record_decode.go new file mode 100644 index 0000000000..99550c0bf1 --- /dev/null +++ b/internal/clusterstate/record_decode.go @@ -0,0 +1,134 @@ +package clusterstate + +import ( + "bytes" + "encoding/json" + "reflect" + "strconv" +) + +type recordCounter uint64 + +func (n *recordCounter) UnmarshalJSON(raw []byte) error { + var text string + if json.Unmarshal(raw, &text) != nil { + return ErrCodec + } + value, err := strconv.ParseUint(text, 10, 64) + if err != nil || strconv.FormatUint(value, 10) != text { + return ErrCodec + } + *n = recordCounter(value) + return nil +} + +// Large retained views decode directly into their typed shape. A shadow type +// admits canonical decimal counters without allocating multiple generic JSON +// trees containing the same complete frame plan. +var cellRecordWire = recordWireType(reflect.TypeOf(Cell{})) +var resultRecordWire = recordWireType(reflect.TypeOf(Result{})) + +func recordWireType(original reflect.Type) reflect.Type { + switch original.Kind() { + case reflect.Uint64: + return reflect.TypeOf(recordCounter(0)) + case reflect.Pointer: + elem := recordWireType(original.Elem()) + if elem != original.Elem() { + return reflect.PointerTo(elem) + } + case reflect.Slice: + elem := recordWireType(original.Elem()) + if elem != original.Elem() { + return reflect.SliceOf(elem) + } + case reflect.Array: + elem := recordWireType(original.Elem()) + if elem != original.Elem() { + return reflect.ArrayOf(original.Len(), elem) + } + case reflect.Map: + elem := recordWireType(original.Elem()) + if elem != original.Elem() { + return reflect.MapOf(original.Key(), elem) + } + case reflect.Struct: + fields := make([]reflect.StructField, original.NumField()) + changed := false + for i := range fields { + fields[i] = original.Field(i) + next := recordWireType(fields[i].Type) + changed = changed || next != fields[i].Type + fields[i].Type = next + } + if changed { + return reflect.StructOf(fields) + } + } + return original +} +func copyRecordValue(dst, src reflect.Value) { + if dst.Type() == src.Type() { + dst.Set(src) + return + } + switch dst.Kind() { + case reflect.Uint64: + dst.SetUint(src.Uint()) + case reflect.Pointer: + if !src.IsNil() { + dst.Set(reflect.New(dst.Type().Elem())) + copyRecordValue(dst.Elem(), src.Elem()) + } + case reflect.Struct: + for i := 0; i < dst.NumField(); i++ { + copyRecordValue(dst.Field(i), src.Field(i)) + } + case reflect.Slice: + if !src.IsNil() { + dst.Set(reflect.MakeSlice(dst.Type(), src.Len(), src.Len())) + for i := 0; i < src.Len(); i++ { + copyRecordValue(dst.Index(i), src.Index(i)) + } + } + case reflect.Array: + for i := 0; i < src.Len(); i++ { + copyRecordValue(dst.Index(i), src.Index(i)) + } + case reflect.Map: + if !src.IsNil() { + dst.Set(reflect.MakeMapWithSize(dst.Type(), src.Len())) + iter := src.MapRange() + for iter.Next() { + value := reflect.New(dst.Type().Elem()).Elem() + copyRecordValue(value, iter.Value()) + dst.SetMapIndex(iter.Key(), value) + } + } + } +} +func decodeRetainedRecord(raw []byte, destination reflect.Value) (bool, error) { + var wire reflect.Type + switch destination.Type().Elem() { + case reflect.TypeOf(Cell{}): + wire = cellRecordWire + case reflect.TypeOf(Result{}): + wire = resultRecordWire + default: + return false, nil + } + shadow := reflect.New(wire) + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if decoder.Decode(shadow.Interface()) != nil { + return true, ErrCodec + } + candidate := reflect.New(destination.Type().Elem()) + copyRecordValue(candidate.Elem(), shadow.Elem()) + comparison := canonicalComparison{want: raw} + if encodeCanonicalValueTo(&comparison, candidate.Elem().Interface(), false) != nil || comparison.offset != len(raw) { + return true, ErrCodec + } + destination.Elem().Set(candidate.Elem()) + return true, nil +} diff --git a/internal/clusterstate/record_encoding.go b/internal/clusterstate/record_encoding.go new file mode 100644 index 0000000000..184a6a8703 --- /dev/null +++ b/internal/clusterstate/record_encoding.go @@ -0,0 +1,76 @@ +package clusterstate + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "io" +) + +const maxJSONExpansion = 6 + +var compressedRecordPrefix = []byte{0, 'c', 'a', 'f', 'r', 1} + +// Large opaque metadata stays lossless inside the physical row envelope. +// Compression changes neither the canonical logical record nor its fields. +func encodeRecord(v any) ([]byte, error) { + tree, plainSize, err := canonicalTree(v) + if err != nil { + return nil, err + } + if plainSize < MaxEvaluationBytes/(2*maxJSONExpansion) { + return json.Marshal(tree) + } + var out bytes.Buffer + out.Write(compressedRecordPrefix) + compressed := gzip.NewWriter(&out) + counted := countingRecordWriter{writer: compressed} + if err := encodeCanonicalTreeTo(&counted, tree, false); err != nil { + compressed.Close() + return nil, err + } + if err := compressed.Close(); err != nil { + return nil, err + } + if out.Len() < counted.bytes { + return out.Bytes(), nil + } + return json.Marshal(tree) +} + +type countingRecordWriter struct { + writer io.Writer + bytes int +} + +func (w *countingRecordWriter) Write(p []byte) (int, error) { + n, err := w.writer.Write(p) + w.bytes += n + return n, err +} +func recordBytes(b []byte) ([]byte, error) { + if !bytes.HasPrefix(b, compressedRecordPrefix) { + return b, nil + } + r, err := gzip.NewReader(bytes.NewReader(b[len(compressedRecordPrefix):])) + if err != nil { + return nil, ErrCodec + } + // JSON escaping can expand each legacy byte to six ASCII bytes; record framing remains small. + limit := maxJSONExpansion*MaxV1BodyBytes + MaxCommandBytes + out, err := io.ReadAll(io.LimitReader(r, int64(limit+1))) + closeErr := r.Close() + if err != nil || closeErr != nil || len(out) > limit { + return nil, ErrCodec + } + return out, nil +} + +// EncodeApplicationResult bounds retained response bytes without changing the +// result's logical fields. Public projection always decodes the original type. +func EncodeApplicationResult(result Result) ([]byte, error) { + if err := result.Validate(); err != nil { + return nil, err + } + return encodeRecord(result) +} diff --git a/internal/clusterstate/types.go b/internal/clusterstate/types.go index b2d5c63738..629e80c1ea 100644 --- a/internal/clusterstate/types.go +++ b/internal/clusterstate/types.go @@ -47,6 +47,7 @@ type Command struct { // Unknown fields fail decoding on older members. // There is no arbitrary JSON payload in which task text or secrets can hide. type Mutation struct { + StageV1 *V1Stage `json:"stage_v1,omitempty"` Assign *Assign `json:"assign,omitempty"` Reassign *Assign `json:"reassign,omitempty"` ClaimAssignment *ClaimAssignment `json:"claim_assignment,omitempty"` @@ -158,6 +159,7 @@ type DeviceSeen struct { // no free-form diagnostic text, so logging a refusal cannot disclose payloads. // Now and Version are application time/version, never a local clock/Raft index. type Result struct { + ViewIndex uint64 `json:"view_index,omitempty"` Assignment *Assignment `json:"assignment,omitempty"` Workers []WorkerResult `json:"workers,omitempty"` Index uint64 `json:"index,omitempty"` diff --git a/internal/clusterstate/v1.go b/internal/clusterstate/v1.go index 0a4adb8b14..bddb4a697c 100644 --- a/internal/clusterstate/v1.go +++ b/internal/clusterstate/v1.go @@ -38,6 +38,17 @@ const V1SkewMillis int64 = 5 * 60 * 1000 // V1Command translates only the request the device actually signed. It never // reads a new holder to manufacture an expectation or signs on the device's behalf. func V1Command(cluster string, now int64, p V1Request) (Command, error) { + c, err := translateV1(cluster, now, p) + if err != nil { + return Command{}, err + } + c.RequestDigest, err = Digest(c) + return c, err +} +func translateV1(cluster string, now int64, p V1Request) (Command, error) { + if len(p.Body) > MaxV1BodyBytes { + return Command{}, ErrCodec + } m, err := p.Mutation() if err != nil { return Command{}, err @@ -52,8 +63,7 @@ func V1Command(cluster string, now int64, p V1Request) (Command, error) { return Command{}, ErrCodec } c := Command{V1: &p, Schema: SchemaVersion, Cluster: cluster, Actor: DeviceIDOfKey(pub), Session: "v1", Sequence: 1, Now: now, Change: m} - c.RequestDigest, err = Digest(c) - return c, err + return c, nil } // V1Authority checks the root, certificate, exact HTTP signing domain and the @@ -67,6 +77,19 @@ func V1Authority(c Command, identity IdentityRec) (Authority, bool) { // time has advanced. A sender cannot make stale ingress evidence fresh again // by placing its original timestamp in an unsigned host field. func V1AuthorityAt(c Command, identity IdentityRec, committedNow int64) (Authority, bool) { + if c.V1 == nil { + return Authority{}, false + } + h := sha256.Sum256(c.V1.Body) + auth, ok := v1ProofAuthority(c, identity, committedNow, hex.EncodeToString(h[:])) + m, err := c.V1.Mutation() + if !ok || err != nil || !reflect.DeepEqual(m, c.Change) || m.PutDevice != nil && m.PutDevice.ID != c.Actor { + return Authority{}, false + } + return auth, true +} + +func v1ProofAuthority(c Command, identity IdentityRec, committedNow int64, bodyHash string) (Authority, bool) { p := c.V1 if p == nil || c.WorkerIncarnation != 0 || c.Session != "v1" || c.Sequence != 1 || c.Ack != 0 || len(c.Signature) != 0 || !bytes.Equal(p.IdentityKey, identity.SigningKey) { return Authority{}, false @@ -89,9 +112,8 @@ func V1AuthorityAt(c Command, identity IdentityRec, committedNow int64) (Authori if err != nil { return Authority{}, false } - msg := V1SigningMessage(p.Method, p.URI, p.At, p.Body) - m, err := p.Mutation() - if err != nil || !reflect.DeepEqual(m, c.Change) || m.PutDevice != nil && m.PutDevice.ID != c.Actor || !ed25519.Verify(pub, msg, p.Signature) { + msg := []byte("codeaf-req-v1\n" + p.Method + "\n" + p.URI + "\n" + p.At + "\n" + bodyHash) + if !ed25519.Verify(pub, msg, p.Signature) { return Authority{}, false } return Authority{Cluster: c.Cluster, Key: pub, Personal: true}, true @@ -119,7 +141,7 @@ func ClientCommand(c Command) error { return ErrCodec } } - if c.V1 != nil || c.Change.Create != nil && c.Change.Create.V1 || c.Change.Publish != nil && c.Change.Publish.V1 || c.Change.Acquire != nil && (c.Change.Acquire.V1 || c.Change.Acquire.Legacy) { + if c.Change.StageV1 != nil || c.V1 != nil || c.Change.Create != nil && c.Change.Create.V1 || c.Change.Publish != nil && c.Change.Publish.V1 || c.Change.Acquire != nil && (c.Change.Acquire.V1 || c.Change.Acquire.Legacy) { return ErrCodec } return nil diff --git a/internal/clusterstate/v1_stage.go b/internal/clusterstate/v1_stage.go new file mode 100644 index 0000000000..425e55c6f0 --- /dev/null +++ b/internal/clusterstate/v1_stage.go @@ -0,0 +1,291 @@ +package clusterstate + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "strings" +) + +// MaxV1BodyBytes preserves the legacy HTTP envelope independently of the +// smaller consensus command envelope. Staging reuses a single bounded slot. +const MaxV1BodyBytes = 1 << 20 +const V1PageBytes = MaxCommandBytes / 2 +const MaxV1StagingBytes = MaxV1BodyBytes + MaxCommandBytes +const V1StagePages = (MaxV1BodyBytes + V1PageBytes - 1) / V1PageBytes + +type V1Stage struct { + Hash string `json:"hash"` + Total uint32 `json:"total"` + Offset uint32 `json:"offset"` + Data []byte `json:"data"` + Commit bool `json:"commit,omitempty"` +} +type v1Staging struct { + ID string `json:"id"` + Next uint32 `json:"next"` +} + +func V1StageKey() RecordKey { return RecordKey{"outbox", "v1-stage/header"} } +func V1PageKey(page int) RecordKey { + return RecordKey{"outbox", fmt.Sprintf("v1-stage/page/%02d", page)} +} + +// V1StagingKeys fixes the temporary slot's complete physical extent. Retained +// capacity excludes this bounded command bookkeeping, including in snapshots. +func V1StagingKeys() []RecordKey { + keys := []RecordKey{V1StageKey()} + for page := 0; page < V1StagePages; page++ { + keys = append(keys, V1PageKey(page)) + } + return keys +} + +// V1StagingBytes derives temporary accounting from immutable records rather +// than maintaining a second durable counter that could disagree after a crash. +func V1StagingBytes(read ReadRecord) (uint64, error) { + var size uint64 + for i, key := range V1StagingKeys() { + raw, err := read(key) + if err != nil { + return 0, err + } + if i == 0 && raw == nil { + return 0, nil + } + limit := V1PageBytes + if i == 0 { + limit = MaxCommandBytes + } + if len(raw) > limit { + return 0, ErrDependency + } + if raw != nil { + size += RecordBytes(key, raw) + } + } + if size > MaxV1StagingBytes { + return 0, ErrDependency + } + return size, nil +} + +// V1RequestCommands translates a legacy body directly to supported proposals. +// A proof that cannot fit one command never allocates that unused command's +// expanded digest representation before producing its bounded body pages. +func V1RequestCommands(cluster string, now int64, proof V1Request) ([]Command, error) { + c, err := translateV1(cluster, now, proof) + if err != nil { + return nil, err + } + if base64.StdEncoding.EncodedLen(len(proof.Body)) <= MaxCommandBytes { + c.RequestDigest, err = Digest(c) + if err != nil { + return nil, err + } + } + return V1Commands(c) +} + +// V1Commands preserves the original proof while each proposal remains within +// the command envelope. Only the final proposal can change directory records. +func V1Commands(c Command) ([]Command, error) { + var err error + // The proof's base64 body alone can establish that a command cannot fit. + // Avoid materializing an oversized duplicate of the decoded mutation. + if c.V1 == nil || base64.StdEncoding.EncodedLen(len(c.V1.Body)) <= MaxCommandBytes { + raw, e := EncodeCommand(c) + if e != nil { + return nil, e + } + if len(raw) <= MaxCommandBytes { + return []Command{c}, nil + } + } + if c.V1 == nil || len(c.V1.Body) > MaxV1BodyBytes || c.Change.StageV1 != nil { + return nil, ErrCodec + } + proof := *c.V1 + body := proof.Body + proof.Body = nil + hash := sha256.Sum256(body) + part := func(offset int, data []byte, commit bool) (Command, error) { + v := c + v.V1 = &proof + v.Change = Mutation{StageV1: &V1Stage{Hash: hex.EncodeToString(hash[:]), Total: uint32(len(body)), Offset: uint32(offset), Data: bytes.Clone(data), Commit: commit}} + v.RequestDigest, err = Digest(v) + if err != nil { + return Command{}, err + } + b, e := EncodeCommand(v) + if e != nil || len(b) > MaxCommandBytes { + return Command{}, ErrCodec + } + return v, nil + } + out := []Command{} + for offset := 0; offset < len(body); offset += V1PageBytes { + v, e := part(offset, body[offset:min(len(body), offset+V1PageBytes)], false) + if e != nil { + return nil, e + } + out = append(out, v) + } + v, e := part(len(body), nil, true) + if e != nil { + return nil, e + } + return append(out, v), nil +} +func v1StageID(c Command) string { + v := c.Change.StageV1 + raw, _ := Encode(struct { + Proof *V1Request `json:"proof"` + Hash string `json:"hash"` + Total uint32 `json:"total"` + }{c.V1, v.Hash, v.Total}) + h := sha256.Sum256(raw) + return hex.EncodeToString(h[:]) +} + +func evaluateV1Stage(c Command, meta ApplicationMeta, read ReadRecord) (Evaluation, error) { + out := Evaluation{Now: meta.Now, Version: meta.Version, Writes: []RecordWrite{}, Result: Result{Schema: SchemaVersion, Code: Unauthorized, Now: meta.Now, Version: meta.Version}} + v := c.Change.StageV1 + hash, err := hex.DecodeString(v.Hash) + digest, e := Digest(c) + if err != nil || len(hash) != sha256.Size || e != nil || digest != c.RequestDigest || c.Cluster != meta.Cluster || c.V1 == nil || len(c.V1.Body) != 0 || v.Total > MaxV1BodyBytes || v.Total == 0 || v.Offset > v.Total || c.Certificate != "" { + return out, nil + } + var identity IdentityRec + raw, err := read(IdentityKey()) + if err != nil { + return out, err + } + if Decode(raw, &identity) != nil { + return out, ErrDependency + } + if _, ok := v1ProofAuthority(c, identity, meta.Now, v.Hash); !ok { + return out, nil + } + raw, err = read(DeviceKey(c.Actor)) + if err != nil { + return out, err + } + if raw != nil { + var device Device + if Decode(raw, &device) != nil { + return out, ErrDependency + } + if device.Revoked { + out.Result.Code = Revoked + return out, nil + } + } + if identity.Rotation != nil && !(c.V1.Method == "POST" && strings.TrimPrefix(c.V1.URI, "/fabric") == "/v1/identity/rotation") { + out.Result.Code = Rotated + return out, nil + } + raw, err = read(V1StageKey()) + if err != nil { + return out, err + } + var stage v1Staging + if raw != nil && Decode(raw, &stage) != nil { + return out, ErrDependency + } + id := v1StageID(c) + conflict := func() (Evaluation, error) { out.Result.Code = Conflict; return out, nil } + if !v.Commit { + if v.Offset%V1PageBytes != 0 || len(v.Data) != min(V1PageBytes, int(v.Total-v.Offset)) || len(v.Data) == 0 { + return out, nil + } + if v.Offset != 0 && (stage.ID != id || stage.Next != v.Offset) { + return conflict() + } + stage = v1Staging{ID: id, Next: v.Offset + uint32(len(v.Data))} + header, _ := Encode(stage) + writes := []RecordWrite{{RecordKey: V1StageKey(), Data: header}, {RecordKey: V1PageKey(int(v.Offset) / V1PageBytes), Data: bytes.Clone(v.Data)}} + out.Result.Code = OK + out.Writes = writes + return out, nil + } + if len(v.Data) != 0 || v.Offset != v.Total || stage.ID != id || stage.Next != v.Total { + return conflict() + } + body := make([]byte, 0, int(v.Total)) + scratch := RecordBytes(V1StageKey(), raw) + cleanup := []RecordWrite{{RecordKey: V1StageKey(), Delete: true}} + for page := 0; page < V1StagePages; page++ { + key := V1PageKey(page) + b, err := read(key) + if err != nil { + return out, err + } + if len(b) > V1PageBytes { + return out, ErrDependency + } + if b != nil { + scratch += RecordBytes(key, b) + cleanup = append(cleanup, RecordWrite{RecordKey: key, Delete: true}) + } + if page*V1PageBytes < int(v.Total) { + want := min(V1PageBytes, int(v.Total)-page*V1PageBytes) + if len(b) != want { + return conflict() + } + body = append(body, b...) + } + } + actual := sha256.Sum256(body) + if !bytes.Equal(actual[:], hash) { + return conflict() + } + proof := *c.V1 + proof.Body = body + original, err := translateV1(c.Cluster, c.Now, proof) + if err != nil { + out.Result.Code = BadRequest + out.Writes = cleanup + return out, nil + } + // Resolve immutable dependencies before computing the expanded operation + // digest. A paged host may retry for each missing record; those retries must + // not retain several unused expanded command representations. + for _, key := range Dependencies(original) { + if _, err := read(key); err != nil { + return out, err + } + } + original.RequestDigest, err = Digest(original) + if err != nil { + return out, err + } + if scratch > meta.Bytes { + return out, ErrDependency + } + meta.Bytes -= scratch + evaluated, err := evaluateCommand(original, meta, read, true) + if err != nil { + return out, err + } + evaluated.Writes = append(evaluated.Writes, cleanup...) + return evaluated, nil +} + +// BoundedV1Result forwards a durable apply receipt when the original view +// cannot fit a normal response. The ingress resolves the exact result from its +// own application of that same entry, rather than reading a newer cell. +func BoundedV1Result(command []byte, result Result, index uint64) Result { + c, err := DecodeCommand(command) + if err != nil || c.V1 == nil || result.View == nil || index == 0 { + return result + } + raw, err := EncodeResult(result) + if err == nil && len(raw) > MaxCommandBytes { + result.View = nil + result.ViewIndex = index + } + return result +} diff --git a/internal/clusterstate/v1_stage_test.go b/internal/clusterstate/v1_stage_test.go new file mode 100644 index 0000000000..75eee7eddf --- /dev/null +++ b/internal/clusterstate/v1_stage_test.go @@ -0,0 +1,365 @@ +package clusterstate_test + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "os" + "reflect" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +func largeV1(t *testing.T) (cs.Command, map[cs.RecordKey][]byte, cs.ApplicationMeta) { + t.Helper() + raw, err := os.ReadFile("../raftdirectory/testdata/v1-large-requests.json") + if err != nil { + t.Fatal(err) + } + var f struct { + Now int64 + Cases []struct { + Method, URI, Body string + Headers map[string]string + } + } + if json.Unmarshal(raw, &f) != nil { + t.Fatal("fixture decode") + } + one := f.Cases[1] + h := one.Headers + root, err := base64.RawURLEncoding.DecodeString(h["Codeaf-Identity"]) + if err != nil { + t.Fatal(err) + } + sig, err := base64.RawURLEncoding.DecodeString(h["Codeaf-Sig"]) + if err != nil { + t.Fatal(err) + } + c, err := cs.V1Command("large", f.Now, cs.V1Request{Method: one.Method, URI: one.URI, Body: []byte(one.Body), At: h["Codeaf-Time"], Certificate: h["Codeaf-Cert"], IdentityKey: root, Signature: sig}) + if err != nil { + t.Fatal(err) + } + identity, err := cs.Encode(cs.IdentityRec{V: 1, Identity: cs.IdentityID(root), SigningKey: root}) + if err != nil { + t.Fatal(err) + } + rows := map[cs.RecordKey][]byte{cs.IdentityKey(): identity} + meta := cs.ApplicationMeta{Cluster: "large", Now: f.Now, Ceiling: cs.DefaultApplicationBytes, Bytes: cs.RecordBytes(cs.IdentityKey(), identity)} + return c, rows, meta +} +func applyV1Page(t *testing.T, c cs.Command, rows map[cs.RecordKey][]byte, meta *cs.ApplicationMeta) cs.Result { + t.Helper() + raw, err := cs.EncodeCommand(c) + if err != nil { + t.Fatal(err) + } + if len(raw) > cs.MaxCommandBytes { + t.Fatal("proposal exceeds command cap") + } + out, err := cs.Evaluate(raw, *meta, func(k cs.RecordKey) ([]byte, error) { return rows[k], nil }) + if err != nil { + t.Fatal(err) + } + for _, w := range out.Writes { + if old := rows[w.RecordKey]; old != nil { + meta.Bytes -= cs.RecordBytes(w.RecordKey, old) + } + if w.Delete { + delete(rows, w.RecordKey) + } else { + if len(w.Data) > cs.MaxEvaluationBytes { + t.Fatal("physical record exceeds evaluation cap") + } + rows[w.RecordKey] = w.Data + meta.Bytes += cs.RecordBytes(w.RecordKey, w.Data) + } + } + meta.Now, meta.Version = out.Now, out.Version + meta.Index++ + return out.Result +} + +// R25/R16 require the complete original proof at final commitment, including +// unchanged time, current revocation and ordered, hash-matching body pages. +func TestR25V1StagingProofAndAtomicity(t *testing.T) { + for _, fault := range []string{"none", "changed-page", "missing-page", "reordered", "stale-proof", "revoked", "wrong-cluster", "wrong-root", "public-v2"} { + t.Run(fault, func(t *testing.T) { + c, rows, meta := largeV1(t) + parts, err := cs.V1Commands(c) + if err != nil { + t.Fatal(err) + } + if cs.ClientCommand(parts[0]) == nil { + t.Fatal("public v2 accepted staging") + } + if fault == "public-v2" { + return + } + for i, part := range parts[:len(parts)-1] { + if fault == "missing-page" && i == len(parts)-2 { + continue + } + if fault == "reordered" && i == 1 { + continue + } + if fault == "changed-page" && i == 1 { + part.Change.StageV1.Data = bytes.Clone(part.Change.StageV1.Data) + part.Change.StageV1.Data[0] ^= 1 + part.RequestDigest, _ = cs.Digest(part) + } + r := applyV1Page(t, part, rows, &meta) + if fault != "reordered" && r.Code != cs.OK { + t.Fatal(r.Code) + } + if rows[cs.CellKey(c.Change.Create.Cell)] != nil || meta.Version != 0 { + t.Fatal("partial request became visible") + } + } + switch fault { + case "stale-proof": + meta.Now += cs.V1SkewMillis + 1 + case "revoked": + b, _ := cs.Encode(cs.Device{V: 1, Revoked: true}) + rows[cs.DeviceKey(c.Actor)] = b + meta.Bytes += cs.RecordBytes(cs.DeviceKey(c.Actor), b) + case "wrong-cluster": + meta.Cluster = "other" + case "wrong-root": + seed := sha256.Sum256([]byte("different root")) + root := ed25519.NewKeyFromSeed(seed[:]).Public().(ed25519.PublicKey) + b, _ := cs.Encode(cs.IdentityRec{V: 1, Identity: cs.IdentityID(root), SigningKey: root}) + meta.Bytes -= cs.RecordBytes(cs.IdentityKey(), rows[cs.IdentityKey()]) + rows[cs.IdentityKey()] = b + meta.Bytes += cs.RecordBytes(cs.IdentityKey(), b) + } + r := applyV1Page(t, parts[len(parts)-1], rows, &meta) + if fault == "none" { + if r.Code != cs.OK { + t.Fatal(r.Code) + } + var cell cs.Cell + if cs.Decode(rows[cs.CellKey(c.Change.Create.Cell)], &cell) != nil || cell.Title != c.Change.Create.Init.Title || cell.Head != c.Change.Create.Init.Head || cell.Lease.Fence != 1 || !reflect.DeepEqual(cell.Frames, c.Change.Create.Init.Frames) || !reflect.DeepEqual(cell.Keys, c.Change.Create.Init.Keys) { + t.Fatal("large request lost fields") + } + if meta.Version != 1 { + t.Fatal("final request did not publish exactly once") + } + for k := range rows { + if k == cs.V1StageKey() || bytes.HasPrefix([]byte(k.Key), []byte("v1-stage/page/")) { + t.Fatal("scratch survived commit") + } + } + } else if r.Code == cs.OK || rows[cs.CellKey(c.Change.Create.Cell)] != nil { + t.Fatal("invalid staged proof changed directory") + } + }) + } +} + +// R25 keeps the old HTTP refusal boundary rather than raising a command cap. +func TestR25V1BodyLimitStillRefusesOversize(t *testing.T) { + c, _, _ := largeV1(t) + p := *c.V1 + p.Body = append(bytes.Clone(p.Body), []byte(" ")...) + if len(p.Body) <= cs.MaxV1BodyBytes { + t.Fatal("fixture not above old limit") + } + if _, err := cs.V1Command(c.Cluster, c.Now, p); err == nil { + t.Fatal("request above old limit admitted") + } +} + +// R25/R27 keep retained application growth a refusal without losing records. +func TestR25V1StagingCapacityPreservesRecords(t *testing.T) { + c, rows, meta := largeV1(t) + parts, err := cs.V1Commands(c) + if err != nil { + t.Fatal(err) + } + meta.Ceiling = meta.Bytes + for _, part := range parts[:len(parts)-1] { + if r := applyV1Page(t, part, rows, &meta); r.Code != cs.OK { + t.Fatal(r.Code) + } + } + r := applyV1Page(t, parts[len(parts)-1], rows, &meta) + if r.Code != cs.Capacity || len(rows) != 1 || rows[cs.V1StageKey()] != nil { + t.Fatal("capacity changed committed records or retained temporary pages") + } +} + +// R25/R21 preserve legal rotation requests even while ordinary writes freeze. +func TestR25V1StagingAllowsLargeThaw(t *testing.T) { + c, rows, meta := largeV1(t) + var identity cs.IdentityRec + if cs.Decode(rows[cs.IdentityKey()], &identity) != nil { + t.Fatal("identity") + } + identity.Rotation = &cs.Rotation{State: "frozen", By: c.Actor, At: c.Now} + raw, _ := cs.Encode(identity) + meta.Bytes -= cs.RecordBytes(cs.IdentityKey(), rows[cs.IdentityKey()]) + rows[cs.IdentityKey()] = raw + meta.Bytes += cs.RecordBytes(cs.IdentityKey(), raw) + proof := *c.V1 + proof.URI = "/v1/identity/rotation" + proof.Body = append([]byte(`{"op":"thaw"}`), bytes.Repeat([]byte(" "), cs.MaxV1BodyBytes-14)...) + seed := sha256.Sum256([]byte("p17-a")) + proof.Signature = ed25519.Sign(ed25519.NewKeyFromSeed(seed[:]), cs.V1SigningMessage(proof.Method, proof.URI, proof.At, proof.Body)) + c, err := cs.V1Command(c.Cluster, c.Now, proof) + if err != nil { + t.Fatal(err) + } + parts, err := cs.V1Commands(c) + if err != nil { + t.Fatal(err) + } + for _, part := range parts { + if r := applyV1Page(t, part, rows, &meta); r.Code != cs.OK { + t.Fatal(r.Code) + } + } + if cs.Decode(rows[cs.IdentityKey()], &identity) != nil || identity.Rotation != nil { + t.Fatal("legal thaw did not commit") + } +} + +// R25/R14 preserve every logical field while malformed compressed storage +// remains an error, rather than a missing record or a successful empty result. +func TestR25LargeResultRepresentationIsLossless(t *testing.T) { + want := cs.Result{Schema: cs.SchemaVersion, Code: cs.OK, Version: ^uint64(0), View: &cs.CellView{Cell: cs.Cell{Lease: cs.Lease{Fence: ^uint64(0)}, Keys: map[string]map[string]string{"wrapped": {"fence": "0001", "size": "1e3"}}, Title: string(bytes.Repeat([]byte("<"), cs.MaxV1BodyBytes-1024))}}} + raw, err := cs.EncodeApplicationResult(want) + if err != nil { + t.Fatal(err) + } + if len(raw) > cs.MaxEvaluationBytes { + t.Fatal("physical result exceeds existing envelope") + } + var got cs.Result + if cs.Decode(raw, &got) != nil || !reflect.DeepEqual(got, want) { + t.Fatal("logical result was truncated") + } + for _, bad := range [][]byte{raw[:len(raw)-1], append(bytes.Clone(raw), 1), func() []byte { b := bytes.Clone(raw); b[len(b)-5] ^= 1; return b }()} { + if cs.Decode(bad, &got) == nil { + t.Fatal("corrupt compressed result accepted") + } + } +} + +// R25/R27 keep a retained physical row at the legacy limit usable through +// paged evaluation, without dropping its opaque metadata or raising the limit. +func TestR25PagedV1UpdatesLegacyLimitRow(t *testing.T) { + c, rows, meta := largeV1(t) + cell := cs.Cell{V: 1, Head: "saved", Lease: cs.Lease{Device: c.Actor, Fence: 7, Expires: c.Now + cs.LeaseTTLMillis}, Title: "x"} + raw, _ := cs.Encode(cell) + cell.Title = string(bytes.Repeat([]byte("e"), cs.MaxEvaluationBytes-len(raw)+1)) + raw, _ = cs.Encode(cell) + if len(raw) != cs.MaxEvaluationBytes { + t.Fatal("retained row is not at limit") + } + k := cs.CellKey(c.Change.Create.Cell) + rows[k] = raw + meta.Bytes += cs.RecordBytes(k, raw) + proof := *c.V1 + proof.URI += "/acquire" + proof.Body = append([]byte(`{}`), bytes.Repeat([]byte(" "), 65535)...) + seed := sha256.Sum256([]byte("p17-a")) + proof.Signature = ed25519.Sign(ed25519.NewKeyFromSeed(seed[:]), cs.V1SigningMessage(proof.Method, proof.URI, proof.At, proof.Body)) + c, err := cs.V1Command(c.Cluster, c.Now, proof) + if err != nil { + t.Fatal(err) + } + parts, err := cs.V1Commands(c) + if err != nil { + t.Fatal(err) + } + for _, part := range parts { + if r := applyV1Page(t, part, rows, &meta); r.Code != cs.OK { + t.Fatal(r.Code) + } + } + var got cs.Cell + if cs.Decode(rows[k], &got) != nil || got.Title != cell.Title || got.Head != cell.Head || got.Lease.Fence != 8 { + t.Fatal("retained row was lost or stranded") + } +} + +// R25/R6 keep large original views off the bounded forwarding wire. The +// receipt names the exact durable application result, never a newer cell read. +func TestR25ForwardedLargeViewsUseAppliedReceipts(t *testing.T) { + c, _, _ := largeV1(t) + parts, err := cs.V1Commands(c) + if err != nil { + t.Fatal(err) + } + raw, err := cs.EncodeCommand(parts[len(parts)-1]) + if err != nil { + t.Fatal(err) + } + original := cs.Result{Schema: cs.SchemaVersion, Code: cs.OK, Now: c.Now, Version: 7, View: &cs.CellView{Now: c.Now, Cell: cs.Created(c.Change.Create.Init, c.Actor, c.Now)}} + receipt := cs.BoundedV1Result(raw, original, 42) + wire, err := cs.EncodeResult(receipt) + if err != nil { + t.Fatal(err) + } + if receipt.View != nil || receipt.ViewIndex != 42 || receipt.Code != original.Code || receipt.Now != original.Now || receipt.Version != original.Version || len(wire) > cs.MaxCommandBytes { + t.Fatal("large view exceeded the forwarding envelope or lost its receipt") + } + if original.View == nil || !reflect.DeepEqual(original.View.Cell.Frames, c.Change.Create.Init.Frames) { + t.Fatal("receipt creation changed the original result") + } + small := cs.Result{Schema: cs.SchemaVersion, Code: cs.OK, View: &cs.CellView{Cell: cs.Cell{Head: "saved"}}} + kept := cs.BoundedV1Result(raw, small, 43) + if kept.View == nil || kept.ViewIndex != 0 { + t.Fatal("small result no longer crosses directly") + } + receipt.View = original.View + if _, err = cs.EncodeResult(receipt); err == nil { + t.Fatal("ambiguous full view and receipt accepted") + } +} + +// R25/R27 allow a same-size or shrinking legacy update at the retained ceiling. +// Temporary command staging cannot consume the capacity of existing records. +func TestR25V1StagingAtRetainedCapacity(t *testing.T) { + c, rows, meta := largeV1(t) + init := c.Change.Create.Init + init.Frames = init.Frames[:len(init.Frames)-8] + cell := cs.Created(init, c.Actor, c.Now) + raw, err := cs.Encode(cell) + if err != nil { + t.Fatal(err) + } + if len(raw) > cs.MaxEvaluationBytes { + t.Fatal("legacy row exceeds old limit") + } + k := cs.CellKey(c.Change.Create.Cell) + rows[k] = raw + meta.Bytes += cs.RecordBytes(k, raw) + meta.Ceiling = meta.Bytes + proof := *c.V1 + proof.URI += "/acquire" + proof.Body = append([]byte(`{}`), bytes.Repeat([]byte(" "), cs.MaxV1BodyBytes-3)...) + seed := sha256.Sum256([]byte("p17-a")) + proof.Signature = ed25519.Sign(ed25519.NewKeyFromSeed(seed[:]), cs.V1SigningMessage(proof.Method, proof.URI, proof.At, proof.Body)) + c, err = cs.V1Command(c.Cluster, c.Now, proof) + if err != nil { + t.Fatal(err) + } + parts, err := cs.V1Commands(c) + if err != nil { + t.Fatal(err) + } + for _, part := range parts { + if r := applyV1Page(t, part, rows, &meta); r.Code != cs.OK { + t.Fatalf("legal update at retained ceiling: %s", r.Code) + } + } + var got cs.Cell + if cs.Decode(rows[k], &got) != nil || got.Lease.Fence != 2 || !reflect.DeepEqual(got.Frames, cell.Frames) || meta.Bytes > meta.Ceiling { + t.Fatal("retained update lost metadata or raised its ceiling") + } +} diff --git a/internal/directory/handler.go b/internal/directory/handler.go index e30d714a55..dfe7cc3a26 100644 --- a/internal/directory/handler.go +++ b/internal/directory/handler.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "github.com/Agent-Field/codeaf/internal/clusterstate" "io" "net/http" "strconv" @@ -15,7 +16,7 @@ import ( // MaxBody caps a request body. The largest honest body is a CellInit with its // key map, far below this; the cap keeps an anonymous caller from making the // relay buffer more, since the body must be read whole before it can be verified. -const MaxBody = 1 << 20 +const MaxBody = clusterstate.MaxV1BodyBytes // call is everything one route needs: the Client of the verified device and // the request it answers. Routes never see an identity or a device to trust. @@ -145,7 +146,7 @@ func send(w http.ResponseWriter, status int, v any) { _ = json.NewEncoder(w).Encode(v) } -// routes is the §4.4 table, one entry per method and path. +// Each method and path has one route so legacy request decoding stays closed. var routes = map[string]route{ "GET " + dirBase + "/list": func(c call) (any, error) { return pair(c.cl.List(c.ctx())) }, "GET " + dirBase + "/presence": func(c call) (any, error) { return pair(c.cl.Presence(c.ctx())) }, diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index 3e0d5dabf5..c315026b6f 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "errors" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/raftcore" pb "go.etcd.io/raft/v3/raftpb" "google.golang.org/protobuf/proto" @@ -118,11 +119,15 @@ func (e *encoder) flag(v bool) { // so a delayed completion cannot accidentally address a replacement member. // Hosts still serialize each member across asynchronous durability operations. type Runtime struct { - nodes map[uint32]*raftcore.Core - serial uint32 + evaluationSerial uint32 + evaluations map[uint32]*pagedEvaluation + nodes map[uint32]*raftcore.Core + serial uint32 } -func New() *Runtime { return &Runtime{nodes: make(map[uint32]*raftcore.Core)} } +func New() *Runtime { + return &Runtime{nodes: make(map[uint32]*raftcore.Core), evaluations: make(map[uint32]*pagedEvaluation)} +} // Call consumes one complete frame and returns owned bytes. The header is a // version and operation, followed by length-prefixed binary/protobuf fields. @@ -164,6 +169,27 @@ func (r *Runtime) Call(b []byte) (out []byte) { func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { e := encoder{} switch op { + case 46: + if d.end() != nil { + return nil, errFrame + } + e.u64(cs.MaxV1StagingBytes) + keys := cs.V1StagingKeys() + e.u32(uint32(len(keys))) + for i, k := range keys { + e.str(k.Namespace) + e.str(k.Key) + limit := cs.V1PageBytes + if i == 0 { + limit = cs.MaxCommandBytes + } + e.u32(uint32(limit)) + } + return e.b, nil + case 40: + return v1Commands(d) + case 41, 42, 43, 44, 45: + return r.pagedApplication(op, d) case 39: return hostedForward(d) case 36: diff --git a/internal/raftabi/application_pages.go b/internal/raftabi/application_pages.go new file mode 100644 index 0000000000..8ff995e0bd --- /dev/null +++ b/internal/raftabi/application_pages.go @@ -0,0 +1,169 @@ +package raftabi + +import ( + "encoding/binary" + "errors" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" + pb "go.etcd.io/raft/v3/raftpb" +) + +var errReadPage = errors.New("raftabi: next immutable record page") + +// A cursor belongs to one synchronous application transaction. Each supplied +// record page and returned byte page keeps the existing evaluation envelope. +// Dropping it on every exit prevents abandoned evaluations retaining buffers. +type pagedEvaluation struct { + command []byte + meta cs.ApplicationMeta + rows map[cs.RecordKey][]byte + need *cs.RecordKey + output []byte + offset int + projection bool +} + +func (r *Runtime) pagedApplication(op uint32, d *decoder) ([]byte, error) { + e := encoder{} + if op == 45 { + if len(r.evaluations) >= MaxHandles { + return nil, errFrame + } + raw, err := hostedProjectionBytes(d) + if err != nil { + return nil, err + } + r.evaluationSerial++ + id := r.evaluationSerial + v := &pagedEvaluation{output: raw, projection: true} + r.evaluations[id] = v + e.u32(id) + e.b = append(e.b, v.outputPage()...) + return e.b, nil + } + if op == 41 { + if len(r.evaluations) >= MaxHandles { + return nil, errFrame + } + v := &pagedEvaluation{command: d.blob(), rows: map[cs.RecordKey][]byte{}} + v.meta = cs.ApplicationMeta{Cluster: d.str(), Now: int64(d.u64()), Version: d.u64(), Bytes: d.u64(), Ceiling: d.u64(), Index: d.u64(), Term: d.u64()} + conf := new(pb.ConfState) + d.message(conf) + v.meta.Configuration = raftcore.Configuration(conf) + if d.end() != nil || len(v.command) > cs.MaxCommandBytes { + return nil, errFrame + } + r.evaluationSerial++ + id := r.evaluationSerial + r.evaluations[id] = v + payload, err := v.advance() + if err != nil { + delete(r.evaluations, id) + return nil, err + } + e.u32(id) + e.b = append(e.b, payload...) + return e.b, nil + } + id := d.u32() + if op == 44 { + if d.end() != nil { + return nil, errFrame + } + delete(r.evaluations, id) + return e.b, nil + } + v := r.evaluations[id] + if v == nil { + return nil, errFrame + } + if op == 42 { + present, b := d.u32(), d.blob() + if d.end() != nil || present > 1 || present == 0 && len(b) > 0 || len(b) > cs.MaxEvaluationBytes || v.need == nil { + return nil, errFrame + } + if present == 0 { + b = nil + } + v.rows[*v.need] = b + v.need = nil + payload, err := v.advance() + if err != nil { + delete(r.evaluations, id) + } + return payload, err + } + if d.end() != nil || v.output == nil || v.offset >= v.outputSize() { + return nil, errFrame + } + return v.outputPage(), nil +} +func (v *pagedEvaluation) advance() ([]byte, error) { + result, err := cs.Evaluate(v.command, v.meta, func(k cs.RecordKey) ([]byte, error) { + b, ok := v.rows[k] + if !ok { + v.need = &k + return nil, errReadPage + } + return b, nil + }) + if errors.Is(err, errReadPage) { + e := encoder{} + e.u32(0) + e.str(v.need.Namespace) + e.str(v.need.Key) + return e.b, nil + } + if err != nil { + return nil, err + } + e := encoder{} + e.u64(uint64(result.Now)) + e.u64(result.Version) + raw, err := cs.EncodeApplicationResult(result.Result) + if err != nil { + return nil, err + } + e.blob(raw) + e.u32(uint32(len(result.Writes))) + for _, w := range result.Writes { + e.str(w.Namespace) + e.str(w.Key) + e.flag(w.Delete) + e.blob(w.Data) + } + v.output = e.b + v.rows = nil + return v.outputPage(), nil +} +func (v *pagedEvaluation) outputSize() int { + size := len(v.output) + if v.projection { + size += 4 + } + return size +} +func (v *pagedEvaluation) outputPage() []byte { + end := min(v.offset+cs.MaxEvaluationBytes/4, v.outputSize()) + var page []byte + if !v.projection { + page = v.output[v.offset:end] + } else { + // Prefix the logical blob virtually, without copying its entire contents. + page = make([]byte, end-v.offset) + prefix := [4]byte{} + binary.LittleEndian.PutUint32(prefix[:], uint32(len(v.output))) + for pos := v.offset; pos < min(end, 4); pos++ { + page[pos-v.offset] = prefix[pos] + } + start := max(v.offset, 4) + copy(page[start-v.offset:], v.output[start-4:end-4]) + } + e := encoder{} + e.u32(1) + e.flag(end < v.outputSize()) + e.blob(page) + v.offset = end + return e.b +} diff --git a/internal/raftabi/application_pages_test.go b/internal/raftabi/application_pages_test.go new file mode 100644 index 0000000000..dd5a1724eb --- /dev/null +++ b/internal/raftabi/application_pages_test.go @@ -0,0 +1,72 @@ +package raftabi + +import ( + "bytes" + "encoding/json" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +// R25/R14 preserve a logical legacy view larger than one ABI buffer while +// keeping each returned page bounded and retiring its cursor on completion. +func TestR25PagedProjectionPreservesExpandedLegacyView(t *testing.T) { + runtime := New() + first := create(t, runtime) + result := cs.Result{Schema: cs.SchemaVersion, Code: cs.OK, View: &cs.CellView{Cell: cs.Cell{Title: string(bytes.Repeat([]byte("<"), cs.MaxV1BodyBytes-1024))}}} + raw, err := cs.EncodeApplicationResult(result) + if err != nil { + t.Fatal(err) + } + want, err := json.Marshal(result.View) + if err != nil { + t.Fatal(err) + } + if len(want) <= MaxBuffer { + t.Fatal("fixture does not exceed the ABI envelope") + } + request := request(45) + request.str("view") + request.blob(raw) + response := answer(t, runtime.Call(request.b)) + id := response.u32() + var output []byte + for { + if response.u32() != 1 { + t.Fatal("projection requested a record") + } + more, page := response.u32(), response.blob() + if len(page) > cs.MaxEvaluationBytes/4 || response.end() != nil { + t.Fatal("unbounded or malformed result page") + } + output = append(output, page...) + if more == 0 { + break + } + next := requestFrame(43, id) + response = answer(t, runtime.Call(next)) + } + decoded := decoder{b: output} + if !bytes.Equal(decoded.blob(), want) || decoded.end() != nil { + t.Fatal("expanded view changed across pages") + } + answer(t, runtime.Call(requestFrame(44, id))) + answer(t, runtime.Call(requestFrame(44, id))) + if len(runtime.evaluations) != 0 { + t.Fatal("completed cursor retained bytes") + } + stale := decoder{b: runtime.Call(requestFrame(43, id))} + stale.u32() + if stale.str() == "" { + t.Fatal("retired cursor still served bytes") + } + if next := create(t, runtime); next != first+1 { + t.Fatal("application cursor consumed a consensus handle identity") + } +} + +func requestFrame(operation, id uint32) []byte { + e := request(operation) + e.u32(id) + return e.b +} diff --git a/internal/raftabi/hosted.go b/internal/raftabi/hosted.go index da91e92a73..4fe58527d7 100644 --- a/internal/raftabi/hosted.go +++ b/internal/raftabi/hosted.go @@ -42,6 +42,15 @@ func hostedEffect(d *decoder) ([]byte, error) { // Projection uses the declared Go shape so sealed map values named fence or // size remain opaque and v1 integer fields keep their original numeric wire. func hostedProjection(d *decoder) ([]byte, error) { + raw, err := hostedProjectionBytes(d) + if err != nil { + return nil, err + } + e := encoder{} + e.blob(raw) + return e.b, nil +} +func hostedProjectionBytes(d *decoder) ([]byte, error) { kind, raw := d.str(), d.blob() if d.end() != nil { return nil, errFrame @@ -54,7 +63,7 @@ func hostedProjection(d *decoder) ([]byte, error) { value = new(cs.Device) case "cell", "list_cell": value = new(cs.Cell) - case "result", "view": + case "result", "view", "typed_result": value = new(cs.Result) default: return nil, errFrame @@ -68,13 +77,53 @@ func hostedProjection(d *decoder) ([]byte, error) { if kind == "view" { value = value.(*cs.Result).View } - wire, err := json.Marshal(value) + var wire []byte + var err error + if kind == "typed_result" { + wire, err = cs.ProjectionJSON(value) + } else { + wire, err = legacyProjectionJSON(value) + } if err != nil { return nil, err } - e := encoder{} - e.blob(wire) - return e.b, nil + return wire, nil +} + +// Preserve legacy field order and escaping without retaining an expanded +// encoder buffer alongside the complete public response. +func legacyProjectionJSON(value any) ([]byte, error) { + var compact bytes.Buffer + encoder := json.NewEncoder(&compact) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(value); err != nil { + return nil, err + } + raw := bytes.TrimSuffix(compact.Bytes(), []byte("\n")) + count := bytes.Count(raw, []byte{'<'}) + bytes.Count(raw, []byte{'>'}) + bytes.Count(raw, []byte{'&'}) + if count == 0 { + return raw, nil + } + var out bytes.Buffer + out.Grow(len(raw) + 5*count) + for len(raw) > 0 { + at := bytes.IndexAny(raw, "<>&") + if at < 0 { + out.Write(raw) + break + } + out.Write(raw[:at]) + switch raw[at] { + case '<': + out.WriteString("\\u003c") + case '>': + out.WriteString("\\u003e") + case '&': + out.WriteString("\\u0026") + } + raw = raw[at+1:] + } + return out.Bytes(), nil } // Ingress proves the original signature before external I/O. Every later diff --git a/internal/raftabi/v1.go b/internal/raftabi/v1.go index 84bd3183be..e9a25eda65 100644 --- a/internal/raftabi/v1.go +++ b/internal/raftabi/v1.go @@ -26,3 +26,33 @@ func v1Command(d *decoder) ([]byte, error) { e.blob(command) return e.b, nil } + +// Large legacy proofs use bounded proposals without changing their signed body. +func v1Commands(d *decoder) ([]byte, error) { + cluster, now, raw, body := d.str(), d.u64(), d.blob(), d.blob() + if d.end() != nil || now > clusterstate.MaxSafeInteger { + return nil, errFrame + } + var proof clusterstate.V1Request + if err := clusterstate.Decode(raw, &proof); err != nil { + return nil, err + } + if proof.Body != nil || len(body) > clusterstate.MaxV1BodyBytes { + return nil, errFrame + } + proof.Body = body + commands, err := clusterstate.V1RequestCommands(cluster, int64(now), proof) + if err != nil { + return nil, err + } + e := encoder{} + e.u32(uint32(len(commands))) + for _, c := range commands { + b, err := clusterstate.EncodeCommand(c) + if err != nil { + return nil, err + } + e.blob(b) + } + return e.b, nil +} diff --git a/internal/raftdirectory/testdata/v1-large-requests.json b/internal/raftdirectory/testdata/v1-large-requests.json new file mode 100644 index 0000000000..0700442194 --- /dev/null +++ b/internal/raftdirectory/testdata/v1-large-requests.json @@ -0,0 +1,44 @@ +{ + "provenance": "Original pre-conditional Go client CellInit bytes, retained with the original certificate and HTTP signing domain. The executable archive test checks every emitted body and proof against these bytes. Independently signed with Node Ed25519; sealed metadata is deterministic opaque fixture data. The bodies are 65,537 and 1,048,575 bytes. No archived executable is claimed.", + "now": 1000000, + "cases": [ + { + "name": "over-command", + "method": "POST", + "uri": "/v1/dir/cells/over-command", + "body": "{\"head\":\"5ff439c2766abbb75b98b4253a085b7fc65b8f6d397ae5ae3132449768c906bd\",\"class\":\"work\",\"title\":\"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\",\"parent_cell\":\"parent\",\"size\":1234,\"keys\":{\"key\":{\"id_52dbd9ad90fc949eb796c5b421936cb4\":\"d3JhcHBlZA==\"}},\"orphan_turns\":2,\"frames\":[\"3df88793d2d654fe9b5b944e3b1d1aa570b5f9327efb9fe66377d4034ed0df1b\",\"60e4a469d7a9da45e3353cae1093808195d1b05ef4c56f26499adee704446fe0\",\"7b61e8b92a6df343ff2bfd681e1d25ef0b4ee95507a363515d3d29287ee1a19c\",\"a0573555e1aac0b5b735049f671ffcd06ea237b3d9a913b44d179669aa7b0d0f\",\"f00d75ee8615eda53533b0b44a45b4ae24321cb296a314639b4241534219cd40\",\"f4de91dd8f155f8a3ebca51c8357c6daea5943f1dc7faace685ca62bc50c0d44\",\"d5d5b1e13ccd0b913d0fdf328495de5de940caf5e856d774a577084a073f4a73\",\"9c43d4677f74712c1d6e2011a7039ad104798ed25465baade2e8ebf47bbe4a36\",\"c54006495ff81f8ba9ba794b2a1ebba979d40ce0c9fe998d3816fe6fa5e5a031\",\"9d914dd470d75252abf3503f0b502297e2e8c2346057ec67abf08d0e40120bba\",\"b8054aa8590ff3e2d8e456005a2ba4237efd88836981ec1bf6845b246371bebf\",\"cc12dfb3fca6345112bc5f15b68ee42abf3bd4af64611bdfb48b9572ab65d9aa\",\"f2289d58821c10b1ce2ee6ead7e35966782c98fe3c83754875b63c1bef6dc5a8\",\"7b076d57ca08cf548b0be8d62df0ef044e28096b0a0a6d84ddc73c7b780dbf9c\",\"ad1cea7a60e8f5d92c521ad19d5dc0d3e10fc356c6b58cc35067cc0b2edfaf96\",\"2e02cfdef3ca08f81ee12d738057cd3fdf205b239efd293e307a77a229455538\",\"cac8bb65da7901ed678e8d3f509f54036a641a4146536a5d2436b417b2c37c48\",\"050b975eb7df60dfa6d2a6bb83d3ead9e3e3b41b8e3655323a9bcba538652adb\",\"784a416448c44748fd9808773f86534059dfbb4a56a942da0af9854dc670f5f6\",\"e2f39c4615fbe5203c2ce90a6074cace51c50baf5305db0f9406bb58e4c83942\",\"4e428d38991c1ab31084718c6b3882b57f7a3c4ebaf3385a5e08d67b957782d7\",\"7afb0e0bd220adfcd95ede5050e1b6d9f3ece49cc155bad9e4036190456bd48a\",\"8b3914081536729ee7d09acd43e40b790367d24392bfbb3691e0a2f491409f2c\",\"a15c4345a158f2935111bd1cef8251c7c20562c69d3366cd67abe4dd6988627e\",\"316a996c6ad959b513ef2ddb832ead7b2b65739d43d85e246115a31f3b40599c\",\"3859b564254c89e41ef7843a166860371698deb415b882e761d71401a85df49a\",\"a18774800d14e47746886dd6ce513250eca3f1e646631fb97196f0e590cccbce\",\"d9d009cc6e20ff0590b42fe18e2649e638f061dcb4accc7b3e7e396bf71bcb95\",\"1bf01d0f413fea6b20930e3f774650c9de4422e5f76f7542c167eb30aaf89600\",\"7c94f34e738f2b50980a53006e20b6ffacfcca5e862cc1c24d32c03889263dfb\",\"f6d777079fa02e59fe91bcf94103cd31d4b06b3e9ec74d2e1da98ccc34736f51\",\"26324a1a611643f39bb9f5777274f46dc3bb92d147bae30828b1aa909f1da3b8\",\"60bbfad4a2301835c8d12250fd0989008d955615f03018cb6fc2e1cfdfee22ab\",\"bd2dfcaf5df67b22f02981304efdfe8259ed89c1a6cd661bfec70cde1e2f7d40\",\"92ab593fea2ddd5c4d88ed4af1fa9f5a42fcb0310dfb5c5b45f8d832c87bfd7c\",\"3b8581620acc98ad8ebf9a8456e8be3a23f68fea8e892716f40326d2b81a4552\",\"3d3d11a7329aade00667568e2daccd1cf259e76bb85974331070558aab1affde\",\"d8a6009fff09fcddbab8fbf79c052527981236156c6b3bc0da4122a8e2fb93bb\",\"5d0a4e8736c470e5683ac9a45862c50b9c7eb8a5828344a87c1508d11f82da0f\",\"fc1a23a833330c0638ad19e06f1a04df4c4846c2685a5c3d6bbb15ec7c202cb9\",\"80307f529a51c4cd947b59a432f3325d02ea21583ceb3e7d6afe55fa353c7ede\",\"fea68b75953010d9029ec1ce781602d8332f572f8054ced96b516523ed4ce69c\",\"4cd62030f6200eff762d23abe8e24560a3d0b1233e04a3d054b8e55493c3f3d1\",\"f0cb5442fd953bea693be8f1af68c6d0c0429c720b8b0ae96348e3e235894602\",\"ed7220deb2c5f05839128393e315a2f11e511b9e6b49705fd131cf68d60a698d\",\"2e01cec7c8059af8118b2dc13a65c77289908d1e899a7ab554b48ee19ecc2578\",\"d3ee73540791a41c2f052c598f2a9df66964a40ceea9e92a2e4487833a51da9a\",\"8c3c82ef108c9f4fcb4e26b326e50fef8ad12e7a227b35143bb2389c1dedb1d4\",\"50d606b3f6d3ff844eda58417283e0cdf52d3c059da397bd7abb09a6574f4ad7\",\"a8b527e6b27c5e4dcda754ee3130907e2de9673e1b755bbac085ec0600aab82d\",\"b11be166a7326811294a8b9387bab1312eefe5cf8a2014a60b8b4284114c0d93\",\"2b5c1e71432d629e7cd53c37786b38073f97851e95f4a9433dc9f65e404f59fb\",\"aaad2886f150ebb409f0ed511f664177d32bb5a381b8de5718303bda398a8986\",\"53318e1b7193530d76bed1d3ec43c31fc1fa1f931eb5dd9611ca17f1fbda4c49\",\"a295009191699a3219247d949fcda59518dcc639381e887f95e77aafc1e30bca\",\"724f8bbcfdf673eb2a531afa7280e0421346b39b57d84d659310b255668cefeb\",\"e1741629f3cf05b2913266e3d44dfde5aeaa187b2514f27610f741defa145a0d\",\"2c8d9d6c49375684cc444c28ed83a6379e3175235992af7a1920cf008604dbf7\",\"7d2f86888185dcd55f30ce47f31e7c51f3e0e298461c2e3e50ec5aeebcc93259\",\"6591f400f0c407eaf0b2fe7092111bf1149378d1fff9aa136d42992a45a7d4d4\",\"cb28a1fc0ca69067f4a5229d703557b41fabdbcd40740e528c12b57f4f621196\",\"291cf8077eb6836688cdb1cc0465a6586b1fb04d26e75c1060fe12dca59b955f\",\"22e49c8a2de22cadb99d6fa036efcf3357a44faec0399fdcd9395d318a710825\",\"fba76deb52bc2ca358dc341c3c21caf298fe64245441080d5a53d7ae42650f84\",\"f03b5f9e68c35da80b8623d837030e7f72e05612aaf11ebdc2ad2bc8d1408ad9\",\"42c0f786892ed36d71375b59985767044eefc39a9eb3fb464856fa108b129fe2\",\"701ea9b145f283737a8ecd117102dc2cb61a4e0195a28b4c384012f90384a916\",\"0a7b5adcdf16ff8633344105e937eae1b727f86e326c298f10d90d423f2c7c57\",\"940fa22ad97864aaa609cf54dd1f78cb05219cbb5dc956d61b92632a22986b95\",\"f0b930eeb723bc8d044470972d61d48504992ce4e3ffdd9efeab56dbfac9a23f\",\"8d262b857504acfacaab41ec3b608440813f0887b7fa638a62a51e6ca8cde84c\",\"75cdee612214e8d9a79690a30fd878ac09f0865c88ff9657d4925ad66c1542d6\",\"346f3c460d9222168e38666193c0fc6a2ff61f154ff757f8578e0a49bfd61c35\",\"10db741665fea833ed9aee63d9345e4544397b676d028685474df0d8d6a7d142\",\"6f2e8c2825bbda275f30c464baf5854346708e9c3b2ff1ffcc20d21790005222\",\"2718fdeb0b55b0441c7fdd9fd7c050155a013fd868c564d9182777afc1257396\",\"910286f4545dcb12c83270b9d2a953bc5fbe9884175c547c8041db4f34128d8a\",\"e6a9ce45f0f2a3dd0a8ca11cb68765e140c5d79adadee5dbd6d51d9fa032d365\",\"69bd4f4c11c8e6b0047f28041e5e7778060b8a940c989bee68d470d4d01ad701\",\"dcba91d6cdf340f29ff93905fbae87884802abe620f8679bbfe35ad26d618d24\",\"a9362371715731b3773185c22cae2a2ef1a48496ab8d5056a40b8a58651d7c4f\",\"d0e4f86f7764bcde341864fc222b3d94563a85c574c1b8f8173d89129f09e32f\",\"4d00079e5dd5a73736306735d606270048996719a21635bb01b092a0bb207eb6\",\"5add5880bdbfeb285a412e6df095478be864e82c1955ea02a2748df58327af66\",\"ed68a74f3629d59fc7c5952cc2522cf949842c1be014cc9e460410e7d47773fe\",\"5ab75a4cbc0f67b237f980d0f79a50861dc8014f3322c3aa563a7eb1f476969c\",\"75bfed73e3ebf7abd75ed79ba1440cd703406a0655426fce05f330a41304aa2f\",\"528c753bd65ee44d64d21628e000e4be1e1667a88afcbb06d07b3dfd5f525de1\",\"2a141b6e0c7cfcdbe731129bfcb469966504e45ca3feaa94e065cd4acf4c8e5a\",\"42eed38026ca1fada99381c52dfbec618edd8c33a41282b76a973683b83a21dc\",\"9c0e23e75e952a0c4f99bdb4ade75cb445d90f04265c065505977fbcab987f4d\",\"95fcd6554e607ee0805e51d277f0d8f6651cf6ebe181a5ee0ff521a4973ab25a\",\"59f1c37af9794d6a560f81764ed6486d1698221a1a2dcd372da4534c1bab87c0\",\"d68282635c619f756e5a3d92703a871c96b1f1870f6d9509dc4d6c8ba634d358\",\"e3773078865355dbea6bc655dc8374daf6d267492f3a531489b17b86afccc190\",\"d73cead92c1575c732ba10897f7689b8187a7bd107bcd83f37aac51769c276e8\",\"70d85ba528bc96d57618a3dbdc891b789e10b37bb8b0a49fea1a459249875568\",\"23180f51f33287aadad3e66b4b9b045a67afc89c4e715dac0924254731391dc6\",\"868b80625f7c582eb82c5f047e290eea37ffd44aafc0d2f74f8e5df1b41be1f8\",\"194ca04a7255093c0af1471973865f77b2b3242fc0b9e8449152002d74005a8a\",\"4776106e42647e200238a244da1cb03714fde929d8f1b9fde26245f577eca2c5\",\"f526c96b3da45b336e4d3ac16f41395c99464583fd2f7a0c5360c82d37de490c\",\"9df7b98b05f9f89ffdafc643f9fe6bb16d176642815d34d528ee76d495f335a4\",\"23a5e79f4fb37b5a82e994295077a722f5dd6c128814cac9040834046f66292a\",\"53866eddba65f237c1029fb4ca2fe5b40ae0dcfe751213c1bd0dff9cc34a739f\",\"429ac35a7cf3dfa715878d5ca35c582db0e4e8c5d626b807037365046ceac64c\",\"cb66c4faeebf5dbc053c411c542c201b47d23984b9a0feac3b142f9cd02f44e6\",\"898cb6d2eb21f697c9f52d8d0d6480abdcc432ecacfc7665bb283f897504c153\",\"c12f829453093a59141fc6f9e4330fda49879bbe65bba9bd219acaabefb3a121\",\"7c1de54e3754e8292dd67dd91fe3ce24ef8cb819a1d8340ee90a0f81df3e75aa\",\"363e517cf25d21d1f931ea39a3b5c856802a5c2ac174e25449549e718dce4ef6\",\"e5b23267509b36028ada1874b1169985f756d2c01ad509fa9d23d0e7e2410179\",\"f718e3cf8f20e58bfc9383d877a408f80515667da9db60981a2db56b735f9a3e\",\"e7fd7c77e790d40ee7eca5c44fcf97d322be086750e5725d557c4763f0cfbc88\",\"0404de210115f9e250e28aeff6ba039d2ed55c02e12fb0f77804ef3c2b9f2df6\",\"f8f2bc838f29f7ac9d3a5a316aa1e3d8cebbe47c6c54a450e9b1bab5f287226d\",\"703d29db3feeb72f9068157e984d2591f20b71cf571f4f24fcc46459059cc281\",\"8a7fd73f59dd054422c2302527a442bd8e9a35f81d5fa5a95785c7783a04af93\",\"5465e56a648b8be3ed53e8808fd550cf2cd719d01e4d8cf4842c9f322163ce97\",\"6261802126e0f4fb7d45ccd809ff9b7609fd36cd93aa184681ba164f685145e9\",\"1f43255649fdb5a197dad1059b0faf351589e5c255b59d01c4c3a3052b72ba51\",\"e4a9ebc383ec3041a5a31aec0f6b31b6577e9ae532b76fcb75ac9d804ee7509e\",\"c76fa80e7b1ef8bc45addf2e8839bdf0bcee473992e5a03ac0d5ded4573039fb\",\"07c66baa8d0212052b80508cb6ebc18bdfd6555203445eb18a66984a51322961\",\"cf773c8f0b05c64a10716b03d449c4a55c732147e65a5c2e51bb78238a98c97d\",\"820e31577aa284df8a7f9aa1b3b699157d66b64624b635e8d06ca623e69331f9\",\"13343b27e0628ff0a14031b6d06af158037cc5d006dd2c288ef20d1b393524aa\",\"7c9e2c84a1bb8a16bd2bcd82f2e9f79038c48f363153b710fc1c715690b3f775\",\"fce5e4cba71173df5b52f464d0d5f42e984b771ef67c7e5afbd2815d07156edd\",\"7e8bb193c0afecdfc5e26d7e98e3e7d1f4e0e39964e26f252caaa58f46b8b6c6\",\"360bb1dc3672b01db86bb061e08b66bc79af5d2fbd3d4dd9949ae404c8d90e25\",\"7c1518f9f7f0055f17fbe4e6d10c918f1994c0fcd7487598a6462f75406481f0\",\"a4077ff967c789567a296f0010ee5e7827cfaee933b1080a53078c470ecb6237\",\"6877178bdadaafae8e3129e1534c0473f735b3b8538b1be33f118d473a9260a1\",\"7b65bd5bc769d25923bb0a601896f1c4f0e67d53a57f196c97a5676a5b6f47b9\",\"a8839f708a699e123daacbb195ca6bd36339a301c4f487c2f105bf58c3dbc1a5\",\"b7d4ba0b7aaaec98e71eb9930eb24e927c8a7b91d429a76f6e0f2090002dab77\",\"32d41a01189cdf55e9160e238b829133a7f8a6fa4c2014a6b385ad372c6827c1\",\"98e66e80901d172886c541284f27dc9a2a15410898963c093e69a3234c0b53f1\",\"4340dd271c9bfd011ce2cd0e9af344aa83140036ba3adf593ae8d38aadce7199\",\"29f09682f7713495702d5be40bfe895c323ab28c3d6d47453c3f23db3dabd479\",\"e77ecc5a1b075e602d0dbd9ebb7cdf23a440e0ce18a4b74c6a9ee55243ca2390\",\"ab3807d7d568842bc4d2a783e2cf6fb99c5918bfc9a7273051d1c6933e266c5f\",\"81c62da9658ceb6ec3e586d361032adbbc32df5fe59e4fab93eb9422645311a3\",\"ebd6a4bce4e06d2f9fe1cf0ee2f6eb97d6ef3bdc03624eb6eb95aaea0b84a0c8\",\"fcb720aed6b3b9869bed29963ea8dce422e77ed0cee6c372a1f1aae249e6190c\",\"22a165cfe9d2f1f151776f901a425909b7b96ebc916563c601ba88bd466a8fa2\",\"423298fd7f2ffd46d4c16905da600030f8df4f5f85a902266fcbf4786b3df91b\",\"a0b03167c3fb0e50be18ede40b7bcb4ae3a305eae0c8530674b0df07b54c3ce4\",\"ff0c04b7cf0d619cbeaab8084be1393338fee8e20001777e540feb7e79993a34\",\"65d8721e979650b5de12fee86ee7050d5061766530ecbdea986254f2fd535112\",\"7b8857ad302e9a552c708663ea8de41044f8d23fe0054780aef1bbe58e5d1983\",\"d9e19f8fa43d207c8f97eca1fc9168f8c95635e108d51bd97337cdfaf6e2b403\",\"1550b7498396b10e4fbe6487bd9860fb9be81bc80f695bd9747a57d007b7b607\",\"ebb60d7b79d52b0f757f5258cc6733867e9558d6b95bd122a70d795e1558d251\",\"486d98c731356394f8cb5ad099770d367bdff61e75149f3d97871e75357a47c2\",\"1375765851cc86bc62595dab00b7eb4d7fd06d6a99fdcc7ce20b62e589a764c3\",\"9f6abbb10ee449f2dcdc77e326935b1503302ff127e467a3550d9680c3cf7a92\",\"abcf023462268803120d38fe107926ad9b7644e0c38c567889b5a4320ed68c43\",\"eefd41a66504aaf854050464bd6da21f630ee48c586f8b03d9ef831a53bda3f1\",\"310d4b4e55e76f22a7e91c28c6d94765888effb0a507078b3020cbe2f424eec4\",\"7d1de62899513c6e0890de5fcef8de5c6e5001416eca91522007b630eb6b2312\",\"902aeefc651fabbf8a9b752d442e9718b922cda5f2996dad8a2b18e549217bcc\",\"48ef08a4a6848723d4fab1931369285c57e04c502b4e6ce15ab105eba5b6ad37\",\"638ce37ebb5059e39c88e3a6a04b170b77338f8c911b53f307ea74915cac6a0a\",\"0235095fc23ddeba5ff86ef4f2a06bf540f6f807e52b991f03acf9b13049d069\",\"4dfe73309816a03687f3ca0bd78b0eb7e2cd9f084fa7af9bff54b131c34f4724\",\"5d8a42d13b9f57ebb9e67279d52b95d192c5b0f067bbf702aafaadcf8b417c56\",\"c4522f63aa28a8752a4fecf4178c830ba56ea9d939ebdd28c78b80cefc68e1e8\",\"6b82c2e2e50b89150091422197de5200b9bb7394e23fd3862a216b9ed9582576\",\"c2092178f31c258cab11c57a0f34e9297f920fa0d074005041faed4f51c60164\",\"da5afdee2f31ba49f217465cf8126431acb78c59c376bf496c1f31267e69ae8f\",\"033c6d0ff0889f144e8d58846a376e99aa14f21ef2f3cf4fa0278c9ffe183c84\",\"18e6867108ff489a416a242c50a7d9a30cc1227c013c3c73fe2b310fd1930e7c\",\"fc607ba674c6c6cbf2518217e5a9606cc04bc8c821c1cc7afcd5d5dc24d89004\",\"931a02ea8fd627705f435036f161dcbec1bfad6402208cca6e1cc8476ad2d773\",\"563a608c3949452e072eadc0bd4f4cdef85710227b11e218441a4b5ca137d48d\",\"49bd1ed99c122c1f78671dcbbe39118c6ba3f297d33821e2d31a919a02ba1714\",\"18de1545873139f6bb2cfa236a26d73ae1d521d1ee436d55fffce9545f835270\",\"bf602f7652e67b9f525c6fa90e6ef26cd29b0e20c08b0bb353f5bd62af14d10a\",\"1bfc8498f8a609b7068b33062337b6e8f2006de7e1daef022a3236dc5281862f\",\"c79da0cd84c24faa3c7ed3965593373029877d34450bc2636fa76f289b77ad12\",\"9dd2aee966d2f47f5dadc40331d5729f5268610bb2dac4f144ed2a2faea6436f\",\"b4ea2664b412d3f9a992c7a418418dc642bf15fa9ed3c72166c8a5e5a4324159\",\"767d0422b159ef20a4d76c826867d67525dcd27dd331bbe218f5a5f5a6b39a48\",\"516a2914d0f058b88d32fa4c1373c3d0288814dd14719fc6bc8de4f02d1e4cfc\",\"8a013fff8d93a6594cec24c070de72c8fb6bfee0642512c3c5a987e1e01a2036\",\"9fb2be8d9d580540c169011b5885a7093b93ed78461723c0714ea342a1f47f71\",\"3c062316bcc098d2997e19368e09cd487bbe14e6573eee9b8f07e173a3190461\",\"71fddf6262bb3ace2870267bb66281e19e1e4abf17f3e762d7c75d0c328439c7\",\"b2dda505b979c15e4c563f03d20d3f1c0faac478af037d37a9e6ae4950228800\",\"3ab0dbc9b2ae6195686ccfeaa3b3b3d7e7d8261e29125025a8c5b36e7a5909cc\",\"3b8836d48c0a3b7f7f10dff6599dce3573732f6a49a7b23f8c45d614f23ba67a\",\"e463186823789585d145882a43f36b56c227d1043ed0f3e70a4f004c1d0647ff\",\"ce145b57d79d5a0f50e578388692da604c82ba1b8f1bc96c85ebe1fbeb8ef186\",\"4dcdea0f09dc2dc481a6ebc606110645389ac81660e0be3f6e0ebffac21f2a81\",\"78b55e0a09e25fd5c858499a56e927f204bcd4131d75cb1b753c8f9ac8f6f20c\",\"e9669692234881fc4411dc0012a74380598679701c73c6da6e1aeaaa09b0a7b1\",\"0782b804af1ee77701ff8811287280f55d432d4404c88eb5cea8eed6aba12c3d\",\"ea3cf49f07ead0ec44b70b2a79ec0fa1f31706b5cb32359bdc25d0769b499c48\",\"4d1f9832b5adcdfd86383bf427def71090f9852e2785f35d4d88b56d67e6c39d\",\"cb772eaef170c4c4104fddd76b557feff76039250b42b533980fa7fdf8b234fb\",\"aa2fdbb1a600c7084f4ce0ad5b3f452df00c8e5279ad7894b2147ca85fb45794\",\"a8d6ad6694a039a8bcd22666949341488969bf978e3da14ea419975c8e240ae1\",\"075e6271e16c752de9c099b9a55b5174e904bb4b2559a22f8a972df350ed8d3f\",\"dd6e31642ebc23622f9666b200a10552372f27db326a33b03e59d7d31792a3a7\",\"5544b5711521c1a53dfb9d35efde80ca4d6abcfebedadf8c8b07a47f5611cce7\",\"b82e95e3869e0c9ad670d19f6d3e5d50496eeb459dcc6cdc359e119d5210b720\",\"5c590179593caaa854c4a431dcb11d68d0cd7a77cb1ea070c352a5451de1c8c4\",\"addd2aea08f32de7cfe03d4185d78754c12754752d2ef751fb34a8c61a2e32f3\",\"41ad6ce101dcd8a36a4eab86c6f5626a1ece943fbbba2c62e6cf8f52b4d78f51\",\"0b83f729f1b6e544b0672118675c2e5dc4ee1fdfb13ff2ec242dba3aa2443f4e\",\"4b01c20f8e798e02d4f1a5e67c823a2c0e6d48fdc99dbc5e76864e3f8f318a27\",\"790bc45bd277b9e190ddaba5f32b769042415d4b211d62ccb13c09ffdc377be1\",\"97bcfaed4997cfbfbfca0bd7c66e546e4ad837afdc62fa79220f061471451ab6\",\"1c733f658c6e657d0896c23bccf917a28bf1439c3f8eada584a2f26c55260cf0\",\"e7307e4ffb82d1c05082e2af4764e29740b1f6bfbbc58e6a8abd02c398961aee\",\"f1809173ab4b119145bc28a1406c158dd7f0a3165ce3e14f35191e64597bf18f\",\"60ae6b663f8eb259050910a85ffa9759b1e528b4d6f73863c01a14ae2471b1ad\",\"72a186ed193c6110e45b21d580e0ed58582252a61987f83fad4a0d4e7f4206f6\",\"06ad5b36abd988d05f016b8bb6d1912dc084002438f53f7bc18ff0895a207bb7\",\"5aa09f31d38133a3fee1ff1aa32871760a7389eafd906b497053eabb53587d4a\",\"92ac66cf16544e766c63a97c9b1d6531f3b23b10142dad22c1401bc20b2d8052\",\"decf516dc231fef215702a7b04780d695550f151b36dcfe87ca207695b4455a3\",\"f99baa62005a72c1b66bf6054b04c5f0100622bacd741e951e8d38372edf1eb3\",\"4415488441d6ecce6161301336c093b6384952317c364e48bb67fff00f0d1304\",\"d3b0dfb4bd9d54ea7cfb611b561d06654aa5487b1281a62623de390ea14f37e6\",\"eb7ec59154de674928d43bb7c20209560fc769946cb2b9dea9126036eef4e358\",\"1cf3b2eea03956083f5bb8d0a74d37656571f2a135af32d3c960e8e1d8b2bc4a\",\"919709dc0d25553cbbc9bb651eb094ab349ce87d373cdb5029fa998f833bfa16\",\"614d65a0cb2553b14a8d635ce82b53028b9c865d110285d439808e67dfd71e6f\",\"79214a95125221fef61e2451744de6711e3d4c9b806b11a48828b6bd9adc1408\",\"0883db9bdd789e6f035829d8086646372f1a9cd9524c8bf49e9c0fbc05b658ad\",\"5ea0cb04c339dc231ad68d7116366b777d945ea5556a1b310c3cada7b24d3d63\",\"7e187d3fa0e1d9a42d9e609c34ed89c267c417831a2b3fd25028509723d56c9e\",\"71b8c6dc5b7f464091cf430bfd5c23d109ddb1ed64197e2bda1f0ad5488ba5bd\",\"9be7d4de680d7119bd630426d34015a43c194e2fa5c5c16b2094ffe44d1f0e62\",\"1a1646e07f2e9b69a39016536d210eefa4ae53baed1ce3841442bff9699237fa\",\"2d3940c7812f1a224571792dff569eae8621de11dddca878cf0ecb82a37edbf4\",\"b7419556766a493d8d68ac84c3304313cc9e80002fe6beed1513eadb62b21635\",\"b4d8a3605aa73a2af638df2bb47ecfbf229bed0d01ab2ede26268293ca2f7fc0\",\"c86ec92c331553a2aac7a2fc09ca2f658ba5f59f89cbdd366da41c2d615766bb\",\"5811c2b0e32bbd9118e03ddc14e076f937a581f7ed80b585898f6faebcca497e\",\"7454b97bf09c9d99ee38e30086d3c4a093a4ac1c8c3e35b2e334905071f92272\",\"73c457815bdbdabd1ff2eeedb12a708c365e9937c3d7fa236a6bf07b8d7b3b1c\",\"f8018787f32f3c379ec10735abf93a02b7f8cd551b07fe32531855f778cc3b1a\",\"7854312e4f5bdc027290f7ed8e5cff31c5792f7e612d9fcb82bb6c9159a919ab\",\"57ea8fa61ede8a1de2bd66390692321f579835afa30d6bf5e5f95e5cd526da05\",\"3277cfcafc6b2ae2941721d3eed15af5c1a0a51f3927eea18a0cc8348e2fc3af\",\"fa0cbeb33bc87b099fe9a3d4c7b3172a287f12093d7227fb1b675ace38b6cfa8\",\"160027b5f5b0c77ff7c41e92ee032f02f0b0777d5003b7e43463006b84c5c15a\",\"8fd71d3a57012b0de3f2be8f7d4a8f439d56f661a6131f394aa033b19ab4e930\",\"a4253a41acbfad309a89c650f122c4fa071460fe86d04ae38da1f2cc3ae50d94\",\"76b931b53bd5b618acb7b495649b1452496a38a7e8314a5782f89385258d1403\",\"bc286a98df13f08288858808bed75e5d6a0483d7b166146c654c12e1930c248d\",\"2a7c573195093e92400d01396718213e7a6f7cf27dbe13c5df5a349d85b4f012\",\"188469feb05b110518707b26a50d23933708820da3f272a69b82362976387a5b\",\"62163e29d37d5e301b03913813b7d0cf6034f389bdb07228d1f6beb497ade18c\",\"1f08516958e5d69bc70ffc5743f499137758e2bec66667989c8f225de99438d6\",\"13ec5345dd56787b7d93285a1fc14626037a782e102ed11663bf553e2ed99c36\",\"774a95867877ebb14ad3ae8ac1336689404d2267b8f434370870992ade7f4594\",\"4383954016c5672377430a44aff37d38af4d8dc11d4e27b43c521c6dc61c667d\",\"7438224fa60eb36d3158fa025da2383b2bba0497b99f4987e096816e31109183\",\"c06740dc970829d273c91bdf9f287d0b123e10b13d27e50597c4c5fd8b6ec528\",\"070b768bf9416671fc3c55def106518378a878c3f4056d4a85e0e2baf6cacabc\",\"f0c9ba5a070976ba50257e8e2700cc00d2c859aaaf47d18b3932d673e01930d9\",\"6cb55f2ec8403ad8638e82e25cb52cf1d4d27fd437ddbd8516d9118ef9ca3a4d\",\"fd5ac2f6281af86af218736495ca74866c927113cc71430a63a3edcc3e955bfb\",\"6be5ba56e1d68bbdd304c3ba0194db87c687b8b8f82ffd2486f464bd01aa9558\",\"3705cd1e3e319fd5ec836dbe19022de3b7a6037ab4751d92776358f7918dceff\",\"0ddedf1dd86783094a62489b6cf015ecd6315bc6d09871da7c953184505a6ad8\",\"993cf7bb6a314b55aeb5b28d3c28bb47c47793be1bf684850b4647e13d5a1387\",\"ce1c45bbf16fe84808b9f3857763ff7e1db995682e45f34fd5bd5843e4364146\",\"04e2032a8e0ac506765883957d062f6f8b566a7a9c3530d3d5fce9ddb9c6ea79\",\"aaf604f04ee6a0b7727bd4978649c98223bce584128d0c5ad8b5ff1d4039e67f\",\"d3bb7716e499d5100b40f98c7feeddc0973d26f6b7857311d4117da047f2808f\",\"ece0335865493e10f4b47b111997b16a937409fe1028551c7cbd861ff038cad2\",\"4ed280b61d743284d0fd6e470401424c9fa37d7d08b90e13156eba3fef0b357c\",\"3f46b0cefb570c32c34bf0ea92f6f786df6bb4217f349ed2344acc47ebff9852\",\"7e10db5758772de4da425bbde74ac1208ad540c2da49acd1ee6b3b50452f5b82\",\"8e306e872c2ff38230a0e604f1568515f89547a3568d0b02384892b715558344\",\"7e0ab1feb6e226d835bf16968a7d7741c4096dd206042a89fa4c3bc5f58e794a\",\"18d0a06f93820d59d925d695cc958a4bf02caf10dd3bac985bd46b63ff921e94\",\"91f6a3f9152fb826b059df2c2d805bf44c6851b5d72043a2e9c172b32a1c41f1\",\"3990614c0b2d3a8d5f64a20aea2c8c828afa0ca676576805dfb6df68df7c1952\",\"593348c631d1fb88a03f8923e43cda1f551d7382e8d22b5675b5f6c9ab340b11\",\"a3f1ef424c4ae8e6a6b53e27e499abd5d914c955bd5008d317c1033583acca1b\",\"66b9df245f462bdf0f8c3c1166c3aee3beddfc569476a1aea6d0f738cf804992\",\"8ffb437db3491f2eea6e85af3feb88b6e659bc2a9092c165973b31f189f52ff6\",\"17e79882f7967b92373b314a16f979d18b69cda0f4203e17113528824b3aa03b\",\"4c89adcf320bb3006493029857c66281355fdded282cad5b4df1337a3df5d25b\",\"f8c8a66bf360e703520298c7b0607f6d6a605aadc7b7739dc552fbc825b9bc91\",\"6a1053e2914add11d7b3a9886aec2521d3aaf2511047eb14efba7a950812a34b\",\"708d092c10765a2742b49e31c9213fea4d26da485b2e659cbb1825a2f12b5506\",\"0ba8e20684871ac3e4ba06b8fd3c2d4d9e26e74b085d281fc769fa232e0860b9\",\"99aa299cc53ff2427770750e11e904a9e8100b69b19e1e95cc1baaa7c80dc142\",\"e611cffd1aa619d525cde7ad82192fdd10fc6b0e8b5ed7fc8072da88310c47f9\",\"2c495e8938d1321bf64b03889b56e658476a4838f469d5600b8085ce0c5f6619\",\"da30324f62bba968c347592e55b800d9684109e22b2588251bbcfff77b8841e2\",\"0a490c175b893f11fcb181503c2ca94b99d99e0984786b0de872c664cf9e9156\",\"4bf9ff29872d0801c25c1ef7c77f1dcafe227381dc18f5d697c9e1e3a1389780\",\"c4fc2b5f97ea6fb0f7957c036b6a0d3449d1e0dd96a062732e49ab748648ed4b\",\"2e797d7a5171d839037ed7be72ce7fa7733480fec4752262add4c18047ed5c22\",\"10090c6aa31ce9356de7081ca0725c3ac43032290b87a1b4cf198a920e1c26b3\",\"1f97833450f109530bcbccdbe304d1945e94db18774184495ce26691a208a2d4\",\"a2cbd58895e0ab2c5b70b2e6464902538aa211931205bdf80cf87e9b4acde419\",\"074bc0eb0835dd816f3666feef2d529a769d026c920968933b51fbc07f859049\",\"fe09b10e3f9eb5f80cc43e680ed8f6dcb2696cb503d16c5cfd77dc5350c3383c\",\"8b73c6fb7947891be38d88dd62107e04b3c698f37eb5ce9e702dc8617576ec85\",\"7c91378febde3d07f82209e5e762ec780807fe105648ea051ed364905f14861d\",\"0ff191bc43df995082d41e741daef41e974e00d31ba4549ec9d5c4d176dfcfaf\",\"1ec70938cfe58596cc5ae10948dd7835a69a8110b179ec17a6adbce6bb113c0e\",\"761e7f9d9bb2d85c6498822d01675fd6af434fb8ba48fd78c7932466452d08d2\",\"1c5072bafdf5ff5500a6eb27ac56974ffd89d104cbe86e7e388653f2842a8706\",\"b87f1599f5f8d86096bd6e22edcece306ea919146f68016952e43bd6dfe0d805\",\"5628d748ccc4f06af8bbdd78a129c883b23580d75787cae4a244373c4f9b8cae\",\"290b10a3241c241e9ee48873a99793c9972bd47d8fc03eac607c539815fa9208\",\"db8108407685e9690ef02cf4833cd48722f60c7373d059ce53d1be1fbc85696e\",\"57ca84bfa363070b56addf2a6a0960d6146d05df53cf2c54167e29294beb6120\",\"9f8afd6febb3686aca98b11a36a6b10041532a5be7d160046bf7a3525461af9e\",\"be5d0ebb925299d37593a32be3b88a8e0761489b031efb66cc877353128a5a6d\",\"c76f11b8c6c556dd06215657ea1673418f853d1f6f50b5da5ca8d8c5afed5e8b\",\"b7da5fe688a8031a0d4ada50b665c97aeb1afc1b5ebe62f046c3bbb976d186fd\",\"247481cf407ae6a5afdf4a0327ecc431261a31079500bce5c00d053c4513d38a\",\"da0eab86c4a5bd8498d32a85134e3c0d0e9fceeca6fda7511b68038a105ddafe\",\"1a70736324fe0f9f46a4bde1988ba170b0ae952958efb7fb17b398c95d7bcf99\",\"dbbfea8db69878978ddb63272116ebfb94e3bfe2a94dede112a0184a4cb16f9d\",\"8182780d3231726360329d058541ca6489aa62e2a9d1552cc17847b5bd23f4fa\",\"695fdcd752e0adaa9369c888154369c6319096dea1f04c67420e78bbe5ea96ba\",\"43028fb77affe77529267ea158fdaf6186dc8200d57d4ef236f16da1f814abee\",\"37c2eb6debd9bf1ddb4674805ba1f6b7f49cd1343dbfc3e375ca560441a1c40c\",\"2dcead43394c725947efa9531a7e2dd80a8f444eba40e67cf1f61761a46c8e6b\",\"4f676fa302d3f823bec1939e6d299716523a6e7243ef5eeda64cb9ec0595b8f4\",\"461cb3ea67519f7b2c79a981827c9df292e596856ef766db6a32d5f58c6715a0\",\"192a73eb1e81056cd73fdd164fd793d5ab165377e2d12e4d86c2f19eff172043\",\"09bc26d56928bd80fd779a066bdc9ff4e43573df1d2d74add682d5e090cf2be9\",\"2134a3f9589f25a54f62cb7080e7d28f2db242f41dbb2534e34d82eaa7f05b93\",\"b05da2db9f2aa151b42c19c77b34ca3f3bf8517dbf4201eadadf01ed82f559f4\",\"2eadd0d3aaf596c2755c06b9558ea785d14ca6b029b104912c8fd6b1a97129a9\",\"efe35fcbc23ed0de40f98f878ced8d72dd94cae82e97c10baf9d045b6c2de838\",\"d9b60d16ecf231e27b000a4cbfc18d5bd43f98a9f4e6119b83450413a1bef501\",\"74148e704c93e5998b893714e36e821345af7d7cf896fd1cd2ad69ae13f08adc\",\"dcb392a8f0310789b8d3e2c51b6756b967e5b6d9c760760c4bbebc57405b8dc1\",\"f4966009d2293b2b95d050aeeebfcd28cab29fd52ed0952235bafbc2eee87de6\",\"9dedb18d8d569690236ea54fc06f4504980781bcc0b769171947d438ffa4a500\",\"883e366ba08b99372e3ac06889d19e41dba9aa4c420858cfa6c9949f243af29b\",\"2db9c3f47eaca26ce8134b73ce68682b253f3ca7affd0c19bd136679b3ca5954\",\"cd1b73be09f8c5e04aa0fb87f1b8a2055e61722280272c4c143a67a13ad5cf6c\",\"1a22aa03733cfcde25b9587b42eb43bb1394214e4f0829fe83e0fd15be3dc762\",\"912b80abaeae9f45de822519661de16c30d85a68967e6557df668109caed9908\",\"962a28081d01aad134242e614fd85ecfe84a73116e16e344a8cac3eba165aa46\",\"adffded71ebe018fec757e725fd001ddc8837edf6f1405ff5cd723ff53be4f1c\",\"3ac0d5393337cc879f0459f885097fd48e884ae12d189bda1c77323c81758f87\",\"a430c838d4c84312d56e53fb3ece147d1db200003148e31ab945891e4dc79ad7\",\"0633d79ae88637697727c3e4fdfc91392d8a7ff3305cb910cd9ab06d1c75c73c\",\"0fdb190628d2fcc45647365fc1afa185109823c508863408ac4ae72562253780\",\"d8c9806c35cbfe2ff0c5318c922c5752d5ff287ca0dcb79b1a1bf7b59c7aa0e1\",\"2644095e5020c3af80b567f8d385efca3745426e05b777329eba91e9c4029217\",\"0d3edcb9d72e12e7b12f5a6d183c14db88dc60a3fda1fe698f3d1bbea6a7b652\",\"a699d9afbca659fc021d4a167ce15e5b7f9413b18367e012c764991dd937a585\",\"da95ae5ec6b377f2837749f128833b9631e7a12f15e0ff95e6999d4cfd99f1fd\",\"e1d9c6de18c4a1f75ed38d02a7ae4d5751be0991844c9b11eb7f035bc75c3848\",\"263af2eef2bb8fa9c5055b62ac4a301d2572e358b3cfb4e4223f796db1577c7f\",\"d068de9d40f9c13160748d55debf0ad2ff424570d56425cdf218de20669e5599\",\"2b3bcf13927383267841a7ae85ba4908279cc36a4e4b840fa1a2689b69fc0951\",\"a7ec03436450945b9af9e93ad691e8dec132bd986f49870ccba8345efb1e4fd9\",\"2d62bd2756973b90270a58a0b6cf3babb2c2530bfda8a1df1c0b878fe8822c39\",\"4ef5072bb9cfc79f32a41be777f2d73da746e0c51fb052ad9e9fe6d69486ff6e\",\"b7fad51c5f14979c87aa0abc5905076eeb9b72c497a428f89fafc337cae90180\",\"c553c553c8a25002423ea9922fc6ccb4604e5a9f2418370634ad746f6b6518a1\",\"18550556185e287a5bb5aa2a52c377d72d08fa445a67d359be20167215b4af67\",\"d86cab365f256e788eb08739fec137744cd7ad4e7e06cac54332f0fbeed1ef22\",\"3f269ebdbb53d47a1620c218fc7348a47089e70334edafd2c71f03d132f3f846\",\"a2a1d02c4e51c754d42e8ec73c06a21a2cf339594cf67322e1b0121a69576b0d\",\"75f4da042bf1b01d04cf861bdfede7de2b27a36bd7730982c09a275126f7305f\",\"55f1f9a50be97f916eb9f573c2651100acacab974793a5477dde805ee3dd0501\",\"1a4ebbc0275306a0e6cc62ee830ac31ae7600b3a5f015a296c0ac4dbc6c5daad\",\"8605e6f5d0a34a89b48e6437200e4ced15565f748a62df57072aa84881493483\",\"7e83bda68f2f556c99301cc0ecd35a5c0cdff709b6b277f07535a871cd773fee\",\"2dea0909f7bf5b1b86918c89382340536cdcf402f1ca1e11add170e7be9465be\",\"81b11500eacb9dd2f14e9e4845e3ed0859cd73dd11c73f5a028bb5191e62d991\",\"f7e7ba4f10fe6efef5c1f57ea9f91ae350859aa30d80e656007c797cbe949c55\",\"697166f18747239265f9c46d1c721ac30d35f9344ae41d56e56a1b3f5e05d897\",\"ed80833d2153597e09bd20f5a5c3623d32f5613a3be109f3c3322c38149378cd\",\"5b0151819fce889db356852354174a4cd719cefa77070a0a862df63ef9550f14\",\"61bd3e04f0102ca10e2ad8d44213c10b0ed420faf072f34f9c3f0d5128a01e2b\",\"e13fa949e000f74f9049279d4a5dcd6cdd7b0cf21c4dbfbc94ec2a730d96378f\",\"b75bedc7a7b8d6b4347755c1dfda80a28033d6ef9ad31fdafc72a14f3cac903d\",\"560b30e0c2f17740c1615aa7319e2582631ae4302ce5bbb73599e891f85496ae\",\"90126cb29e3513fe54357954b9263a9ecba1235640b29f3146c3de38e7e25238\",\"cfb9cbcaa5bdc56404f64500024b5f6d1a483c6fb6f0db8dcf2e3a77bcb79eb7\",\"aeb1debfa41b170c55b8b3327847c0691bcf9dc1d01e135f1216176ce207b8ed\",\"a6342fa0919c61bbfb719cf8a52234aef80c75eae45a7023f21054f2c1b1aecd\",\"6a2db120b51648b00f984dbf18363687bdfe910ccb80523e934209311b8081cf\",\"0b2356509bf1c4eaf39e0ba55f1212fc6eda7519dbb194272ac91f7b5b8fbd83\",\"b74616e6d28f89b1b2ef3efc94c989d8fab278c3a5f8273a01337878aafbd882\",\"e614a2898b5971d25e7a032df09472cad22f864aa773dc87a561a833a2befa51\",\"6926967dc13f4d1d1285263f7cf81f649455ad04e024fc7b2112bbd2b327789e\",\"cf7c4e7123faf48f673558ccf1918b8194e8c3316cfa0d327f390114ebeaa15e\",\"3ee8f7d8c33e32776e2b3ac6a7e67c6e40185d1cd00a4c794edf99fa6402bee7\",\"a8a7b7f3c4b76bdadd0e9496ca2bd416eec3f9458452d93800e9307811ca8f1e\",\"9d1ad3d539cf30b65ae175868bedb0a384d5599cdb06393e241d5bb3fa6ba04e\",\"289fa4d901cfbabbfa536d10af2518bf0e296fb5d96f66b904442b84274602f1\",\"034bcf384e3ffe53833f34673390eb9b94cd3610dfdab9a9382a6fbf94321bea\",\"e9568557fe8f6f7cabcad29a0b1ccfa98b9709359d11d4835235ea3a104abac1\",\"53371a43b530c3e66b2159962f49d0f0b01729261fcfe7e79d6d6f237b382fab\",\"a05968a82437e487cc9160c728f33e9886e3f39717f63ca6a8414efc0c9d28e5\",\"beb8bef8e248832380809092cf81001dbfc000a75bb6f7a7179c9e1e0754c406\",\"edc6a964f4cdf4de30f6e0c61d3c9982ae8cf29e33cd5f7a4ec192312fb2062f\",\"734b747fa5c8320c9ff46488bd01be2fff597cf1d6c60c3c273d15ebae49da6c\",\"cfae882d5452d2086ce0b897e3c4f2dd3f50fd74aeb42760f4e045372d0d828b\",\"631b3565d1ac3ae76c7ec9fc8c0f8e31b39209d76b6c2d586625593a00835fa0\",\"22351b684c5fcc1f1f15910e387005a1416fea2d632e1f6213ed606ba474366a\",\"9886d1a9d0f7947b93e1f0be6feb290a9d444d82827d4d16d709d6104b114e82\",\"361578a8e9049ba2a5bb2a6241f5d32a569f62bdc5e4758bd6fea386f982fe37\",\"190fe4bdaff8c8617d4e57fbe2b25219170d95413bf8d4d24660328d766389ef\",\"91c83fc5b6e83267d97987def4d2d60d54bf3c7bda472c7039897bdf70700d2b\",\"b2acf1154ef0bc46cc6ca47b7681e57d413e2ead031ab5ae7a44a2814efa5008\",\"6671db8f8ad187b7cae14d2a1e658648b8d6d223f24d318fffe1a75bd121c7d6\",\"510b11dd680d60c3ee019b93ce2a59427bf86aa8781061e2e98d7945b44aa383\",\"981a8e48ef4b0652fe997fcb4aac9d444d52bf6c0144b6554807be73113d997c\",\"bb6eb02c0005b0f8a22b45374bddc17da8e015af3be665bb3a43065e157af8fe\",\"daf5acd1c303de700f2fb638705e972408c3076419a96f28c1d8214f2111beba\",\"7459678487e742f1e7e0785bd223b0124cca8fde67c79a6cc6671185e324fd12\",\"89dffb82d0a1539f44e56d1831d1816793a4af09728cbf7d08c407c6b7f4b73e\",\"5a48d96dc1922c43cd55a3e32f5dc3e3fdedc104406aa7da2d0f4822acdfe6c7\",\"8d9b165a3414dcc3f1468a601ccabe966b8294277c74ece438df322d97f574c0\",\"c290aa67dcf8df535585c5052169ea8dcba7c5ef4e1888f95a28d52e50177b50\",\"1b8800d7450b723caf337d4003c816351b22ba356403a717cde18e77ce3ad8a1\",\"2290d5119a950eda1308c8337a60ab2e5ac9dfe00f84f723a858dda41d9b9523\",\"068db26c8a90b3528caa6aad95ece6f102522dc9a9b0445a0e78f2e0f76aba4a\",\"775ec87b66937d9a5606f5ac7461a2f4335bcb5d1a561a168a311c17e2c0bbbf\",\"dfd7f10d97a7cf287be7b418c711a8c845127ef80c9f428cf656d079139cbd24\",\"98850a43e119964c59458c6d459b2fea1e47b0f97afc4ceb6852b0a5226fbd2b\",\"81a0af2bfe56fa64cf1f6b9267b3d83ca741cb1cbaad138f60f358ad03b994cf\",\"db8126f98995575c8261e581d94533884c24420075ffbe3cf0efab3e3a744747\",\"fd475e92064a11dc494c7d889ac351819c280301e5f4b54853b0d6da86affc01\",\"b6b0c62acbedf50f3a9cb739c00ec3de6969d8a468755dd121c170fabd13106a\",\"722c44d52b25ae3e7165e8c53068b21dfb9d9f59ae4fc4c8034ac07c176b48c2\",\"23ffa45d44752ac107bd0c6d60743fb600d0a7acc4794ab6dbdf15d9e1a6c019\",\"ff5f10dfdd1184fa0e3374e31287d1c5f5a84ce06b533844fe12dce5087d4aed\",\"56951728515b7ce85bafc304d6ffe2d39acd97f48d714cef72a2dddc7749d2dc\",\"e63f9255d33c0658f40196c0b8caa8c6aeeaf44ae168997463cc9c202ed31f72\",\"fab4d67ee74161fccec886bd32aace6e7e1fc43b93e47a3ca81e1a33fd86502f\",\"6417502de875ac61cb7eeacd26409ce30b99286bdd213fdb529300f0f66019f1\",\"d1af52817e813f8e64a6f5fb98a5b27a7ed16885f37d808a5352d7e568db010b\",\"fe526d860f42dcb804b420f7573c08bced1230facaa3d007a9a8d91023b5b3e4\",\"fcc668f3dbda3b2e72c7281c91c01370af8e19dd861e1e5b5636986cc7d3fa6c\",\"c297514ffe6e04cc20b146b5ec012faf9748b9cd74a3bf22f2cc78b67986a4b4\",\"ba5da041106f2a373d7b8b7d81269212193fdb1392471a40aef9bc2315b4e2e6\",\"d42bf2890a3d2f7bb6e20770c859863ac725a4c2df672c6a917b25d2c932326e\",\"cd404f0721aedb400b1a5dce026023fe3249b02cc3e241cda5b6517090e9ec54\",\"64580834ade402cd8718da0edb4465b126c39ebcc0aca01a64d013dacbf56744\",\"a420d436a0ee86031299266e0ee4c6ede5c64ec98ba6103d52b9e1e55af117e8\",\"1ed17538eaa2a4a5ea996e5e1bfd07dee0f3b7ab2134e2e35cdf34f75878a5e9\",\"a50b85524f287c3228db110cd8ecb238faec2a0227d62d8cdb20a449a3c33bf2\",\"7a10dc0bf8b1667d8501e83331e515d12ebbd9e2dde5be1d9eceb3175495ac00\",\"967d8d43896ae651c2aa7112f36bbf0c455497c6c90c9d73d790958643a554dd\",\"83428ddecabfedf57cd3684bb2a8f955faae05a4ef46527c32881f12e14d06c3\",\"88e098cd3a1307eb710e5e1326dfc6f7f7bddbd32b1f3e385dc843946f95269c\",\"b4bc35038659fab1bdeb95ab5bd3b11384e6f1f15254703148c1588fcec9835e\",\"082114aa6e02d7922d1636a9dfcb954421cc8ed1e75c9bd7eeedc0e70831135c\",\"15e4f43768950c159bb3f0c1c5aa9950a88021ca57f8564e81c70dfae7aee422\",\"aa617aaccf1a988d0cafefbaee6cb368a48a0cef2ddc0305593df897a22b955c\",\"4b16bcd5c1190cfd8fe3230eaa7c9d521fb3b3ab18e12b7886aeec94e04c4b4b\",\"41c8d1c5c975d9ea6de050261822afeb29a332b9bf5d39ee2f89ec094fd13879\",\"bab2b716e9c4f900712f924de2a4a32f99b5dc07b48e70a9df322f38b27f20c4\",\"101aa16698d8c3a835eabb1c17faa15c2dbbab0b4a276c037821b27f847c17aa\",\"a826c5a93ae95137096b96ab6ce8b72e8c50a93ffbb745c26c00afe69dc330a1\",\"0976bd9c2bfad2107244658aef43aff5a3865245f7b978a7f9516cfb9ec6ef90\",\"aaa5b2e608137f405344ff92bca1458019bf9dd2afdc295cc750db63335e86b6\",\"5950ee978be44f9f0be1f3740023fd7fa0a3737325ce2607deae112530a4a964\",\"8a8ac3835f62bc7ec3d4f123e92ed668f268689a29bc85a3c50a6b68635e1b9c\",\"976139ab73124de7444b8388e35b9db5a118726bf9ef00270f37be27493b2410\",\"81a409ffa4171555d5b25127f324ceca3fa0524a61c1f0c3f9fa067e6032ad3f\",\"df20eb3b068de94e0c5e91edf2f680e856f2938b22bdf3cf16718c0b0a1b6ebb\",\"73a61e99652b2da594c93dd0cf7033581c599180b15bda2eb790c92be2d69aac\",\"4cba8d32cb75cfb20caa002791e6f23cc815b8739864f13e8888040d80b62a2a\",\"e08f0daa9bda4c49808dd3650503ecf7f1eadbeba75b60709282cdfca17c9141\",\"7bb6cdfbafdc5760bcac18a84e349519f235cb0263198d40650577eb887af469\",\"3c15ea262333ca9729bd9bddd92cb9308d11b3ed6082eb433db7afb6a6ece913\",\"9629a3ad92c7317da008a1e10070f506ec8eb6f6b4230b1add263ad67414035a\",\"21f9da12a2070fcade2b77f599d690dda0a49d2cfd0faa9e18186d88b128d27f\",\"e4089410abefe743017f2543b94689d90ed8ff03d34c52d18084e7bd8b23ad08\",\"ce42bd406158ed18d483f253b0c09ff36791eae8df7bf500cf475ba184aeefac\",\"4c8f64b0e3f715034de8bef3e32439ff96a317ffa2b0b484974a449b67435ef7\",\"657decda4deee0c917ea94b95ffb71febe5387c8d849f1613b13b3ed0803714f\",\"6f4f6b7b85592d68215b2adc8d64f6d7cae30ce9a4058149303d81b30fd71947\",\"bec123746368a9f4a14300cc30a1cf035d60ecf3eb27cc8092bcd69dea585f61\",\"82646b17bc0ff5770d1c7d0e14511ae587b1d2edda3d4033c22b96f0c28af869\",\"bc85645437178886a7e95b737071163ef33597464a5c2dc2c0d0824a6841bcaa\",\"699bac8acbe1503152ca89806a4860a2f9e7af60e8475dab8fda368aec35b1c0\",\"c3c06bbfb833c4916ff9e90e429294ede06d863e5dc4494f3ee440eade544073\",\"1fad7f249c20dce799467d5aff258535de4fdbdb8a2b96c31e5272c03046a403\",\"81544b10037e944c8ac23b58122c9ed5474125a1f7a10aa3c311bb76871b1633\",\"ba23d74d3f3c25c0f72f348312bcf027ffae546593013fa196474a08e0a9cc02\",\"3c95653f4f7374ec5c65023497c50e60a897fca94646b64dd7bc2bd1a2059d30\",\"28b72bf686e08379cabb7ba1862d7e735ed8c8973d485e9c95bd5697c0eef8c7\",\"ac67258ca7157e3bd43d695539307d3956a7273fd9a663dcb53500f152faefcb\",\"da13a0f017f29c05856530be1d94a57065856bac37bfa23ddd8744566a246e93\",\"640fc6185b13bb9b6414fc922c73254eaf942b8bf9c2eac7e83135b0ea83fa14\",\"fbc36151040f35d228a7e597d92f93f7656fdd8cd3accbcdc989c006c8363c0b\",\"919886f3ed4b8cbfd227a75fe5ce78db77ec212faf2c330f97dbeb0ee5aadfff\",\"92dbc59a9f6d55efa332722464806e07597312067099f294e11ded344f52ecd4\",\"19f5f895083b2f940074cc56799928dd18de6d8fb3bd191d2f02c031b9c51b18\",\"827c83a0015cae333c6e64860b4431641e56bcaef1e184fec76d1871367c99cd\",\"c02a85b0405f8a665e0724a1588d0778526169abc69e11ca956260610f91cfdb\",\"8894f9d29cc5a03ab092a1f29e5cbccc5e2156aa55346fa5bf095a91fdf08164\",\"9922b3bd08e867472cad2fb3f1c3a0df39fbcde1361c7b4e459374e6fa85af33\",\"d54ef138840d0ac633e4fba269bac65cd0612cd0d68129033274bfb2acf333f2\",\"a25f061bf43f492b57c37d6c857ec964f385fce15d0e6a7110fb44d456f0eff1\",\"6e62105a7e197688c5740aec9748c8c1c95607339c3985515dfb143a5a2d5ca7\",\"0a02ea962a34727a3537d5594bd7661b73e73c4b651d82db277142bb796a90e2\",\"a5368c791b170c1fb75b722a15061d6145798dd3e65ad4e924b9da174ed9ffe0\",\"0453c1696becb3d8f9c8cb2f6bb33cb6b726fc53577599ac6d66c17a8a3b1b1a\",\"72555ce6f458c7e017cb33c2b90644e8ff01c1e8577684a31ee3692ba839ccfd\",\"16638326daf96c856ee55dffd397a973d18aee03d4613aaf77c15935049ee027\",\"aa9815aaa8a43b566969f8b061550eb60c70f9dfaa803e7f199836803cdf610a\",\"6e1b9707118bea1cb658cf294f9cffd08e788bc208e117e46885884201910643\",\"7858907870b5834f9f38c2bee7648b30bcbac41accadeacb97daee77b89a5395\",\"cfdbaaeb17f23b3062da08cb4dc72585d486120d518f98a6d941cd0d74e3e936\",\"fbfd9ec810dd7378d6256d1cc1c3b69413271ae0a7e0082762e266a334203e0a\",\"eae575f8fb25743db4576edce4cb9e7c2a22c9b5693239bd50b1d374c04a5173\",\"13feee41836ff007e886021e92b96e9f358b284a191303996fbe8f27087467a9\",\"a4182090ec65d3f673a57e0b7c1bb5dd5dff79a4c47539e17d927beb33511f04\",\"6e28e654754364c016b53ca4d905369d0fd89addd3f28046c2c2e9e0267d5c65\",\"2291e9e9aea4aa017c6673c3363b894c1f9c2f397ef3236e8a7d448391ea4f5e\",\"fa9380f08c65314331055bd1b0892092acc0b00da38a2b4d4213261c8f7c8363\",\"012d1caf3fd16213f527ee2cdef7e7383c5b879ba6e5aca75be0567f3ee79611\",\"86ec85998aea4e7ffa4d1fc2a006923fd31027633aebf3cc0810626d855a7781\",\"e32d5c1b3b7d4c2cc2ef27026915abc8bc0e8fdb3794f7e7b66eac5ae1db7a91\",\"4bc56193584bacabb18a48b40eb7642ef6b9862999cca188cb46e88b0859d9c9\",\"5d9f9c9778baacb6f33baf117a21a28a7144c4d68515b64f53596f60b373f09b\",\"8153ebd3387f5f8699c6909066ed762fc3a2e250c07ceeeaf1c5c7d4caca816b\",\"861dba58b80692f1a8c07ba5c16e9359aac5877263b843ed01d77c5d03c13e80\",\"a3fdac7c6c67d7654bef1c2f8f9424efc0e3efe6beeae3f44c4bb7dcd94f8869\",\"df5dcfa96beb4bfca7cce3b5d168105c433df4c32a8f75a58cb3d8a110583264\",\"bae7cbcac1a78c942fc060e03f22be53ea52e843effe4e08cd7b22475afaeac2\",\"c5635beb04f41b240a9396ae78894c62ac57b799982f24cf15ca2949b7ab33d7\",\"d2facc4f0d2b0e27fa2108fa72b287bbdd340ed269f33e6bfe5a71e3317c1e7a\",\"69000d6209c693dede57286c711a7d84fd85840c9875be3dde9b0c96ec22833f\",\"44133dd3e6f3abf57a6e77a7359b6ab2a59aa3b9216ad4ffa3d14f0e7ae36187\",\"cab3448564dca321809bf8f77c87bd6380c93ea36c42ea0bb7071051e1953bf8\",\"713f3489d9f49e23f3401ac3d1536cae860d534b26efbb0a6b65ed453a6654a1\",\"fd2ae4b3465b6419e102fe7402e6c5d2e8e5041520abefca2562b625584fed9d\",\"035d6192473f2975ee9ba089fd544460932a1d1af0455d26c32dba528a2172e9\",\"3be517bd50903361181335d186e87c431c669ca9e1e4bde43208a0dcbe768e4a\",\"8fd2ff597ed85f5fba7c3ea91d95af1d6e80881cea223093c3810eaed1671c73\",\"08e78aa4e8dd2ebfcc3e70a558080761fdab3462c520791b08c988cbb78e151c\",\"d9bfdc4c8940f1b148dfcb94d39287df3cf9009da742825d3a907dae57c94166\",\"53fc7639224544be7e3a25212784c690f163ba78dce4840c775876d5ddd67eb8\",\"e3f066563cf2ca25a04dedbda41f12b998323f20f8cef5978cb1bdcbf83582ce\",\"fb6b07cfa14bf07417a132e51fc0616ddc14a3c6eebcb26c9085944c81fb06ed\",\"a91d57949525262b3e3b6b64632dce3dc06e3adbaec9f9088c5063059972b05d\",\"cf7b9c251186631f67e8d5dac70f1c68bcf5fc1da437211f55ddb87569ca6742\",\"e2e24239dbdf0c7378028319ac49d7dcfcd83326bf5a9532428823f8601b4544\",\"07b6213261e98956fa1bb5c4df07de33f22fddbb219b0d7777ff70124190f99b\",\"fc2b425bacbc318daf7d17ce2df24f1f5a5331a6257975f677b76588f6f0067d\",\"4b286c6f049383a2896c4d9829d5ff9e1c3368c82862aa65cf3ded896cdc6b35\",\"9cd957db4613b27149e9826acd61f5287e706f807e188e26d43677319eb96e08\",\"2b76bfda817c61cdaba3c9da14aac729953cb0c8621de47452a1edab7fdd862b\",\"469b0ad8a943d6a1e9d30bec0c1e1c15debb75f60686756664e17dbb94dd89db\",\"fa024be78f3126bbc9ec87cbba10f0bf0bae56f353ee799bb2d03dc6b23a1ba6\",\"04f27c05f8f0c074afd2391b8f20a6931dc1117c57a67ccc8d1af05905f6b1b3\",\"8baa7fa0fe752a2527e5498eb3a90ba9c8b98f5face92b5f8d113e758d74cc01\",\"eb471b0846654f4a4b6fa3765d943f97daf08baf2d437111f1d6bf3006cae67c\",\"989c66ab82e739c8c988d4fb5fcc68ef04de4238d20fc8fbb914357a28f3bef1\",\"4e34b8cd4de41b9d317f807b3ac37e326650e55b812c230f411c02b85f58797c\",\"74b8dcf4db82d846d757d8c2a64b87c0ed6d887ae4ec7ea4e7360f48dae63fe8\",\"3d10ebc5ca90a7093c6977d2be83d7fff2db6483eddddc0517a4ab95bb3ef032\",\"c801b0ed15c16fd43558b2973cadc524c7082c7cfb6d07ba35f7ec35b1ec08b9\",\"db500406d8c0bec10a84c7cf8b917bfbddf84da440c1e270a66c1e66344aa740\",\"62fd9b2b8e3b456d6b93f9de867540b8e2398cacba8f9d7e26815767c56ac5e9\",\"5e13f4db861baf936f70931575fb62c5d162332b6992a9dc7b45d898fad6705a\",\"f8566dc3f3324ef26b1192a2a8093766503a2297e0888e4eb931359c6af8f155\",\"d9b21bfa8330e0a72512796d5d2a88ad7c0359f61c66212e073086d1fb559131\",\"4d2cb27feff141605a9c62bfa62de3067b05e1757b74a7f92a8e66e625f709e5\",\"c8201794d4b8117046470311602166dd8845010ede65155f93a97eb05632e2f1\",\"7aa4982a2225cae7734470a77269511c626da6fbac47dbf74ca59d7b0e2bb1f5\",\"935717adf9ef78e810976a74d74ee5cd1ab98f783a23cf432a896cb1bc7fb1b6\",\"b686be2b1d79c5047dbb41efe214d850f6c137356beec2d7e3a0a6ef7a2afacf\",\"488c108f9bd74ba49080d31ac72769ba1c7dd12d5bf1b9f750ffd1ea4a3f4499\",\"a12e355fcb50a1c25c67c4b8be0cbfe3525d5dc6218b471dd41a7b73a748c9e7\",\"5d6bc773dc7b2278af8b3d4c9bfe2bd58c6426f98c1592cc690d727289576438\",\"2419ddf58169d1e6cbebda43367e7d5867be3e3eb1b2f2394ee51dc8e407b5d2\",\"5d77f787c7a32b6b49471f93a8ceadcf79ab0b05fd9bfde07e0a80a2fc121bdf\",\"b13e4b0a2043935c84acb8c9a009cfe87152c89123cf8283fad49353d4b5e1f4\",\"42cef517805baf57d317ebd0327118572ddac21b0fbab1f31174bef3e190324b\",\"25c016e15d6fba8b3f1754c181e2d24c7e74b8766ead33f301dbfc962729e162\",\"7bd8fbf8a906f36c5f9bbf65a92453dc8193b03be15bcc6b557c17e1c580fbdd\",\"b37e506c9b588f8cf9eee1635a6633140da202bac43c394e732f9a6105d8d7c1\",\"3e7808383e15f05e4e11e3cc7ce2817efadbb0a19745909bf6b9449b65d9cd6d\",\"3682504a79aee8931aa6946b03affee84ba8d01978d7162e19b2733d689823dd\",\"0521820cf4515ef62f47b478bb40f4fa8cf19650f5d611ab9d6edb9107bfa42e\",\"e1441af62e0b61890f96ca7177ae4ee91cdc29c7dc2fbb56febd4eb3b76c9e7d\",\"18356f7437e7d2f6e270804522f16504838c8588e0893f8f255016968b1a0ed0\",\"94d188d9d892fec8f42ba2894e24159d53a1e26f030e33bf68cc631287ea4dff\",\"14d7d53e3d80eb586adee50b4650269da4ecde3e74a63a6f9ef6d28a30c91cd1\",\"55108f6040395e518691685941309426e1dd87a85fbe86b8810a9f9c1d26f5c2\",\"ecc045b0ae17393a7a62d2d650f6fc9a387f5b639da91469a2fb889acf1b0ce5\",\"9445f63ffabd70337ee706b3327de6d693af482711ab66fa63429b0102b8be06\",\"f97b3bdc8d6f2d79206daa3814174f5b0316fe991951c99e6bd22d650ec04e07\",\"399e5a5d04fb1db8f314a798fe885005282cda4b11759d1192eed063834806b2\",\"79861e5d0283a663d0ec4db5515885f2c4cd87f76842c9a947374ace776669e8\",\"0414312809ce23eb8c0b6cb38df71443488b8cd5f155d8a0fe3b92deb738b661\",\"b6fb01e885a0de750258fa450715e8f556680e0d92143c01bb5cac90e860f40b\",\"4359bac547137687e0ed5049d2d7273734813e9bb376c6ea9f20b2cd1c7355d6\",\"787f9e98b4920dbcbcd68d776898e138bc70a991dfba4379f8ea8476d403cb76\",\"f23215efa4ce54c46d68d21a326d96e7e47fe47c4ce6e83978d36d6fe8863070\",\"d8b6c044f049b6dffee25160e1682981288ad8670c9b680b3db5de77cd9caf98\",\"daedd9805719ac3fd689800f0da2a22f0a89773b45527eab0380c475b2607597\",\"8d328418d8f8d1e1ee6831a330dfa447e7e2a1e9068be66474865b991eaff33c\",\"76d785ad564a2103a72588092530597086274412c27ca81290188ebaa1285d62\",\"d30cccfef97fac890f8253ff3e495c50c0673c9441b0a408d86a64ee7f7bec19\",\"803b127fd129133ad5660aa13d787d8bcbb9f6a4d8a0d69a2161136467b667bc\",\"c7af546a6dd0a6017f268e1934509c8c76afe80f2daa8722050067d5d393535f\",\"eb2e6765935f53de024592ddf4c0405365e21efd31bc3a7be766f131bc3ed0da\",\"f66f450c78517d6c7e6f54d8d72fd3ada15d91133884f46babb7176677a3fa8e\",\"a1686745e147bb6993246516ccbd0f4238d3611b33e20549427cc5d7befab681\",\"4de922d6b48e480d59115aeb7cab962f2b06515614d36926e70b62168db92758\",\"598caff0a50734b405b6892a15ccd441ce397189f9d08dd76f2ca7b730c5ddad\",\"c31b6d614e11dc3c930fa95b116d811cb2aa54a32998d04a956fda900c40de19\",\"d51cef4e51a8e445abc76eb9e761b0b286c8cdb779f117af4d483a6057904eda\",\"1aa00e0fd1647d79d3b1dc0f18ad12d07ce693e962fa538fbc1205128e2abf40\",\"9afb699a86ff5027862ebe4542e70a75e716d8c37a7600668288884811a5c119\",\"c6ae623da78c5089acb1265a60f991377eda9d5e5f6039810392b0830b0c0c63\",\"f3462af96fc2304cd070185180d94bd2d4aa16af940e8e93f9d5cd4f150d2489\",\"6eaab47a0277d06af1d27d250cf0629ddaccb107b92645e3cac80f8eaec2fe78\",\"4b5d18c134886ae2f2c798266c88dbcd8350b1d8ea111f0810b8d39b8f468193\",\"7566799d6a63a806780ba3dd5785ebb2b9a95524884838d03bc5fdfb6a605d4b\",\"08dc1a1bd3d53c5eb83b3a77dfe85d1a0c7c3d8ce0f198dbb174f606cb7bc1bc\",\"822eb8b056ae5865f4d46a9e38de0676b0a53f5fed47f769785054515f776a89\",\"fc9cf3bc92e83b3e84973c4fefc7fcbbdf70745990c09e80c9f98b71cdded240\",\"71100b27cc2bb792337cc26780e268d384a2ea9be6a0f025496c5d7dd053bf51\",\"4f762dd86d7a4319868ca893a1fe7d1f3c864d724cf4af18eea69fe9fe889b86\",\"61392bbc007388fba8414ccd81e4b22dbd2bf1d8774c46cfb1b1adb706418680\",\"3de477c1e73ed6d9d964a0cf5c226b449b30176b211d9c01c7b20df9ff3ab5aa\",\"957977f40ca162a6d398451a3506ed9467e0c0b200b72f267a7a7c987bd480d5\",\"d6b160d31bdffa59179946d6505258f86f2ec2f623dcc6cfbe9615d20c71ddbf\",\"cf9eb842e13963b93a69779ae1668f1524971a71a43a1c219347860274d64531\",\"933efa4e75a8a6221f3d6d3258316e41f79cc6514bedbfde36a5f614a38a5c1c\",\"e5f5e7573ca9aaedb1ba2e5ee02d6be4a5a0d90dbb01f33c2b04684ebbcd4dfa\",\"d566cf0f5431db3d9ee838c7c1269b870ef2d46c1dffebaa4b3d7ace80b5310f\",\"789c6f2326f620d025941b742ad559057d6d7f7ba2bf6d9be2d7acfcb4e1fff0\",\"d13f6233d8f8a96b66aa4207250a29857db372ec9b2a1de359016c471e17732c\",\"4fe9892cd002d64c838829ce8761b2e69e053481463b9c031c58e25dec005a95\",\"aab74fbb1ab783e01eae23d3ffc739f2ec99a1c760df0059fcbd31f64630417f\",\"be50491482c3aa4a81ee79be607e99ab390526c307ef7e7071b093b8659687ed\",\"28ada92647842a77ff1b8b6d23dc699992ac03a090ffd38553a1cde5947375ea\",\"dcaebcdc2924160e3c6aaae5fdaf3d76b89e7fc6936e1228d1bb9a553d59b01d\",\"0847caf0e7f5f4f46edf00de2f81cafc1331b2f295cd525171af7aaa24ff540e\",\"31667477c845a07f3175f1bcfb3deebb66cd8f064264925f27def516a3b93f40\",\"1c207191667252b40d3e15c7c2cbbdcea252f58fb129c757449ccf6d02f62895\",\"0d1ba5f087e3165ec33511627c7bf4f7507452512523a5dfde173c2a091b6887\",\"576fcfca00e0636481c9a1d5b6eee72c83009e580caeb22aefb1f6fd1f295f4e\",\"32314eeb24a1e9bc836eaaa305a6064c875fb40c8cd30dc0d15693472d7e5bb5\",\"5dbbc3f7016fe4d851adae9b9c6ee83c9da841fd0c532913e8a6388ca572d6f0\",\"f2aa401bd8f231bc4305b0e7a294e0c9c19ba328d891be284764d2eb5be7e31b\",\"3e5a41b512f8c1a86b3b5fa66e88c38a4eaadaf786eeeb8977f6acf12b94e7d1\",\"4b587bf5502cb85215ba58baa1b01cdc61d29be6b54cb8f78a18c2e8ababef67\",\"fb864e05daf12711d9ce37a5855c5ba5d5645d65348adbd1242d334a29277338\",\"cc9f58f68cb44810379a7ab4e86709744580102d4518d6bf183e12d131cd4fbf\",\"1bad3cce699985e4bd0ee5b47e232dc0c356b15fdd831d1a3c7756bb20718d32\",\"2fb2ceb6e794fb123461add01186355109401af98be2aa4ec2bf20c0b2b05e3b\",\"5065528b8d066dbc06d938e26884b870414c2d3e64e474ade5cab2ae4921d4f9\",\"ec3e004da8bfeb0ab7a4d9bc6430e16dc9f913757638c69813da8daad7381e15\",\"6d44dea3e09c1cdd5071e7c4b053d70683e8a860ddafce197588a6ed5dd71645\",\"b4160999c68d574ac803e03f3d02c886a95d3a6f2f56d13bd4a7a70fcd4897ec\",\"8c1bf303f51bcdc37f75acadd46472aa407cab196b8a8142f98286250899f211\",\"4c3692973789accf42b1fe590889227cb84d22cc760b758fb60e7f6af1d9105f\",\"ac56723bfc5a87e502952c3540309f87d51c8cbac180fddb6af8b9c237fc0eca\",\"73dfb20a81480aaf47b95a646d3635630e86370d18bf7e5b68e71efb3a8ab8f4\",\"c35ad8a27a172562fa9525f5777dfee1911c78afc9e925d4609b2e4e23e4cb45\",\"4969da23b5f74a6adf89b90737f3edb7ccadf670133a88efd0b43ba784ce57a0\",\"7ae3497e153df1cd7568f2d19dd423b680c80dfc6b447c890032abcef44d402e\",\"f1a5ee283390c0b239eec94087ecf2244a8e4ef7768b67c58a6b7808d00c60ee\",\"48d730dad7ec8ef8e3d8cdbb799f9abb96d318f16bd598598c9d263e76f08cc2\",\"db42a273b6d7b7866bb3ee7668f86611fcee7d6da070b54a1f6d83565a41605d\",\"6a497ef48a4150e0e6d877a54389e96dba6c29526e4c436b6f9068622c70a761\",\"728132d171732e39d0c2ad0e8dde4209c40e43ff8e3b6439a5c8e4c6ec224545\",\"a9769ee0312109b6c77997727ccd9ead9ec9cebd1aa43fbd18eb895e9043b085\",\"d7b2ac3be884f20298cd8d5b0e3c66b5a8b2760c439be27af14e117b0663efdb\",\"7f8df4ae89d9f9b2ad336ea0c82ebffcdcf112ffc92dc9906ce25bd2dd9bf16f\",\"cd32f7ddc5a0558f4614abda0ac653981dace4cbd0496c8044571400bed7c3bd\",\"1acb355bb1b78e29985ff00d575f71b0bc01a815ed342b1c482e83a4d4ba34de\",\"00105dd45bb57c2f9a1f5be70471f06418923d3557f59661fb982567e1c9b3d9\",\"ed4a8f11dcfc6ba88ce4f175eb231d76308b88cbefad459b94622cb40717f7a1\",\"814449caecb2294a739e093b15a840a19a73cea1e815d51919134af4b7c57555\",\"12505ecde302a9d2d86129f529a7009867d1295a9e1d2697b6e8e9a515395fa4\",\"9f153e8a7eeea2c367f8f87a0a9b53fecd0fb704e820a57ecdfa7e1e68d7e39e\",\"d7435200eccbe9336d7534e01fbcc1a846cef2d751cb6a1af61d09454e13d566\",\"7d760b1a5256328183ef3bffa3fd820a500b9b1ab6b80f244841bdb8c5d1f250\",\"73cb80077f1e6bc529a8bd0a3de01bb5dda6657ee875a9b87f40664aa02a66d7\",\"bc6990bec17c92551bf6b92f1ef452585be7ecfe9c72a733ac0583df51cccba1\",\"8fc6b05ab1f07fd260f5e2c2aabc4dc846552d62a3478af60f07382a71993eaa\",\"a7f3144b1620c60db76a4e37d2f3c79874c540ae0454d1b32cfa0af4e4e1e615\",\"9f72c4e28c8f94faa7efe724d01bf20520258a2fe5186ab70ecb78b777ad8e43\",\"7e13b6a9c0a07ad1f4c79672f1d244913d4c89b9a227e310d56077cd71feaaea\",\"00b20be05f23f0b053be069a1c3f318095749e38787f0373aed9fde50309f9a9\",\"79e43fcb6c998c2ad7cdf400a4fff2431dc2d8e7f1390eb5396ba2b152c6dbd0\",\"9fe09fcdb2da20aafaafb64760c6fe2baafd1990d5c97968b8e7987f86e3df40\",\"75fb9584a1d3c9777856d91e3cc6cb93e35dbf5e02f9724e35eafdc49696b244\",\"0c00f2c1722f122578da394859af33b717601b51373fbd43fcaa612ce06e1776\",\"3cc0aaf92c1abda502a5e5d27f3b1f6da3401aea640d6211742ca525a192a065\",\"4b1cb26064b8ffee958fa818be353f62986d87c91a94f7fc3d4da02baf0b75da\",\"e37073aa56ad8013677e5371196c3b76317ddb0322322e55c9a6e6a2140c42ee\",\"d97381b17ac671edef649b1f6746a319bc2aa805e1b54b698c0a413cee20e245\",\"a00a726ab08595f7f32d0ac4e6ad1f3e089da238c5cd6a2cf2708944cf49360e\",\"39deff47043558cd308d16409e80f94cccfbfad9ee38969881754de5001a18be\",\"5a0621fa310fa361913f46de95fe1aa34eb43d4d798a11a410521dca53a5e98a\",\"71087281ad14377140ec683de366073701315c974a7ef7435022be9f6ed5caac\",\"175c2943be56d5f4c4f1a209207a9df585a86c4e0731beda31e87d6a3bb650b2\",\"83f025683240e3d8437432db7f0187489963adac7c7913a66f53f780e64911cf\",\"c91374266da007a2fca9e1a5e6c42bb09091b9a2eb4da0d8452cd7c0465bcf6b\",\"8e28528331ae06fa1ed5fc777069e7989582f685726ef2025cded5e3a7c6380d\",\"36084c5f96b459269335a7e2b5199f7d386575262c24aed43e410c2b7e0a1871\",\"c282948935c6c181250bb713bb7b5331eafa33b091bc501c1d182dce00ada9d1\",\"6b76d3f9c9acb98c922470eb8cac7c71837a75c2254ce82556b243783b8ba232\",\"bbe0388956568e31ab5fbab16f78b6e594598f8310e7dbecff1b35f84d72aacd\",\"c8ebd18a97587fc747f2ac5af911a1150c8e433472770a75ec9c4e95039024b2\",\"7fa97ddcf0fb972083a0b15c714c0cee6d9e3694df26de9fb0dc785a9edb8cd2\",\"462a3940dce0543f010b210bb53ce4e8f25c83d692f70e58c668bf037f3b6762\",\"43020566e7d7c2d956ef33c81e880e1d9cd35eb1f7f5a01a902faf0b8d8f1b13\",\"f35e1782e29adcaaa98c361303e89f1ca63da5b183fd0bd81eb5368a15f00ed0\",\"74b34bca72f6e3694af76972de12208bcde69b8fc59059a09991289b5182623c\",\"cd725247fdc8987dadf9d7c7b566675d6219d183e5f2faca7fa086b2dc8908a5\",\"be726b25c377abab943771e48b73a5c5f820dcb0eedd14d1f0693163137515f2\",\"8cd7bdbe1dd221c5cd66f8217dbee0560a43e6dafaab5d6bb0b92b521113a1d5\",\"c9995ad2fe17d0b0f6ae1561082e556fe680ac44d10c98cbd28d15cd547be19e\",\"389c9b6e304341ce22eae90e3c118791c04d05f2a8b846c44211acb85ac6d55b\",\"366c72aecd2878920a94645601d7201527a7f5955909cd833dbd89e72cfd6d46\",\"43e4a6ab2de843f4498810c90a34b79179e407d0720d27e88e14a86ef4917f0b\",\"4930309bab3dcd951a5e29e80cac3335d4452b1fe0dffabcd16091fd17bb128f\",\"ff3da0d9d97054b9a107f7da1f8d9e17d655ca522b2341222e494dbd5a2d270b\",\"50f209f6b86d5e6b4d2a00e9aa764984b970657158b38b03ef1a5e7c2bc2d4ab\",\"d82a29915b267b9c980006f7085bf2c7aac0714175af811668d990a5817cb8ef\",\"ec2e0809c9865e4a1859f91a21214b32ce46a5428bbe017d0c43bad2511418ad\",\"d6aad93b29cdc8ce53691552c8a4822c87d78438d3df6d6ea68025994dad8fea\",\"0ddba101cba2ed06e1c601461d8e9a817700bb9d216fffaa1c966b00074cbb37\",\"36cc019d67d0097e4296aa7cfb7ec6ca2044925e094c52f84c5901a7f4c47b81\",\"145515f5e3cd986102fc662fdfa6bb49f3bca18fa3f67bcc2028c8482b818989\",\"a2dc0862632cda41c1b63fa5ce9b2aa6ffe3f4ba7d96e56a31c938394cd8dc55\",\"2c5956bacec675a76762bd95e912bcc795d6d80b807336267c58a0a4a36da036\",\"efd26fdcb05183775da26701768579d9a92fe9814718307a91985ae56e59b156\",\"01b1d5b0c1c84fd9c89c5e655dfbbbb355333451b49b241ca32845e6664c8c49\",\"19c0187cfbf82eae0cc8dcacc4440b0138d8cdffc0f4b29334cb325262c609ff\",\"b7a54ad1ab889bb9e4de19136ba315f8cc6c8321237e55bb7639dd5c371fa6ce\",\"0094e1bd120028e1fe00b9ba220ef93df0c55621922bd1eda33edff9613119ba\",\"ac9e8bd12ee6f3de99b18ebfd70fdfb401d7a3bb6ae735eebbe3e934de2d39a9\",\"3a3ede840a466d69c9f5309ba948cba073c84af1a1f8622e0851c24a825cb143\",\"dfc8fd265348bdc0784821c4ce64e728c52cd8ac8a25065e666671ce9d9b27c1\",\"1ba24399bc114d690e22c1808d81d1288578d07d5175facd8ccbff5d48a4762c\",\"f71d0a5f9345fe5b2b494df283c8748e4e8860c9763e0b911e4162582089bfa4\",\"aed7b87ce7ff7caf854f04054de921818792266284985ccdf3a72f47a4fd2806\",\"1a31393db1ec37bd7dc5e648fe19c53939a6caf9e0bab25867a28f4d3616f612\",\"4a86acc92d0aab14042878edadb4d81e26190f4d4820b05ba92f012c3ba5f856\",\"f4953e86478b547db627a7a57e18af65cc4ed8bc0a85200ee4fda706715443ec\",\"93333037aa8c97edc1f19d485fb0ada27932e882d75e0e7ef2cb37022fc21ef0\",\"c4e1de0ab5ec031e317cceeb98896d0cdb5d197a796bc684d083145c4c507548\",\"f0351e7c5e7de4ef856061877668414e920c40b24172414a3d7318d29e348e7f\",\"9209b1dcff484565dda584389c3ac7737065bc990b52114b1b0bd81b9b495a03\",\"06a94d92a97a68cd7b864940be1f44acb51968d0662ffc98f3fa49a04ef9b991\",\"cdf9312cc201a2ab8ffd5c894ce91ba6357fc48c946c04e49e4eae3b9021791b\",\"b819332234517979014a58d8e1df07eb05af458c3a285d1024f9f8b81e996b19\",\"40e4f0a5b619f887a68acc2905c38faa7f93aa37b81a16e6347e8c3489f6c8ee\",\"6b984e91b72585507acc2568d01bdca5728fed173f574cfc2df6307c3cd15fab\",\"0c97a26e646b61815336118090b526664c90a9b86500324ab871864e899fb280\",\"70ab24e905053339aead889d07a230cc218efc964fc99e1ee3ce6bf23b1fdc2e\",\"366523cffa7b7936ddb05fb735f5c6c6a9ed7a9b59cee2e22688c4f2a2744718\",\"dfb6c5917c0dda38a055d23de67434a9035a4ec2658f7125d7cd26bf7026dc89\",\"6cb01b68109a039e81fa25f22c7028a1ab10bdb569d8bf4029ffc38fb7bf9037\",\"e5edd3a56a5dce5205097687aeccc84b9d1a9a35cdebdbb1f8672500a11f3301\",\"86ea71d59f637c422a401cbda3379c63a45c221e57b4c9440350f21dc4511f2e\",\"d4698f48505c6fc63f46d53f09d190d074aaa6baad8de8658a3f465dc9d38c6c\",\"29ae046f4b2c312344273552466c22d2303ecaaa7e502db2b7caa6aec5188c38\",\"b27cb1268e904717db8c84f8fc87ca4d92e98d30e15494d1e43484062d24bfb2\",\"247438ce221a4e5a8fdb978c607c9ca37d0d7943a74ff08fe93a4c057f06abfe\",\"dc1356aee0da5b7d98cc18dcf13b454c07780df2ff7ed433d613236a5fca7970\",\"27ac4deed031a12956670a67799d7f5801ff85d0bbf59599580477a2f4fdf6f9\",\"b5793719130a50c41d25ac31e4d8b8e49aeb5478eab26314a8641326ef2d96d9\",\"a63864d2c685a07ee49db53316c0377003d2bffa4d983f1913d30f6a8438e93d\",\"76dbb69fc7218b21e88a26c02bbddd02ae0cd527fab29a7072f9f0faf6e221ab\",\"78920d6590f05ea30aadd6d13c83e19bc5f0edfbd3eebca6829528507d3bcaea\",\"fa53b5c6b87380f0420a8975d8a7fedef046e61bf52af54ae35bc13f65de9db5\",\"e4e9879d10874972dfa94842adb574e47c27763bf7af24c448baa59bc6fc4118\",\"5d3ed5f709d7516e7cd4ef57ad13ed5755c1d91f40cc1782c045283b981dd961\",\"1564453a1d439f5265d6f9970a1f984f16b25a4c292cf4f3654b0ccacb011450\",\"89aa928abed121ea7cb9b2091fee3fd3268bc8940f760767e46dfa9717023d81\",\"38c0c2dec216efec4bdf392606b620d6ae7594b3e0bcc01e82caf1800157390b\",\"05965789c9b13864b2c943127ed9762d8cc04c36f95e20715cf7979647923e94\",\"b84b00c2025457c4d576f62b20a9c0b1b6881ba85f6e17e8d2e99d75a1351008\",\"7f30e194e5e5994e9969bc215999789a00a0083623b51468439a7f7d7e0b5148\",\"2c9ab2447187c4d44156a4320a74cb2df265f5808e8919322bc13b2367027513\",\"8a8f918c3f397891448000be782904ff1a89fa92d0c83ff0cd58de6356baaef3\",\"b1c44a74e0ae90291f48c8d0178a9eb963605870403705e59ee6a310c04663a3\",\"4e08101933a796c006d82832e643ac85fa2ed29ac0ba1e6ac7ac7772a20a5370\",\"8f810347167688471c88901d410dbaebf29f3b93b7ed60ecfe774b96dc228ea1\",\"48f92c12b8a67d74df00574fa279d87180295fc6d9a5bca615d4aaf2384237ea\",\"ebcd4adeac3b6fbe06d63a22437520599a3af0cd8a38dcd6f90dba8ab9fd28bf\",\"adaabba9007cf794d482cccf9cc86eb8a5d649b98320df45fb6cec95c6bc70e8\",\"c6acd1ced05765ab8f8dbfcdccc68dd3d800cc9c72bdfabdc1b2eb9934795f62\",\"c1e59c516b12f4aa294b68d675e43704b94f07b02ceb653cc677fe47bfe3903f\",\"cc8b6420b459cae8f6d3a43a8d8be552d570ab457e69ad09d43efe08723327eb\",\"891f322c302970e785d6909ffc937006412263fe80dcac938a90ea5cb53f149f\",\"c563cfb8dab33835bb1b82d4449996fb66f3c2b89e71f70a90964f2199065825\",\"be37c5b7169210bae01bebc99976059bf2af4facdb22b345a584630d7e89c1dd\",\"4044f4bd39215cfbef661230be458be0671570f84c195aa986e4bf2b7dc2a98e\",\"f020c267e1ab5b7af9fe08dff7ad770acc7a225df05d6b2772454795a9f8db7d\",\"e6442621d6c9e1e06f92b12e807dad79466dae42d0a1b4f82f3361c45b2d6d01\",\"90aa290005a799379e6149610926588d645c000aa0a7740be109581359925d8f\",\"7a90a30176911436c45589e1c89c8fda48aeabcd47e3b0800ba1d13c01d509df\",\"04c0149c881e9c67c5c8c6e82c62da6ce4af4d03f7a746369a184e868c3515cb\",\"4d151cd9e3e3e5957ed65a72d427b5d4a4d8f169477049f395a26acd2d9ed684\",\"11ff03b3c151a0f9d359a01d16ea1e04776b679168d68fad8e0172f0e0e54dc5\",\"feb3f26e99cd1536d554b3e6afbab131dd9b1ef3380e29d52550117d7d97ea6a\",\"259eeda027547d8a9c3dd01ac774d3dc6e6316b440b1de6723070d948f678f11\",\"368b0c5152b315175721e5bcb8e076d12695ce30a0625a6bff9e1eb8415333e4\",\"6630da655be8afa869c6e0a19c65878b4e6f4f51643bc4b23808f63f807cfdaf\",\"c386533ae9f6b124f41a6f2f5ff16fcaef01b370da91ed3b1e120c228443d8ca\",\"6b3e5ca1b2fcadb30432c8d2e77b6f0f30c8e1cafb1fc2e44e73d314c7c4f4c2\",\"7a8bcaf3f79d9dfcf4332a80bb8997810640fe49bd73521559ded985324023c6\",\"57624fd3c4b4cf1fa7a72dcb603133f3f369f5087f9e6da944f69ada346d637e\",\"1be4a575169d0ec90acaeb04373ccd4325107c1b0deaef1fd07a25fffdcd0039\",\"91ec01b456e5148c1aecd0605674e75ab890cc7e1be403986df56d141ca1e5e5\",\"216446889a5d974f59d9e19a5bb675ed164cab6b9aec5286d0c972c5564b46d8\",\"cb2d59fc5ad4d20a649defe8c8ca09b3e65fcde7e307b94ce5d0a25a36e6cebf\",\"b9beb4b3d9db260388e1e3788f35a60e90cc8944ef504e8424a22967d86c20e4\",\"f85082ddd973269048d4b8e51806baa1562a51b31f640a6594b9ac6532b6862f\",\"d7960f229d954615ed5a8cac6b238de06de655c89eba25c7c6d2c00a6b7d2a3b\",\"9f0a511d4f1ef05babdf5e63b01766f7d2ff1f05e124992336e4ebab8cca45c9\",\"dbb47c6417e1da679952828465fcf9497f07e4a866f936ba11a0fb8f285f42bd\",\"5038ce634e87a8fd5191f1465c513d0911097b6c02027d01f0d7955c5e28cbe2\",\"eaa350c61dbc3239bc4cf74fd7da76c6b797291b46dfdc8fc13e46115ce6b468\",\"1af3245ac79facda710b38b230df48d8134264035978aff7f3d40def6b426a0e\",\"e7d79a9393b7f39ffc6c3a6e5e6ab1415fdf5e59e8729533acb5b6535be418c9\",\"fab07384844d33c67e5063e7454e88940ed038d51868dad65bc99312a6e6d433\",\"4a3cc7865f8eede56a48030d69bb40635b8a32e85e983cc84d19562b9dac62dd\",\"42669d428bca5fbef61dc1c1facd33c1d88c27a1c832f34646111ed9b094adbf\",\"9bc01d1aadc6c368c4622ab1c61ee43740409fb7851c05d765901d9e75726f23\",\"d54f9ec98f0ce3267caa3fb88589df854a4e80f0ae1c549c62c52f2fc7547f11\",\"a66f132b4539fe0e89612e22712763979a6b6cd40be8380879b340db5c0efab4\",\"460ddb60fc79c5d630773914bf809803d99d07c1912f7cde6e4248885eb1b631\",\"8e7cea33fd4c3596fd8d78967e2e67f7d23f0d2c73163828d6e35e24489b45a8\",\"4d049dfd8109fc82c1c040439e956d1bbf60d4a37a0cdbb4b4f907ef40fc541c\",\"736ff7a27fdc94e44bf14515e369756b367cb830db40400677ace5df8573c439\",\"a40cf2f9f4d002afbaa868d551fb6aa90f7751d8074c1cf9fa2fb4f3f273c90e\",\"8661ec06f1658e9f5f6f87eb37055154b8a3818690bbaf930c3996c022312d42\",\"77c1b6f399913671c61a034ec61af9175a2a60ba82659f4fcf2687d846de568b\",\"70ced97390fefc77251f98209dbf7338e68ab846d622d4f6cd7b64594ced83e9\",\"af1baf96843344d67ae7dd9072017364232cc22bb679f63ce423a8ba24527303\",\"e9da890e820fd6ad3e82e273ab7826078bd0544f543d36dbc6bf85b1e0b0f4ca\",\"c8860d7fec08a340c7345923e22d5ddaa03b7e298e7d5fca3bd981790d0e8261\",\"c8f595e7543dae99d05a16cf8d34c828ddddb29759775060830cfbe6d769403a\",\"2496151721a961ba107b63fdbc05878b8edbdfe4f8423931f64cc9e57f796fbf\",\"f276dd31c6915fecb1e563cbdec4983206a25524930fca5d11f340f0a2c00223\",\"c8e956b28f782780f3467043c5fd5bc5a68fcb9ab9e6c07733d4136c2232a127\",\"db3d3d6810c08bab2370d3fbe104aeea802f92eb4d50ce7e6b44a69c6a0c13dc\",\"88b1c97baea2d938ef1a345741bdd5f41e1eb8f45fe4e3271839603b17b9256d\",\"fc2e1f25003b7b1cd708f43c5793d612f20ad3fed733763016ee81ebc462869c\",\"ebaae482127b2aa99dde2679cd74e37ec42b9fb78dd8e0bdcec32be69a422476\",\"69ba4d69bc82fac91b29ebd5766c046c501f2e991c2e41f5d3f23a9080728ada\",\"4a1952295de91bd745488e4d5f09cb7e0cf907222350ba21135a71a56aa5226d\",\"ce7f03b1653e0fe9f47df3eb6d8b1276c6e0b4e2b7b74f83066de8e430d37109\",\"d600178b16cd5d9296a4be19dce980fd46560648cb436fa9c4bc23f87a75e23a\",\"522f511816e7c1fbf35672c8d54abc6fad3749116ac2b8ca856ceeebffafb530\",\"2ce014bbdde6cbb635c932de62650328545d5e998e098551a5c076c48ef04a87\",\"ae0ac11034d570591ca997406f33b469991835e32e9483a01812eab54ede2359\",\"20998e9504bed563cf9e8aa6895ee4cfb3788dcf92cfa340bf2dcaa426831a14\",\"7679b5c63c9539ab50db95086445d91922ad406a1e14822fc0cb2ac54d4864f6\",\"abe9bd2be32f82d9f68b6512803ee57dda51bede5f3ac9154da2d28e0d3b4e1d\",\"8cc10e4958129416a1835096cd0b3586319bf444c2bb417b5513dec839eabec9\",\"2fa86ec070063fd5cc91a9b7cd14115207f5cf0429c7a40eaf1c669d49072b1a\",\"074e010df884c2ad44dddb185f4f0167929daa5de4d0b35843d0f871f90d41dc\",\"b6f0853078a138f74ac72a62672c85444f1cf1f79fd419d75bfd3ce259377af5\",\"1b1e86eb94e3dded8e48659543bdc13b59ae5628e51d0c0678154c4c87a7f67a\",\"37924a9fec610828f70b83efc5c46293dd48f9d46200bd1ef50fe46a9686d93a\",\"fedb662ec3a22f710a7c5f096f0946710d2689838c68de4e52b885bb5b0a2eba\",\"f62923f5b703b335a98c58eaa3f55bf07b5209bcdc61a41d4fb1f209dd009621\",\"336c1c7a2739e58cbe366c73474450930f9bd0380c1a97912b9bfacfb26ae742\",\"aa7931e80cfe3e0e7c211d5fc1a46e6548d863e39de8298047a66deb452312a5\",\"1c70b2839b262e55dc8bdacd03a45c53c4340b001bb468db3e73e0d3d51c2d2b\",\"430faa6ec89a4b37e6422d2befd7ec9b10f2760101e96ba67ebe077834df4855\",\"547ac8db3a6debef316fe3f01dae6cc7cd3747bf7e6a539d7bc3f8eb094bf61c\",\"57dcbfa5ce4d60c49c181023cba412d0a75cb437f7d38476f2ba591befe0d8ce\",\"e066fe79c99e952ffc6c4857c32becebe740ead25b2c776bb9d496c65831eda0\",\"1867a0078a16ea7490e23da5b3f440f813c0dbf416827572f34ad904129a7aae\",\"b6917dde70dfdb75f623a46b1b00942f1b5e96cc988720cd950b6b8f22a5cfdc\",\"4847f7971a6e628f1f34bd28d703b14a0598395cb97ed48b0f4fb87496e9bcb2\",\"b50a5b641a028f052f0e3d92dda53cff1ef284c55a57c9e0b0625736f079aa43\",\"33b9b784c9c9e331c4cb3c4deb92561266491358b64cd887fd76a8a7e0cc7f02\",\"9a858e6bcce73fb0a421a4bff630c6727cdb72bf568303139a45d6820168faf9\",\"12d006ad3d904bf193adc210980b158bf3e80499d608cf537778ca493217b3d9\",\"4b252ed3a73ca59776bc2f24daf3d4c6e7dca9ac052d1bf828469d3bf4deda34\",\"9c5d6007d7264af2f70f3d1e8c2245ca68542c01d5b1bd74c2a63fc3e27abccc\",\"ea8c379b858d758c47e76c9f6a78075ad5390f771ca5730e5ecb73fd13216d2a\",\"806fa6490dc74b07356185ca501806486bed13f023704866d6bb5cd070c39d64\",\"d8cf2eaffdf5822b3b509ec1ab93cfc6a1821281b961cdd20a19fb505a6e332d\",\"a97921844b08c4b8f025fb7566c22fd17e6bbe72fab4e3a65fbf880209c402cf\",\"81a23339419ff60541bfb42e3c2da94e10d42d85f2f8beaea5aebaa163f31938\",\"3e1f0f3aefae25da5d7ddb714d4dbae6e7ba26374a8a9587c437475aaf6ad866\",\"ed1248183b6dae2bf7c58e60f4e7ddd8695231ff42e97d2b12d83c1a4c351ee9\",\"fc40c2cb0b477b738717c84d69e75e4d48f84f4d1348e14e98fc5575d4468487\",\"113dabf87d7cae48d3bd244c08959cfd5676186118d74e96bb148424763a63fc\",\"84210bccf8957303eb5e0bc59b64d432cf085c66b22a73689b89101072ab369c\",\"86a65005921a3326d551adae26dc0c5204f139178abcb6ad7676445f99a7414e\",\"b3725062ffff9c4f33ab70de52e238fcfff712de9d78da61efd4def5c419663a\",\"9023165fd2deb87b4f0cf08b662b50e863354fc150d5675e87d30b6739670716\",\"216bf2aaf9cc5ff6cbc841c1eb2ad0196391f1338932197d58cde1122b8241f9\",\"a6da53217a10ce5e70373ec6907b0c55bb38f3cd85e2d812f20eb05c302e909a\",\"d950f0c5589a82cb8dcfbd801298ceb9f5415ebd7a3219d2ecddefbe17145c49\",\"ed42d91a6e437e4f550b1f4d94459bef3e7b7c628554138221487d657c08fe18\",\"a75cf6fd1a5913b73e5963f6eddbc3c6372d6fb147358fea3fb7ad837386556e\",\"1f949295b0f9de044620bed464d959cfd17ab1844a75165ab787426790ec3040\",\"ab3fd531952557e3fbebe851c7e5bb8c31be12759802ab2b9f28872c5c897d99\",\"4d75e25d668240fde6c7a6cee59eaf6d67780c98edc3fb6191b38036ebedbca4\",\"e60a354b2e9256f0258f4f63a1895a6bd8a36a4418d18f24c4b471831efb62bc\",\"b7f8a32c8a7ee22fe2af9408283e913daeb7ad1d1dc7db67e04e3b1babd576d7\",\"c11e1c5ffeef06c6bb12c52e6dfb7dc33c58d20f2935ab979cb69d0cb6de5637\",\"74f629c41d81bd3b22b9d0452377d7af89c4fc7cb6c2783739eff3948287d499\",\"435f86c1f382f3d647276fb195a9384783feecb402587f88cc7591a51d2ebbf3\",\"6dae77e59b43225a8a2389859f6d8a6919696729187f6c10132b29481e0b1627\",\"69dae446646567014d224c05b21bb8749a506d8df6b4c9093f5c17cbec173a4e\",\"e22efb8314bab857c531a1ae7b3ade6d53f6a6241215aaf73d72e03eeae27d16\",\"bc251207e594d57fc98f7b5a5708d22ebea0e1c9c684aa1aa6a60204e91c1f32\",\"0cd261c3d30eeaa3d960d97392009608f59bc2ffb85b25992819226407d460df\",\"3a7d33261b2829171c167bad4bbe314524b61a9a7bb37c2ebf6e5b1107ff17fd\",\"38a5a2a992b778d887aab1f9b3988cd71736fc07dc289739ebcf2dceb6b57d60\",\"64ed30a9e9842516163299ba2a09b082554b31b6c6b657cc8c671dba49979619\",\"4e72307696efc80b4950c4baec63d2e946f10ee7d9ed560b8460a4f9fceb583f\",\"33037686dcde4dd3303bc0885b4aa7e0caaec98d1d0c39d0aa7e176de58a1a4a\",\"7f32847a6cbdcef3202cef84f9fc790e27ae35f50bf005645d21d72147ce1627\",\"64000dd7a7f342609eda70eed0888e6d07cb11be9c4e5732e6ba5a0ea53ee1bb\",\"5d714cd04f1108107c3036b169ec1856f043505cd4474c33bf3c19ad3aeb07a2\",\"8dc3526044033bd87b818020c4177935e5fea29bc009f8239a9b63bb25a72770\",\"b40e6cce9af9d4ea0ed3f090e7f3e6b010a0326cae39bba70d5d838dedc9144e\",\"1c3db7c24b42576aab6e6346b496efddef9a6860d02f982097d0568a24eefcb3\",\"f2d20bdf2f3a4ae5513a31dcb9ff5aae8e81e408b967464b596b2345101ce0bd\",\"46c8d6d741f9ac6272ec3ca7ac2eb44000ae89fa1a7c03ff337036c2e9ef8279\",\"6299972028babf87e69af701ebc49d11d8d83c2dbe600b3796c75ce44bd2e61d\",\"daa7cea7a91271633649b31cc644435dc51de46cb5036ea1a32a524de6977026\",\"65bee8b460de9b96646533eeec9fda16ed57b29fc04678d3ad75e7ee5a9b12d0\",\"cfc6f00dfa1280c0341df44afc2e0d4e05f8eb76ac211f06edd964c3b83956ad\",\"1c1e2b3cbc290e8670397690aa751a2865a4ec362fdd7cda25613d987aa8bc78\",\"add3784560aae09a19c385a482848f9c3804fd194b801004fab7ec5a6504b6a1\",\"ee76ddab7452c0ff53b4d1e06c2ac5fe881fa8af0ec05efc459c3ccb54b683ea\",\"064f79da2f4da0fcba008098b81c895b302c83abca9f0fc0ca6cb28c51b6cf1e\",\"a3ffd760fb8bc1270f103d922a9a85f235a88017eeb8ad3cae0295b24bc15c2b\",\"d42d5b35fcc4e2b87d447dde462f43c4e098d5b000c1c681a3081bc643d01f5e\",\"8d518c03a2b6793fbfa74b9987644d72db2c17f7268164b065d3c0da8b568320\",\"9802abecea24bd661f310f2f407d1bfaef1fa58f685f540cd15d85a3c9c7be96\",\"c209c3e656e74fa9b9f268356f9f63e1c0686ffa7b833e478c90e61621242ad7\",\"d79cb2c1317f8a348dffa3842309c4023ef74d98ec1f626ce12961dd8ad8b97a\",\"3a8f35fb1c4189c59085f552c903ac11c413d81f59b08f716199c59e3c6e3c4f\",\"4019ece76fd5f5770adb3864a71c23f599aae7fa1b5d0cc8c09ebfcb6cf99f15\",\"ab369ab51da9622512db205887ced4dc2411d6daca4ddce9c88ac6c63ec61cc2\",\"51b03fe9ec1ca8a06985730a47499c4f1dab23fe3163fc006e3bb0706c40c342\",\"54720e6f130d347bef92035e47b8876677042144b839a17ed354bf276d3cc964\",\"d813e64a1848bd848e075146d38eb7a11634c0d36cb54721706cf152cf6f8692\",\"74bb510634f65b2bc0e4c9ada97affe9c31ec1a8a4504daa162f986ec897be21\",\"8402874dbe01f6d92d969c339c60b3d7bf1f85c1d8dc337c560bf8d81f9e1876\",\"d05afc754709e6088e95df9717a6d79ec487f545b87303b3f96176569815cfbc\",\"56ea9a63bf7f958aa55c7ed99dce31d9cd18c6eeca2d029f6bc875c032159a30\",\"ad6e774a9bb0a8a1253fc19006d20b3d3affaa4ad35c7d44e6b703c51ad3939d\",\"8a19f8ac52c47d13f4452ae9be74c0c5df948499e58d38df4d19cbc02270be18\",\"66118ef815007395a8a00335cfcb24001fdea23ad7a515c08514a6ba0a8fa545\",\"fe19c2af5717313438147377a67d393db89ab470c6e24956eb46ba8c99640dc1\",\"84399218f676a580eff1f998db6c1c893c20757ee3f8585d61f5833bcbb65fb9\",\"80590225a2bb7fd7b3f94660750f2aad73936183bf8e56f3738853e44a5d333b\",\"f11af9d337e2b433c97078b2845731bb3686dae1a3a918e75de6253e7fd78e21\",\"04000959601933dec40891adcf7c64e7e22b252c8cffc76b74157e93a7c86ede\",\"18b077f3b499795e5d7cb295b1f5a2286f7f221b0c0ebd3929d060c908cfc440\",\"5cf920b1aa1b71892ba93477a21b77689edc7806e73e47db918c3f791f463bf1\",\"4aadc0ff97d0c71410fd0e5bbf0259516bb62be20b98d6e72876a115e197a918\",\"720d95abbc6bf8ac12c13ed54ab8d5d407d249b32d0aaaf869c7629a445b4a23\"]}", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "QlGPP-Woy2_romW5blE3tyf479cSMpIw2PBl7xgJaqf9T3QOGBJ5vtd-2Q92Ikd860WPSsNX6grpC5e0qQNRAw" + }, + "read_headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "C1j9IA9BFFwChXXHn9Rit5NKomAdIJtqLj9IJc1MOhr6X3SHdPcCsEbYWpiiX9H0NsJOFpQa9SJ_98fjYa-yBQ" + }, + "body_sha256": "5a4c2d6a905dce47ef2d183c015f293c50669e15a7d80d4cecc6da078b211bd1" + }, + { + "name": "near-http-limit", + "method": "POST", + "uri": "/v1/dir/cells/near-http-limit", + "body": "{\"head\":\"b3e20030ca6a85824904d74739f99cf1598b68ff00abeef54c974f00c900012b\",\"class\":\"work\",\"title\":\"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\",\"parent_cell\":\"parent\",\"size\":1234,\"keys\":{\"key\":{\"id_52dbd9ad90fc949eb796c5b421936cb4\":\"d3JhcHBlZA==\"}},\"orphan_turns\":2,\"frames\":[\"3df88793d2d654fe9b5b944e3b1d1aa570b5f9327efb9fe66377d4034ed0df1b\",\"60e4a469d7a9da45e3353cae1093808195d1b05ef4c56f26499adee704446fe0\",\"7b61e8b92a6df343ff2bfd681e1d25ef0b4ee95507a363515d3d29287ee1a19c\",\"a0573555e1aac0b5b735049f671ffcd06ea237b3d9a913b44d179669aa7b0d0f\",\"f00d75ee8615eda53533b0b44a45b4ae24321cb296a314639b4241534219cd40\",\"f4de91dd8f155f8a3ebca51c8357c6daea5943f1dc7faace685ca62bc50c0d44\",\"d5d5b1e13ccd0b913d0fdf328495de5de940caf5e856d774a577084a073f4a73\",\"9c43d4677f74712c1d6e2011a7039ad104798ed25465baade2e8ebf47bbe4a36\",\"c54006495ff81f8ba9ba794b2a1ebba979d40ce0c9fe998d3816fe6fa5e5a031\",\"9d914dd470d75252abf3503f0b502297e2e8c2346057ec67abf08d0e40120bba\",\"b8054aa8590ff3e2d8e456005a2ba4237efd88836981ec1bf6845b246371bebf\",\"cc12dfb3fca6345112bc5f15b68ee42abf3bd4af64611bdfb48b9572ab65d9aa\",\"f2289d58821c10b1ce2ee6ead7e35966782c98fe3c83754875b63c1bef6dc5a8\",\"7b076d57ca08cf548b0be8d62df0ef044e28096b0a0a6d84ddc73c7b780dbf9c\",\"ad1cea7a60e8f5d92c521ad19d5dc0d3e10fc356c6b58cc35067cc0b2edfaf96\",\"2e02cfdef3ca08f81ee12d738057cd3fdf205b239efd293e307a77a229455538\",\"cac8bb65da7901ed678e8d3f509f54036a641a4146536a5d2436b417b2c37c48\",\"050b975eb7df60dfa6d2a6bb83d3ead9e3e3b41b8e3655323a9bcba538652adb\",\"784a416448c44748fd9808773f86534059dfbb4a56a942da0af9854dc670f5f6\",\"e2f39c4615fbe5203c2ce90a6074cace51c50baf5305db0f9406bb58e4c83942\",\"4e428d38991c1ab31084718c6b3882b57f7a3c4ebaf3385a5e08d67b957782d7\",\"7afb0e0bd220adfcd95ede5050e1b6d9f3ece49cc155bad9e4036190456bd48a\",\"8b3914081536729ee7d09acd43e40b790367d24392bfbb3691e0a2f491409f2c\",\"a15c4345a158f2935111bd1cef8251c7c20562c69d3366cd67abe4dd6988627e\",\"316a996c6ad959b513ef2ddb832ead7b2b65739d43d85e246115a31f3b40599c\",\"3859b564254c89e41ef7843a166860371698deb415b882e761d71401a85df49a\",\"a18774800d14e47746886dd6ce513250eca3f1e646631fb97196f0e590cccbce\",\"d9d009cc6e20ff0590b42fe18e2649e638f061dcb4accc7b3e7e396bf71bcb95\",\"1bf01d0f413fea6b20930e3f774650c9de4422e5f76f7542c167eb30aaf89600\",\"7c94f34e738f2b50980a53006e20b6ffacfcca5e862cc1c24d32c03889263dfb\",\"f6d777079fa02e59fe91bcf94103cd31d4b06b3e9ec74d2e1da98ccc34736f51\",\"26324a1a611643f39bb9f5777274f46dc3bb92d147bae30828b1aa909f1da3b8\",\"60bbfad4a2301835c8d12250fd0989008d955615f03018cb6fc2e1cfdfee22ab\",\"bd2dfcaf5df67b22f02981304efdfe8259ed89c1a6cd661bfec70cde1e2f7d40\",\"92ab593fea2ddd5c4d88ed4af1fa9f5a42fcb0310dfb5c5b45f8d832c87bfd7c\",\"3b8581620acc98ad8ebf9a8456e8be3a23f68fea8e892716f40326d2b81a4552\",\"3d3d11a7329aade00667568e2daccd1cf259e76bb85974331070558aab1affde\",\"d8a6009fff09fcddbab8fbf79c052527981236156c6b3bc0da4122a8e2fb93bb\",\"5d0a4e8736c470e5683ac9a45862c50b9c7eb8a5828344a87c1508d11f82da0f\",\"fc1a23a833330c0638ad19e06f1a04df4c4846c2685a5c3d6bbb15ec7c202cb9\",\"80307f529a51c4cd947b59a432f3325d02ea21583ceb3e7d6afe55fa353c7ede\",\"fea68b75953010d9029ec1ce781602d8332f572f8054ced96b516523ed4ce69c\",\"4cd62030f6200eff762d23abe8e24560a3d0b1233e04a3d054b8e55493c3f3d1\",\"f0cb5442fd953bea693be8f1af68c6d0c0429c720b8b0ae96348e3e235894602\",\"ed7220deb2c5f05839128393e315a2f11e511b9e6b49705fd131cf68d60a698d\",\"2e01cec7c8059af8118b2dc13a65c77289908d1e899a7ab554b48ee19ecc2578\",\"d3ee73540791a41c2f052c598f2a9df66964a40ceea9e92a2e4487833a51da9a\",\"8c3c82ef108c9f4fcb4e26b326e50fef8ad12e7a227b35143bb2389c1dedb1d4\",\"50d606b3f6d3ff844eda58417283e0cdf52d3c059da397bd7abb09a6574f4ad7\",\"a8b527e6b27c5e4dcda754ee3130907e2de9673e1b755bbac085ec0600aab82d\",\"b11be166a7326811294a8b9387bab1312eefe5cf8a2014a60b8b4284114c0d93\",\"2b5c1e71432d629e7cd53c37786b38073f97851e95f4a9433dc9f65e404f59fb\",\"aaad2886f150ebb409f0ed511f664177d32bb5a381b8de5718303bda398a8986\",\"53318e1b7193530d76bed1d3ec43c31fc1fa1f931eb5dd9611ca17f1fbda4c49\",\"a295009191699a3219247d949fcda59518dcc639381e887f95e77aafc1e30bca\",\"724f8bbcfdf673eb2a531afa7280e0421346b39b57d84d659310b255668cefeb\",\"e1741629f3cf05b2913266e3d44dfde5aeaa187b2514f27610f741defa145a0d\",\"2c8d9d6c49375684cc444c28ed83a6379e3175235992af7a1920cf008604dbf7\",\"7d2f86888185dcd55f30ce47f31e7c51f3e0e298461c2e3e50ec5aeebcc93259\",\"6591f400f0c407eaf0b2fe7092111bf1149378d1fff9aa136d42992a45a7d4d4\",\"cb28a1fc0ca69067f4a5229d703557b41fabdbcd40740e528c12b57f4f621196\",\"291cf8077eb6836688cdb1cc0465a6586b1fb04d26e75c1060fe12dca59b955f\",\"22e49c8a2de22cadb99d6fa036efcf3357a44faec0399fdcd9395d318a710825\",\"fba76deb52bc2ca358dc341c3c21caf298fe64245441080d5a53d7ae42650f84\",\"f03b5f9e68c35da80b8623d837030e7f72e05612aaf11ebdc2ad2bc8d1408ad9\",\"42c0f786892ed36d71375b59985767044eefc39a9eb3fb464856fa108b129fe2\",\"701ea9b145f283737a8ecd117102dc2cb61a4e0195a28b4c384012f90384a916\",\"0a7b5adcdf16ff8633344105e937eae1b727f86e326c298f10d90d423f2c7c57\",\"940fa22ad97864aaa609cf54dd1f78cb05219cbb5dc956d61b92632a22986b95\",\"f0b930eeb723bc8d044470972d61d48504992ce4e3ffdd9efeab56dbfac9a23f\",\"8d262b857504acfacaab41ec3b608440813f0887b7fa638a62a51e6ca8cde84c\",\"75cdee612214e8d9a79690a30fd878ac09f0865c88ff9657d4925ad66c1542d6\",\"346f3c460d9222168e38666193c0fc6a2ff61f154ff757f8578e0a49bfd61c35\",\"10db741665fea833ed9aee63d9345e4544397b676d028685474df0d8d6a7d142\",\"6f2e8c2825bbda275f30c464baf5854346708e9c3b2ff1ffcc20d21790005222\",\"2718fdeb0b55b0441c7fdd9fd7c050155a013fd868c564d9182777afc1257396\",\"910286f4545dcb12c83270b9d2a953bc5fbe9884175c547c8041db4f34128d8a\",\"e6a9ce45f0f2a3dd0a8ca11cb68765e140c5d79adadee5dbd6d51d9fa032d365\",\"69bd4f4c11c8e6b0047f28041e5e7778060b8a940c989bee68d470d4d01ad701\",\"dcba91d6cdf340f29ff93905fbae87884802abe620f8679bbfe35ad26d618d24\",\"a9362371715731b3773185c22cae2a2ef1a48496ab8d5056a40b8a58651d7c4f\",\"d0e4f86f7764bcde341864fc222b3d94563a85c574c1b8f8173d89129f09e32f\",\"4d00079e5dd5a73736306735d606270048996719a21635bb01b092a0bb207eb6\",\"5add5880bdbfeb285a412e6df095478be864e82c1955ea02a2748df58327af66\",\"ed68a74f3629d59fc7c5952cc2522cf949842c1be014cc9e460410e7d47773fe\",\"5ab75a4cbc0f67b237f980d0f79a50861dc8014f3322c3aa563a7eb1f476969c\",\"75bfed73e3ebf7abd75ed79ba1440cd703406a0655426fce05f330a41304aa2f\",\"528c753bd65ee44d64d21628e000e4be1e1667a88afcbb06d07b3dfd5f525de1\",\"2a141b6e0c7cfcdbe731129bfcb469966504e45ca3feaa94e065cd4acf4c8e5a\",\"42eed38026ca1fada99381c52dfbec618edd8c33a41282b76a973683b83a21dc\",\"9c0e23e75e952a0c4f99bdb4ade75cb445d90f04265c065505977fbcab987f4d\",\"95fcd6554e607ee0805e51d277f0d8f6651cf6ebe181a5ee0ff521a4973ab25a\",\"59f1c37af9794d6a560f81764ed6486d1698221a1a2dcd372da4534c1bab87c0\",\"d68282635c619f756e5a3d92703a871c96b1f1870f6d9509dc4d6c8ba634d358\",\"e3773078865355dbea6bc655dc8374daf6d267492f3a531489b17b86afccc190\",\"d73cead92c1575c732ba10897f7689b8187a7bd107bcd83f37aac51769c276e8\",\"70d85ba528bc96d57618a3dbdc891b789e10b37bb8b0a49fea1a459249875568\",\"23180f51f33287aadad3e66b4b9b045a67afc89c4e715dac0924254731391dc6\",\"868b80625f7c582eb82c5f047e290eea37ffd44aafc0d2f74f8e5df1b41be1f8\",\"194ca04a7255093c0af1471973865f77b2b3242fc0b9e8449152002d74005a8a\",\"4776106e42647e200238a244da1cb03714fde929d8f1b9fde26245f577eca2c5\",\"f526c96b3da45b336e4d3ac16f41395c99464583fd2f7a0c5360c82d37de490c\",\"9df7b98b05f9f89ffdafc643f9fe6bb16d176642815d34d528ee76d495f335a4\",\"23a5e79f4fb37b5a82e994295077a722f5dd6c128814cac9040834046f66292a\",\"53866eddba65f237c1029fb4ca2fe5b40ae0dcfe751213c1bd0dff9cc34a739f\",\"429ac35a7cf3dfa715878d5ca35c582db0e4e8c5d626b807037365046ceac64c\",\"cb66c4faeebf5dbc053c411c542c201b47d23984b9a0feac3b142f9cd02f44e6\",\"898cb6d2eb21f697c9f52d8d0d6480abdcc432ecacfc7665bb283f897504c153\",\"c12f829453093a59141fc6f9e4330fda49879bbe65bba9bd219acaabefb3a121\",\"7c1de54e3754e8292dd67dd91fe3ce24ef8cb819a1d8340ee90a0f81df3e75aa\",\"363e517cf25d21d1f931ea39a3b5c856802a5c2ac174e25449549e718dce4ef6\",\"e5b23267509b36028ada1874b1169985f756d2c01ad509fa9d23d0e7e2410179\",\"f718e3cf8f20e58bfc9383d877a408f80515667da9db60981a2db56b735f9a3e\",\"e7fd7c77e790d40ee7eca5c44fcf97d322be086750e5725d557c4763f0cfbc88\",\"0404de210115f9e250e28aeff6ba039d2ed55c02e12fb0f77804ef3c2b9f2df6\",\"f8f2bc838f29f7ac9d3a5a316aa1e3d8cebbe47c6c54a450e9b1bab5f287226d\",\"703d29db3feeb72f9068157e984d2591f20b71cf571f4f24fcc46459059cc281\",\"8a7fd73f59dd054422c2302527a442bd8e9a35f81d5fa5a95785c7783a04af93\",\"5465e56a648b8be3ed53e8808fd550cf2cd719d01e4d8cf4842c9f322163ce97\",\"6261802126e0f4fb7d45ccd809ff9b7609fd36cd93aa184681ba164f685145e9\",\"1f43255649fdb5a197dad1059b0faf351589e5c255b59d01c4c3a3052b72ba51\",\"e4a9ebc383ec3041a5a31aec0f6b31b6577e9ae532b76fcb75ac9d804ee7509e\",\"c76fa80e7b1ef8bc45addf2e8839bdf0bcee473992e5a03ac0d5ded4573039fb\",\"07c66baa8d0212052b80508cb6ebc18bdfd6555203445eb18a66984a51322961\",\"cf773c8f0b05c64a10716b03d449c4a55c732147e65a5c2e51bb78238a98c97d\",\"820e31577aa284df8a7f9aa1b3b699157d66b64624b635e8d06ca623e69331f9\",\"13343b27e0628ff0a14031b6d06af158037cc5d006dd2c288ef20d1b393524aa\",\"7c9e2c84a1bb8a16bd2bcd82f2e9f79038c48f363153b710fc1c715690b3f775\",\"fce5e4cba71173df5b52f464d0d5f42e984b771ef67c7e5afbd2815d07156edd\",\"7e8bb193c0afecdfc5e26d7e98e3e7d1f4e0e39964e26f252caaa58f46b8b6c6\",\"360bb1dc3672b01db86bb061e08b66bc79af5d2fbd3d4dd9949ae404c8d90e25\",\"7c1518f9f7f0055f17fbe4e6d10c918f1994c0fcd7487598a6462f75406481f0\",\"a4077ff967c789567a296f0010ee5e7827cfaee933b1080a53078c470ecb6237\",\"6877178bdadaafae8e3129e1534c0473f735b3b8538b1be33f118d473a9260a1\",\"7b65bd5bc769d25923bb0a601896f1c4f0e67d53a57f196c97a5676a5b6f47b9\",\"a8839f708a699e123daacbb195ca6bd36339a301c4f487c2f105bf58c3dbc1a5\",\"b7d4ba0b7aaaec98e71eb9930eb24e927c8a7b91d429a76f6e0f2090002dab77\",\"32d41a01189cdf55e9160e238b829133a7f8a6fa4c2014a6b385ad372c6827c1\",\"98e66e80901d172886c541284f27dc9a2a15410898963c093e69a3234c0b53f1\",\"4340dd271c9bfd011ce2cd0e9af344aa83140036ba3adf593ae8d38aadce7199\",\"29f09682f7713495702d5be40bfe895c323ab28c3d6d47453c3f23db3dabd479\",\"e77ecc5a1b075e602d0dbd9ebb7cdf23a440e0ce18a4b74c6a9ee55243ca2390\",\"ab3807d7d568842bc4d2a783e2cf6fb99c5918bfc9a7273051d1c6933e266c5f\",\"81c62da9658ceb6ec3e586d361032adbbc32df5fe59e4fab93eb9422645311a3\",\"ebd6a4bce4e06d2f9fe1cf0ee2f6eb97d6ef3bdc03624eb6eb95aaea0b84a0c8\",\"fcb720aed6b3b9869bed29963ea8dce422e77ed0cee6c372a1f1aae249e6190c\",\"22a165cfe9d2f1f151776f901a425909b7b96ebc916563c601ba88bd466a8fa2\",\"423298fd7f2ffd46d4c16905da600030f8df4f5f85a902266fcbf4786b3df91b\",\"a0b03167c3fb0e50be18ede40b7bcb4ae3a305eae0c8530674b0df07b54c3ce4\",\"ff0c04b7cf0d619cbeaab8084be1393338fee8e20001777e540feb7e79993a34\",\"65d8721e979650b5de12fee86ee7050d5061766530ecbdea986254f2fd535112\",\"7b8857ad302e9a552c708663ea8de41044f8d23fe0054780aef1bbe58e5d1983\",\"d9e19f8fa43d207c8f97eca1fc9168f8c95635e108d51bd97337cdfaf6e2b403\",\"1550b7498396b10e4fbe6487bd9860fb9be81bc80f695bd9747a57d007b7b607\",\"ebb60d7b79d52b0f757f5258cc6733867e9558d6b95bd122a70d795e1558d251\",\"486d98c731356394f8cb5ad099770d367bdff61e75149f3d97871e75357a47c2\",\"1375765851cc86bc62595dab00b7eb4d7fd06d6a99fdcc7ce20b62e589a764c3\",\"9f6abbb10ee449f2dcdc77e326935b1503302ff127e467a3550d9680c3cf7a92\",\"abcf023462268803120d38fe107926ad9b7644e0c38c567889b5a4320ed68c43\",\"eefd41a66504aaf854050464bd6da21f630ee48c586f8b03d9ef831a53bda3f1\",\"310d4b4e55e76f22a7e91c28c6d94765888effb0a507078b3020cbe2f424eec4\",\"7d1de62899513c6e0890de5fcef8de5c6e5001416eca91522007b630eb6b2312\",\"902aeefc651fabbf8a9b752d442e9718b922cda5f2996dad8a2b18e549217bcc\",\"48ef08a4a6848723d4fab1931369285c57e04c502b4e6ce15ab105eba5b6ad37\",\"638ce37ebb5059e39c88e3a6a04b170b77338f8c911b53f307ea74915cac6a0a\",\"0235095fc23ddeba5ff86ef4f2a06bf540f6f807e52b991f03acf9b13049d069\",\"4dfe73309816a03687f3ca0bd78b0eb7e2cd9f084fa7af9bff54b131c34f4724\",\"5d8a42d13b9f57ebb9e67279d52b95d192c5b0f067bbf702aafaadcf8b417c56\",\"c4522f63aa28a8752a4fecf4178c830ba56ea9d939ebdd28c78b80cefc68e1e8\",\"6b82c2e2e50b89150091422197de5200b9bb7394e23fd3862a216b9ed9582576\",\"c2092178f31c258cab11c57a0f34e9297f920fa0d074005041faed4f51c60164\",\"da5afdee2f31ba49f217465cf8126431acb78c59c376bf496c1f31267e69ae8f\",\"033c6d0ff0889f144e8d58846a376e99aa14f21ef2f3cf4fa0278c9ffe183c84\",\"18e6867108ff489a416a242c50a7d9a30cc1227c013c3c73fe2b310fd1930e7c\",\"fc607ba674c6c6cbf2518217e5a9606cc04bc8c821c1cc7afcd5d5dc24d89004\",\"931a02ea8fd627705f435036f161dcbec1bfad6402208cca6e1cc8476ad2d773\",\"563a608c3949452e072eadc0bd4f4cdef85710227b11e218441a4b5ca137d48d\",\"49bd1ed99c122c1f78671dcbbe39118c6ba3f297d33821e2d31a919a02ba1714\",\"18de1545873139f6bb2cfa236a26d73ae1d521d1ee436d55fffce9545f835270\",\"bf602f7652e67b9f525c6fa90e6ef26cd29b0e20c08b0bb353f5bd62af14d10a\",\"1bfc8498f8a609b7068b33062337b6e8f2006de7e1daef022a3236dc5281862f\",\"c79da0cd84c24faa3c7ed3965593373029877d34450bc2636fa76f289b77ad12\",\"9dd2aee966d2f47f5dadc40331d5729f5268610bb2dac4f144ed2a2faea6436f\",\"b4ea2664b412d3f9a992c7a418418dc642bf15fa9ed3c72166c8a5e5a4324159\",\"767d0422b159ef20a4d76c826867d67525dcd27dd331bbe218f5a5f5a6b39a48\",\"516a2914d0f058b88d32fa4c1373c3d0288814dd14719fc6bc8de4f02d1e4cfc\",\"8a013fff8d93a6594cec24c070de72c8fb6bfee0642512c3c5a987e1e01a2036\",\"9fb2be8d9d580540c169011b5885a7093b93ed78461723c0714ea342a1f47f71\",\"3c062316bcc098d2997e19368e09cd487bbe14e6573eee9b8f07e173a3190461\",\"71fddf6262bb3ace2870267bb66281e19e1e4abf17f3e762d7c75d0c328439c7\",\"b2dda505b979c15e4c563f03d20d3f1c0faac478af037d37a9e6ae4950228800\",\"3ab0dbc9b2ae6195686ccfeaa3b3b3d7e7d8261e29125025a8c5b36e7a5909cc\",\"3b8836d48c0a3b7f7f10dff6599dce3573732f6a49a7b23f8c45d614f23ba67a\",\"e463186823789585d145882a43f36b56c227d1043ed0f3e70a4f004c1d0647ff\",\"ce145b57d79d5a0f50e578388692da604c82ba1b8f1bc96c85ebe1fbeb8ef186\",\"4dcdea0f09dc2dc481a6ebc606110645389ac81660e0be3f6e0ebffac21f2a81\",\"78b55e0a09e25fd5c858499a56e927f204bcd4131d75cb1b753c8f9ac8f6f20c\",\"e9669692234881fc4411dc0012a74380598679701c73c6da6e1aeaaa09b0a7b1\",\"0782b804af1ee77701ff8811287280f55d432d4404c88eb5cea8eed6aba12c3d\",\"ea3cf49f07ead0ec44b70b2a79ec0fa1f31706b5cb32359bdc25d0769b499c48\",\"4d1f9832b5adcdfd86383bf427def71090f9852e2785f35d4d88b56d67e6c39d\",\"cb772eaef170c4c4104fddd76b557feff76039250b42b533980fa7fdf8b234fb\",\"aa2fdbb1a600c7084f4ce0ad5b3f452df00c8e5279ad7894b2147ca85fb45794\",\"a8d6ad6694a039a8bcd22666949341488969bf978e3da14ea419975c8e240ae1\",\"075e6271e16c752de9c099b9a55b5174e904bb4b2559a22f8a972df350ed8d3f\",\"dd6e31642ebc23622f9666b200a10552372f27db326a33b03e59d7d31792a3a7\",\"5544b5711521c1a53dfb9d35efde80ca4d6abcfebedadf8c8b07a47f5611cce7\",\"b82e95e3869e0c9ad670d19f6d3e5d50496eeb459dcc6cdc359e119d5210b720\",\"5c590179593caaa854c4a431dcb11d68d0cd7a77cb1ea070c352a5451de1c8c4\",\"addd2aea08f32de7cfe03d4185d78754c12754752d2ef751fb34a8c61a2e32f3\",\"41ad6ce101dcd8a36a4eab86c6f5626a1ece943fbbba2c62e6cf8f52b4d78f51\",\"0b83f729f1b6e544b0672118675c2e5dc4ee1fdfb13ff2ec242dba3aa2443f4e\",\"4b01c20f8e798e02d4f1a5e67c823a2c0e6d48fdc99dbc5e76864e3f8f318a27\",\"790bc45bd277b9e190ddaba5f32b769042415d4b211d62ccb13c09ffdc377be1\",\"97bcfaed4997cfbfbfca0bd7c66e546e4ad837afdc62fa79220f061471451ab6\",\"1c733f658c6e657d0896c23bccf917a28bf1439c3f8eada584a2f26c55260cf0\",\"e7307e4ffb82d1c05082e2af4764e29740b1f6bfbbc58e6a8abd02c398961aee\",\"f1809173ab4b119145bc28a1406c158dd7f0a3165ce3e14f35191e64597bf18f\",\"60ae6b663f8eb259050910a85ffa9759b1e528b4d6f73863c01a14ae2471b1ad\",\"72a186ed193c6110e45b21d580e0ed58582252a61987f83fad4a0d4e7f4206f6\",\"06ad5b36abd988d05f016b8bb6d1912dc084002438f53f7bc18ff0895a207bb7\",\"5aa09f31d38133a3fee1ff1aa32871760a7389eafd906b497053eabb53587d4a\",\"92ac66cf16544e766c63a97c9b1d6531f3b23b10142dad22c1401bc20b2d8052\",\"decf516dc231fef215702a7b04780d695550f151b36dcfe87ca207695b4455a3\",\"f99baa62005a72c1b66bf6054b04c5f0100622bacd741e951e8d38372edf1eb3\",\"4415488441d6ecce6161301336c093b6384952317c364e48bb67fff00f0d1304\",\"d3b0dfb4bd9d54ea7cfb611b561d06654aa5487b1281a62623de390ea14f37e6\",\"eb7ec59154de674928d43bb7c20209560fc769946cb2b9dea9126036eef4e358\",\"1cf3b2eea03956083f5bb8d0a74d37656571f2a135af32d3c960e8e1d8b2bc4a\",\"919709dc0d25553cbbc9bb651eb094ab349ce87d373cdb5029fa998f833bfa16\",\"614d65a0cb2553b14a8d635ce82b53028b9c865d110285d439808e67dfd71e6f\",\"79214a95125221fef61e2451744de6711e3d4c9b806b11a48828b6bd9adc1408\",\"0883db9bdd789e6f035829d8086646372f1a9cd9524c8bf49e9c0fbc05b658ad\",\"5ea0cb04c339dc231ad68d7116366b777d945ea5556a1b310c3cada7b24d3d63\",\"7e187d3fa0e1d9a42d9e609c34ed89c267c417831a2b3fd25028509723d56c9e\",\"71b8c6dc5b7f464091cf430bfd5c23d109ddb1ed64197e2bda1f0ad5488ba5bd\",\"9be7d4de680d7119bd630426d34015a43c194e2fa5c5c16b2094ffe44d1f0e62\",\"1a1646e07f2e9b69a39016536d210eefa4ae53baed1ce3841442bff9699237fa\",\"2d3940c7812f1a224571792dff569eae8621de11dddca878cf0ecb82a37edbf4\",\"b7419556766a493d8d68ac84c3304313cc9e80002fe6beed1513eadb62b21635\",\"b4d8a3605aa73a2af638df2bb47ecfbf229bed0d01ab2ede26268293ca2f7fc0\",\"c86ec92c331553a2aac7a2fc09ca2f658ba5f59f89cbdd366da41c2d615766bb\",\"5811c2b0e32bbd9118e03ddc14e076f937a581f7ed80b585898f6faebcca497e\",\"7454b97bf09c9d99ee38e30086d3c4a093a4ac1c8c3e35b2e334905071f92272\",\"73c457815bdbdabd1ff2eeedb12a708c365e9937c3d7fa236a6bf07b8d7b3b1c\",\"f8018787f32f3c379ec10735abf93a02b7f8cd551b07fe32531855f778cc3b1a\",\"7854312e4f5bdc027290f7ed8e5cff31c5792f7e612d9fcb82bb6c9159a919ab\",\"57ea8fa61ede8a1de2bd66390692321f579835afa30d6bf5e5f95e5cd526da05\",\"3277cfcafc6b2ae2941721d3eed15af5c1a0a51f3927eea18a0cc8348e2fc3af\",\"fa0cbeb33bc87b099fe9a3d4c7b3172a287f12093d7227fb1b675ace38b6cfa8\",\"160027b5f5b0c77ff7c41e92ee032f02f0b0777d5003b7e43463006b84c5c15a\",\"8fd71d3a57012b0de3f2be8f7d4a8f439d56f661a6131f394aa033b19ab4e930\",\"a4253a41acbfad309a89c650f122c4fa071460fe86d04ae38da1f2cc3ae50d94\",\"76b931b53bd5b618acb7b495649b1452496a38a7e8314a5782f89385258d1403\",\"bc286a98df13f08288858808bed75e5d6a0483d7b166146c654c12e1930c248d\",\"2a7c573195093e92400d01396718213e7a6f7cf27dbe13c5df5a349d85b4f012\",\"188469feb05b110518707b26a50d23933708820da3f272a69b82362976387a5b\",\"62163e29d37d5e301b03913813b7d0cf6034f389bdb07228d1f6beb497ade18c\",\"1f08516958e5d69bc70ffc5743f499137758e2bec66667989c8f225de99438d6\",\"13ec5345dd56787b7d93285a1fc14626037a782e102ed11663bf553e2ed99c36\",\"774a95867877ebb14ad3ae8ac1336689404d2267b8f434370870992ade7f4594\",\"4383954016c5672377430a44aff37d38af4d8dc11d4e27b43c521c6dc61c667d\",\"7438224fa60eb36d3158fa025da2383b2bba0497b99f4987e096816e31109183\",\"c06740dc970829d273c91bdf9f287d0b123e10b13d27e50597c4c5fd8b6ec528\",\"070b768bf9416671fc3c55def106518378a878c3f4056d4a85e0e2baf6cacabc\",\"f0c9ba5a070976ba50257e8e2700cc00d2c859aaaf47d18b3932d673e01930d9\",\"6cb55f2ec8403ad8638e82e25cb52cf1d4d27fd437ddbd8516d9118ef9ca3a4d\",\"fd5ac2f6281af86af218736495ca74866c927113cc71430a63a3edcc3e955bfb\",\"6be5ba56e1d68bbdd304c3ba0194db87c687b8b8f82ffd2486f464bd01aa9558\",\"3705cd1e3e319fd5ec836dbe19022de3b7a6037ab4751d92776358f7918dceff\",\"0ddedf1dd86783094a62489b6cf015ecd6315bc6d09871da7c953184505a6ad8\",\"993cf7bb6a314b55aeb5b28d3c28bb47c47793be1bf684850b4647e13d5a1387\",\"ce1c45bbf16fe84808b9f3857763ff7e1db995682e45f34fd5bd5843e4364146\",\"04e2032a8e0ac506765883957d062f6f8b566a7a9c3530d3d5fce9ddb9c6ea79\",\"aaf604f04ee6a0b7727bd4978649c98223bce584128d0c5ad8b5ff1d4039e67f\",\"d3bb7716e499d5100b40f98c7feeddc0973d26f6b7857311d4117da047f2808f\",\"ece0335865493e10f4b47b111997b16a937409fe1028551c7cbd861ff038cad2\",\"4ed280b61d743284d0fd6e470401424c9fa37d7d08b90e13156eba3fef0b357c\",\"3f46b0cefb570c32c34bf0ea92f6f786df6bb4217f349ed2344acc47ebff9852\",\"7e10db5758772de4da425bbde74ac1208ad540c2da49acd1ee6b3b50452f5b82\",\"8e306e872c2ff38230a0e604f1568515f89547a3568d0b02384892b715558344\",\"7e0ab1feb6e226d835bf16968a7d7741c4096dd206042a89fa4c3bc5f58e794a\",\"18d0a06f93820d59d925d695cc958a4bf02caf10dd3bac985bd46b63ff921e94\",\"91f6a3f9152fb826b059df2c2d805bf44c6851b5d72043a2e9c172b32a1c41f1\",\"3990614c0b2d3a8d5f64a20aea2c8c828afa0ca676576805dfb6df68df7c1952\",\"593348c631d1fb88a03f8923e43cda1f551d7382e8d22b5675b5f6c9ab340b11\",\"a3f1ef424c4ae8e6a6b53e27e499abd5d914c955bd5008d317c1033583acca1b\",\"66b9df245f462bdf0f8c3c1166c3aee3beddfc569476a1aea6d0f738cf804992\",\"8ffb437db3491f2eea6e85af3feb88b6e659bc2a9092c165973b31f189f52ff6\",\"17e79882f7967b92373b314a16f979d18b69cda0f4203e17113528824b3aa03b\",\"4c89adcf320bb3006493029857c66281355fdded282cad5b4df1337a3df5d25b\",\"f8c8a66bf360e703520298c7b0607f6d6a605aadc7b7739dc552fbc825b9bc91\",\"6a1053e2914add11d7b3a9886aec2521d3aaf2511047eb14efba7a950812a34b\",\"708d092c10765a2742b49e31c9213fea4d26da485b2e659cbb1825a2f12b5506\",\"0ba8e20684871ac3e4ba06b8fd3c2d4d9e26e74b085d281fc769fa232e0860b9\",\"99aa299cc53ff2427770750e11e904a9e8100b69b19e1e95cc1baaa7c80dc142\",\"e611cffd1aa619d525cde7ad82192fdd10fc6b0e8b5ed7fc8072da88310c47f9\",\"2c495e8938d1321bf64b03889b56e658476a4838f469d5600b8085ce0c5f6619\",\"da30324f62bba968c347592e55b800d9684109e22b2588251bbcfff77b8841e2\",\"0a490c175b893f11fcb181503c2ca94b99d99e0984786b0de872c664cf9e9156\",\"4bf9ff29872d0801c25c1ef7c77f1dcafe227381dc18f5d697c9e1e3a1389780\",\"c4fc2b5f97ea6fb0f7957c036b6a0d3449d1e0dd96a062732e49ab748648ed4b\",\"2e797d7a5171d839037ed7be72ce7fa7733480fec4752262add4c18047ed5c22\",\"10090c6aa31ce9356de7081ca0725c3ac43032290b87a1b4cf198a920e1c26b3\",\"1f97833450f109530bcbccdbe304d1945e94db18774184495ce26691a208a2d4\",\"a2cbd58895e0ab2c5b70b2e6464902538aa211931205bdf80cf87e9b4acde419\",\"074bc0eb0835dd816f3666feef2d529a769d026c920968933b51fbc07f859049\",\"fe09b10e3f9eb5f80cc43e680ed8f6dcb2696cb503d16c5cfd77dc5350c3383c\",\"8b73c6fb7947891be38d88dd62107e04b3c698f37eb5ce9e702dc8617576ec85\",\"7c91378febde3d07f82209e5e762ec780807fe105648ea051ed364905f14861d\",\"0ff191bc43df995082d41e741daef41e974e00d31ba4549ec9d5c4d176dfcfaf\",\"1ec70938cfe58596cc5ae10948dd7835a69a8110b179ec17a6adbce6bb113c0e\",\"761e7f9d9bb2d85c6498822d01675fd6af434fb8ba48fd78c7932466452d08d2\",\"1c5072bafdf5ff5500a6eb27ac56974ffd89d104cbe86e7e388653f2842a8706\",\"b87f1599f5f8d86096bd6e22edcece306ea919146f68016952e43bd6dfe0d805\",\"5628d748ccc4f06af8bbdd78a129c883b23580d75787cae4a244373c4f9b8cae\",\"290b10a3241c241e9ee48873a99793c9972bd47d8fc03eac607c539815fa9208\",\"db8108407685e9690ef02cf4833cd48722f60c7373d059ce53d1be1fbc85696e\",\"57ca84bfa363070b56addf2a6a0960d6146d05df53cf2c54167e29294beb6120\",\"9f8afd6febb3686aca98b11a36a6b10041532a5be7d160046bf7a3525461af9e\",\"be5d0ebb925299d37593a32be3b88a8e0761489b031efb66cc877353128a5a6d\",\"c76f11b8c6c556dd06215657ea1673418f853d1f6f50b5da5ca8d8c5afed5e8b\",\"b7da5fe688a8031a0d4ada50b665c97aeb1afc1b5ebe62f046c3bbb976d186fd\",\"247481cf407ae6a5afdf4a0327ecc431261a31079500bce5c00d053c4513d38a\",\"da0eab86c4a5bd8498d32a85134e3c0d0e9fceeca6fda7511b68038a105ddafe\",\"1a70736324fe0f9f46a4bde1988ba170b0ae952958efb7fb17b398c95d7bcf99\",\"dbbfea8db69878978ddb63272116ebfb94e3bfe2a94dede112a0184a4cb16f9d\",\"8182780d3231726360329d058541ca6489aa62e2a9d1552cc17847b5bd23f4fa\",\"695fdcd752e0adaa9369c888154369c6319096dea1f04c67420e78bbe5ea96ba\",\"43028fb77affe77529267ea158fdaf6186dc8200d57d4ef236f16da1f814abee\",\"37c2eb6debd9bf1ddb4674805ba1f6b7f49cd1343dbfc3e375ca560441a1c40c\",\"2dcead43394c725947efa9531a7e2dd80a8f444eba40e67cf1f61761a46c8e6b\",\"4f676fa302d3f823bec1939e6d299716523a6e7243ef5eeda64cb9ec0595b8f4\",\"461cb3ea67519f7b2c79a981827c9df292e596856ef766db6a32d5f58c6715a0\",\"192a73eb1e81056cd73fdd164fd793d5ab165377e2d12e4d86c2f19eff172043\",\"09bc26d56928bd80fd779a066bdc9ff4e43573df1d2d74add682d5e090cf2be9\",\"2134a3f9589f25a54f62cb7080e7d28f2db242f41dbb2534e34d82eaa7f05b93\",\"b05da2db9f2aa151b42c19c77b34ca3f3bf8517dbf4201eadadf01ed82f559f4\",\"2eadd0d3aaf596c2755c06b9558ea785d14ca6b029b104912c8fd6b1a97129a9\",\"efe35fcbc23ed0de40f98f878ced8d72dd94cae82e97c10baf9d045b6c2de838\",\"d9b60d16ecf231e27b000a4cbfc18d5bd43f98a9f4e6119b83450413a1bef501\",\"74148e704c93e5998b893714e36e821345af7d7cf896fd1cd2ad69ae13f08adc\",\"dcb392a8f0310789b8d3e2c51b6756b967e5b6d9c760760c4bbebc57405b8dc1\",\"f4966009d2293b2b95d050aeeebfcd28cab29fd52ed0952235bafbc2eee87de6\",\"9dedb18d8d569690236ea54fc06f4504980781bcc0b769171947d438ffa4a500\",\"883e366ba08b99372e3ac06889d19e41dba9aa4c420858cfa6c9949f243af29b\",\"2db9c3f47eaca26ce8134b73ce68682b253f3ca7affd0c19bd136679b3ca5954\",\"cd1b73be09f8c5e04aa0fb87f1b8a2055e61722280272c4c143a67a13ad5cf6c\",\"1a22aa03733cfcde25b9587b42eb43bb1394214e4f0829fe83e0fd15be3dc762\",\"912b80abaeae9f45de822519661de16c30d85a68967e6557df668109caed9908\",\"962a28081d01aad134242e614fd85ecfe84a73116e16e344a8cac3eba165aa46\",\"adffded71ebe018fec757e725fd001ddc8837edf6f1405ff5cd723ff53be4f1c\",\"3ac0d5393337cc879f0459f885097fd48e884ae12d189bda1c77323c81758f87\",\"a430c838d4c84312d56e53fb3ece147d1db200003148e31ab945891e4dc79ad7\",\"0633d79ae88637697727c3e4fdfc91392d8a7ff3305cb910cd9ab06d1c75c73c\",\"0fdb190628d2fcc45647365fc1afa185109823c508863408ac4ae72562253780\",\"d8c9806c35cbfe2ff0c5318c922c5752d5ff287ca0dcb79b1a1bf7b59c7aa0e1\",\"2644095e5020c3af80b567f8d385efca3745426e05b777329eba91e9c4029217\",\"0d3edcb9d72e12e7b12f5a6d183c14db88dc60a3fda1fe698f3d1bbea6a7b652\",\"a699d9afbca659fc021d4a167ce15e5b7f9413b18367e012c764991dd937a585\",\"da95ae5ec6b377f2837749f128833b9631e7a12f15e0ff95e6999d4cfd99f1fd\",\"e1d9c6de18c4a1f75ed38d02a7ae4d5751be0991844c9b11eb7f035bc75c3848\",\"263af2eef2bb8fa9c5055b62ac4a301d2572e358b3cfb4e4223f796db1577c7f\",\"d068de9d40f9c13160748d55debf0ad2ff424570d56425cdf218de20669e5599\",\"2b3bcf13927383267841a7ae85ba4908279cc36a4e4b840fa1a2689b69fc0951\",\"a7ec03436450945b9af9e93ad691e8dec132bd986f49870ccba8345efb1e4fd9\",\"2d62bd2756973b90270a58a0b6cf3babb2c2530bfda8a1df1c0b878fe8822c39\",\"4ef5072bb9cfc79f32a41be777f2d73da746e0c51fb052ad9e9fe6d69486ff6e\",\"b7fad51c5f14979c87aa0abc5905076eeb9b72c497a428f89fafc337cae90180\",\"c553c553c8a25002423ea9922fc6ccb4604e5a9f2418370634ad746f6b6518a1\",\"18550556185e287a5bb5aa2a52c377d72d08fa445a67d359be20167215b4af67\",\"d86cab365f256e788eb08739fec137744cd7ad4e7e06cac54332f0fbeed1ef22\",\"3f269ebdbb53d47a1620c218fc7348a47089e70334edafd2c71f03d132f3f846\",\"a2a1d02c4e51c754d42e8ec73c06a21a2cf339594cf67322e1b0121a69576b0d\",\"75f4da042bf1b01d04cf861bdfede7de2b27a36bd7730982c09a275126f7305f\",\"55f1f9a50be97f916eb9f573c2651100acacab974793a5477dde805ee3dd0501\",\"1a4ebbc0275306a0e6cc62ee830ac31ae7600b3a5f015a296c0ac4dbc6c5daad\",\"8605e6f5d0a34a89b48e6437200e4ced15565f748a62df57072aa84881493483\",\"7e83bda68f2f556c99301cc0ecd35a5c0cdff709b6b277f07535a871cd773fee\",\"2dea0909f7bf5b1b86918c89382340536cdcf402f1ca1e11add170e7be9465be\",\"81b11500eacb9dd2f14e9e4845e3ed0859cd73dd11c73f5a028bb5191e62d991\",\"f7e7ba4f10fe6efef5c1f57ea9f91ae350859aa30d80e656007c797cbe949c55\",\"697166f18747239265f9c46d1c721ac30d35f9344ae41d56e56a1b3f5e05d897\",\"ed80833d2153597e09bd20f5a5c3623d32f5613a3be109f3c3322c38149378cd\",\"5b0151819fce889db356852354174a4cd719cefa77070a0a862df63ef9550f14\",\"61bd3e04f0102ca10e2ad8d44213c10b0ed420faf072f34f9c3f0d5128a01e2b\",\"e13fa949e000f74f9049279d4a5dcd6cdd7b0cf21c4dbfbc94ec2a730d96378f\",\"b75bedc7a7b8d6b4347755c1dfda80a28033d6ef9ad31fdafc72a14f3cac903d\",\"560b30e0c2f17740c1615aa7319e2582631ae4302ce5bbb73599e891f85496ae\",\"90126cb29e3513fe54357954b9263a9ecba1235640b29f3146c3de38e7e25238\",\"cfb9cbcaa5bdc56404f64500024b5f6d1a483c6fb6f0db8dcf2e3a77bcb79eb7\",\"aeb1debfa41b170c55b8b3327847c0691bcf9dc1d01e135f1216176ce207b8ed\",\"a6342fa0919c61bbfb719cf8a52234aef80c75eae45a7023f21054f2c1b1aecd\",\"6a2db120b51648b00f984dbf18363687bdfe910ccb80523e934209311b8081cf\",\"0b2356509bf1c4eaf39e0ba55f1212fc6eda7519dbb194272ac91f7b5b8fbd83\",\"b74616e6d28f89b1b2ef3efc94c989d8fab278c3a5f8273a01337878aafbd882\",\"e614a2898b5971d25e7a032df09472cad22f864aa773dc87a561a833a2befa51\",\"6926967dc13f4d1d1285263f7cf81f649455ad04e024fc7b2112bbd2b327789e\",\"cf7c4e7123faf48f673558ccf1918b8194e8c3316cfa0d327f390114ebeaa15e\",\"3ee8f7d8c33e32776e2b3ac6a7e67c6e40185d1cd00a4c794edf99fa6402bee7\",\"a8a7b7f3c4b76bdadd0e9496ca2bd416eec3f9458452d93800e9307811ca8f1e\",\"9d1ad3d539cf30b65ae175868bedb0a384d5599cdb06393e241d5bb3fa6ba04e\",\"289fa4d901cfbabbfa536d10af2518bf0e296fb5d96f66b904442b84274602f1\",\"034bcf384e3ffe53833f34673390eb9b94cd3610dfdab9a9382a6fbf94321bea\",\"e9568557fe8f6f7cabcad29a0b1ccfa98b9709359d11d4835235ea3a104abac1\",\"53371a43b530c3e66b2159962f49d0f0b01729261fcfe7e79d6d6f237b382fab\",\"a05968a82437e487cc9160c728f33e9886e3f39717f63ca6a8414efc0c9d28e5\",\"beb8bef8e248832380809092cf81001dbfc000a75bb6f7a7179c9e1e0754c406\",\"edc6a964f4cdf4de30f6e0c61d3c9982ae8cf29e33cd5f7a4ec192312fb2062f\",\"734b747fa5c8320c9ff46488bd01be2fff597cf1d6c60c3c273d15ebae49da6c\",\"cfae882d5452d2086ce0b897e3c4f2dd3f50fd74aeb42760f4e045372d0d828b\",\"631b3565d1ac3ae76c7ec9fc8c0f8e31b39209d76b6c2d586625593a00835fa0\",\"22351b684c5fcc1f1f15910e387005a1416fea2d632e1f6213ed606ba474366a\",\"9886d1a9d0f7947b93e1f0be6feb290a9d444d82827d4d16d709d6104b114e82\",\"361578a8e9049ba2a5bb2a6241f5d32a569f62bdc5e4758bd6fea386f982fe37\",\"190fe4bdaff8c8617d4e57fbe2b25219170d95413bf8d4d24660328d766389ef\",\"91c83fc5b6e83267d97987def4d2d60d54bf3c7bda472c7039897bdf70700d2b\",\"b2acf1154ef0bc46cc6ca47b7681e57d413e2ead031ab5ae7a44a2814efa5008\",\"6671db8f8ad187b7cae14d2a1e658648b8d6d223f24d318fffe1a75bd121c7d6\",\"510b11dd680d60c3ee019b93ce2a59427bf86aa8781061e2e98d7945b44aa383\",\"981a8e48ef4b0652fe997fcb4aac9d444d52bf6c0144b6554807be73113d997c\",\"bb6eb02c0005b0f8a22b45374bddc17da8e015af3be665bb3a43065e157af8fe\",\"daf5acd1c303de700f2fb638705e972408c3076419a96f28c1d8214f2111beba\",\"7459678487e742f1e7e0785bd223b0124cca8fde67c79a6cc6671185e324fd12\",\"89dffb82d0a1539f44e56d1831d1816793a4af09728cbf7d08c407c6b7f4b73e\",\"5a48d96dc1922c43cd55a3e32f5dc3e3fdedc104406aa7da2d0f4822acdfe6c7\",\"8d9b165a3414dcc3f1468a601ccabe966b8294277c74ece438df322d97f574c0\",\"c290aa67dcf8df535585c5052169ea8dcba7c5ef4e1888f95a28d52e50177b50\",\"1b8800d7450b723caf337d4003c816351b22ba356403a717cde18e77ce3ad8a1\",\"2290d5119a950eda1308c8337a60ab2e5ac9dfe00f84f723a858dda41d9b9523\",\"068db26c8a90b3528caa6aad95ece6f102522dc9a9b0445a0e78f2e0f76aba4a\",\"775ec87b66937d9a5606f5ac7461a2f4335bcb5d1a561a168a311c17e2c0bbbf\",\"dfd7f10d97a7cf287be7b418c711a8c845127ef80c9f428cf656d079139cbd24\",\"98850a43e119964c59458c6d459b2fea1e47b0f97afc4ceb6852b0a5226fbd2b\",\"81a0af2bfe56fa64cf1f6b9267b3d83ca741cb1cbaad138f60f358ad03b994cf\",\"db8126f98995575c8261e581d94533884c24420075ffbe3cf0efab3e3a744747\",\"fd475e92064a11dc494c7d889ac351819c280301e5f4b54853b0d6da86affc01\",\"b6b0c62acbedf50f3a9cb739c00ec3de6969d8a468755dd121c170fabd13106a\",\"722c44d52b25ae3e7165e8c53068b21dfb9d9f59ae4fc4c8034ac07c176b48c2\",\"23ffa45d44752ac107bd0c6d60743fb600d0a7acc4794ab6dbdf15d9e1a6c019\",\"ff5f10dfdd1184fa0e3374e31287d1c5f5a84ce06b533844fe12dce5087d4aed\",\"56951728515b7ce85bafc304d6ffe2d39acd97f48d714cef72a2dddc7749d2dc\",\"e63f9255d33c0658f40196c0b8caa8c6aeeaf44ae168997463cc9c202ed31f72\",\"fab4d67ee74161fccec886bd32aace6e7e1fc43b93e47a3ca81e1a33fd86502f\",\"6417502de875ac61cb7eeacd26409ce30b99286bdd213fdb529300f0f66019f1\",\"d1af52817e813f8e64a6f5fb98a5b27a7ed16885f37d808a5352d7e568db010b\",\"fe526d860f42dcb804b420f7573c08bced1230facaa3d007a9a8d91023b5b3e4\",\"fcc668f3dbda3b2e72c7281c91c01370af8e19dd861e1e5b5636986cc7d3fa6c\",\"c297514ffe6e04cc20b146b5ec012faf9748b9cd74a3bf22f2cc78b67986a4b4\",\"ba5da041106f2a373d7b8b7d81269212193fdb1392471a40aef9bc2315b4e2e6\",\"d42bf2890a3d2f7bb6e20770c859863ac725a4c2df672c6a917b25d2c932326e\",\"cd404f0721aedb400b1a5dce026023fe3249b02cc3e241cda5b6517090e9ec54\",\"64580834ade402cd8718da0edb4465b126c39ebcc0aca01a64d013dacbf56744\",\"a420d436a0ee86031299266e0ee4c6ede5c64ec98ba6103d52b9e1e55af117e8\",\"1ed17538eaa2a4a5ea996e5e1bfd07dee0f3b7ab2134e2e35cdf34f75878a5e9\",\"a50b85524f287c3228db110cd8ecb238faec2a0227d62d8cdb20a449a3c33bf2\",\"7a10dc0bf8b1667d8501e83331e515d12ebbd9e2dde5be1d9eceb3175495ac00\",\"967d8d43896ae651c2aa7112f36bbf0c455497c6c90c9d73d790958643a554dd\",\"83428ddecabfedf57cd3684bb2a8f955faae05a4ef46527c32881f12e14d06c3\",\"88e098cd3a1307eb710e5e1326dfc6f7f7bddbd32b1f3e385dc843946f95269c\",\"b4bc35038659fab1bdeb95ab5bd3b11384e6f1f15254703148c1588fcec9835e\",\"082114aa6e02d7922d1636a9dfcb954421cc8ed1e75c9bd7eeedc0e70831135c\",\"15e4f43768950c159bb3f0c1c5aa9950a88021ca57f8564e81c70dfae7aee422\",\"aa617aaccf1a988d0cafefbaee6cb368a48a0cef2ddc0305593df897a22b955c\",\"4b16bcd5c1190cfd8fe3230eaa7c9d521fb3b3ab18e12b7886aeec94e04c4b4b\",\"41c8d1c5c975d9ea6de050261822afeb29a332b9bf5d39ee2f89ec094fd13879\",\"bab2b716e9c4f900712f924de2a4a32f99b5dc07b48e70a9df322f38b27f20c4\",\"101aa16698d8c3a835eabb1c17faa15c2dbbab0b4a276c037821b27f847c17aa\",\"a826c5a93ae95137096b96ab6ce8b72e8c50a93ffbb745c26c00afe69dc330a1\",\"0976bd9c2bfad2107244658aef43aff5a3865245f7b978a7f9516cfb9ec6ef90\",\"aaa5b2e608137f405344ff92bca1458019bf9dd2afdc295cc750db63335e86b6\",\"5950ee978be44f9f0be1f3740023fd7fa0a3737325ce2607deae112530a4a964\",\"8a8ac3835f62bc7ec3d4f123e92ed668f268689a29bc85a3c50a6b68635e1b9c\",\"976139ab73124de7444b8388e35b9db5a118726bf9ef00270f37be27493b2410\",\"81a409ffa4171555d5b25127f324ceca3fa0524a61c1f0c3f9fa067e6032ad3f\",\"df20eb3b068de94e0c5e91edf2f680e856f2938b22bdf3cf16718c0b0a1b6ebb\",\"73a61e99652b2da594c93dd0cf7033581c599180b15bda2eb790c92be2d69aac\",\"4cba8d32cb75cfb20caa002791e6f23cc815b8739864f13e8888040d80b62a2a\",\"e08f0daa9bda4c49808dd3650503ecf7f1eadbeba75b60709282cdfca17c9141\",\"7bb6cdfbafdc5760bcac18a84e349519f235cb0263198d40650577eb887af469\",\"3c15ea262333ca9729bd9bddd92cb9308d11b3ed6082eb433db7afb6a6ece913\",\"9629a3ad92c7317da008a1e10070f506ec8eb6f6b4230b1add263ad67414035a\",\"21f9da12a2070fcade2b77f599d690dda0a49d2cfd0faa9e18186d88b128d27f\",\"e4089410abefe743017f2543b94689d90ed8ff03d34c52d18084e7bd8b23ad08\",\"ce42bd406158ed18d483f253b0c09ff36791eae8df7bf500cf475ba184aeefac\",\"4c8f64b0e3f715034de8bef3e32439ff96a317ffa2b0b484974a449b67435ef7\",\"657decda4deee0c917ea94b95ffb71febe5387c8d849f1613b13b3ed0803714f\",\"6f4f6b7b85592d68215b2adc8d64f6d7cae30ce9a4058149303d81b30fd71947\",\"bec123746368a9f4a14300cc30a1cf035d60ecf3eb27cc8092bcd69dea585f61\",\"82646b17bc0ff5770d1c7d0e14511ae587b1d2edda3d4033c22b96f0c28af869\",\"bc85645437178886a7e95b737071163ef33597464a5c2dc2c0d0824a6841bcaa\",\"699bac8acbe1503152ca89806a4860a2f9e7af60e8475dab8fda368aec35b1c0\",\"c3c06bbfb833c4916ff9e90e429294ede06d863e5dc4494f3ee440eade544073\",\"1fad7f249c20dce799467d5aff258535de4fdbdb8a2b96c31e5272c03046a403\",\"81544b10037e944c8ac23b58122c9ed5474125a1f7a10aa3c311bb76871b1633\",\"ba23d74d3f3c25c0f72f348312bcf027ffae546593013fa196474a08e0a9cc02\",\"3c95653f4f7374ec5c65023497c50e60a897fca94646b64dd7bc2bd1a2059d30\",\"28b72bf686e08379cabb7ba1862d7e735ed8c8973d485e9c95bd5697c0eef8c7\",\"ac67258ca7157e3bd43d695539307d3956a7273fd9a663dcb53500f152faefcb\",\"da13a0f017f29c05856530be1d94a57065856bac37bfa23ddd8744566a246e93\",\"640fc6185b13bb9b6414fc922c73254eaf942b8bf9c2eac7e83135b0ea83fa14\",\"fbc36151040f35d228a7e597d92f93f7656fdd8cd3accbcdc989c006c8363c0b\",\"919886f3ed4b8cbfd227a75fe5ce78db77ec212faf2c330f97dbeb0ee5aadfff\",\"92dbc59a9f6d55efa332722464806e07597312067099f294e11ded344f52ecd4\",\"19f5f895083b2f940074cc56799928dd18de6d8fb3bd191d2f02c031b9c51b18\",\"827c83a0015cae333c6e64860b4431641e56bcaef1e184fec76d1871367c99cd\",\"c02a85b0405f8a665e0724a1588d0778526169abc69e11ca956260610f91cfdb\",\"8894f9d29cc5a03ab092a1f29e5cbccc5e2156aa55346fa5bf095a91fdf08164\",\"9922b3bd08e867472cad2fb3f1c3a0df39fbcde1361c7b4e459374e6fa85af33\",\"d54ef138840d0ac633e4fba269bac65cd0612cd0d68129033274bfb2acf333f2\",\"a25f061bf43f492b57c37d6c857ec964f385fce15d0e6a7110fb44d456f0eff1\",\"6e62105a7e197688c5740aec9748c8c1c95607339c3985515dfb143a5a2d5ca7\",\"0a02ea962a34727a3537d5594bd7661b73e73c4b651d82db277142bb796a90e2\",\"a5368c791b170c1fb75b722a15061d6145798dd3e65ad4e924b9da174ed9ffe0\",\"0453c1696becb3d8f9c8cb2f6bb33cb6b726fc53577599ac6d66c17a8a3b1b1a\",\"72555ce6f458c7e017cb33c2b90644e8ff01c1e8577684a31ee3692ba839ccfd\",\"16638326daf96c856ee55dffd397a973d18aee03d4613aaf77c15935049ee027\",\"aa9815aaa8a43b566969f8b061550eb60c70f9dfaa803e7f199836803cdf610a\",\"6e1b9707118bea1cb658cf294f9cffd08e788bc208e117e46885884201910643\",\"7858907870b5834f9f38c2bee7648b30bcbac41accadeacb97daee77b89a5395\",\"cfdbaaeb17f23b3062da08cb4dc72585d486120d518f98a6d941cd0d74e3e936\",\"fbfd9ec810dd7378d6256d1cc1c3b69413271ae0a7e0082762e266a334203e0a\",\"eae575f8fb25743db4576edce4cb9e7c2a22c9b5693239bd50b1d374c04a5173\",\"13feee41836ff007e886021e92b96e9f358b284a191303996fbe8f27087467a9\",\"a4182090ec65d3f673a57e0b7c1bb5dd5dff79a4c47539e17d927beb33511f04\",\"6e28e654754364c016b53ca4d905369d0fd89addd3f28046c2c2e9e0267d5c65\",\"2291e9e9aea4aa017c6673c3363b894c1f9c2f397ef3236e8a7d448391ea4f5e\",\"fa9380f08c65314331055bd1b0892092acc0b00da38a2b4d4213261c8f7c8363\",\"012d1caf3fd16213f527ee2cdef7e7383c5b879ba6e5aca75be0567f3ee79611\",\"86ec85998aea4e7ffa4d1fc2a006923fd31027633aebf3cc0810626d855a7781\",\"e32d5c1b3b7d4c2cc2ef27026915abc8bc0e8fdb3794f7e7b66eac5ae1db7a91\",\"4bc56193584bacabb18a48b40eb7642ef6b9862999cca188cb46e88b0859d9c9\",\"5d9f9c9778baacb6f33baf117a21a28a7144c4d68515b64f53596f60b373f09b\",\"8153ebd3387f5f8699c6909066ed762fc3a2e250c07ceeeaf1c5c7d4caca816b\",\"861dba58b80692f1a8c07ba5c16e9359aac5877263b843ed01d77c5d03c13e80\",\"a3fdac7c6c67d7654bef1c2f8f9424efc0e3efe6beeae3f44c4bb7dcd94f8869\",\"df5dcfa96beb4bfca7cce3b5d168105c433df4c32a8f75a58cb3d8a110583264\",\"bae7cbcac1a78c942fc060e03f22be53ea52e843effe4e08cd7b22475afaeac2\",\"c5635beb04f41b240a9396ae78894c62ac57b799982f24cf15ca2949b7ab33d7\",\"d2facc4f0d2b0e27fa2108fa72b287bbdd340ed269f33e6bfe5a71e3317c1e7a\",\"69000d6209c693dede57286c711a7d84fd85840c9875be3dde9b0c96ec22833f\",\"44133dd3e6f3abf57a6e77a7359b6ab2a59aa3b9216ad4ffa3d14f0e7ae36187\",\"cab3448564dca321809bf8f77c87bd6380c93ea36c42ea0bb7071051e1953bf8\",\"713f3489d9f49e23f3401ac3d1536cae860d534b26efbb0a6b65ed453a6654a1\",\"fd2ae4b3465b6419e102fe7402e6c5d2e8e5041520abefca2562b625584fed9d\",\"035d6192473f2975ee9ba089fd544460932a1d1af0455d26c32dba528a2172e9\",\"3be517bd50903361181335d186e87c431c669ca9e1e4bde43208a0dcbe768e4a\",\"8fd2ff597ed85f5fba7c3ea91d95af1d6e80881cea223093c3810eaed1671c73\",\"08e78aa4e8dd2ebfcc3e70a558080761fdab3462c520791b08c988cbb78e151c\",\"d9bfdc4c8940f1b148dfcb94d39287df3cf9009da742825d3a907dae57c94166\",\"53fc7639224544be7e3a25212784c690f163ba78dce4840c775876d5ddd67eb8\",\"e3f066563cf2ca25a04dedbda41f12b998323f20f8cef5978cb1bdcbf83582ce\",\"fb6b07cfa14bf07417a132e51fc0616ddc14a3c6eebcb26c9085944c81fb06ed\",\"a91d57949525262b3e3b6b64632dce3dc06e3adbaec9f9088c5063059972b05d\",\"cf7b9c251186631f67e8d5dac70f1c68bcf5fc1da437211f55ddb87569ca6742\",\"e2e24239dbdf0c7378028319ac49d7dcfcd83326bf5a9532428823f8601b4544\",\"07b6213261e98956fa1bb5c4df07de33f22fddbb219b0d7777ff70124190f99b\",\"fc2b425bacbc318daf7d17ce2df24f1f5a5331a6257975f677b76588f6f0067d\",\"4b286c6f049383a2896c4d9829d5ff9e1c3368c82862aa65cf3ded896cdc6b35\",\"9cd957db4613b27149e9826acd61f5287e706f807e188e26d43677319eb96e08\",\"2b76bfda817c61cdaba3c9da14aac729953cb0c8621de47452a1edab7fdd862b\",\"469b0ad8a943d6a1e9d30bec0c1e1c15debb75f60686756664e17dbb94dd89db\",\"fa024be78f3126bbc9ec87cbba10f0bf0bae56f353ee799bb2d03dc6b23a1ba6\",\"04f27c05f8f0c074afd2391b8f20a6931dc1117c57a67ccc8d1af05905f6b1b3\",\"8baa7fa0fe752a2527e5498eb3a90ba9c8b98f5face92b5f8d113e758d74cc01\",\"eb471b0846654f4a4b6fa3765d943f97daf08baf2d437111f1d6bf3006cae67c\",\"989c66ab82e739c8c988d4fb5fcc68ef04de4238d20fc8fbb914357a28f3bef1\",\"4e34b8cd4de41b9d317f807b3ac37e326650e55b812c230f411c02b85f58797c\",\"74b8dcf4db82d846d757d8c2a64b87c0ed6d887ae4ec7ea4e7360f48dae63fe8\",\"3d10ebc5ca90a7093c6977d2be83d7fff2db6483eddddc0517a4ab95bb3ef032\",\"c801b0ed15c16fd43558b2973cadc524c7082c7cfb6d07ba35f7ec35b1ec08b9\",\"db500406d8c0bec10a84c7cf8b917bfbddf84da440c1e270a66c1e66344aa740\",\"62fd9b2b8e3b456d6b93f9de867540b8e2398cacba8f9d7e26815767c56ac5e9\",\"5e13f4db861baf936f70931575fb62c5d162332b6992a9dc7b45d898fad6705a\",\"f8566dc3f3324ef26b1192a2a8093766503a2297e0888e4eb931359c6af8f155\",\"d9b21bfa8330e0a72512796d5d2a88ad7c0359f61c66212e073086d1fb559131\",\"4d2cb27feff141605a9c62bfa62de3067b05e1757b74a7f92a8e66e625f709e5\",\"c8201794d4b8117046470311602166dd8845010ede65155f93a97eb05632e2f1\",\"7aa4982a2225cae7734470a77269511c626da6fbac47dbf74ca59d7b0e2bb1f5\",\"935717adf9ef78e810976a74d74ee5cd1ab98f783a23cf432a896cb1bc7fb1b6\",\"b686be2b1d79c5047dbb41efe214d850f6c137356beec2d7e3a0a6ef7a2afacf\",\"488c108f9bd74ba49080d31ac72769ba1c7dd12d5bf1b9f750ffd1ea4a3f4499\",\"a12e355fcb50a1c25c67c4b8be0cbfe3525d5dc6218b471dd41a7b73a748c9e7\",\"5d6bc773dc7b2278af8b3d4c9bfe2bd58c6426f98c1592cc690d727289576438\",\"2419ddf58169d1e6cbebda43367e7d5867be3e3eb1b2f2394ee51dc8e407b5d2\",\"5d77f787c7a32b6b49471f93a8ceadcf79ab0b05fd9bfde07e0a80a2fc121bdf\",\"b13e4b0a2043935c84acb8c9a009cfe87152c89123cf8283fad49353d4b5e1f4\",\"42cef517805baf57d317ebd0327118572ddac21b0fbab1f31174bef3e190324b\",\"25c016e15d6fba8b3f1754c181e2d24c7e74b8766ead33f301dbfc962729e162\",\"7bd8fbf8a906f36c5f9bbf65a92453dc8193b03be15bcc6b557c17e1c580fbdd\",\"b37e506c9b588f8cf9eee1635a6633140da202bac43c394e732f9a6105d8d7c1\",\"3e7808383e15f05e4e11e3cc7ce2817efadbb0a19745909bf6b9449b65d9cd6d\",\"3682504a79aee8931aa6946b03affee84ba8d01978d7162e19b2733d689823dd\",\"0521820cf4515ef62f47b478bb40f4fa8cf19650f5d611ab9d6edb9107bfa42e\",\"e1441af62e0b61890f96ca7177ae4ee91cdc29c7dc2fbb56febd4eb3b76c9e7d\",\"18356f7437e7d2f6e270804522f16504838c8588e0893f8f255016968b1a0ed0\",\"94d188d9d892fec8f42ba2894e24159d53a1e26f030e33bf68cc631287ea4dff\",\"14d7d53e3d80eb586adee50b4650269da4ecde3e74a63a6f9ef6d28a30c91cd1\",\"55108f6040395e518691685941309426e1dd87a85fbe86b8810a9f9c1d26f5c2\",\"ecc045b0ae17393a7a62d2d650f6fc9a387f5b639da91469a2fb889acf1b0ce5\",\"9445f63ffabd70337ee706b3327de6d693af482711ab66fa63429b0102b8be06\",\"f97b3bdc8d6f2d79206daa3814174f5b0316fe991951c99e6bd22d650ec04e07\",\"399e5a5d04fb1db8f314a798fe885005282cda4b11759d1192eed063834806b2\",\"79861e5d0283a663d0ec4db5515885f2c4cd87f76842c9a947374ace776669e8\",\"0414312809ce23eb8c0b6cb38df71443488b8cd5f155d8a0fe3b92deb738b661\",\"b6fb01e885a0de750258fa450715e8f556680e0d92143c01bb5cac90e860f40b\",\"4359bac547137687e0ed5049d2d7273734813e9bb376c6ea9f20b2cd1c7355d6\",\"787f9e98b4920dbcbcd68d776898e138bc70a991dfba4379f8ea8476d403cb76\",\"f23215efa4ce54c46d68d21a326d96e7e47fe47c4ce6e83978d36d6fe8863070\",\"d8b6c044f049b6dffee25160e1682981288ad8670c9b680b3db5de77cd9caf98\",\"daedd9805719ac3fd689800f0da2a22f0a89773b45527eab0380c475b2607597\",\"8d328418d8f8d1e1ee6831a330dfa447e7e2a1e9068be66474865b991eaff33c\",\"76d785ad564a2103a72588092530597086274412c27ca81290188ebaa1285d62\",\"d30cccfef97fac890f8253ff3e495c50c0673c9441b0a408d86a64ee7f7bec19\",\"803b127fd129133ad5660aa13d787d8bcbb9f6a4d8a0d69a2161136467b667bc\",\"c7af546a6dd0a6017f268e1934509c8c76afe80f2daa8722050067d5d393535f\",\"eb2e6765935f53de024592ddf4c0405365e21efd31bc3a7be766f131bc3ed0da\",\"f66f450c78517d6c7e6f54d8d72fd3ada15d91133884f46babb7176677a3fa8e\",\"a1686745e147bb6993246516ccbd0f4238d3611b33e20549427cc5d7befab681\",\"4de922d6b48e480d59115aeb7cab962f2b06515614d36926e70b62168db92758\",\"598caff0a50734b405b6892a15ccd441ce397189f9d08dd76f2ca7b730c5ddad\",\"c31b6d614e11dc3c930fa95b116d811cb2aa54a32998d04a956fda900c40de19\",\"d51cef4e51a8e445abc76eb9e761b0b286c8cdb779f117af4d483a6057904eda\",\"1aa00e0fd1647d79d3b1dc0f18ad12d07ce693e962fa538fbc1205128e2abf40\",\"9afb699a86ff5027862ebe4542e70a75e716d8c37a7600668288884811a5c119\",\"c6ae623da78c5089acb1265a60f991377eda9d5e5f6039810392b0830b0c0c63\",\"f3462af96fc2304cd070185180d94bd2d4aa16af940e8e93f9d5cd4f150d2489\",\"6eaab47a0277d06af1d27d250cf0629ddaccb107b92645e3cac80f8eaec2fe78\",\"4b5d18c134886ae2f2c798266c88dbcd8350b1d8ea111f0810b8d39b8f468193\",\"7566799d6a63a806780ba3dd5785ebb2b9a95524884838d03bc5fdfb6a605d4b\",\"08dc1a1bd3d53c5eb83b3a77dfe85d1a0c7c3d8ce0f198dbb174f606cb7bc1bc\",\"822eb8b056ae5865f4d46a9e38de0676b0a53f5fed47f769785054515f776a89\",\"fc9cf3bc92e83b3e84973c4fefc7fcbbdf70745990c09e80c9f98b71cdded240\",\"71100b27cc2bb792337cc26780e268d384a2ea9be6a0f025496c5d7dd053bf51\",\"4f762dd86d7a4319868ca893a1fe7d1f3c864d724cf4af18eea69fe9fe889b86\",\"61392bbc007388fba8414ccd81e4b22dbd2bf1d8774c46cfb1b1adb706418680\",\"3de477c1e73ed6d9d964a0cf5c226b449b30176b211d9c01c7b20df9ff3ab5aa\",\"957977f40ca162a6d398451a3506ed9467e0c0b200b72f267a7a7c987bd480d5\",\"d6b160d31bdffa59179946d6505258f86f2ec2f623dcc6cfbe9615d20c71ddbf\",\"cf9eb842e13963b93a69779ae1668f1524971a71a43a1c219347860274d64531\",\"933efa4e75a8a6221f3d6d3258316e41f79cc6514bedbfde36a5f614a38a5c1c\",\"e5f5e7573ca9aaedb1ba2e5ee02d6be4a5a0d90dbb01f33c2b04684ebbcd4dfa\",\"d566cf0f5431db3d9ee838c7c1269b870ef2d46c1dffebaa4b3d7ace80b5310f\",\"789c6f2326f620d025941b742ad559057d6d7f7ba2bf6d9be2d7acfcb4e1fff0\",\"d13f6233d8f8a96b66aa4207250a29857db372ec9b2a1de359016c471e17732c\",\"4fe9892cd002d64c838829ce8761b2e69e053481463b9c031c58e25dec005a95\",\"aab74fbb1ab783e01eae23d3ffc739f2ec99a1c760df0059fcbd31f64630417f\",\"be50491482c3aa4a81ee79be607e99ab390526c307ef7e7071b093b8659687ed\",\"28ada92647842a77ff1b8b6d23dc699992ac03a090ffd38553a1cde5947375ea\",\"dcaebcdc2924160e3c6aaae5fdaf3d76b89e7fc6936e1228d1bb9a553d59b01d\",\"0847caf0e7f5f4f46edf00de2f81cafc1331b2f295cd525171af7aaa24ff540e\",\"31667477c845a07f3175f1bcfb3deebb66cd8f064264925f27def516a3b93f40\",\"1c207191667252b40d3e15c7c2cbbdcea252f58fb129c757449ccf6d02f62895\",\"0d1ba5f087e3165ec33511627c7bf4f7507452512523a5dfde173c2a091b6887\",\"576fcfca00e0636481c9a1d5b6eee72c83009e580caeb22aefb1f6fd1f295f4e\",\"32314eeb24a1e9bc836eaaa305a6064c875fb40c8cd30dc0d15693472d7e5bb5\",\"5dbbc3f7016fe4d851adae9b9c6ee83c9da841fd0c532913e8a6388ca572d6f0\",\"f2aa401bd8f231bc4305b0e7a294e0c9c19ba328d891be284764d2eb5be7e31b\",\"3e5a41b512f8c1a86b3b5fa66e88c38a4eaadaf786eeeb8977f6acf12b94e7d1\",\"4b587bf5502cb85215ba58baa1b01cdc61d29be6b54cb8f78a18c2e8ababef67\",\"fb864e05daf12711d9ce37a5855c5ba5d5645d65348adbd1242d334a29277338\",\"cc9f58f68cb44810379a7ab4e86709744580102d4518d6bf183e12d131cd4fbf\",\"1bad3cce699985e4bd0ee5b47e232dc0c356b15fdd831d1a3c7756bb20718d32\",\"2fb2ceb6e794fb123461add01186355109401af98be2aa4ec2bf20c0b2b05e3b\",\"5065528b8d066dbc06d938e26884b870414c2d3e64e474ade5cab2ae4921d4f9\",\"ec3e004da8bfeb0ab7a4d9bc6430e16dc9f913757638c69813da8daad7381e15\",\"6d44dea3e09c1cdd5071e7c4b053d70683e8a860ddafce197588a6ed5dd71645\",\"b4160999c68d574ac803e03f3d02c886a95d3a6f2f56d13bd4a7a70fcd4897ec\",\"8c1bf303f51bcdc37f75acadd46472aa407cab196b8a8142f98286250899f211\",\"4c3692973789accf42b1fe590889227cb84d22cc760b758fb60e7f6af1d9105f\",\"ac56723bfc5a87e502952c3540309f87d51c8cbac180fddb6af8b9c237fc0eca\",\"73dfb20a81480aaf47b95a646d3635630e86370d18bf7e5b68e71efb3a8ab8f4\",\"c35ad8a27a172562fa9525f5777dfee1911c78afc9e925d4609b2e4e23e4cb45\",\"4969da23b5f74a6adf89b90737f3edb7ccadf670133a88efd0b43ba784ce57a0\",\"7ae3497e153df1cd7568f2d19dd423b680c80dfc6b447c890032abcef44d402e\",\"f1a5ee283390c0b239eec94087ecf2244a8e4ef7768b67c58a6b7808d00c60ee\",\"48d730dad7ec8ef8e3d8cdbb799f9abb96d318f16bd598598c9d263e76f08cc2\",\"db42a273b6d7b7866bb3ee7668f86611fcee7d6da070b54a1f6d83565a41605d\",\"6a497ef48a4150e0e6d877a54389e96dba6c29526e4c436b6f9068622c70a761\",\"728132d171732e39d0c2ad0e8dde4209c40e43ff8e3b6439a5c8e4c6ec224545\",\"a9769ee0312109b6c77997727ccd9ead9ec9cebd1aa43fbd18eb895e9043b085\",\"d7b2ac3be884f20298cd8d5b0e3c66b5a8b2760c439be27af14e117b0663efdb\",\"7f8df4ae89d9f9b2ad336ea0c82ebffcdcf112ffc92dc9906ce25bd2dd9bf16f\",\"cd32f7ddc5a0558f4614abda0ac653981dace4cbd0496c8044571400bed7c3bd\",\"1acb355bb1b78e29985ff00d575f71b0bc01a815ed342b1c482e83a4d4ba34de\",\"00105dd45bb57c2f9a1f5be70471f06418923d3557f59661fb982567e1c9b3d9\",\"ed4a8f11dcfc6ba88ce4f175eb231d76308b88cbefad459b94622cb40717f7a1\",\"814449caecb2294a739e093b15a840a19a73cea1e815d51919134af4b7c57555\",\"12505ecde302a9d2d86129f529a7009867d1295a9e1d2697b6e8e9a515395fa4\",\"9f153e8a7eeea2c367f8f87a0a9b53fecd0fb704e820a57ecdfa7e1e68d7e39e\",\"d7435200eccbe9336d7534e01fbcc1a846cef2d751cb6a1af61d09454e13d566\",\"7d760b1a5256328183ef3bffa3fd820a500b9b1ab6b80f244841bdb8c5d1f250\",\"73cb80077f1e6bc529a8bd0a3de01bb5dda6657ee875a9b87f40664aa02a66d7\",\"bc6990bec17c92551bf6b92f1ef452585be7ecfe9c72a733ac0583df51cccba1\",\"8fc6b05ab1f07fd260f5e2c2aabc4dc846552d62a3478af60f07382a71993eaa\",\"a7f3144b1620c60db76a4e37d2f3c79874c540ae0454d1b32cfa0af4e4e1e615\",\"9f72c4e28c8f94faa7efe724d01bf20520258a2fe5186ab70ecb78b777ad8e43\",\"7e13b6a9c0a07ad1f4c79672f1d244913d4c89b9a227e310d56077cd71feaaea\",\"00b20be05f23f0b053be069a1c3f318095749e38787f0373aed9fde50309f9a9\",\"79e43fcb6c998c2ad7cdf400a4fff2431dc2d8e7f1390eb5396ba2b152c6dbd0\",\"9fe09fcdb2da20aafaafb64760c6fe2baafd1990d5c97968b8e7987f86e3df40\",\"75fb9584a1d3c9777856d91e3cc6cb93e35dbf5e02f9724e35eafdc49696b244\",\"0c00f2c1722f122578da394859af33b717601b51373fbd43fcaa612ce06e1776\",\"3cc0aaf92c1abda502a5e5d27f3b1f6da3401aea640d6211742ca525a192a065\",\"4b1cb26064b8ffee958fa818be353f62986d87c91a94f7fc3d4da02baf0b75da\",\"e37073aa56ad8013677e5371196c3b76317ddb0322322e55c9a6e6a2140c42ee\",\"d97381b17ac671edef649b1f6746a319bc2aa805e1b54b698c0a413cee20e245\",\"a00a726ab08595f7f32d0ac4e6ad1f3e089da238c5cd6a2cf2708944cf49360e\",\"39deff47043558cd308d16409e80f94cccfbfad9ee38969881754de5001a18be\",\"5a0621fa310fa361913f46de95fe1aa34eb43d4d798a11a410521dca53a5e98a\",\"71087281ad14377140ec683de366073701315c974a7ef7435022be9f6ed5caac\",\"175c2943be56d5f4c4f1a209207a9df585a86c4e0731beda31e87d6a3bb650b2\",\"83f025683240e3d8437432db7f0187489963adac7c7913a66f53f780e64911cf\",\"c91374266da007a2fca9e1a5e6c42bb09091b9a2eb4da0d8452cd7c0465bcf6b\",\"8e28528331ae06fa1ed5fc777069e7989582f685726ef2025cded5e3a7c6380d\",\"36084c5f96b459269335a7e2b5199f7d386575262c24aed43e410c2b7e0a1871\",\"c282948935c6c181250bb713bb7b5331eafa33b091bc501c1d182dce00ada9d1\",\"6b76d3f9c9acb98c922470eb8cac7c71837a75c2254ce82556b243783b8ba232\",\"bbe0388956568e31ab5fbab16f78b6e594598f8310e7dbecff1b35f84d72aacd\",\"c8ebd18a97587fc747f2ac5af911a1150c8e433472770a75ec9c4e95039024b2\",\"7fa97ddcf0fb972083a0b15c714c0cee6d9e3694df26de9fb0dc785a9edb8cd2\",\"462a3940dce0543f010b210bb53ce4e8f25c83d692f70e58c668bf037f3b6762\",\"43020566e7d7c2d956ef33c81e880e1d9cd35eb1f7f5a01a902faf0b8d8f1b13\",\"f35e1782e29adcaaa98c361303e89f1ca63da5b183fd0bd81eb5368a15f00ed0\",\"74b34bca72f6e3694af76972de12208bcde69b8fc59059a09991289b5182623c\",\"cd725247fdc8987dadf9d7c7b566675d6219d183e5f2faca7fa086b2dc8908a5\",\"be726b25c377abab943771e48b73a5c5f820dcb0eedd14d1f0693163137515f2\",\"8cd7bdbe1dd221c5cd66f8217dbee0560a43e6dafaab5d6bb0b92b521113a1d5\",\"c9995ad2fe17d0b0f6ae1561082e556fe680ac44d10c98cbd28d15cd547be19e\",\"389c9b6e304341ce22eae90e3c118791c04d05f2a8b846c44211acb85ac6d55b\",\"366c72aecd2878920a94645601d7201527a7f5955909cd833dbd89e72cfd6d46\",\"43e4a6ab2de843f4498810c90a34b79179e407d0720d27e88e14a86ef4917f0b\",\"4930309bab3dcd951a5e29e80cac3335d4452b1fe0dffabcd16091fd17bb128f\",\"ff3da0d9d97054b9a107f7da1f8d9e17d655ca522b2341222e494dbd5a2d270b\",\"50f209f6b86d5e6b4d2a00e9aa764984b970657158b38b03ef1a5e7c2bc2d4ab\",\"d82a29915b267b9c980006f7085bf2c7aac0714175af811668d990a5817cb8ef\",\"ec2e0809c9865e4a1859f91a21214b32ce46a5428bbe017d0c43bad2511418ad\",\"d6aad93b29cdc8ce53691552c8a4822c87d78438d3df6d6ea68025994dad8fea\",\"0ddba101cba2ed06e1c601461d8e9a817700bb9d216fffaa1c966b00074cbb37\",\"36cc019d67d0097e4296aa7cfb7ec6ca2044925e094c52f84c5901a7f4c47b81\",\"145515f5e3cd986102fc662fdfa6bb49f3bca18fa3f67bcc2028c8482b818989\",\"a2dc0862632cda41c1b63fa5ce9b2aa6ffe3f4ba7d96e56a31c938394cd8dc55\",\"2c5956bacec675a76762bd95e912bcc795d6d80b807336267c58a0a4a36da036\",\"efd26fdcb05183775da26701768579d9a92fe9814718307a91985ae56e59b156\",\"01b1d5b0c1c84fd9c89c5e655dfbbbb355333451b49b241ca32845e6664c8c49\",\"19c0187cfbf82eae0cc8dcacc4440b0138d8cdffc0f4b29334cb325262c609ff\",\"b7a54ad1ab889bb9e4de19136ba315f8cc6c8321237e55bb7639dd5c371fa6ce\",\"0094e1bd120028e1fe00b9ba220ef93df0c55621922bd1eda33edff9613119ba\",\"ac9e8bd12ee6f3de99b18ebfd70fdfb401d7a3bb6ae735eebbe3e934de2d39a9\",\"3a3ede840a466d69c9f5309ba948cba073c84af1a1f8622e0851c24a825cb143\",\"dfc8fd265348bdc0784821c4ce64e728c52cd8ac8a25065e666671ce9d9b27c1\",\"1ba24399bc114d690e22c1808d81d1288578d07d5175facd8ccbff5d48a4762c\",\"f71d0a5f9345fe5b2b494df283c8748e4e8860c9763e0b911e4162582089bfa4\",\"aed7b87ce7ff7caf854f04054de921818792266284985ccdf3a72f47a4fd2806\",\"1a31393db1ec37bd7dc5e648fe19c53939a6caf9e0bab25867a28f4d3616f612\",\"4a86acc92d0aab14042878edadb4d81e26190f4d4820b05ba92f012c3ba5f856\",\"f4953e86478b547db627a7a57e18af65cc4ed8bc0a85200ee4fda706715443ec\",\"93333037aa8c97edc1f19d485fb0ada27932e882d75e0e7ef2cb37022fc21ef0\",\"c4e1de0ab5ec031e317cceeb98896d0cdb5d197a796bc684d083145c4c507548\",\"f0351e7c5e7de4ef856061877668414e920c40b24172414a3d7318d29e348e7f\",\"9209b1dcff484565dda584389c3ac7737065bc990b52114b1b0bd81b9b495a03\",\"06a94d92a97a68cd7b864940be1f44acb51968d0662ffc98f3fa49a04ef9b991\",\"cdf9312cc201a2ab8ffd5c894ce91ba6357fc48c946c04e49e4eae3b9021791b\",\"b819332234517979014a58d8e1df07eb05af458c3a285d1024f9f8b81e996b19\",\"40e4f0a5b619f887a68acc2905c38faa7f93aa37b81a16e6347e8c3489f6c8ee\",\"6b984e91b72585507acc2568d01bdca5728fed173f574cfc2df6307c3cd15fab\",\"0c97a26e646b61815336118090b526664c90a9b86500324ab871864e899fb280\",\"70ab24e905053339aead889d07a230cc218efc964fc99e1ee3ce6bf23b1fdc2e\",\"366523cffa7b7936ddb05fb735f5c6c6a9ed7a9b59cee2e22688c4f2a2744718\",\"dfb6c5917c0dda38a055d23de67434a9035a4ec2658f7125d7cd26bf7026dc89\",\"6cb01b68109a039e81fa25f22c7028a1ab10bdb569d8bf4029ffc38fb7bf9037\",\"e5edd3a56a5dce5205097687aeccc84b9d1a9a35cdebdbb1f8672500a11f3301\",\"86ea71d59f637c422a401cbda3379c63a45c221e57b4c9440350f21dc4511f2e\",\"d4698f48505c6fc63f46d53f09d190d074aaa6baad8de8658a3f465dc9d38c6c\",\"29ae046f4b2c312344273552466c22d2303ecaaa7e502db2b7caa6aec5188c38\",\"b27cb1268e904717db8c84f8fc87ca4d92e98d30e15494d1e43484062d24bfb2\",\"247438ce221a4e5a8fdb978c607c9ca37d0d7943a74ff08fe93a4c057f06abfe\",\"dc1356aee0da5b7d98cc18dcf13b454c07780df2ff7ed433d613236a5fca7970\",\"27ac4deed031a12956670a67799d7f5801ff85d0bbf59599580477a2f4fdf6f9\",\"b5793719130a50c41d25ac31e4d8b8e49aeb5478eab26314a8641326ef2d96d9\",\"a63864d2c685a07ee49db53316c0377003d2bffa4d983f1913d30f6a8438e93d\",\"76dbb69fc7218b21e88a26c02bbddd02ae0cd527fab29a7072f9f0faf6e221ab\",\"78920d6590f05ea30aadd6d13c83e19bc5f0edfbd3eebca6829528507d3bcaea\",\"fa53b5c6b87380f0420a8975d8a7fedef046e61bf52af54ae35bc13f65de9db5\",\"e4e9879d10874972dfa94842adb574e47c27763bf7af24c448baa59bc6fc4118\",\"5d3ed5f709d7516e7cd4ef57ad13ed5755c1d91f40cc1782c045283b981dd961\",\"1564453a1d439f5265d6f9970a1f984f16b25a4c292cf4f3654b0ccacb011450\",\"89aa928abed121ea7cb9b2091fee3fd3268bc8940f760767e46dfa9717023d81\",\"38c0c2dec216efec4bdf392606b620d6ae7594b3e0bcc01e82caf1800157390b\",\"05965789c9b13864b2c943127ed9762d8cc04c36f95e20715cf7979647923e94\",\"b84b00c2025457c4d576f62b20a9c0b1b6881ba85f6e17e8d2e99d75a1351008\",\"7f30e194e5e5994e9969bc215999789a00a0083623b51468439a7f7d7e0b5148\",\"2c9ab2447187c4d44156a4320a74cb2df265f5808e8919322bc13b2367027513\",\"8a8f918c3f397891448000be782904ff1a89fa92d0c83ff0cd58de6356baaef3\",\"b1c44a74e0ae90291f48c8d0178a9eb963605870403705e59ee6a310c04663a3\",\"4e08101933a796c006d82832e643ac85fa2ed29ac0ba1e6ac7ac7772a20a5370\",\"8f810347167688471c88901d410dbaebf29f3b93b7ed60ecfe774b96dc228ea1\",\"48f92c12b8a67d74df00574fa279d87180295fc6d9a5bca615d4aaf2384237ea\",\"ebcd4adeac3b6fbe06d63a22437520599a3af0cd8a38dcd6f90dba8ab9fd28bf\",\"adaabba9007cf794d482cccf9cc86eb8a5d649b98320df45fb6cec95c6bc70e8\",\"c6acd1ced05765ab8f8dbfcdccc68dd3d800cc9c72bdfabdc1b2eb9934795f62\",\"c1e59c516b12f4aa294b68d675e43704b94f07b02ceb653cc677fe47bfe3903f\",\"cc8b6420b459cae8f6d3a43a8d8be552d570ab457e69ad09d43efe08723327eb\",\"891f322c302970e785d6909ffc937006412263fe80dcac938a90ea5cb53f149f\",\"c563cfb8dab33835bb1b82d4449996fb66f3c2b89e71f70a90964f2199065825\",\"be37c5b7169210bae01bebc99976059bf2af4facdb22b345a584630d7e89c1dd\",\"4044f4bd39215cfbef661230be458be0671570f84c195aa986e4bf2b7dc2a98e\",\"f020c267e1ab5b7af9fe08dff7ad770acc7a225df05d6b2772454795a9f8db7d\",\"e6442621d6c9e1e06f92b12e807dad79466dae42d0a1b4f82f3361c45b2d6d01\",\"90aa290005a799379e6149610926588d645c000aa0a7740be109581359925d8f\",\"7a90a30176911436c45589e1c89c8fda48aeabcd47e3b0800ba1d13c01d509df\",\"04c0149c881e9c67c5c8c6e82c62da6ce4af4d03f7a746369a184e868c3515cb\",\"4d151cd9e3e3e5957ed65a72d427b5d4a4d8f169477049f395a26acd2d9ed684\",\"11ff03b3c151a0f9d359a01d16ea1e04776b679168d68fad8e0172f0e0e54dc5\",\"feb3f26e99cd1536d554b3e6afbab131dd9b1ef3380e29d52550117d7d97ea6a\",\"259eeda027547d8a9c3dd01ac774d3dc6e6316b440b1de6723070d948f678f11\",\"368b0c5152b315175721e5bcb8e076d12695ce30a0625a6bff9e1eb8415333e4\",\"6630da655be8afa869c6e0a19c65878b4e6f4f51643bc4b23808f63f807cfdaf\",\"c386533ae9f6b124f41a6f2f5ff16fcaef01b370da91ed3b1e120c228443d8ca\",\"6b3e5ca1b2fcadb30432c8d2e77b6f0f30c8e1cafb1fc2e44e73d314c7c4f4c2\",\"7a8bcaf3f79d9dfcf4332a80bb8997810640fe49bd73521559ded985324023c6\",\"57624fd3c4b4cf1fa7a72dcb603133f3f369f5087f9e6da944f69ada346d637e\",\"1be4a575169d0ec90acaeb04373ccd4325107c1b0deaef1fd07a25fffdcd0039\",\"91ec01b456e5148c1aecd0605674e75ab890cc7e1be403986df56d141ca1e5e5\",\"216446889a5d974f59d9e19a5bb675ed164cab6b9aec5286d0c972c5564b46d8\",\"cb2d59fc5ad4d20a649defe8c8ca09b3e65fcde7e307b94ce5d0a25a36e6cebf\",\"b9beb4b3d9db260388e1e3788f35a60e90cc8944ef504e8424a22967d86c20e4\",\"f85082ddd973269048d4b8e51806baa1562a51b31f640a6594b9ac6532b6862f\",\"d7960f229d954615ed5a8cac6b238de06de655c89eba25c7c6d2c00a6b7d2a3b\",\"9f0a511d4f1ef05babdf5e63b01766f7d2ff1f05e124992336e4ebab8cca45c9\",\"dbb47c6417e1da679952828465fcf9497f07e4a866f936ba11a0fb8f285f42bd\",\"5038ce634e87a8fd5191f1465c513d0911097b6c02027d01f0d7955c5e28cbe2\",\"eaa350c61dbc3239bc4cf74fd7da76c6b797291b46dfdc8fc13e46115ce6b468\",\"1af3245ac79facda710b38b230df48d8134264035978aff7f3d40def6b426a0e\",\"e7d79a9393b7f39ffc6c3a6e5e6ab1415fdf5e59e8729533acb5b6535be418c9\",\"fab07384844d33c67e5063e7454e88940ed038d51868dad65bc99312a6e6d433\",\"4a3cc7865f8eede56a48030d69bb40635b8a32e85e983cc84d19562b9dac62dd\",\"42669d428bca5fbef61dc1c1facd33c1d88c27a1c832f34646111ed9b094adbf\",\"9bc01d1aadc6c368c4622ab1c61ee43740409fb7851c05d765901d9e75726f23\",\"d54f9ec98f0ce3267caa3fb88589df854a4e80f0ae1c549c62c52f2fc7547f11\",\"a66f132b4539fe0e89612e22712763979a6b6cd40be8380879b340db5c0efab4\",\"460ddb60fc79c5d630773914bf809803d99d07c1912f7cde6e4248885eb1b631\",\"8e7cea33fd4c3596fd8d78967e2e67f7d23f0d2c73163828d6e35e24489b45a8\",\"4d049dfd8109fc82c1c040439e956d1bbf60d4a37a0cdbb4b4f907ef40fc541c\",\"736ff7a27fdc94e44bf14515e369756b367cb830db40400677ace5df8573c439\",\"a40cf2f9f4d002afbaa868d551fb6aa90f7751d8074c1cf9fa2fb4f3f273c90e\",\"8661ec06f1658e9f5f6f87eb37055154b8a3818690bbaf930c3996c022312d42\",\"77c1b6f399913671c61a034ec61af9175a2a60ba82659f4fcf2687d846de568b\",\"70ced97390fefc77251f98209dbf7338e68ab846d622d4f6cd7b64594ced83e9\",\"af1baf96843344d67ae7dd9072017364232cc22bb679f63ce423a8ba24527303\",\"e9da890e820fd6ad3e82e273ab7826078bd0544f543d36dbc6bf85b1e0b0f4ca\",\"c8860d7fec08a340c7345923e22d5ddaa03b7e298e7d5fca3bd981790d0e8261\",\"c8f595e7543dae99d05a16cf8d34c828ddddb29759775060830cfbe6d769403a\",\"2496151721a961ba107b63fdbc05878b8edbdfe4f8423931f64cc9e57f796fbf\",\"f276dd31c6915fecb1e563cbdec4983206a25524930fca5d11f340f0a2c00223\",\"c8e956b28f782780f3467043c5fd5bc5a68fcb9ab9e6c07733d4136c2232a127\",\"db3d3d6810c08bab2370d3fbe104aeea802f92eb4d50ce7e6b44a69c6a0c13dc\",\"88b1c97baea2d938ef1a345741bdd5f41e1eb8f45fe4e3271839603b17b9256d\",\"fc2e1f25003b7b1cd708f43c5793d612f20ad3fed733763016ee81ebc462869c\",\"ebaae482127b2aa99dde2679cd74e37ec42b9fb78dd8e0bdcec32be69a422476\",\"69ba4d69bc82fac91b29ebd5766c046c501f2e991c2e41f5d3f23a9080728ada\",\"4a1952295de91bd745488e4d5f09cb7e0cf907222350ba21135a71a56aa5226d\",\"ce7f03b1653e0fe9f47df3eb6d8b1276c6e0b4e2b7b74f83066de8e430d37109\",\"d600178b16cd5d9296a4be19dce980fd46560648cb436fa9c4bc23f87a75e23a\",\"522f511816e7c1fbf35672c8d54abc6fad3749116ac2b8ca856ceeebffafb530\",\"2ce014bbdde6cbb635c932de62650328545d5e998e098551a5c076c48ef04a87\",\"ae0ac11034d570591ca997406f33b469991835e32e9483a01812eab54ede2359\",\"20998e9504bed563cf9e8aa6895ee4cfb3788dcf92cfa340bf2dcaa426831a14\",\"7679b5c63c9539ab50db95086445d91922ad406a1e14822fc0cb2ac54d4864f6\",\"abe9bd2be32f82d9f68b6512803ee57dda51bede5f3ac9154da2d28e0d3b4e1d\",\"8cc10e4958129416a1835096cd0b3586319bf444c2bb417b5513dec839eabec9\",\"2fa86ec070063fd5cc91a9b7cd14115207f5cf0429c7a40eaf1c669d49072b1a\",\"074e010df884c2ad44dddb185f4f0167929daa5de4d0b35843d0f871f90d41dc\",\"b6f0853078a138f74ac72a62672c85444f1cf1f79fd419d75bfd3ce259377af5\",\"1b1e86eb94e3dded8e48659543bdc13b59ae5628e51d0c0678154c4c87a7f67a\",\"37924a9fec610828f70b83efc5c46293dd48f9d46200bd1ef50fe46a9686d93a\",\"fedb662ec3a22f710a7c5f096f0946710d2689838c68de4e52b885bb5b0a2eba\",\"f62923f5b703b335a98c58eaa3f55bf07b5209bcdc61a41d4fb1f209dd009621\",\"336c1c7a2739e58cbe366c73474450930f9bd0380c1a97912b9bfacfb26ae742\",\"aa7931e80cfe3e0e7c211d5fc1a46e6548d863e39de8298047a66deb452312a5\",\"1c70b2839b262e55dc8bdacd03a45c53c4340b001bb468db3e73e0d3d51c2d2b\",\"430faa6ec89a4b37e6422d2befd7ec9b10f2760101e96ba67ebe077834df4855\",\"547ac8db3a6debef316fe3f01dae6cc7cd3747bf7e6a539d7bc3f8eb094bf61c\",\"57dcbfa5ce4d60c49c181023cba412d0a75cb437f7d38476f2ba591befe0d8ce\",\"e066fe79c99e952ffc6c4857c32becebe740ead25b2c776bb9d496c65831eda0\",\"1867a0078a16ea7490e23da5b3f440f813c0dbf416827572f34ad904129a7aae\",\"b6917dde70dfdb75f623a46b1b00942f1b5e96cc988720cd950b6b8f22a5cfdc\",\"4847f7971a6e628f1f34bd28d703b14a0598395cb97ed48b0f4fb87496e9bcb2\",\"b50a5b641a028f052f0e3d92dda53cff1ef284c55a57c9e0b0625736f079aa43\",\"33b9b784c9c9e331c4cb3c4deb92561266491358b64cd887fd76a8a7e0cc7f02\",\"9a858e6bcce73fb0a421a4bff630c6727cdb72bf568303139a45d6820168faf9\",\"12d006ad3d904bf193adc210980b158bf3e80499d608cf537778ca493217b3d9\",\"4b252ed3a73ca59776bc2f24daf3d4c6e7dca9ac052d1bf828469d3bf4deda34\",\"9c5d6007d7264af2f70f3d1e8c2245ca68542c01d5b1bd74c2a63fc3e27abccc\",\"ea8c379b858d758c47e76c9f6a78075ad5390f771ca5730e5ecb73fd13216d2a\",\"806fa6490dc74b07356185ca501806486bed13f023704866d6bb5cd070c39d64\",\"d8cf2eaffdf5822b3b509ec1ab93cfc6a1821281b961cdd20a19fb505a6e332d\",\"a97921844b08c4b8f025fb7566c22fd17e6bbe72fab4e3a65fbf880209c402cf\",\"81a23339419ff60541bfb42e3c2da94e10d42d85f2f8beaea5aebaa163f31938\",\"3e1f0f3aefae25da5d7ddb714d4dbae6e7ba26374a8a9587c437475aaf6ad866\",\"ed1248183b6dae2bf7c58e60f4e7ddd8695231ff42e97d2b12d83c1a4c351ee9\",\"fc40c2cb0b477b738717c84d69e75e4d48f84f4d1348e14e98fc5575d4468487\",\"113dabf87d7cae48d3bd244c08959cfd5676186118d74e96bb148424763a63fc\",\"84210bccf8957303eb5e0bc59b64d432cf085c66b22a73689b89101072ab369c\",\"86a65005921a3326d551adae26dc0c5204f139178abcb6ad7676445f99a7414e\",\"b3725062ffff9c4f33ab70de52e238fcfff712de9d78da61efd4def5c419663a\",\"9023165fd2deb87b4f0cf08b662b50e863354fc150d5675e87d30b6739670716\",\"216bf2aaf9cc5ff6cbc841c1eb2ad0196391f1338932197d58cde1122b8241f9\",\"a6da53217a10ce5e70373ec6907b0c55bb38f3cd85e2d812f20eb05c302e909a\",\"d950f0c5589a82cb8dcfbd801298ceb9f5415ebd7a3219d2ecddefbe17145c49\",\"ed42d91a6e437e4f550b1f4d94459bef3e7b7c628554138221487d657c08fe18\",\"a75cf6fd1a5913b73e5963f6eddbc3c6372d6fb147358fea3fb7ad837386556e\",\"1f949295b0f9de044620bed464d959cfd17ab1844a75165ab787426790ec3040\",\"ab3fd531952557e3fbebe851c7e5bb8c31be12759802ab2b9f28872c5c897d99\",\"4d75e25d668240fde6c7a6cee59eaf6d67780c98edc3fb6191b38036ebedbca4\",\"e60a354b2e9256f0258f4f63a1895a6bd8a36a4418d18f24c4b471831efb62bc\",\"b7f8a32c8a7ee22fe2af9408283e913daeb7ad1d1dc7db67e04e3b1babd576d7\",\"c11e1c5ffeef06c6bb12c52e6dfb7dc33c58d20f2935ab979cb69d0cb6de5637\",\"74f629c41d81bd3b22b9d0452377d7af89c4fc7cb6c2783739eff3948287d499\",\"435f86c1f382f3d647276fb195a9384783feecb402587f88cc7591a51d2ebbf3\",\"6dae77e59b43225a8a2389859f6d8a6919696729187f6c10132b29481e0b1627\",\"69dae446646567014d224c05b21bb8749a506d8df6b4c9093f5c17cbec173a4e\",\"e22efb8314bab857c531a1ae7b3ade6d53f6a6241215aaf73d72e03eeae27d16\",\"bc251207e594d57fc98f7b5a5708d22ebea0e1c9c684aa1aa6a60204e91c1f32\",\"0cd261c3d30eeaa3d960d97392009608f59bc2ffb85b25992819226407d460df\",\"3a7d33261b2829171c167bad4bbe314524b61a9a7bb37c2ebf6e5b1107ff17fd\",\"38a5a2a992b778d887aab1f9b3988cd71736fc07dc289739ebcf2dceb6b57d60\",\"64ed30a9e9842516163299ba2a09b082554b31b6c6b657cc8c671dba49979619\",\"4e72307696efc80b4950c4baec63d2e946f10ee7d9ed560b8460a4f9fceb583f\",\"33037686dcde4dd3303bc0885b4aa7e0caaec98d1d0c39d0aa7e176de58a1a4a\",\"7f32847a6cbdcef3202cef84f9fc790e27ae35f50bf005645d21d72147ce1627\",\"64000dd7a7f342609eda70eed0888e6d07cb11be9c4e5732e6ba5a0ea53ee1bb\",\"5d714cd04f1108107c3036b169ec1856f043505cd4474c33bf3c19ad3aeb07a2\",\"8dc3526044033bd87b818020c4177935e5fea29bc009f8239a9b63bb25a72770\",\"b40e6cce9af9d4ea0ed3f090e7f3e6b010a0326cae39bba70d5d838dedc9144e\",\"1c3db7c24b42576aab6e6346b496efddef9a6860d02f982097d0568a24eefcb3\",\"f2d20bdf2f3a4ae5513a31dcb9ff5aae8e81e408b967464b596b2345101ce0bd\",\"46c8d6d741f9ac6272ec3ca7ac2eb44000ae89fa1a7c03ff337036c2e9ef8279\",\"6299972028babf87e69af701ebc49d11d8d83c2dbe600b3796c75ce44bd2e61d\",\"daa7cea7a91271633649b31cc644435dc51de46cb5036ea1a32a524de6977026\",\"65bee8b460de9b96646533eeec9fda16ed57b29fc04678d3ad75e7ee5a9b12d0\",\"cfc6f00dfa1280c0341df44afc2e0d4e05f8eb76ac211f06edd964c3b83956ad\",\"1c1e2b3cbc290e8670397690aa751a2865a4ec362fdd7cda25613d987aa8bc78\",\"add3784560aae09a19c385a482848f9c3804fd194b801004fab7ec5a6504b6a1\",\"ee76ddab7452c0ff53b4d1e06c2ac5fe881fa8af0ec05efc459c3ccb54b683ea\",\"064f79da2f4da0fcba008098b81c895b302c83abca9f0fc0ca6cb28c51b6cf1e\",\"a3ffd760fb8bc1270f103d922a9a85f235a88017eeb8ad3cae0295b24bc15c2b\",\"d42d5b35fcc4e2b87d447dde462f43c4e098d5b000c1c681a3081bc643d01f5e\",\"8d518c03a2b6793fbfa74b9987644d72db2c17f7268164b065d3c0da8b568320\",\"9802abecea24bd661f310f2f407d1bfaef1fa58f685f540cd15d85a3c9c7be96\",\"c209c3e656e74fa9b9f268356f9f63e1c0686ffa7b833e478c90e61621242ad7\",\"d79cb2c1317f8a348dffa3842309c4023ef74d98ec1f626ce12961dd8ad8b97a\",\"3a8f35fb1c4189c59085f552c903ac11c413d81f59b08f716199c59e3c6e3c4f\",\"4019ece76fd5f5770adb3864a71c23f599aae7fa1b5d0cc8c09ebfcb6cf99f15\",\"ab369ab51da9622512db205887ced4dc2411d6daca4ddce9c88ac6c63ec61cc2\",\"51b03fe9ec1ca8a06985730a47499c4f1dab23fe3163fc006e3bb0706c40c342\",\"54720e6f130d347bef92035e47b8876677042144b839a17ed354bf276d3cc964\",\"d813e64a1848bd848e075146d38eb7a11634c0d36cb54721706cf152cf6f8692\",\"74bb510634f65b2bc0e4c9ada97affe9c31ec1a8a4504daa162f986ec897be21\",\"8402874dbe01f6d92d969c339c60b3d7bf1f85c1d8dc337c560bf8d81f9e1876\",\"d05afc754709e6088e95df9717a6d79ec487f545b87303b3f96176569815cfbc\",\"56ea9a63bf7f958aa55c7ed99dce31d9cd18c6eeca2d029f6bc875c032159a30\",\"ad6e774a9bb0a8a1253fc19006d20b3d3affaa4ad35c7d44e6b703c51ad3939d\",\"8a19f8ac52c47d13f4452ae9be74c0c5df948499e58d38df4d19cbc02270be18\",\"66118ef815007395a8a00335cfcb24001fdea23ad7a515c08514a6ba0a8fa545\",\"fe19c2af5717313438147377a67d393db89ab470c6e24956eb46ba8c99640dc1\",\"84399218f676a580eff1f998db6c1c893c20757ee3f8585d61f5833bcbb65fb9\",\"80590225a2bb7fd7b3f94660750f2aad73936183bf8e56f3738853e44a5d333b\",\"f11af9d337e2b433c97078b2845731bb3686dae1a3a918e75de6253e7fd78e21\",\"04000959601933dec40891adcf7c64e7e22b252c8cffc76b74157e93a7c86ede\",\"18b077f3b499795e5d7cb295b1f5a2286f7f221b0c0ebd3929d060c908cfc440\",\"5cf920b1aa1b71892ba93477a21b77689edc7806e73e47db918c3f791f463bf1\",\"4aadc0ff97d0c71410fd0e5bbf0259516bb62be20b98d6e72876a115e197a918\",\"720d95abbc6bf8ac12c13ed54ab8d5d407d249b32d0aaaf869c7629a445b4a23\",\"8d3c66facbad0db5a5aa3e1257b66fa2367e4f0e0e36d8b220707579a7e7d12a\",\"4b1c1baa3ede31fd47b05b326c8e4c53950cfb5533b4617110e0ea04e6b93f37\",\"9a9a07099da2e486de6045d33ba5d20f3e7af2d6f73e9aa301d8ce98f60a8b98\",\"2ff18ff49f3c1c80f1a8af126594e5532381afd81a5a9a0722a7beaabd9b2d90\",\"c7bfe31f4b2f80fa8fffe5b715e9b7ce66a09ca141c43a3ea7a407fc42bb28d9\",\"50449c1da043f8b2cfe3d3fa71011851e8575737b2fcd32a3bfa0c33b38b95b9\",\"b626f4d3a51c67ebad5ceae56071dfec0d0ad3315244da6dde2738ccb3a52039\",\"20338050bf2736b0c73a7cb05b04066622536618f7049d98bdd08067c6337699\",\"236968cc541ca5ca522a7c8d3578754f2682ac580a26887b9c2fcab5c2943265\",\"80d0e4119cc1d412b56fe1cb62660e7041523b5f9e8246ee4a25f4f746147dc0\",\"e476e9fe7a794714cca37c86a58476a0cf73f35a0364c721e9e8ff4f6b0d1be2\",\"0b1217a23e3a2ddbf779acbf998718fbf81f7fbc315588d46dbea5c7732aa175\",\"e7cadf17649a12db6f6fb80d87518cc8b42b5190bae254917aad3f5e5381680b\",\"97782490447aee8a3138e3011ccd55d42754e97b293b0dbad25bfe6679604a54\",\"6137d5fd352eb0532e735db9dba39f01f394c26ee4a8161fd4e62448f183a68d\",\"03f3c09a7e6bc813b861412d5b31434e5ee38a00373fa2967cdb3e479fe65604\",\"f3d415165475b13b3d9c285e5732f03b9f0535b4f3b62a9a2ae2da0f3cddb0d8\",\"58869dc36612ff6f3d9aeba2f51cb1bd104eee528a11cb43185b42a879a270be\",\"b311673bb5357606129525241a24bf1aed0f0067cb8703d8636fb9d63d165d1c\",\"0f80b334684b757b3862c472cef22ebf348473c4463237def5687138a5aa20c3\",\"15cfa00990b76d94183743a17dceef6cf4297382a4792b8c4d57cfca5ff9f8e8\",\"2a9969c389fdaa6388acea9317487710e52d35822e4f2ec8641868019b63ae58\",\"55200f04adb1bd7b7aa942cfde4392226904b09afd2301604b421f50fb0a7eac\",\"b93b6bc4dfc30d1200e8dbd5c92b9a772b4faa23a9e71cb1aec0daf7012266d8\",\"7590d355762e56951705a3c1679b26710d394978472c49e2baf1de1fe5f617de\",\"905af1f9d3a5974b4df38a4c5c1e03f031495d87ba0d68a52fa2a25364ef75b6\",\"d9094caca91a9aa564c532ae44ad8d452e77ad02129f06fce94e24f60e73e55d\",\"75a93f3f51e99554ad8f9952d9c5d9b1dd1c4d9291c7e24f1eeb9730a0d043e1\",\"79a37adfd86cf5312b1c137ae7d89091f75153749e78c6ba78564ab0631a22de\",\"73f76423234ca38ba9f40988f11db5e5c2c9cdfa07c14ee3b9b909218ad044f2\",\"a7fec776810495749b266f97e117e53e87715829fe850c4523282e9267fe0cdd\",\"3243b3d361bc4a52320f0631c743849589b47c27eaac4fa04acb1c2b3973ab82\",\"248f821b62665ee8ecadb5376b547d5a24b15cd9d28a485604a8e4ff7436cd44\",\"16d6e06d0706a55f58f463862a8c4038d0a0b892ede88dd3ac88b7ca28ea46cb\",\"6a27e0f64d6b535ae123e4ef7b2936b2b67b0ae1c62d494e75ddf2cf7589d7fe\",\"8ef78bd64f2f2af7131b0020b702aa333210240bde1e2792dffa83b9cfb64720\",\"146ac271efdcf22efde0408762bf41e37588e9ee0b08033063746d79d2a37586\",\"a4c2b44505ee24a072f5fa1b2856a83ecd3d77b155f6a4a71b851a315fde78b3\",\"e4203214966d71a320b410c7322baefb7e7e0b68757e4423fa1a9d926c54391d\",\"28f543f86fe62ac39356444bbddd8e63fd837a08b56a2b50d32973114a8a4329\",\"c7e3975cbed02ce9adc7f56f223a394fb3913b20d5dc9c7247255568207dc999\",\"e7259b1b67b4e9b25262fcdaf2548fddfdff1e3a3fc97eec2a9900bdd545a425\",\"1a7875f1906e72b16ed34011651969e8ca92f42587d0f559636ac0ae6b7e935c\",\"891b9b69c83ab7daedef1eaec7940513553d8c324a7f4919aaa8021b684afc46\",\"8ad1abf1af0f834fa0192aebbaba4b70e1d473a3bde5b1b8fe7a32e8d7006cf0\",\"a9fa539e3fe25a79b0ad30237f4bd5141ba6cdbc77b64cc11a8e215285942740\",\"11507f6436623ba7db35fe40b0337152fd3019654814297db673952f041f2581\",\"3fd57a7964303a7da572524ac23875b9fbd3f3f07db9855b73ff361370f9980f\",\"daa6381cea737ed8f3201c0064c3c0919fa5c46a5e5bc0e5fb58af6f3ec5bb2b\",\"913c009dc88c9911e07a51e75187b8320bf45a31f45c4bd71d96cefa75864e2c\",\"11dcded515ca9dc6594b4132e8801c141ad8abe9e31a901989d938162e6cc6ff\",\"cab6b09664bd36985d0754eca90f61f056ac74e16101827a92be25ee6798bb06\",\"ded93550928af29b06a1d2640c104bd978b8baa602bca13070a412c12b5fbca5\",\"4e7c864052ce8c8e2f862e8fc684aab2fd959cadb89924f4f8a6b9a4a6d8af6f\",\"8b392f248e6470e3a379280374bd590709810cb63cf6e71ccf10c854a76f662c\",\"81d95469e46ce1f4412b4f59caeacd1bdc6a3df4d23150f021de06f1356a44e2\",\"1ecdca581dcff9d7c8ba1330615e4ec32849f29ead7d6242a0c36bbe5119117d\",\"8c279a31777e21a35f6330862df8aca441e2fcc2082051781d92abad8d7e949c\",\"06211e4ebe79edc312815adc9eb1d8a04c04e7dc7a38986da3c2ec631e764c4c\",\"ec1b158910b1ca2fa0a17808e6353bcda98709ea6cbe7e1d1003604f1f9ccf1d\",\"f363a071701ffd9e460cffc5dc48f08f59a7b81351bc81ca7427e2042f0ed42e\",\"5565103b424f0b0ce71eb6bab5edd63679e96b748d716629eea7ed839e5b6de4\",\"b487abbce126b5d67af1d0bfd9e4b60f3a00bb607ba050d0fc109edea2fa96fa\",\"33fe77ef1720a1d79048293b7b815beb312cd9772e743f0075cb49a0eabc0336\",\"7c371968fe7b357ce5e885e2bae7612d6044d8589a2530b5a0806ec372c1bfb1\",\"5a7c9c8a14e7fb5b414c9c4f557b6ebff2c59ac4796dd6a7ac813970a0112331\",\"04eb2a3db394e0865b5f59a5e15b46a55a8c08dff3e51b3bbc8a619b4335e474\",\"43aa375d55e75bdadc7dbfe95c0faa9f1447d13e43c2a66b78ad51040cad345b\",\"89c511c027f812319e85c6d081ea969898adbef7da89715510f8261d470a00f3\",\"c4b33a0cd9f6443309c17dc5480badbfd3c55c35fb1fceb379493c375604f891\",\"aca34668bb24ae1d9aeae6032bf442443bc9fa370a31546afcdd7963ec4c7b70\",\"736d0eca87250ea95e98329515a4ad39487b56d58c3047b323cf29ad86038d26\",\"47738829e14b3f02620c2b0a2f7a505cec14984ab2fe152165d414e52826e8f4\",\"fc74016095be63063f1c24fc0a56b1661fb1107ffcd75d2537ca3480c6b8cc7b\",\"5e35438758f4f59aa5e29c50cb784a4447a75c8e3f5c5f7c97691fd95372ad00\",\"565d24549d4f6381a17eb38baefe4669fff09d4a72b7e115304d3cac8cf89693\",\"5976d40283652e0fb55c6bff0c84440ced7cbefa03485df68e777a4900437681\",\"f49fa0299fba67577d4aa37356eaa3bc6ae2151391f4950d0d8bb4cb4409f241\",\"e4bcfd338b6be166429bbf560ebccc0f76fce86e2acd61ee128ddcfdb239a922\",\"2fe64850619a324fd3f28535a8a48a8f29b8c3f2f3d9650b5a9f18be2d858c55\",\"5be9fd6c65a19fe98b4781cbcc240ad55ccf1bc477e853db054ad73657f342d9\",\"a1e8d3919431b5f8fa34eb81063fe032046eb80f043951b114dbae0015629df9\",\"6be38c6e096c031fccc3ccf7bcfbcd7ddf7c1ef54c1ef1ffa00af6fffa1ea1aa\",\"8ac8a79aff403a03fb967467752d58f915af2f8c66885657294ddb4e6099d6c8\",\"9bd5836bee24777622625239e7e52d5982854c7eb7420e8c5ae8c8b629a06cb6\",\"af92340d75593701802b5646510505e539f2eefeb3a33c4bc31f3634532217e6\",\"64fc882538012f22b68f21fa9884dfc79409b77883b069b0ebb6916936d9a4fe\",\"23624a339de61627a553c82e8f1f5a0d383e6b1324f8a8bf64c26c1342778356\",\"f0ef5b0060ed842b1cdebd635092ceacbf6cd032be90b509bc715fe2cfdcbd44\",\"81cf7937afbf81202216ae0ad9a195009e54404559c2051323dbaa8675aef2b9\",\"c75e9cf3d2a8661c8ecdbf71bc4ca4b5352e7e6d8df79747e57cf07a778adb86\",\"81836239b1561ea1090364ee6905859a658f57e386777c46f9411259b669e4ca\",\"f387d9f1773ac1991d8c657870c6cb2568b9b915d7e64bb06a5c060ba3c55429\",\"691c66ef69002c446a822b2900e8a6333588f4212a27f27379fb758644c3e8cd\",\"6f60650f8ba9f8403ba43ab6b83e9be76c8872b2a3845ab891c880e265329caa\",\"94fc2a9edec2077c3db8135df7c7ced3294df63391662bfbfe22fcd481948cd8\",\"257c783229eb3f2cf991350f1729ed1a210578e5e5d4b97ffbbadb7252302b7d\",\"2641f2a5f4a4a28cd191aa91a22713957adf3d730e51e1f3155d71b0380c8796\",\"ef531661d3d93558f3d224a7b7dddbd267cd0a770b9b6953317550124c6183c6\",\"4c135def672329b96918f57a6342a5182c6ca8ab5987b8c2cdca43bb8491070f\",\"1c4fcd49d580176b40a02d186b9110835d532c891037bc08a10301b0c36a3e05\",\"f9a855996944598dd5a1d1d34c89d7809999ba6abfa440812ce0520a6c3da05c\",\"22978ba60f82c236628d0cd04bea591cedbdc79aa675abb8468d439c1a06e5ec\",\"115852e8fe10499c4c9f81f04808d3c1978173e24b89b12babb4ffb449c78ad2\",\"711c8b0eea5ed5c2946fc1ed390a89e609ab69a7c1828c77c8d35dfe28079909\",\"f6ff8795f8b8f7a20cbc1cdc022b624fe9f00410ebd57ec7a7f9b22d2fe36c98\",\"154bd2c985968ba9f7650d9fea6b654a196258c3e6365bdc134fe7cfc12899e2\",\"058788c8658e35a4a630219cdef3f40010a143ec309252fb9e119851217d0ec8\",\"bb9d41f28579bd0b58d30ecd98ed192b437944af2f6be1a70aaf0e2dfa072187\",\"311099866ccaf78c1a883381c46d71c0b364f7d547accd2777b858ccb259c384\",\"a5b703cff7b643dbcaf04549b09764960ac9f0c92fbb5281df9b31d6a8630e35\",\"3545745c8c5bcdd6f0afca2ff2faf3b30fe19b6535ed3dec2f3d26f9b7fefe14\",\"e77323dff92dbae1f5be925ad0f2ce5b97150d0833219f47edeed3bb5bb6b934\",\"0a8a5b36568701a6306b6e7502d3dfa4b61c8bc77e4ba83398ccb09745f8d7da\",\"dda936ee83742e46070d4e2a1386785707a1b9fea8599994464db997e533e858\",\"453ce0a1a3f7708c51377bea79ca3e3aa0250d58caf5347297d567dd6611bd88\",\"6f1464cf81643888008142cb719f607f65b111692a2d2cc2d4ec46cb4d4d6b39\",\"284164d17e92d0119e98fd65c1fa8882c34905fd9943abcdd3ff7b14ab541f68\",\"19244a1c87174228430f659aa1d7a80a04c959586b83adaeaa3abe40750c1860\",\"55081a297faf653fa409e2487bec17727b46e9cf3240ed63be8a514a20a3f167\",\"fb45081f62551455c46545aaf0e1c9a45b186c202fc1f3461e2c50973ae6b0cd\",\"7cf55d33012f62547c96b9cc0f09db111a4983d69f3bee13dc3884622468ac89\",\"cb420221fd1653fbd0e60da8d777be33ac80cb33d648b559de570c62616012ac\",\"ea02a6d440b0d2854f1b02c7c282e47495f1293d15d1dfd1d1527e4b3fcfb630\",\"10e5d7370f74c2150631e09dba8e8db79362aa28ecda19fc19e54e0e19ce84f1\",\"4d30aabfa65f5a2c9b98468851315ed14e60786cd2e1bd594276ab0466ff79b2\",\"85120239b3bd88b38ab89de19fae65e06438f5c69b9dd3ed12bbc98319025722\",\"c23cb290719c96b6e043cba4dda4a601113a112d8f820d865dc7d720a8352d14\",\"509348a134e8670c663001d2aecff09de8174acafa9398aab241025d57a5ab6a\",\"8344e225e1fe80548b648c12e9f5aeabbe5ea26a7647b3dfe9a4a96d39cf475e\",\"3bbc75f39feed22d1b49bb45fbe507949c75ec2749b567a1c27807fb61a6371a\",\"e8aa550838494eecdc020e654373ac37254344607d4740d6a4713f4e82ddfbbf\",\"d6aa71d348456210f35467a7458d4a7a8e4fff76a507c71b4478478d7109d25c\",\"5f095714e071379490cd55249dc373c56890a22585c7bcdc2c0e350de55a18a5\",\"1d3cd71119d23be4ae066d98e95ad92d631b63f983d33049b7e2921701cfc92f\",\"1affcb2feecf58759b5e88920a3e242bf758b4245d97c56e5ddace805ad980c4\",\"b21b097adf87595cf6a585b45a490b7179b09d382641784741a5fe90a11d1fac\",\"df3e3d4099d364a909564467b06b96d88a5f4c2b8287afe304798ead135626b9\",\"894cff57ae222ab2feeb15fe823e711bfc4c7c47439426705f751db6da48c98d\",\"713ab5714efd1313d8c59e5c4f027715bbfef2d0018f522bd7d6b361a9caf272\",\"2d9ed492f043e57e66b5bb64aa2d719f648637ab7400a63011763940f75f4441\",\"aef33f5e10acc8c2c823d1b4d3aab88cc0ee2917a3f1e42410335d4df863ac39\",\"45ab3b246350e4bd0e7be54009232b36a57aa10b61d4a1ee9d85181b56888857\",\"e61c80e2f039349ad7a06c9098e5c238812d2940b3c8c1fa8166a1d68845d06a\",\"9869f854c517a386227b83d998eb08732fd3ef04c327a71de3f79a711b4c1b11\",\"2b0ea5a236f0d15563f194c9e3204e8cb6668739d85467f30aa45d397f10a8df\",\"4655256a3a1936928b8d8aa156c326e5d86f3b39b36bdd7406befe660cd3039e\",\"4c6a94b64a7ab0e403a5139b6b5a8d3a61cec51acb6319fb7b7bb8c1e166002d\",\"fefff8120abce755fb83379ac12f47b79f4b95fc207d22cfd9601eeef7762b54\",\"b38955dfb977042db4305d1bdb2f972d1230c47ccde88fc6dab4c1a7382947c1\",\"8a4a0290e3ad240d80743b10975bc3c8abbf34fbd7902ad942fce09782b537a6\",\"8d03a837a43aee70e7fa9c148a054ab7ff32dcd9c8d6187b3068c7a6ec198d4c\",\"51ecb1b3bfa12c36367c9bec1819aedda00663ed0298e84237441e83955a5812\",\"37f7069aaf77be3a1fa533679f40bb19acef6da81929ab75b89d3901ad3b830f\",\"8cfe3b724db3e96b09fffcb62195b4166d1346ffa13ef02ce47f7a3be289f787\",\"66b8b6335dc247a2c014846de948c3ff8abe18a5dde5577819e4d918f8200466\",\"5d5647e294cd8772e14142760afcda2aec9b2c6f809ed4291c1fad8b0a53f462\",\"0a4cf24a5908bef8618b2666ecc51584698dbbbe05b3d8faeb0730f3ea46a7dc\",\"2f28bef9b680c6a614b9e580ca4aaaa91724a63ffa352924d49b41cfa82dfe11\",\"09685664144110eebff61f3a90fba938178198f2456e52cb8fe04ce0d42026e8\",\"25638278274f171644c49ce2a1d80e116e1251f175ec0c9e3980604bf7536728\",\"1d309ac0073b1a36c98a6eb983b1c54c7a5eb969e40aee1ddb4e70abb958c944\",\"9126923f9568b70b34a441542356eb41f9489f6c99681d3cbd8d905cb714050e\",\"86cc066add6572072350b46a5213f950fa814aaabb6ea92ebaa2dee65075fe68\",\"3071dd61874e841214b626499b3e3a97c72f8e27133807742b2e9bf4d38ac9d1\",\"a41f563785fa2c39aa72806c7dbfbded23233303025f2028de53ed01ae885124\",\"54cb3a64c83556d56bd110f8915afb41289187c09a5d9c8f745030c9e8bc7f9d\",\"65fd1c21d03c33d45163efa965f3949f44e8598669cf5ac5ac21bdfc441dfeff\",\"dbb4b0691273758a71ad958763df7d73d312740cc4316b2b8fd67cad711b086b\",\"6841819961348bb0cd7accc777386470fbffeea4dc101b7404320e857969be65\",\"eaa263ffe8ecb1be7fcb5ba88bc96dc172196f6031bd92aec5f8a26d480cdf91\",\"0b38b7c16e9afa67c060cc280ddc4f6b81b84e66ac6f693433fe94868531ff4e\",\"8f77f4c44f955b6d799bcbe16246e17a95629b610a0cf1e7a9513249e7826e77\",\"f190ad22aad953410ecbbfafec22f84e9a8854b0f8b9583ccc9786adbe2c7fad\",\"a85a39e2f1db54da20bd7c6a47858d3db0629f0660bff7dcb0cdc78204a7aa53\",\"86ad331d7dbbf1662a7cb0c621ec5939e218c03bec63e66d552cd88bf281c7eb\",\"305f542128f984d50033094ddaae3fa2952bd94f2c4c6679d6f13075eb85d690\",\"e9fde556d169ca74d9e6b8409f461eeacbb8f971274217af877b06759ee6720e\",\"62e42f7b02b1c91cb3b1f2c01acd89e168c9043d1686d603d302a8154bc0d83a\",\"55eb133840de3b3231350584f747b64a71088deb1135c522d65f5e91973d36d7\",\"783766ce3a6c4c7d833dc32ccf4dd3c23ae3383ac641d6a2b8c074defead94f9\",\"dca72cbd709e81d8ec00c3e420342c36e283436fd3ba4b07c87acb910e0e7109\",\"5e91ea8f1333ed3c59479d37b6af863dafb67d4da306b90f22cd83a16df389e5\",\"b1d67159ca33143db365402c7a9ffc72ae2282b798748a3b3d37ee2689d8fdcd\",\"55ed9af8276beffb5ec2dcc894355ac3747865c420ceda1bb4e718fb0f0d4ecd\",\"887bee83c68da208405346e51614b42a3f0d5e6d32e3973ae307d4f678ad72cc\",\"359d97efd851a576fa6aa972c309e22ff6034bc7963a33dcbfc36713a6ff4e43\",\"1a19dbc5aaab1a3cfd93b815cceea06f1e8376c6db3729d67d2439f4452e0f9b\",\"33c2a64f67ca1e2fea228fb4af7cbd196f740e2a1c557b5a0c4d1b69c51aa441\",\"8fee877d6395a9b45554b180042fe4871fe3a5f56f08f7a2be42ba70f9c22c62\",\"451e501996410aa5166711c87db64af7488d03e7e15d1998e44ac70175d44bc6\",\"75590addb28bb8a0ab2cd002077716aafa83987131f539c119a42dc6fde20539\",\"bbd2c9168ab9da47d8afe24287e114e685e48a1e642aa45aa2eb389d59de9b5a\",\"27e0986c8895486d26ef01aae2be9c1cd2abb82661e3ff467f54c89a072cb755\",\"7c04c49d6d0e3bcef71dad9ab5cac0beb52a2f5bd5cb765b7f51670ac025388e\",\"9c3851ea953c312d50a40deed29b52e5f6753da46c801e2ed921372d87324469\",\"d69c129c6ab2d7e7579149d53ff4ccff52581c4efb29084384177c7d1d619e6c\",\"4352c77ab1b70573454f8e486abc24179c7675c1a794b5fc600539a272c410c2\",\"23aa9359e4784cab9a460a64a496e4ed005ce3d710e70689549c1f03556e216b\",\"5aa9c486db7c39203163829ce82d9723b054cd81078c959f291b80cfb7b154fa\",\"88da7efa57946f5c06306cc38bb77a5bc8f5881f1ca2306337222d3d8ae00f03\",\"03fd77235863ebb62c94183f3877190e3b8da730893c3361b7995091f317a1ed\",\"7109e8567bfc6267e48203dc29e7e323e8fb17984832f693cb289c0d52a03d52\",\"2e0826368bad82ddb4b19f1d3ce776bdcddd7b9aca96336b666eb65fb5b8b793\",\"a7898b0cbc76d04ff67e4100f87cdac3b38179451e35d3e7d7de8f68a28feb0f\",\"1c00b4b2275b30426a14de0778d03ee1d9adcb535bea64e9f8c6103aa7a62868\",\"6c964dbdfb6c04177f7cea9c2b4509f2f2b7a6f7d39bfbbb9cbab72b0174aa5f\",\"cf7c8c9763fa1b60d363ab811fe3078d5fb6d7d8bd0df8ab1ba2a666e500c235\",\"9b47b3073537f81feb2d5a2fa8cbc48c72ec2709e6ca9873511f81141196ccb1\",\"520340edea88ed80e206730f7e9f2cee3fbd13fe127bc87cfe642ccd07fdc8aa\",\"d7e73a147a759fc7a59ea4283e57e581457508b4bb9e31aa88caa97a619a8a5f\",\"4fd70b9101e5ffa9d8fc460e3d871f3886f4c5b727d0627c44390fdfe23064ed\",\"1009ac08813aa3cd507c676970322a734f3a67f011fe7d2acc204e58eaeb086f\",\"49bcd5acaca6e6560dad84e5527f8c70ce0c3e1dbeb128ffef2ace02877955c6\",\"d3318c1b2637284b7eb5f5cf22ec083fb53cc4fa175103e47ab262f526d25468\",\"d0fcc4f903d2c21c0c47bebea67e770e96b6271932a245393362c0fa52f554f7\",\"a8e70b011aec2ffc669dc717b616e9dc18a110d74b9d84c6134d107f64ab2b52\",\"a03b37bf387d630bc3f3d5a0faadeadc4964ff11dd5460c6e2c4b4091f8ceab9\",\"c8492be7103eaadf1eacb73c2ba981bae0bb5314ca2f27c177b5872f69d459d1\",\"6076452c254b06d53330b4b8996f464067f7de0fd71402644f0f6d99f0772a05\",\"bbca334797b4a3ba25386626547a1973b348ae8e207341265b8a4b99ebfa6eb5\",\"8ecbebfeef98d9ddd57a17ea3722562d9bf086daa0fd4f3442ec791acb44f90a\",\"b3a90725e289c144aee7001798041591799dc95933ddfbc3d98ca60f800bebb2\",\"8afbc768698c07b4c855a5b871b5bbc4427d681dcad724204f240722f1823e4e\",\"57ed8e5f7573b3cd7a45d4831c8ff56c53c2587962c4e6e32cdd8f17940cf8a4\",\"0ccd00926759b0b4bc20303b6387ae01fddccef08d0e791d964c70dbdd154362\",\"dd6a35384258bd0a66a4efbeb819e23832dbb8b8166fe3ec100cdd8af492ca5e\",\"fe71c13ce9a48231e0b15cdd79a7f367465bb41bc09939487494a49c343cf5d7\",\"be44b68f9dc203b636737f4232896bcfa687e044c1c6a0627c45417589fb4a60\",\"a50945209cd87d0d6a5af82ad121cb52e16e0fafc2a09aae3b18c960f9a9b585\",\"ef410f9d360c5bd0cb3ab0d278fd532e8b89bef12373e75e5055ebfc8e59e7e4\",\"bf6d40f25cd59b18d08ba62a0e655b612d307881d5ff5b700d5fdeff10026957\",\"2c2e7a7cce33f70eeed71f4c6c547ebf6398d4d8aee33c39a41ac5343d922a3b\",\"650ab38d55e22f75e4b2e8ec41b2fe1609598efc145451c52d5fa0019c5eb548\",\"ba9a52ac12c4ad36fd9064d731acd0c334b3c217525c0858ffe16583c3cbfe05\",\"1c9cc9d5fa30722dc45977af29ca4aa104cc87be0e4187d7854840e17369f85e\",\"25d1230e34a250efcb4dc83430f8384793d5abb33cce6b5e9b45323b2ee1060a\",\"4717a2fbec0507d8ab0b10a16a33ef777f3f65d65c336aa95c2036ebdb92bfe6\",\"7e89aeab501bb7b39a3fe86375ff8a38452e126466115bec0d2ef8b2484ddacd\",\"07960f754a233a77a398c21733a3c3354976fcb528b9486c6f1f55e42eb509d6\",\"7586136c9cb1fe09161c171796a74a7bd6db74d90291a6dc6f38dd00832995ed\",\"10683317e2ed2f71611f348b89bd7610998dcdbc9932f481f182a0be4b8e9f38\",\"1036e329ddc541304a4736b2f15cc75581f59337a63036244df8348c58ffe2ee\",\"06197ee67799387c84c51bab57916e42439e2018892bfbbae6847a5a794bdb9c\",\"6be4fa37bc5a154348c743eb80483a8d49c96086d075fb843e38d22f72240b16\",\"7010d2840be3df0668c92773c1fd204643e359699d132a2d8d039cc99ff0cfd2\",\"6ffcb09e65434fe5f90ba51807b8e7814fb89a9b03ce56a63cc61734dfc05c01\",\"01e1b117b303a4d981885e1c120e345c1416daec308e4b134ec506663e11f200\",\"78090e547dc9d023879dfc61b9a955ca48d1c3d0827d32aff226c9ddc62078a6\",\"ce283043708e9442f758e8bedd0cf18b356f67a447a591f6ed94f3bb33c03747\",\"f2fb367c34e69e6611f6623d31779293b62b378fb4fca51f667845fe240b21aa\",\"fbbc9b482e7eaddc099d770ad325e8cedca2fe6cddae15c0e6ba9194192a167d\",\"a7389b87d5f42ea847348050778600f469a1b07b23929c1af85bd8e92f5f4c3a\",\"08bf1de9c5a0ceac2f474959fc8f7c41395ba6c31450dba002703a0c7facc581\",\"d18aa7b596be010045ecc9baeb6b376058a5976ce557b2f156a8917a052775bb\",\"0c13612f04ce6fbf5a38bcb815e8f7294a6aaf3a2de9d0117f1c1491d6cbc40a\",\"17fd9e42714750be9786e663986b75a0e4f1ce484449792db16fc40a4f2b90ef\",\"d9d6d66cdd548875c3fb5daa3cb624315feaa5e4b20f584ddefa0547f8dc5085\",\"af3625e44a97908448d2ad8260b53926f9075d93741ad28830df00f51fa0d5c9\",\"8277601d2b50cc010ed31d1ac7ce54aa440c5adf1254e14f6b9370bdd21480ba\",\"3aea4189da60edaac122524bc9e132cecc52d58fdb64d9b0964409516f9848e9\",\"137e7250643e1204ad532ec486028ad0fd1fc1c85aad6748ef3ff673803c3101\",\"ff8723a99640d72192a762d6356b5c05c7ee01205a9b1b2a0cd51fae53265b2d\",\"4895f0c8b39c20efc3d0a255d910aaa6599248457ab7c6f85db3cdf1c98b1ab1\",\"6212b57e10486f57525b2eb3ef3cf4f46b8732a1e1a006a934d8a674f42ae572\",\"8a1c75f1e35fcf6f16f4f05f31d2c39ae49af76656d799fe83b725643daf9f26\",\"954a1128f7d17212f250ce9702065e97aed1f06e9699e2a63b4b31e69c0138ad\",\"6907583d658b5b646b43f8b2e934d561747675e3279cf07590b4445858549164\",\"b3c782c0ebdee4b49f871a9b0cb3c5cba924efd37f9209793139987994f40d29\",\"d048ae2b95ab3514bcec17f5da7d181c037cce71ab06d8c5f009e970a713d211\",\"12cb7c0d7067a08cffa563960fcbccebf6197c1aabfa2a813f9ebb1353fa9153\",\"e42beb2613858a2fad8f435d44961cb5c8e532c6cf52eef3281f22170a8115f6\",\"9fc121dab09f27bc0d9e36874f00c8951854eb14bae27979a76707b0cf9a7b9e\",\"d0fa25fabda5a51f91f8e108b7944b2a2968d83047968f050cb108641653de71\",\"4b4d1ed8be33b61aa30eeed0bc38bbbecc88bb2a0ad6b761556e33ca559baf0a\",\"75fabfa2f8c6d3b166dbfef37c9c8b5350109da83a989af2676e59e0a44a33b7\",\"c0bc5f8f7201b26a9f122eafacf227cdc2aeabf696b8b7b1bacea3eacfed3e6a\",\"08d8ac1ab235a3bf3c81bdf6f7049911cb3aaa8b40bdc3138f607f01001f755f\",\"2ba88d6ed8e8f0d81cacdc48c85cf9af5a596b2083837584799add3eddbd3ef1\",\"4605bf9da238d9a054f47d3406f972bd5305813caa966f406f361fa4bf0cf492\",\"a3b87b6ac36d58583c992bb51b7d25ac9296a27fb6a55716c04e4ec91408f53f\",\"85ea15cd6b170d0a6a98408ccc5c56fe7be371846862c2a8b7789fa1a0afb254\",\"e2f7a83551b3184729a6255e4b84297782a1ff25da9554544739a86aa766a67c\",\"aa9ba83e5094decdd9a140d5f98a03ed842a707feb8eba7b0a9dd15450f3acea\",\"750acdc8cb462bbb68834d96c6416d94484ccc848e9c5002e961d13f2883848c\",\"14f526b1c3076ef249efaf737c8014df92f7b80437871e9b1ca516712fcccff9\",\"da396e0a73f70368c0d876b22757c15fe340b49813573b8747c7e4b645d229b5\",\"74577b04a68c637163fea8169347ed4077966d7c5a6480ccac0f48fcc8ff1c75\",\"1127524b37f2e9691f1c48d6b0f95dfee158cb1352f97e50f42d5ae4eec8ea6a\",\"8e33b89538ff375a35133b48bd253ad96a1eec3bee16e1e2483fbd9e04ca64d7\",\"273e36b25216f1c2d7accf91d4ef83aa6a94c4ec6b4961b0cec6e560bf8f5aca\",\"25695211e85520a971e8f527b80e6abcab1f30fd8e83770496c12f4925a5045f\",\"1318177e7aca708997a4d5a876bd1f4f3d3352688257581bb300edca48569d9b\",\"4371f8612dfada145ee74d09a3f136e66d5a8fc1cbb2179810b67771a8d20cf2\",\"1e654e9e803884d3b14766070de5b731c5d134f96c8876e7b6843b2a42847e50\",\"42073cf4a302e7135229165b9b2f5fd6c53d43a288b669ad2bc51511f0edb71a\",\"41d00d9fbae3524e392a3d5e8facf982175f3addda7d32bd11104aa4948e9620\",\"9f60b433206398ae123a6a30380f7d9f14532051739b5cc6c34ee300f7014e3e\",\"5a8ede6ff12634f0ca5fef549dde62a652b3ca95b9238e4ea39e9d3c0bef6e1e\",\"318514d8e61d89530fbe4a003d46959c37bc0abc3d4f11d1c4fa3fd14f756b3c\",\"18761579c4d2b7dce8a8ef3a22b778cee847edb74fe178fa7104a4c5cb18887c\",\"1653cfdebff9ea5031a1dc1b093094450342bb4a8224472f9042aa3cade6e1c1\",\"344bf230fefb03e7a5b9cd9f4fe21463ea79ee8ac5c2f7cb3b0f6963c5113184\",\"8ff198ae5087c1c4261f587403e00da0bc571b47f6fef23f6dbb3ea89b0992d4\",\"d9344989dc323265e4607292c2b2a5aaf4d2e7c45f6d8dd3200d410d1be8433c\",\"0b78e51eea2dae42012f0378f7d4eff31f4acee1c82f6ab709639d4659a1af2a\",\"087bbfe333359dc6e22e1a12f2f54ea89b2f8913a39634bb87ecf4a85ed33c0b\",\"b69a3ece081c27e736284eaed2ce4bf53b88d24fbe8b40bebf2eac485e537504\",\"4477d3fc499520f8dc7a2d3dbf0d7867316f6bc9709649e61227a4ce48bf9998\",\"3db63be0ad147161126b9e7f95c3a7152a96958a311972534ddd765eed24b40c\",\"953dc04fb84923f3a032e2922cbf0fa9332479615574d39c9236a8793afc326f\",\"ffd1e7cb2d0c23dd8509b0ce161a2c96ba684886f217111e1885fbb24a19847b\",\"422e9a83e6a003f9330424660e9943f719eb4e0c93a54276d216619b4adb5360\",\"e481c455b7d3b56f2407a39456db6d994d7cda36d601d4dcd149c41ae3f68dc9\",\"edbf421cbc7d5987e106e57cf42095a928b38a32beed99f68d398fae69398415\",\"036dab7585c917ac0a5e30dd3933a9e3e9940d815d7f31c8720a1dc9813aff4a\",\"704e3dbdfdf1923dfe72b86df526702f40fb3011b3f7fa2c0df86c0e8df4627f\",\"fdd39b97f73eb98f658c96843285f0843081b1d65100b6a78e30ad383cc72ed5\",\"4fef6071c3f389090f98f6c442c8b4e6a0dcbfefd59ee01ebfd9b88fa331ced3\",\"5478a1da427251e629b76d5b0829f7bbc1cd41e76db22fbf42ae90cc239a1192\",\"3c6389b54ae075e27c31f6fcbc634470aa286f2985c4dacc8a6f1f71b4a4dcde\",\"9851fb5838f0c40f0e683ec06e31c9f8ddfb72657751be5d9e06703cb57885f7\",\"ab8a196cf0abceee59e6aa9003109a1e336cebfe2f7381ae0ca9fe2e5c5bace2\",\"38714409927a7de62888f6ab20e64c3fa774daca31b4ee38cf979a6bcea1d5be\",\"dd0ee48439fb48d09b3baa2819a650d2df7a2b71c3ca1052708d4ead2bf2bf53\",\"4e1762e33721396a868f3bcde9dbc34fd8bb2e139a5e8e4326cd8bf5f55b329a\",\"073c6de1e8e0d29380f1906d871b050455b7454d988f8ab660cb65176e06eb9f\",\"7f3ef052a55eb9974cbd8cc994924a62068b54801bdf4d0372cf67dbacf1ec90\",\"b203939a66b85b5b490df64530a705bd898c02cb1a5fe460a03903248e50b594\",\"dfc20699e9cf7ab8a0c7145adbcea6db945c6483134b9e93a60cdafebabc6700\",\"17ea9890324c2577733ffce99b89d31ae37da8ceda0ae929afddb0850c34761f\",\"2e4d01619001eeb559da368aebc3ce3bf40bbb8ee4c8e7628950977990ac9144\",\"78567d009a0db78e0dc71f40e7fbe372da7925f75294c8d4cab4fda4d556e78f\",\"59fcade47dc837e422a714750f7d1e324844c65aedf80726980345519935f4fd\",\"cdd0abf7cea20b2f059fcbf3cc9d312391a61d3f8cc095649ef1359b4acc0d81\",\"27bd56fb62c2e5a9bb0723e0afd64ab24b95eb9e840b46785cd05ba09172397b\",\"7da3b0166e10a65b701e5cf4b2098402dcde19056cc3f4cf62f2378ff97428ea\",\"59e5abf42a1da6683c1f1588f2f51b732438ca93eb76885b3a7a05dbd24d0933\",\"e97e641bcdb980f4ac02f67fdf36e4d4ae4e66b43f1076903817ae0e620dd040\",\"15c8d5b4950a59fb68e68547ca3152b6045c196beac11b9af70cedd08927a668\",\"a28782dace255610b2b24075080f076975999924ed55c2bc610f76c3bf6a3126\",\"45eebc6cfe660dee167cd3294dd90c6aaf6b527062664d1eac28dcda66476d38\",\"87e30626656bc315d75722848a2d0291719a594e4ddb2cd50dd0be19fa5767ce\",\"094b0cc753a5475f92082d1746f6cfb6a821d6315ce18f9573ae43333d2462d2\",\"4130398ff42d9d3e44f5389c36744eab2fda2f03c3dd7f52a44a3a364a57992d\",\"a5dc9115899b0a82af264986b61c69215fe5544a6b2468cf33bc21b73b47be61\",\"a77d98ffea1e04e5ad1d4cbb34db967f4f226853e4eb2867a31a6753a8b7deb4\",\"91004809293b5b086590e28959958964c15b71453de7199e79e9a5ed3d906509\",\"c25cbe6452c4b4b818b7cda71bcd10b2cb0c32314448ba3b81dfb9d9a584a4c5\",\"3aef2c10aab5193e90d808b300ace80923bbb07e44b69ea39461fac911504353\",\"bab73f982a26068f3b3a31dfceac4e846fd52c3db9a748a8db32ee045314f66c\",\"3f3f1900341c3d9a11dfe44d8a8f01aedbc45a4f87ca8511a3ea00baed67c6e3\",\"d2a6178848a4ddd8a5a1d126b37fe1d15b86b89e1e9947bd5e6dddaeae564a5b\",\"f2c287361d01e2d2f826207c7305ed693a43252d83ca0ef9d12053bf9d5c8610\",\"9e44df1222a3a82b772200a41a60cf73c8cab423fdee27f31479315762b58cfc\",\"629e22a5529303b8587cd21d7d9a7bcb25a2d24fcde3c3d0abc7edf2a55aec5f\",\"91e9366e1af975baa5d6b522a5700671e05e9210c70228b445f6e0540f5b3814\",\"d23b1efbf68f3f8cf7ae22e98a38c384ae66005faa4a7f5b0d4b26b6fe5fe387\",\"d29bf486e7141e8251884be5ef166cdd3d66681a60de26c93a834e16c4de92ae\",\"69bbaa503e5dc622dd6920fb2685e5a1ca7311fa2494929d9b083a30d9b2e84b\",\"ffba1c0ede059424ba6f4795d106804bec075118cd53b7a8f50320c43fc0af58\",\"6babf2b66244b418e40e361a1f2fd1e5366318b62a97c69cd00d4768aef3e86c\",\"47b0532b6f6f8ba03c08b291f8bc3d0d5afc30583a8105e920911c84f39fba80\",\"f4d69c56e3ddbebbad5da40facb816ff3ee282a06af54f908091069b4794bb9d\",\"10d0a2aec8f7481e6899ee2f2c2968f7226e5719cc93def4fc011ce2b253d9c8\",\"eb2ea4aecad569b4ea24f0de02beb419aca891645537fdc7f066ba4a78ca3d0d\",\"4972f4d1047a1b954264ea7dd68eabf368b76a86d42a2ccae15303f9d3003e0c\",\"aa32825d1074ed1fdb8545fec57400b3fb687dc14c6551cec094899b9c16f39c\",\"f7498bdcb2ae8261c718bc5561a6fd70877a0e1ff2a614e9f55aaa8a0a810b6e\",\"98f2aa0c8bafbcb7faded9854e0aff06bac1d03cb838a588c99dba53bbd1f29b\",\"63cffaf15767b08e9a84ba8fb8f6eca58118879dbefdc14ed634bfbf05e668b6\",\"e1292f76e21d88b69d085392dc7f1929ea6e04b238b6af457b2a3b5b3c1ad738\",\"ebaa43d453465e8b24ae5f27cf349d988966b5202aa09a766bb333bfb1f35e48\",\"9cf2081f4961ae19fb427e5ab736793382ba9464997ff6cfe4c0bae476d3052e\",\"32dc9a4cd3981e302f49417a9497ac4c49ed6095f7a14a0fcfa08f98cdb07264\",\"c5caef1d2f63c0df5835ff80aca83375ae6550f414e15ed26a29e44bd4996ded\",\"ebb9760f67784044e4966dc6a6b0fe251d6e755d73798436fd6cb358574c5aa3\",\"4e95e1c8e0b0eec86cb811b3c382fd43c9b60f5e498baba467788901d466ad31\",\"522c26b81bdf68b5095651a4a06d597683aaa73afca3802d21e6af5a659d5f5b\",\"7e13dc49169ab61c08c4cf6a787a64285a898f76271d3ad9f0999c70ab533b35\",\"bfbfcd4e478f22858c246b4b70549716bc944e6b77c88c91b0593806bc62458a\",\"4ae9e0edc0e2f670adf7d904cef53f0c17a1e0faf1382208023de16949cba23f\",\"38afef864979a4d0055f14ec034d2e542507e08a9c45509870b121bb018935fc\",\"a0961e21f5c4b5d27cc9fa2923d094f78af0af2d0e9c17688022120b53c95eb8\",\"e9b6fbd6c525a36f0e9e500c93a966b0cd68f2aaca0bf279e02a6847fd5fd176\",\"0b498ea770c2e18d0ed1564fe6f40cfa38871e1539fa527a31ab3f7e7d0e17e8\",\"89c9420c8721ddc2152ba599d857125b1bdc359bdf125dd6ce1459ab4fdd2abd\",\"d723bb5ceb4985c5368c8ba336362e4af486a02501c469189e96a60cda5a4884\",\"ec3d5cf12d105e74a814800cb6e1406c800dc965aa1d4fc28eb923d05d2ebbb6\",\"c3cf0220b421c7ffaeb853008d7d956b0a02b34f9eb71ca5dcb201e702b941f5\",\"cd7356333e315ba3ac5160b40a1261d73c2e5347afe5d8727a4e2593c6487d97\",\"505fc6a1281e70d4d2ac0b0a31028bd4bfc919797f76cea9d4ac94d8ec9b68ef\",\"e11c474be4c5825eda00307c956437f44ee249a036daa4a667c68caf5893dc8a\",\"c8a5e1af0bd36eaac250f5452a57e8b7688054b1ea0368fd04e4aceee7b2b2da\",\"8f6a36e87a1873f42bf6aea9d9c017d191ea4951cd1dd7b8b5b5c37213e1edb3\",\"7f0f843c0ac79171a56fc11ac5c09d9a4a413c4a41f7b36d4e2c56c2a0840d81\",\"22725480b57b60b60e99a3449f11ff1f161a7b36a70b07fecaa962dca995d9f7\",\"38679e2f1a4b74b4f20e370f0bae0dc7a5f2be10b227bec85d161ecfecc2a0d9\",\"c52b94eaf1ffe3c99434c4dfa62959d51643b63c117e30fc391c2a0226f3ab2e\",\"100ee1527c3b2de72877761b59f118eb1b3b8499101111153487f58d3abbdb83\",\"37071094c74f4d2685e77c18337e90dcf557ef526e8f39d21a58c1ccfdd0082a\",\"cdae5d74c4d08381d2d0ec90a5c93f25775d7616a50f26a674eb240d47b4778e\",\"b199450111b38a9d2c1cd99df704a58a87cb8ef69dea772346bc54eca033bc19\",\"d3c1ce4d0f63de14728434082971dd5c9df10557874826d762065853a8b85b20\",\"75566d7907d0dac4171c31bec5d8244833bb1b7c1c32fc7eb689111f7be7e687\",\"29777aa15ab30432b147004103e59f6836ddbbf4290e3643dacd80fdbe602c42\",\"d4f83387739832174dc4e427f6d7193bff8f66e565e53f0fb58b26f8d709caa0\",\"4361fada35474cc6728a754aa2de768ea6ab0c2200a16fdbb2096c35cf7398e8\",\"13d080eb8c68a6a7b422c923d3ea3ae34c7400f937487979a68caa73068ed943\",\"a411523115b8b72fd55ac47559391b9d220eae86a5cf6ae111f3137ce40287b8\",\"bbd392f8979ba1be53dd96b96a5576124ad420ffc9467258d24fe75f391b05f8\",\"4ff92888159a66eead796b15d03c7e71c67c3cb8e375a07ab24b726c4cb3e3be\",\"1f5af80e26a144528b9b56d96821477d5cf039259be26147ed4a6aecc442e45e\",\"57fb6c6648753b72e5e549b9d481b9abfdc3043219b3758c9a0e60cf4f75abe9\",\"19f94bb87d40b1a5ad98fec9faa7ac2d3e94f2f7019d819f207b8da920d71991\",\"8340b692b37935327d66a9e0b39d25bf7957df650e6230cbc28d4dec31625ba8\",\"1c4de36a4413d3c0ea76f9281da63b1d1435b9e45f6db59f8934571bb8cb62bb\",\"d4a407ddd6911d454c04b6abd66520dbe35161d0a19958d2d15125ee8e5e670c\",\"b017f44c66293fb09398a59f9444f73cdb9c4e8b90f60cffe5d8b526cbc47de9\",\"22c1c66e994110da274fc746f056c052e348b32b7c0abfda5d83d342ceb2a307\",\"f7108219710488d7fc893f9d0bd291dcdce91aa8b026e4b67a1b0d755b5b3c92\",\"325db00787ed40e47fc155c4ea5d6791e5f632144dbb11946fbed5c72f9532c5\",\"dd848128bb568454435ad2d3b19ff7817f951e39dadaf71e4d968883d1db8210\",\"36fba4d9feba64707e3938273c4746bdba723b4d10f4765ecf844acfb5d4390b\",\"d98368c31ce0b0ac76b95101dec8f1d2e90a4d88ed48012722937aacf3b2738f\",\"74957f525e24b238cf77fa2d608b43247a75972d4673885d7f22378e7178226c\",\"e4a4f88c071210cc0a2d8092991a7e34d60b160d3a07f687d9a05f4d2af5dbe5\",\"6a8778f469b12583b6fdda816ea6598d961d3149db402e1676fc882332f53819\",\"0443bb1afae1c4e9783cd0ed40ff605e2ddfabce0ed3aa3f7d0e5642cf6001e1\",\"82f17dba533d55c49bf09456ef7c9f9d9e7c2787453dd3ee709dc8fe89ae8ad5\",\"c45ebaf86fdc97b321061c763eb197461e8d74bc1e4a89b339ebb2f98bc37201\",\"0d50a9738e1cb88df38f867b42258a9e240823f04aa625b691d96973735168c0\",\"581d8f8942232c9ec0b18ca8966bb2b9e1e19224f3eaef89507f46d2117887fc\",\"d615a19f248e35e8fe99b6dba5adf5c81aa0d60188dd0430dc9c475d82ee3dfb\",\"d9417e7f87347fb94e3c2ff17744612e7be370e883f3ae3ec4e17d9886e5b8a3\",\"9b47bfcbe8e9afe39c73f83396a13aa461a382776543ea6d02094300b680fa8d\",\"bfb268fffd17ab501a4f88c132ed337dc4d6f0bf4333a048e0053f47f90f5811\",\"3ba23ef934072d92c4e5d088b61a962641d8855a2fe992a16608cc27c8e8ba85\",\"f69ad7bacc0e3a7a93f32d87023d71c9a7bfd3dc9555ce8ac1ac0bb063853b5d\",\"9a7ee489000dd1f6d99a7281f0a05f657ef38d1509ae10f1ddd7e155290ee97b\",\"9f7e3e8b1871474837c2462ab3c0323ac47ceb9951698848257c09ebf885a4c1\",\"cdd51948b98a40afe5d291fa4e4077ed9a383fd92dad6b6ca1ff277c9cb17708\",\"fe36b85c71b38ccfe77639430395f56a5bba89f3776358f1ea833e7538317f83\",\"edaedf4635f34dfef232b57999b808fdcec74b6dfcf6211f63a4266f00550066\",\"3e57bc6854199c94612d861ef9fa5e6ebe9dc6e150827831d5a1ba81fbc046d9\",\"c1643045ccbe35c177e22ae1c554793a36339db9844de3393f7389a59b50be82\",\"3a46c705107dfaec23cd8d60811005043cf2d76a5ec780aa64705b275fbbdc6e\",\"6c8a814e3d195e2d4c0b2b38a3de0224119685ab8400f74a29f45a72477f078b\",\"fe65b28f66103896985d2f8289d8c9c861add1766e7e698bef48ae4c7aed4a32\",\"6023caadc9504180f3cba067f084add12d4dd7555ade47ce3793557c80b0a26b\",\"a3af58f85e4864d3e687004a3a73e1af1264a4b29c8fd0fbc784e1a2d1156abc\",\"5dcc2f77d3f1f5b6603c3057f79ffeb190148a9ac1ddc6623b6ab1d9d95cad98\",\"b5e1103af7e6ba9ff41b7e77f1a3fc1d873ff6f8673bfb7a1eaf9a4328ab35b8\",\"cb4d9547f67891c2190018d6b68c9e75a825697e73f930c430c6250e77a8a53e\",\"080629abf39b9c740a12ad86b5a59440dd20294b58a49b38df36db81fe47cd90\",\"075f47edac1100547d4569bd875e017b7c409c8ac7e4ff2dec67bad0929c7ea5\",\"bb992c09385329c195ead9549f2ca49709167473a3ad662d767f9163d7866d7f\",\"a20b9c53bd3a2b779fc62571814462f9e9f52f54c06278524611a9fe0203c131\",\"279e8ef447f54db7ac13e4e4e66c3fe7b6d0a7fbe120a351abb17b8bee11b7e9\",\"f9db8eb51be905683ea734e17ee7f6855ad8c5f6fc3264d0b9fcdb4a85c3048e\",\"dc8f2d1057424693103294bb932f1a505c65ae8a1bd9ddc297863a80b7138015\",\"57cf74db63d4f72ea87c01bcf6bd9cb4974523d78966105f4b45a62fe65a003f\",\"82d8a55ee948c507240445274d5876efb113507ab8f95699644e0c0cd17fb16a\",\"0a65429d795e7d37a911688328b5afad7dde7588245e137cbb81e927e22751fe\",\"0a4040b5f9b0ddc7aecfb36d9ad6de94596528d5cb0b6633e173d9b2a9935dd9\",\"dfc1fd9b1640787e7b17e96a0891d1a0f937dd1be7b6e6a09fe6ed46633e9384\",\"d5c7107fc3356302711b37a09f2004960bb0e83d109601aabbc998da74181cff\",\"4aa5191cb2b243ed923a621f34a2164e1abca786d28f3c6c62d7d3b0693bdd62\",\"606a2d9a5eb4563278f792da7c843cd0a52e0b6aa9cbc2e465fdbf9b27185386\",\"c12f3e12768dd6369980718169e0224c984eabae7538ec7a4f2d5cb43b74c597\",\"0dfa0d57c9c3b283dc02d7a7590e2513d8a8608384789572328fb94c415f5784\",\"b526a58a97a27427d3b3be738d2a72a1e81bd15673afdfdfe889cc609698feeb\",\"9e14da578f97b7697a51a415127bf23919ea8ce26b7f30d46e57c017972a2c1d\",\"c6cbfbd7eb8bc1e9904d32ec4b7ff40d4e5ccf0152d06efa409a0fbf03b74d14\",\"3c9d107c755fb5a3f440605be3f311784a32cd3dd3f462afab092a0f062db9bf\",\"224e7bd562da1b9cab1282c2570049840ead4fcfa8d03e4ddc195e385d72cd14\",\"f474a24ad51f5db4b838dbac5ba78772df770e3389a43795976d79d70b733605\",\"660f7151e6169b235cab1ccdf9cfbaa9f47627a5800569984ecd8ed74a809e7f\",\"011306bd770e58d7fb5b0155e278e81d65fd3df14e4cd7bf174d6143b9acadbf\",\"5f0ebf325d1bdb63df6032d5ddac694b99c9ede011a4d1f833f12eda3010c00f\",\"6730972bb86eb902b59c873c884fa6e2f58ebb14bf182457a2b8c7be3fa54b94\",\"33581c5ef8a49643599983e582f3f372f28de6f5238965371ea8dd25c63e7f4a\",\"d8d3880d6ee32c18956e4c6048eb2b598f4a318cb0a5342cdb7444e0d83d9326\",\"8bb6267596bf90bee75f6735f6cbef81014aab4b5ebfcf2882a8495afc53ba94\",\"6bf95a8dffcadeb35b5aa1347730e893c010940d090d2013e04d30c84769ac9b\",\"00440d0a456ed94d2af4d051ade5b92b6b4ad7dbb319c4069108ad1157a42bb5\",\"72fb1faa353b02db36fc3c314069ad3f502bef445b27c8c3cc625f9231295077\",\"8f7cf93d071f59c5fe2367da4feebadf0a9190cbcb270a0ecd24452beb1516f8\",\"10298f29e830c7676607f92b75d639070f4426ad642ecb3966ee487b31fea8e2\",\"dfa21fc6041c9699baafc530c5fcc5437d0e77030037602183e2cd46e5ca446e\",\"480672e53855f84aa6935c803f0e28be9e89c621af7f89e9397ed9bb2f37fa8d\",\"02669878d35fff148772fad36ce07a4185bab49d10a9f56af13580d3c3e6920b\",\"8fea4445215e5aa914cecc707f48e646e59a7df7c5a5305dc3f33c8020ccfd3c\",\"63a3d20939a0984483749cbd46866057c2b55e4767b2cb00e40658bf515483a6\",\"df779279be909137281d5fa76e002164e9fce82ef7b568d23fbc7bbe2f74f7e4\",\"3559e3410bf71661e2044150b8f0b9dd98aa56f670a90823b223b7fb3830c8fe\",\"78cc1b9cb7e36815ac437e96ed34dbcfbff42dd43bfa0d880417e7f206302d28\",\"b53dee36ab2497ccd22ee4784e95aae5b69905d7a820a67b87f0676b9afaecef\",\"fe17cdbbb0da181e24b6e1256128d6407cf43e240a400f69ba58e3f981f02aa0\",\"07258f984609fa5bcea4062deb8ad9e22d9659a9a54bd76434fd25f7adba94a6\",\"2ca385a5eb2af24227d623d7338279fc14f84561e4a4c49ef79e481362c66784\",\"2c1d5ee756f682b688696d99be30e45ed2cc4b508a7bc1738763c8864d522dfb\",\"e20149805f74e907179a94b1261794940879fe4e11031cfc965cda9fc8895038\",\"dcb618fb456bc6bfeb4223c0acbc2f3a403ac4eba9c32806ffcad4488c2b3925\",\"be5a89d8a5360dc2157365320840ed0c126b0ddfc31ef411ff715ee496b9bd6e\",\"33cc1b00f3a5475971bab9b869049d1e8443cfd50f550c7645454cf53eefa229\",\"d625477b533b83a11dc5582956e65f7ac233eda9fe4ba5681b8523a987a3fa28\",\"1aba4a1576b2b1a2a689cda9a16e85a8d5033154d308a59c04023b8ba82c6bbc\",\"da31898f2dd7132385f2bd2b4b25f2b32aa55d514ea7ccf0d94133d2c9128c03\",\"088a8494f8e71fce2051f2df0ff210fe7cc0b40cb15f6deac41285b6c98fd987\",\"97c3e5acc615e7e4353db50673b639f5941ea695a948bdb50ce58fc7a1cbcdf2\",\"385ddb04ab5afb29c62460d825d3249ca3fc05457505759a3f0c84c7c5af65e7\",\"79f5247772544cb96a599d352f14a174c41faa1979380ef826eabf54c7feeee8\",\"650b930713f9723f3fccbc08a25d86eeaa3b76e8803316f27e999f29336f7b85\",\"9acc61e1f9f8b1cd870c71d6345dda0c46c19639649b9ef4b2c4099786b3334d\",\"3ff1c4b5b337bddf955920fc11297e0595d39713e257144aa99beda3af9241c7\",\"8bfd59e9253c1a9d97746d6eaae85a3d8683344f9f4ecb93f211689781244f6d\",\"2f3269077b1599582a11b9f1e894fdd99aae10b16e6b61075be38329875a9440\",\"d4276ad69409abe53d73c344dfc9afd9199026b819dcaaf16c7871a6a74a7aa3\",\"4d01d7e62be5593e1d77afe26ccd9cb8fa18300c706a0ccc53d7087f56c96ab9\",\"a36de0f95afad510e5ff4a5c3b7953af4099ab39883ebcb25fac9cfb1d4e6e8f\",\"ae90d1177c1d60e9a752be56d48107957d5a8b6d8821878145f8637c48003a86\",\"a8d984f55ad38851b2b9899d4f80afb73e53e66572895255bc21f4b678fff1c4\",\"40cf3895c0bb214b74d87d1fd755e54b92c64ada50044681ff11906fd18ffe0d\",\"6d3761e78f9da1c824cfc2e6585a4448f3a142e4ad82743802a0c26ca05ff60b\",\"be67a8e02507b1c985dcbad4e6f5aba1af1a549db08b193f004975bb0fbbf7bb\",\"82ff6ef25cb8cab80fcd20d5a9aa328d6847922f0b658d08d1032d9a4c1e1ae3\",\"5376d4c94f1b10cef818725d910fd86b4673067fb7cb42361e5532b2cabf9583\",\"a28f76cf46bec00b0286f902951232341bfc5e235c15327282cd510fc5cebd28\",\"2fe26fc93a6190385abeb37b66ab457d3f20d0c1ae835960d6b905c2dcd3de8c\",\"2b6ec5e6d2c88875848e240a234209db070875fad75e8b16465bd480fe4dcc2f\",\"311b320208f1e297916707a155299490b747dc10a6dec37e6d6cd7a0588b3f54\",\"96d375c7d2500cad10119bac4a8bdfd601afe47bbdf77f2048d59f838c0ea3c8\",\"69045b1f809cd927956cb2a62581f1b196272adb8fbc13cdfadefca0046b9397\",\"3d7ec1ca693a0becadd255710168f473d5def8a2e91cb17a90c02d0b071c1287\",\"0fa85abf7f22835ffe539cace199f19800e662439080f7869b6b5b8a3793345e\",\"074b1367f1518e0738aecf1513e2a49e14253e6f83480fcbb69248cabc0aea7b\",\"3d032911f7a520b933ecd70179aea528b25be9f55bb47304bf0829ff0a934446\",\"4c558deb26150cf4a0e972ad3f69a1345d275649b0aecf5bcf9424270bc38328\",\"386ecfc70f7dbfa5bd1deb54c2ce982c07605b3d9093bc7e1813605cc739952d\",\"b49dde0a4f44f71051f9a314dfd1ff38f2ee5f8c2489bc00f19b06de2a9cfc0b\",\"2a3f67263f00ee14f7c22ebe81ce3d5a9675989f59d1b261c8639066b3e961ea\",\"186d71f8f87063e61f61bc80bbf7ffb654f1f1d135d65f3c0018570e4e7e51a1\",\"046ff8f9cc43de6217aa3752b90f147c3b653af7de7726b9be15c8e148e5aec7\",\"2b63015487efaa96421a9b1b112eeb42910bcc1cc428bcfca4b2bc30675bab3e\",\"04bbd354a1954c4d8ded26acb9edc48e559da469702fd5f98a3d5b208ff036c9\",\"187b94e1dab9bb13f0c80c92ced894fb905f7c1c2998ccb88366ace9c15e93dc\",\"2bb24a8192590135d15cb9a3dfc1999733546bdc4eaf10cbd650a3b1868c7975\",\"6a71b04232327127eb1a3806fe641dad92c790544e73c084f06289da84bc78da\",\"f30c070a5c19bba934a11ba71a8ca8a6fe39b3e9e17b3fa5787ff97b38fe3fd6\",\"adfddec60a802fd37cf1c8657fd21f9dd0c37327d2524de4fc24c4644335454b\",\"5482b158cb94258253ef1c93eb34c4ca63aa8f09655fcdd4f962176a64cf850c\",\"dbcf5d724439d588e4fb0764b6464b8072f3bbd30b33073431f8a583962d71af\",\"27aaaea108c6903b274d280937442c2fc02d6355d20dfb5460328102544bbadc\",\"9f5c6dc2b37f56868011827e78055590be686ec0893edc0f0baa94c1bd1f75c0\",\"1eddcb314c1971f76381c85380c7ffb82a2b148ffa2142aed76fc11de3cf4827\",\"dd7905190fce09ea497c4040fc17f34f144259c3ab780e7dabf1c305eaec782a\",\"84102935a1266ef34e6814b4d3e22c436fa3fdb1b26a8ffb0a39f8e80d075330\",\"e86ad19e04e7a2454ae1ea154822a29e711b11104321c27b4b8f76340107f31d\",\"a2043da312a7afa0f8caa84b45f4de5c130d5e2165712060cd9396bbc1d22419\",\"4ad35dacd4f89aade652da8ddbef674ec8556bdeb8b2e811d892c82d2e9525ee\",\"d203f28c3f30f84cf755ce4748305893d458f8c4b010f4986529206e254de7d0\",\"e08db5760c0e4fc37c8796f261e632533d00dd05412f5c1819be6694ae476256\",\"39d994926be839f8a23ba2860c2cd3df35bbfbd3f53c4ade0b48e642cd225920\",\"15e9211bf9e7972329147482d9e1ba94e780276001cb19562ca45c6d9753160b\",\"1dfde31028007443c5a00c3bfcb60a883c4550496336e96ef6f56fd25f062b16\",\"141387aa5a8e772ce44bfdbdee819ede93eee49391391487dc661bfbd770c268\",\"e1a9f84cd97c6e605c46d3dfa4e1b2897464e3fb1463e43d9e7ea33b258117b9\",\"63d8f435ba37eb4c8dfb1ae8e969d8a997fd79e1d5f3fb014040f77142c13cef\",\"e683bacc667d05b1c83dd84328456d194b97ce0640e49db10c99f53798dbb9f3\",\"061bdad21a6e3b7b482d69f523245e289bded1cafa20d283398a25ef43235b35\",\"d9d2efd9d51e66835550e2621928aa9301dc2240365df4c8e573ca80de1bdfac\",\"081dc72390d2b3bbcb07ee283cc827b69a1c608f43d245c690f2a14609ea6a19\",\"687a42bdb12fc73d4f9404e7fe95e0b7b5c412328c39c17efe1015698b3fd1c2\",\"7b076ed1229b659f256a576f0e10856d54d2866d07405656a4f833085b1f9614\",\"cffe05e86a3ef85168c5a48d1f26665d2d86f211d5fb97fae6ca44a1dbb23826\",\"17ed3e7c127be727ff78b1a8903706003ec4f2cb628ae0446a19bcde0017d183\",\"52338ee29b241204dd9c0e083d5f1bbff49e2263aafd91479dda0e0dd3b13b45\",\"075200c90b2db1b9ac8c5eb90531c314a89c585c1146cc7d873627b9b3557ed3\",\"45e12acfdb7700e5f8f49e373049ad92cf040dcdb7277f5af06224442ac10925\",\"5386ae07f1c83c5160af068d0b7d595f7c555887fcff2000bdac6e5392095b4e\",\"7c8fb613d7d94e33584e74a3f0f26a9e9626a714b41e010a427ad286498fd7f9\",\"3edff89c69cb6a55d7afb1cee46f908166ff8fd9d9f8b229795dd4d3398131c1\",\"070d6252e72fbc8e20f0076df203b2973c1168de246f1a309efb97f1da6f1268\",\"435994767277cae1357c763dfc3521aab51e5afd90cbe4f8a741a4b9d5a92944\",\"5f31a0cdda9c5e2d1e5eab1b51372a4b8491d00d96248ba4b6763c4b9b6ffa40\",\"0a349abb8c015c4635d8f0c1009a14f6c51b4c511953be2029ea5561c351d7bd\",\"b0aebd4c96087396124bc20be5e8f7604777b82c5d8cffc8f4cee1ec4f74dcb9\",\"48fc18ebd6ce886ec2b7da5f8c2cb502b564cf50ca0c9abd50a2c2892fbd3d90\",\"00ebfa056c36bd0924a301feecbc0924c2282a690738d0d654953dfbcb04bdca\",\"1abb8eff9e031d85f7de59872117cb1288d1a99519db2b140700aac9ae50b64c\",\"7d6ed62c982d37067da0957c26f988227f8c78b84628c075a42897b1091be518\",\"38e042a371c77b7bfb34a1a2c41eafe9a28442131980fdf3b56f89bf1cb63a0f\",\"2cc175daf096d3082b6b825b9bc50507201d1e5d62c98a39952c11a22535e135\",\"e8e4dc455592727d9662158d28b2a6ae2a8156cc516ba792c50d268cbf1734dd\",\"a9c4c59d76e98c00769ee591ab4cb1e0865cd3b3df81c2032dd977ec180dc5da\",\"f23a2f5b047b02243d6e01d9ef4f52f5354246dee079a4fca6a166b19e681be3\",\"5069229ad371421a8f09fea2473acaac63db342116d0c1a30ee23e0e34b2a509\",\"c075fb3f67957b5975a85b1662ff8ff48438283c4a8581f3efc630184da65e74\",\"ffb8af3326e385555a5d130e6067d28b9c6053b226471a6a01d6120abfa9be37\",\"8f73128daa9f857750833ba8faba8656b12dd2415c1306943dd9819c9c546ab1\",\"019c10281acb97f7e3b0ff56f96643e8b59638a27126d1efae8ac6c834750c2a\",\"0e62cf11813cb5f4dd3a5321403e3b0fcc4e8dd15562fa62e2bdcd7407013e48\",\"f3ce87f666b0e4ab5da13d39c3b58b8b48cbeee99769bea798af7b9a6e56fc69\",\"c30252a08aaa6aa7470989f4ba684177bde259a0e5b830f3b0d16d7395d089e9\",\"5492ba0f0dd13eaa98b533f32312a198f44a9eca2c3d1ed3a0f877216abd7d48\",\"8a4637fabb21f0e3ada76037ed042a1a8a3387560172ef1ac55c92dd7f24fcca\",\"f6dd4f222f53ec355b37a2effbc8f5360b352011b6e12634e87c3a6f4600b16f\",\"919b3dbadd7e3b8d44a55c70310e5b8b86ab4dea2123dffee5fa81f62db7b049\",\"416f8087230efbc64680cac7b64596aff96bb3efc2a8a779c61f8480b7879fca\",\"925c3aa6f2e734c2c791f92cd9592b1daaac8e32409a8e16fc2508bbce80eb8f\",\"6b7c8e7d604a7265f2fd075c3a14cb8c8d10bba62c573afc061ed3b13c606fd6\",\"0387a576cfade5443b3f9dea795c4ef1b12f64391672dce0e397a9f95f49019f\",\"a07a0e843a87638abb7186d4a52b93dec15c66fc7b217f097baa843a048cfc7b\",\"365b16c60693d2f213abfcb4cc28d539e7560f080ac0144377616226cdfaa838\",\"67edfef3aedbef2f01b435c749bafd6b5149663af022f37b96b17b9f8da4723e\",\"182bdee91cfee28be5ec7f57cd218c576df981a8596302f0b60e2a221ebd117d\",\"25608f2fd3120a503ab46833a66345ee832922561c50236612ac50f8efbdc493\",\"c8e729828672364d2dc35af6192dbd79129902cdac4810156f58139c3bb4d25a\",\"01df07bf0794f8be704b62ebe6792660277cce759b895fda2812727f78d5d46f\",\"9d0aecdb9d6d0ece267dfb9ff81e17b4dd4facafd2041bdc796aad61bada5d67\",\"d0ce9295b154a883e5f52eb52ea59865ae2f1deb54ebe6059564c46a4af4ff51\",\"acc8405c3fa27d32f4e10f1839dd5245ac7c207973d7fbf80560162a995b7f50\",\"8625aa18dd6d6f9122e651c7e7853101fc608d3f4f5c9fb4efcc93b45e73ca6d\",\"20708e8eb8f08343f1255f9124b2ee134dfbeb8a1a48143b12f5b185a4fc1246\",\"1ccd6b8df1e02d2bb20cd7ca447ad4074962b51063323cf16d3bd9ef2f5964d2\",\"69a8ef44dbfbe35bbb981b9fc101612960d884bf36d36d9832965f3d035890e0\",\"1f0a6a1fd5a6fc0643990ce97dddddbb23548c23b1092f9fee096202d3e68a29\",\"f8ca5331f8f84daf15cf0f379bf5b464550253a9751230de0350d083698c33ab\",\"dfa83a64c99ca17e916463391a2b99fa29813d7ee8351e06f7ad1adf592aec56\",\"131c18102a266ba7070ae8cafe5e2384d11d763ea337c38a2ebe9718c3ec0c76\",\"0f32ddb9bc0f5b77e82fd418736e525f26f59149989cead95206501464f6a45b\",\"a840e350b51d3a98d0788dd7df41eabc070466a1146d7250ad1f11669377a8c7\",\"7d44fd73a4717f925f4d808e63498c645efb6f8f84f88a4058c6dd57ed52626f\",\"e27572fd78d831603e4fb2284e1eef4a78c1a461ec6a2139b11e2292b4931b8c\",\"a1405cd6408719e95a47bf9efd08e2b6af183d15cfc800d971e2174a4957af2d\",\"1c0c63d9d6d69f9f811c898a1bb4b8b48fef3939c9115700d25dd8b79e19abec\",\"da8ccb55e7c88823d61e2107f396ed9233b59195f608a2cbd23515285ab770dd\",\"10b64ab117d194ada60cf5e8c4e12a87f305faa4f3ae4e6c16efcd81043ff594\",\"9247ad41e82f117e7eb53fde10cfbcf5acbafedec6575bc1c3a2f28fd80d8eae\",\"59998d10779d8b4ef5ff0e059147f6bc5a72f81a690ff04ae84b559d596ffb87\",\"fb06c9d700ad1da49765e5f71016d54c74db42cf48cc8337796a3c77932758c6\",\"0cbbff1bba7554c25a4c3aa11bde3a19f7c83c76b197811cca99ef8dd1177260\",\"adf3d0678016f5ce2ca6a102e06e2910e9bf371a85f0ada3033373849858f205\",\"cb3df7fff1f1210d626b2922361d1e69794e61b5ed4063e38416bffb2233d99a\",\"89ceb34ab1ca7838eb965419bcadbcb9c5ae9b22590b9704368b5dceb2fcee3e\",\"248086ce6f8d54f6804f652bed5f91a1d0c1a9819624dbe4bbc31296d60b7912\",\"5788695c984000e88656b0d991a8c930fc2b351bf0835d03e97b2f3a7c628a23\",\"5454676322c781c90763cf177c359458131ce67f31cc158560645400df22cf61\",\"8a7acddc423262166592215e35aabd5c57e3e9218e5c0bc7c17103b870b57ef9\",\"a01799ba49d6f42995f90dd8d50a8b2928cc5f74dffcb0b34499b82e766ceb16\",\"a268ad1adb3cc7785eb36bbcdd214462b7acb8b866374b6ca5e95964dc03ade9\",\"9c3cdf7c09898b4ca725198c846296fbcef0bb3a30ddc906a49aa7dd59cd057e\",\"b895c3b92a6a23b30ca7689c22989e3fabf7129e5acea47c554589ebe023d35a\",\"764d0dd93cd7cd06ce798edfb6d5cf76175ba50f65f2a89967356a97c084133a\",\"347ddf375c213ec0c47a6fe66eb0f985854abcf417933afdcec7b9208938439c\",\"223d49d5b0c1d46a2b7b203f648469196a39c0ae46c739920c077c19bb3489c8\",\"d3166ac2ce46a2bc8086eb931172aabbf873162cefdacdcedb1a8d77d095a5c3\",\"120b960f54733d22a03e49fd0b780f23d361b72a49d73a8eaa3d3131e66955a7\",\"944999b882ee74716a6e6fc2e57722aa8ec1fd8fc4d4a699e5f4e01042582f82\",\"53aefe0ffcb4ea4fda47e282eb727eb79e5eb50bc99be53687ab6e7ee6de191d\",\"3ec211cb18c17ea8c687b39be9d4d0e3935ac755303e978e100880aa14946a64\",\"d6a814811edaea7b9f32423cefac3db4b25e073995ebe86063ba9f8f1f667158\",\"58e6d60231d83929a3352383a16a5dfa8150d609f65db40fe35d440f9343fb2b\",\"1b1bca80b47fe8222a1d03f4a44ab7d8137af34e934db16925d7431872098d88\",\"939526372c9b17d0d0e5d53385f8d8baf9d5abf6450783064eb34753ef39617a\",\"06b67363db28f28092825dc83cc0ecbd0a92a29ccf8a6954fe15e20a1d021261\",\"2b29b0f25fdf1483b82b865d044f0d6d564459544cf29aad0b9b86fd237ada91\",\"e3ebeabd308f71a6ea25ff61b3a53c8fc3df2f56a591372c88be97a5958b8ff7\",\"32559bb7e515fc2b41e5a41d8847f2bbef3f2a0cfbf566cfb33555fc48697659\",\"86b34398a71872077e7065c8f1d41def6f80222e2e15544864d0fd063cae54d7\",\"da0fe317d44ab61357958c3ff92cbbdc019f03b196efafab517d1c377a66a6a0\",\"849ea5232f4273ffc06fcb3ca99d7f0256a5d4628606e8b9fb520a4ee951d9e9\",\"1c8d7f5486f065676418098f4b062ab5d3414e7fead490a9f61fba1a41989b9f\",\"fc087bfd5ddcbd6f2b891029cee1757b5f781ed668376b0a5cadc58cf0f7a249\",\"f6e0207436d7bac34be09c346a3d94be25df5dec1e48b8dcd016ee8d552800d6\",\"bb5347eb735cd2400af8ff7277768af5dafb2aea02ed7072532b4418151c6c85\",\"625af8566ab5de0a7b79689716b86163ee1bf3f1e7b1c6f61ab9e709a930562d\",\"925419c8d1918c26aa5cd9bc3935ea05caec0e1f3d929114dd6d20ff999cdd7e\",\"512107b50ebc99055215a8c6922e904819a4017c82e513cbd44401e11777f77e\",\"33b4feb5555c5d5e406ba4616228d85c9ab8a863a4e5f3082311dd67d86f4229\",\"7522dbdeaf368be34f9f9bf9d892ed4ab147302fdd9a31bee51f7bc226c93030\",\"15d1ef834467757b1dea4408459b3285b42b82e520e276f9e31434ee605b34c2\",\"656534543577ae668beee791a2953b5cd5dae1875c9863cbcd743d66682d9643\",\"50b58e0b156c2d9f5d7b6a969403b8874a3ea41c4037ed27b80b96f0455ccb0b\",\"3586816e5be555d6e99a140a150911340d8816f29b93d895f903dd5c5fa18984\",\"439ecf91c820a1b15ca568ccab120427fb340ad08dd5ea6488aa970924e876c1\",\"282b765c51d031c9be2f71b801b62151e8abe8bcea5a5921bb559b6971103390\",\"125e4e2c4ebac1a4c320016b378775da7fe5cee4e26d92af68189408a3ad8802\",\"d06b78c801e2fef50f9120e4c3f18a60314614c265b5a933ab6f9eebe7f43017\",\"15ed4a0f316e12267f40b40f612f07b9881b78a30550f6cd3070fc58f85d5dd7\",\"2a8765f7e7da125d31a6e5ea674629820a8bd259ec85b8aae5bb2ac8810feba1\",\"4dd704f74a669a9f641e54dad2beb4047a39cfc3a4e590a748a0755b8be0a672\",\"26bb136db9b465064818111520abfe18a57bf26ce2b5035f43118d7cf2f96ec7\",\"7bcab41dcc1215fc41c79020f3419d1655570f9e9a0af9f30c218a5efa998f53\",\"7850ec018460702e85d4f5cb8dd76d3dfa9b2caf8328e1801ec60064738cb4e2\",\"fef997c063fd2777786a3a32f278270e3c3b37b5d816229bd9bb31eccbc6590d\",\"ca1066e8b27bb1bb4a75b5d26f6229fa0fcdf49a978cd051be0b827d2abece78\",\"bc340290df25763e757d483c6eece0a16da29d785bb37885be0b3635bb6b859b\",\"0de73aa01413afb7f57b74aa27a9bd83dd2eb68b49fc4ade73e2d9c4509638e0\",\"67dc424e664162a6a68dee2ef56aa06c74d559bac229e841743fab3cdeaf95ae\",\"20c1d867f8764bd598585fc8b8ebb52457019d48ddeef2a3c246f333076f5146\",\"95fcd99c6e24259f74bff70cc9aaf36ca3673121bd69a59db19a613447350298\",\"8803a59c663bf84e6e849ae4f42438a0041fe5a5f8fbd26cc1a20f49479e0b6f\",\"67a59a5d6ca2ca86e7cbeda5dcf75ff6ebfa53c29d4ee72d48994b459198ca02\",\"936c3dd68d1435439e38a1bb8cd86aeec4412d96f6d94fcbf2c77d19e8262984\",\"9a7e5bacdf3c09354401d9d838075568c47e409f806fbf937960329547d81d19\",\"f1264d93b7bd086e71b6490afe64369786f3ea96e65ee68386d7fcbea65bb43a\",\"df29fece287ac8c5d3a42d88f9917b4c10de0817f40caff2bb2e83157a4f8430\",\"f88e7857d7b06a70c01c3c5f485b3736bab981c1615ff01821169349a5538d95\",\"b6ccd0df5142c8f4006b7bf0236b5e12f34e6fe750c9fe3053f717f3e1573c4e\",\"95a87d5f440429ecd43e83e6ae31c10dc6cf0aed674d2d108d3c84a969cfc4ee\",\"a621373540d216afb19b69907cacffd450be9efd8d98dc21996741713800b151\",\"dde56fa2d67c57236002c138fe91fbb43e62c2e294f408f63314897bee900a87\",\"14f49c37480a5e2e64800d583357599f0f02cc27510e035c4a57bc8ca4f8cec2\",\"2c4859857cb554cba880b40fe75d9118460888dc935fdb8799ce6ed02cf8acea\",\"731ea117dffeabca269d2d08cdf6b19ba26be5bc6f69d6b944158f6306809d35\",\"7da8cf96dc821c7bb7ad3ba23db1e723462edfdef42b31640fa864cc5d2cacf8\",\"146370da6abc6c0d84b92ce662469fde2f14337e831b3fd0c666416c657d630f\",\"716375494d4f3680640f67730b6a6f1a227a6bdf63605f9806b4a20274a9f29b\",\"14becc03dae88d6e4b931b9525106f05f98e26550e30daf2312de3d7d6fcb7b5\",\"18d93bae39603aafa432532d8e823867c35af934c6acebff9b0a71f124744e1f\",\"dc44e9b18643af5aecb9c5e50f3d3d81050e3daced589eadc29f9f99ce969c24\",\"d354e6744356522bab0c18c8128c16abb5b19066c0c903919eeed2b864b59d9f\",\"b67d7ba80b282e3587d6d1e4ef1e5e5c13c6c02c7704fad51db0cf19380dd2dc\",\"8b3d313b428043ba516612252d0dc8978669a0498e3c31e6fcaaa46e4a4c1d4e\",\"267ba992d744a43baf663ba39cf67da9695dc5553ac8bbd15016939e112f190d\",\"481ac2e3200a122eb2ec9df1468448288b51686806e138dd4848e6bf0464da05\",\"0feb1b461cc2801ef7f0eb9f06887090e95363234908109ef4b58ddaf1baf997\",\"882276868829da2a2bf356db787d160b9974940a1a7901bd8cdeba834cdc4477\",\"13b5e2a2f3600961dc245c30977fbcc8c47259ae64a1d089fb105bc5a17fe483\",\"0d4ee6b0561d7eb4c28614ab2afe641423a0aff867538dca88a64acee8f2dbf7\",\"5860c0596d1f5e60344613f1ecaa1fa3b23447c87b667808aa4894a8829dcebb\",\"e0f715c26e66ad297671110495c150fd580d9640f0a41627c8bd6943cd2f10ab\",\"7c929caef3658c3f8dfe880594f4e168333b8cf970e8e849eec92bd75c7510bc\",\"eb9249ed3ffcdbb4659bc91360f0a70a75e6c5617d5efe9cb43d3f4bb0a4ae84\",\"07eac512715d46e57601fbf6b659e5f70b0cea1d4b6aeff57d0f0b13a90cf689\",\"f4dcd9328b3485edf5491acd06ab6e615ddbd138ef05df9130baada313d58231\",\"505f5d5d59acf940044a0166e833691c8999876c10619907376f7eb3f21cbfa3\",\"f3217f3644ac1f78d9aa706b56c7894aaf37ebc7b3dbeb421405dd643485b12d\",\"59e6b5a12416336463651c53b4f436daaf1813c8ebddc859254aad0aa27e38e5\",\"d1163b09c90c7968f2cfa98fdfd915431cb9445b5ee1a7624dc28d7bdd8eb547\",\"2698b2acc04a683fba3d7e8ff1ed4610e5ff5eac2a24ce12fd681e4bba340f16\",\"edfaec756b2e13293bb6e12fecf55d57a35bd92d0c679dd1e7321fd46bedb071\",\"fc8e646f49cf9189c33451de74fdc52eeac3e3cbc6abc92a7546ca926fa84c22\",\"1d6e9fbeea1e2ec6560f632c10dbb57925bbdea3a4e79a43141ddf5917de93e3\",\"5381a9cd43796bbf0f5517e411517d3d3a2bb1b5f18f6512ddd9ae89dee3fcef\",\"82d949d4e672264cbe32b31a17c3168ce361a211fae4951c34bdb915c01a88e4\",\"4ba14a6ee99f1d168cd8751c05b38a9d9873c2bf935e64fa7626014227a915c5\",\"8c3c57bc69c4ee5788006b01eb691568a5e30b7159754d3ff6fc67a214c1f2bc\",\"ab2b6a57c7aa0dc3a38e1552d5a704bab6f210516470a88734f84f4caae46bbc\",\"46f8de5d7448c7b3b65d32af376764d9bb6ab74b20d6188facd8e9dd8b82407c\",\"d6c04d96ac8e0807fa08c752095e3363124bb797069bc26e90e0c7959fb20540\",\"4c0478837c1d50cc4029442bc06907a1718ee023e56d47a4bc83205c23430bb2\",\"8872b723195372c21fc2071cf99763b2e4838166e42acc3c0178314008560450\",\"0f9a14e2f5ebcd024886fea5fa508f43d738f1b00b41786a67c7cbed3b392174\",\"9c1043776b85fa036e4555b00b459ce1806cca7694a9e1d729817eda3f5ce7ee\",\"7b926cd868b4d6d3915750da09c023bbb36e9f9522efa5a9f6f9fa2f96d5a49a\",\"59aba4cccc03fda4364518abad1b3af51accee39ea438613f1f4c55956887f2f\",\"f26a3f018d8ba99c318a4c2bdf69d70d9fda19aab46544871f03137fa04e18d6\",\"dfdb2d0120dd3adf6864d01d36f346a32af8087e13e1f208ba90e3685898ce17\",\"14f66bb8f780aa1daa8406fbd5e5cfa2cbe21cb91fd8f97b6d307cc01a4c1fd1\",\"624b1a09eaae73a4893a980598ce5f29865178c86cdbcd992581d01180b61ade\",\"54b02f2c980e5ad13ad2a17b26e111de35d1b0bc393939fd25203af39065defa\",\"c48ae6158e3a20495d98592ebb563f637e8c0e9b9be1c0a4703a78a903d2bb8f\",\"a8ddcb4f380b21503322a23685315cc72f6b7326cee81f8cd31ddd47b482e269\",\"8e57fd7756879dd7059ddd6afb6c3ea0404a6c106f204b39f4c6b89e5621bf1c\",\"332bcbd75bb382a6a727a9648298353b652653022892c4a15dbfd2e6d0b1eacc\",\"3549216399a969e277cc5d5145eb939edb4b747403a90c89f7642ad6c1925b7e\",\"03a6e6ebf8df335659dcc44c84b67b60993d7a3d8eb1b5a8b64c49f0c37dd11c\",\"a94883aa89fb5f67910a90dd2fdfd98d9a28fc0ce8793af862fa12ee93eb70c5\",\"6a1562b17ae9d982a1690caaaf229fb8b9f958fa816d50ac4e0e8699a44cdd58\",\"ff8c91d1124b78731350a25447674ec5eaa89691186e8027127a1a982ecc4e32\",\"acb49a3c9e7533c13a85151d43a8283bbdd7cccf2a19d98452e714601f773a09\",\"79e9c217843d7bc094bf7eb806df1592c812bcc9e08f849cb5042d285aa99eca\",\"ae8ce3d90caf0a0036559a6057e884bb50c1451fc97f8b2af7a83e23cb5fd763\",\"045514178fc2696552825d9df57f1e34992b56204fa382057f8def274773e069\",\"b093fe10f2b10743d9de79ea4e82c48c3657cabb586cbda28ab28e949af7ef75\",\"ab220c63f152f22de5063624954ac61734b55681666df3f04992ef407ee52955\",\"86d13aa5c770c2e7e6d4e418d756f43ee714fc530bae461c321ea6e33a75e599\",\"20117e96c1cd13ee0eb0fb54ad9514afb9f9c6297345e8d3d1ce17e92a767d26\",\"5fbd096cf72c26ec3b97fd5e2068e5923652022c457e5ab8d7f7230cc1175a64\",\"988ca34e2a63d9e102db4a5a31f113cd7635e7b9ae8c1315c1d6fa4a52a1365a\",\"f2d1da3d1706be8bbf13978aa9ab0f50342fbfeef5f3d67aaf8e9a733379776d\",\"5ad3bc8ad21493003a803514f82f5b7819a1e9cdd544f312c0730adc2053b30c\",\"6ec6e69f2e79fc32d22ce029b23b0dec515cf883a82de55d1d0befc510edc657\",\"576449d1a4b37c0ecd84b78918e26a640ae64b61f2cb34f106a9f136094f3500\",\"3ebcf500886f620c2e753f97b0330026ef625a04092a9099a88cd8b7257c28bb\",\"068970d2abfc6cfb651147c419fc04df458dc1a1d7b75c9ed34bf1768d4d9166\",\"d0c59dc3a6aa071c85136d19fc180269d397acd29fcf64b86d75659010ddce4f\",\"4dd466d4619048059ca290fff90cc4da39e3ac491f3867f5a444f0b77f06e6e5\",\"f0f7f80543259be3e294ded5092d25ebd3546008be2836951ff5217035196974\",\"7bd52b4dae27d4ccdf4455e4827405ce70f5974b02b756dc459f0d7e1816e222\",\"113d87247e339a4afa4c44d5cbeb2d8ffb38ce5f3437bab009238d3ae09d62f3\",\"c4487a111fee430f12566e6d24f7c3f5ef8032183f596b9eff4c49fa1f473409\",\"7b91a01db0f99e901687b4a4a1ed74ea9c175d78658e80d2c0cca67691c98892\",\"b30c758dbfe90449ae453704a0a3ae2e5279a1c779e293346d71496107103bb6\",\"c2db6dbbfa8079885021250ba0988c55ebb033130b089f0297c86e99136d780b\",\"936a1b81b2f633515875d49691e06b8639338683103c518f5916fa0439394a4f\",\"227c5924eb525eabd706ce250af7de3d8ac1d65976eedb2683df40e62bb50c12\",\"9ea107439577b7ae4840410110d54b8d78f675fd1fdf3dac0917caf3c6fdbaee\",\"e22b5c5953ed065c9611066eedf2e52e50bbfb3b481d3a84ee3f2349121550d6\",\"37c5e37968c6be7f8ddee3ec55b628afbf67c68ff2c65da0e80308f7d44b5075\",\"73664f10afebe71d3c838f6734b693ae1830c2608246e84d25e27b613b6a1970\",\"f4b56bea17c075a8260cdf52614782d558d46046f8585f1ad215d1599c992b91\",\"a71e0398d3e53aa2c4238d904dac9816112026f456a15ce6b6c307ef8ca01059\",\"8077fcd59febe505132175a5cb0b379353716e4cdf91cc95e9216e687571b5d6\",\"2381ca82e54107fa73ac4bf1918d2ea2b35d84f82d0607b05d873a8a43604b62\",\"2cf940007ae8c98bc5bf4dab30381ec38f6295891956eed2c6357cba8dc7a57d\",\"d3ad6ac4d3b8f529e979b3a72b7cc95ccc040d2f678e3521e60e4c48b59aa76e\",\"2baf37a5f4e486f513df6d6c1b08b49a95fe6fe2bfe9145a2e482285b763e1d7\",\"7433b3048eea6a17a43fdfbbe463b58468df8a1be273f4e8b1e7f13de8fb15d8\",\"7fffa048287a2617afff141c6b589e0520cd2da2a0dadb0993e32492a6eb307c\",\"6c45ce64beb909af70af0f55baa8fd3aaccade2965c9d17745b9584bd0760a9c\",\"68b45d1af54a90d9d68993387598f5921830121404522b5cfdf291a245e44414\",\"1d78150d7ffcd4329c12100be2cd6a7bac17394b1f132c4d79a0ae443c9feff1\",\"4f6382b29f2b635bf42c2d71d2807cdaa72e6428d699cebf1f5161940ae0425b\",\"dd812756ac028c9a2ed84193a840d20ef4155fe67bf47252a749c74e85af3ecb\",\"57c697655cae9d34f98ec438bff095bc7c9c4c9f94201f9471041dfa21dc94c9\",\"be3dea38681a984cbcac5503ba24e1df286651ce67a9a5435237de0466310a38\",\"2f94d20d8f3d9584254070d2160491de07925bde65bdf8ab752997ac8e5ba3c3\",\"023f535669739aa47bab2cdd63fe537099b4dbe06a8ca19ebd1066bc11f7bbf2\",\"2a342009583f63f4a60b82a64473bf8b94a83e130bdf88e1880c8d89aa0f3419\",\"e9aa0c00f45192576a4edf9330550b3f7bebb86251508606ae4502d9f9808ec5\",\"c7d5a37b6440bf17d03f978eb7a4624cc8f06ed8982ea3fcfe653501afefa57b\",\"3b5455a639a51fcf65e68c9e2c7c5061fa372c5014ae955a4019ded3dbce9da1\",\"0fbd0f3530d47dd227166fc3b020a29ba2aceba796178ad920112a5d8154ae03\",\"3779467aba6e5a5aa7b6599cbef410e5b8f2977c5181a97c7c762277a36862d2\",\"fca0f8ef62a335a6f8c61443d0b117255dfb777f2f929dea06e472ce59e61071\",\"10f164617a725b9427682e0b58c0bd5e2ebc120fbccd0d1146642b2e769c47b5\",\"3ce430983fcdd3c605a6702036f2a88aa70a5b63d020a5ae9dde93f18a3a9289\",\"f06b042b4d7dc5e966f4c331e812e9347026dd4e51717c76e16b5515b39c43c9\",\"dcb0592795f13e16addffc2994fc540c14e5b78d9c3fc5c20ace66ea56aa5ab5\",\"ad683ad41dc379aab05b9d48aae9e871782b28ece9dbb99b2a9b5eeb34103170\",\"756fc58e8a9fcfb8ac1e8efb5515a79765ef1bef9f4a76ed3645cdb8ffb39192\",\"c9fcc428927b36fc3738e15ab60c1e1ab152d91d1b5a0d08386d97309422c004\",\"35701eec12af0eecad072c9a51899b738006cb669ca559d0bcc701caa968ce18\",\"f3b57c424310e566233508583e7ba49135042911281d170c2d35b36f8b4493f5\",\"e584996eb066912ada9100a002439ff1bb2ef60817be922ef0ab64af5e415902\",\"578f504ced1417000599ac6cdaec2e432190fa3b87e136981dfb12ca3ae22040\",\"de0399e6b91cdcf166fbb3299eeab27e3eff2d454bcb7a0e9c15ca319fb81ae2\",\"c5fe7fabebe9c7ac51107c89a409e3da2faa5d1bb84f2fe0a1a261e64bc4c720\",\"48269e9eed1390b6074857f1f1825cf19d8f26c20069b857412b1d1ff901eead\",\"cbcda451232b6fe65ab016e45f0fd6edbad4b1276f8f28784e92928cb7dcf184\",\"a7e52d84a5b6f111d80b91068421c50c3fb537db9ec9b0c261eae1bd21fc6daa\",\"594b19fa1a492f5412f91153e40602e60b3fc254cca87249588b837674b4f5a8\",\"8c7e05ecb57bd235a1d1127397b883e3352af22e8143d67666010c38afc41a33\",\"6a261fb6c6ff5504b255fe1d2f6437150bed7b6142ef9b3c5fad1080f6eab383\",\"c50ba02e6b57fb44e80a5df4c4d6962176c998d1c4c533570ebd06a9dfa6ecad\",\"07dd15a577fa6ef8b93e6262684a3dd8218210b8fdddf4fad456b90d7bf2dc77\",\"bded40240312ede9edcf22caf923522de6febf197eeca5ac2038f805272c8dc6\",\"cf011e101f2b800e8508daa3010d97cc84c036ae24ef03985c9f03f3ac5c4b55\",\"98e534a599fdc710e7a6bca1ec7be6cc1433078c3083e77ea6d6d64b973336f0\",\"21ed90b33189432d149f3ba9e2577bb40dff19625c932e5d414af2e98dfdd56a\",\"a8275fece09da0528d1564d60769ad4e266f96af816f2dcce296c9e22e36fd59\",\"62c74f8a13c6a7818ef1c6dd4aa109280cff127ce0a9c6ec5f9c25dc47de5187\",\"e7a6ba0f7014e9dbab2fcdb5bcdb742aee9eaeac91cb7886804df7096a4e1d3c\",\"a5c5955165b242f5224a42d23c97103e5f041e90c72657343f69ec798a502c68\",\"749ea83b6c58cb03fe120383a249ad157d4548cb1f5233d990be8cc46ea3e66a\",\"9ae53d9d3a5cd09f5ae15628db089fe1dfabf0885984e49380a3b137222073d7\",\"b1122823fd074f7a820c67c54df8dee388859914f7170829b541352fbe16daf6\",\"3bbfc906ff09c38b5281e911d16af48d84944ef548215950d6f4f9fe92aa7ba2\",\"60381eea3c906d2899a37ebf589c1c4c507bf9622654ccf25188b18f2246409a\",\"e2852444a1fa5ce147e6b0fa5c4cd1b94677719877c9ca3de48a568d336fd2eb\",\"b6e15eb7f0ed00a98b8198f9a872ebc3d40fa24228bf3caa9de9ca3367f14c8b\",\"9d99f408f917cb073a6ee976b9b5097e143d49a4c8fb6b9975da49768200e921\",\"28fb66296058ddf37f274a3e8dd4ed3791e58f7612a0f83a2ae2bc82cd38f26e\",\"155691ac0728e86eb81e972be85e289d454005a5a728725108c6ccc6dfda7f01\",\"4772d9ae0a2f59606decc90517fea16c33c94a78acb0f9f7e11b8ff75b8f1b15\",\"1e22d24b1f4e38badb7cea706c3bbd4a7f0b50e1f2515ba3ffb0e7779e895a6c\",\"64a0dfd1b9d7c804dbe69ec40e0c97e3819849944f36a3a07ca3b46d9a00a601\",\"d2af13537ba515b94a5cf8ba5f05bf3d6d55cee2f4c195f8344f718c4a3a8673\",\"ed7710bbea6cce317924345dd094847b35af641cf48b0d8769c57c693c564fc5\",\"17744a208f8386003270b5a545983b725585ffc2f8ccee8787b58e5ad0c581b4\",\"035c7ebddb770ff81eb42013e957e1acf7778adb8c4d1c22b6b9e67aa46d378e\",\"267ffa171f74f16b3ec63624ca762268cc5f5e7dee6e960fed45226a9474fbb0\",\"8f7f3a7bf670a5994f21b0aa354d4781ed538ee8f19a0911141c4c69aec154d9\",\"68fef10d6356d8086172ab32970725ed2ab267e03c16d5ec8e6f758ea51e8032\",\"358698e547ab14159dbe32b5777fd412bec1cae54b6f995e3b1580a915d9f3ef\",\"f52460817a592bc67a673832e385e075c167079b7930d82b804a6d9ff55c1dac\",\"73ddbe9b43c467d424d5017ea305daad826583fca2f0010c676942d18ed059ad\",\"874d35cd7f8b4221284676f4761c5d4c6a732cea2bd72f8d5a4c58c45a2c966e\",\"a8844e859a1d5169f7a962ca71e749a940f62580ae9399448bfc16a6b5ecf3a2\",\"e62dabc6ff0b3810f0f0bc8d8f680a062db5e51ec348998a8055f1d4413d912b\",\"86e8095a7f3716c11679dbd94b0044e2f8f925d6f26d4520eced3c6f4134189e\",\"fcfae9bad8f366028a208b4768bc628ffc5e4b9dcd00fa12e7603d905eb565fb\",\"03bf56b3d9187bc74a0b01848387bbf16ac9698f61ee888c6476b777367cfea3\",\"5e60f3c38ffef631c438cad8b03316ac6fefa23eb01519d6f8ea7168b4d081d4\",\"33b925b28dd7085c97e836e14765a283f25bbe157c65e5e935296e21d19f13d4\",\"0f7d67ce351c728a420655c04070d81bdd3c606cc397c5ca857e32f8a1a12592\",\"506463be277b18d76bf3721078bd7ece9a6c9cd75444c424ab76ff2f337bbdc6\",\"88ed93a59c1d3f2d8eb2d2f39f4aeaefc49471e41553c73501010eefe076b835\",\"bd9f9b3393e01684b77d25ff0cebc18e982f92a7c9c405cf166f80493775df89\",\"f944a9c1f1c10e77f87990794f1ae593453b8d0dbd3d63a2ff839556cb50118f\",\"542fdb9ee1031eb15910e22c2b5ebd7bee6e8bcbf478916c7663b6beded02be5\",\"57acb149486d1f4e12cd0640b0b3ce530f3ce57f0ee868006c989675bf2f9d89\",\"721462ed95fe296336eef8e6a431ebccf311d4edb5ea69cf6a98ae01f600a52c\",\"a9bf93d88f24764554e5dfea2a12f78f3ebfedb6b294417163827ae8590daaf0\",\"8ce3ab4cef3244e87490e86d3120a6107d2e1b82202ed991a494a07c2fc533ba\",\"893f1ec198b1d293da1ba6e675598b2584880d155317efb76367365afaaea9b8\",\"536a20b18ba52cb1c2497e276577d7b9aff30d14054db97a77b5bfd23641f037\",\"d9592ba42c912921374fe6828f7796b2291b51423b9070bbc3c422e544b71153\",\"042e8cb0abedea31ddc8db890c818cf6e461dd156631ea99a4596fdf153a875e\",\"0dc11e67295e7d3d01683f65b48d0fa0585a501912bec210e73e9d24b018ffdd\",\"b62110d99735fcd2522fe9f69ffc8644b29b88d82c99207db23aacbbb6366d0c\",\"209561a761f441fb5a11f81d02f3ba3baa4a50657aaa40390ca37174ccb75237\",\"0d40bbcd2ef3a3a34581fe84f0cc1b2d20bf6bb7f0b7951b74f941012924e045\",\"fcd8f9f45c760955b4e27e2d634077b2325b2911af4a85019d01a4d213a9086c\",\"6ed38007a537c60014c9acecdc04f842d083f2561298ec71cfc185b5ebc4a826\",\"077adb80feb1da41675c4adf4f6e2cbc4dd6a5c001d409dc23e7d3b9b4288ae3\",\"cabd9fed29169a01b2ada2c1811555579b52b66a5a31656862f2aa59ea5951af\",\"380423ac82fe5a57947e7a95520a4e7e0b1192f6769150b1e342a6fcf21a8fe3\",\"8fdad41c578c1054dfc306ff37678b6894edc651f6cd518dc699ba29b8b34d5f\",\"601ecd5a5aff559ae246bb3c9d1786f3c9803f62a90f4c969bcc2aad562f5017\",\"dd9ec1f530739cb5cbf2827cc97fdb163562a96a3e8cd821dac024d71d93340b\",\"073807c3ee508209f6e648886dac917d6df02677a61ce166bd4bf8524f0954cc\",\"a7f2921192dc3c2b75570f75978f56356d39fd545056bcd8a924225747fd0591\",\"1452c30ae5b5a7b90410940de8c65a4540ca770b932921e575cdecfb88ca96af\",\"e7ce4e2a7ce76af53c6ff55e3580dc0fa485c533af1d9878d27c47fdeda92aa7\",\"25dd3a54a23177560a97334284da4e8c578af3ac33123efd670d1efa2820a8aa\",\"4e0dac01c3d23d46f9d5f84d00897c27d0459b7d9453a0f7d5105716ef391e7a\",\"145cbcf42ff743694fce85c4ad863b31b1ac00ebfc2803731e7c69c7d28c6afa\",\"c255b3337859e2aad5eea715b41c568123fcad9d5388992617332cfa23fcb485\",\"fa1c2fd8836fcc88b3185f95696f73bda8f38266d96c597989501890d75f8a93\",\"26cb70e48c6bb18de240b270f9fff94ff0c27dde377cab98a3c62c20e85140f1\",\"60a65710da99ceb097144ccbb61e8c982b94129c5fb4db3f41802cfa1f7b3dc5\",\"32b59241290d0162f17c59581545771859d17b133affba6ce39e608f7ebae941\",\"d39e9b43e2a8e2f9228102cdc5822fe1329fea5d639ce939ae2a7a50449aff77\",\"a3cf6f8f226e3436e648ee23d3e881a257b4d7598de0c4988c85c032dddbedd1\",\"80ff263985ca06792b17c18c69769a5c95825867fdd618a4dae6ea5bae00f1f3\",\"49acaa27047e410bea6dd983c6d3be1bfc6aa39e2cc25936154349d9ccb48c19\",\"c2176a7edc4c989d5260d94323711695caaf60301a73c9084ca7e9df00ec3a42\",\"9739e57f215071607772a7b2d3261b922b1783577d77caf58838a54209ada94b\",\"c1e5ea865552c8a8fc1c8efd98a4a15374b1bbbcac430aa6fb43e9dcef3b7847\",\"58c7224a06a34e60f347e21a4fb4242838f3d1d060d00279cc9814295961ca3b\",\"9d659ab715fdaafd1a29b6a06e7e6bfc572e1182b522a0af6f7dfaf1419fafa7\",\"58633a83660bac0b8c2f922e42ea611f93e79206ed4cd848d5785c2ca384b8ef\",\"feaba5d7d5db6ad472500214446a8bb823f65e5e57005dd0c9f361bc2f941026\",\"d16bb793e3ccd4791104a1c75436e390ecc8ca749a2527e0f05eba3e2cc6e49d\",\"90699447f981f8816da27913efc18cd5327b7fe2216a4880c18b87118122ec80\",\"de4be77312ef22b21504e62c9578c4d4e6aa7e7d3fcdf0e821128b5e5ed7e44d\",\"e9d35ed06150b918ecaf2b52d61d5960896dc5b059b44ba5ffdb8c11940cdcd7\",\"06da7c23de4803360a2879f6dd7f6f4c377cc994d768e1b856323cc5c60fb21b\",\"845f1ebefd8a0ba2fb848d5b49f6f99789b8510935369f965068a2a7a4a6a67f\",\"cf505c3949f842cb8af3645b4dd97df30b1f0f86c56af538f9cfb032f366a69e\",\"f7a0ed3b449833031d7adbc8c2a9233f2f91b1d158ce5c41e32df4c7b58a9434\",\"58207b9c7a4b19b5919f3816fe25bc01ac15f755bf0dfb529c64361cb6b94fab\",\"d29ec02f5cd4be1ec938050b6736cd9cb6dd6bb48d9152f3826dee055621b261\",\"d57131a66bb3f1809e6ec3c8321a39a16e9953e873ba68828ca6f8107a79e244\",\"f5481e782222c51b5e48cd6d21c49d27d787728e1b567db6c5ded6986a0de413\",\"a57b64ad30956a563c611560eec6719de1ca6e4801a7c2dd507fef4d9d46138d\",\"e0c3d57a732a58799ab397c743bdb568ee7cd48f29ab6fd505da153af3dac040\",\"682b250f8299974b2c578ef998bed8a4ebc8f57ee916ab2edab74ab3e2699882\",\"3cd0d2e07ff962d2f44d42eaa4025f75a9d7b3ce22676632f649ec2410be93e0\",\"e6fc97c9f954a42f76014ef118b60105d9332106e3091770e74a125b6d008e9d\",\"f1a15c13f3f65f0ae27de4f96b3dd86b89377089910aca55771e9428256efee1\",\"17fe1efcdbe9224880d90e9ac9870dbdb58ecb8bc88d13117b05a30e1deb9b11\",\"cbbb4d789ea83a2f7498e7b72a3affca3638ecb26c5458c7ee3b5befe279d26e\",\"0a93acf47366db48a36bde5f177f4a3159eefcea607d546eb1d5302bb3d3a82e\",\"5deaf0e76fb1f127abc46fec0e02e7062f769b7ebc8c9a236282cf5246f2a17a\",\"7f4c9698595ca1d7da273df155226c3aba0a48acce12c202b043ca115e7a363b\",\"1ce0fc47f34e3533114b3514aacbebd922623d1007711a7eed62f46f7c4d4bbd\",\"2099aea3507d2913031e6a65733abb3c3f0952ad1b46fa2d39e656e4f622403c\",\"fb81a570f6af708f8bc5ce5b3ee715417fc3eec5cc5eb0bee936c1bdfe211567\",\"8f3c86750b2d6b7863cc35d26ef11d7578e000efcddd427d04ebfc499fb1fe4c\",\"1217e28357bc3c541f0a62add12ae1a014acdffa98bb6f5624086ac5bed28ffd\",\"3fb3d01b7167b52840073270790c673a654e98fe0e55323979b5d175b28c47dd\",\"a038ff571c81a6cb4d4c1a74760688b93cefe7dcbc58b11c91763d679a3fc54c\",\"c372109321cec5fa0cb79b031b805a5772ed37110dd5ab9b8cceb474c27fe88b\",\"be5cf22a7c2f0c73166db51bb0911583f8da5759e6ff992ed2f9f128a005ba3e\",\"e0f1f68469e5a0672de15fc6261e80efdf331e5a25437a3a6311775890ee0d8a\",\"825897978cb749e55c276be8c28f4e1538effed9e2b5156d203db8cdb8d7fb9d\",\"f3beba774f2b258fe0faa13c259cc3a6498dda5458e2effb7a7745cfd4e774ae\",\"b04434cb139a712359b1cc7f82bb7b05ab7d01c41a94c7471e11c74411704111\",\"6a5502edab1d3fc3c531d4af47dec08f12953cf552d20af8d71b947020c1b467\",\"5d05da6e0e64343b0fcc4797e46b66db18f6b4d1a306fd1ce87c0cd38f71179f\",\"9ed934a6ed91d7ad217f71665e264a413ef971474c620d47fdfdc43960ee99b4\",\"d691f6dac3acc857960023f5162caea30c6b67a0ed0038f896c7916544fc9bb0\",\"df8956530441756917179b9ba5e8c4031caa4a1ee72aa70beb85dcaf8e26265c\",\"2e72912ad00ba84ab258a9e4ec42684d9708d5231f2c861d163c31c10e5d2535\",\"2bd608a76c25d26150808055f5d9394cc4b1fce725c97b295c5f2d007792fc3a\",\"2be8880bc914c6661f2b28d8007e50695cac86de6fc8c665af41926731cf2c97\",\"3ff53325c5dd644bdf7aeca3101698e4db53cb42d8153cc6a5efea3306e9225b\",\"9679313e28e34e7a149dd70268c3d25971cd70c774d4b06474bc1f0a5cd4e0be\",\"9d2612344963445a0a1dee8f63d2dd252d4f5b443bd869c813463c0721047ca0\",\"ae958cdccccb0d5b059594caa88fc753fe91d683aca37ecf9f0cdbb69d164d73\",\"d4d10af8110e8917d40eca1b3750301a62729b50b46e23c8b37b7b295b7dadd4\",\"a6c0cc1a294680df8c06721fa7e6dc5dd8d32dcd47d2ef0249e37871fa0d02ad\",\"5d18d6f0a6ef3f1ae9db134497101ad3727d2f28f11de2e3cce4ce5e6af1ec68\",\"2ca8643fb75535c62693b20869c9d940500fffd9c1d551bb67bea2d2a8b785f9\",\"49256d4b2ede2f8d0a240d3b5c66bc05dcb8dd8708d2fdd87d8fe57f6738e225\",\"25d74dac1bbfbf97dc4820531e708ac68b14ab2ff65319a26947c24fd3895ad9\",\"16408a355f7c582ed168f093a430a5bd6f500c83e50fda15b5f49b2b5cddcc93\",\"2ff2a120ea1ebbffa1aea4d33fd176f6778997fe81df23afdb9420ac2a8e51b6\",\"828d1bfed89c7aae28c70f7d4c194dc24e09eee6444ec89c5680be03ba5d1f50\",\"2cf681a75f493b18d9e5f9a4b1a146298d34801fd886bda07e16e443b20070ba\",\"c8e23060cc42de11587edadb9f5503710493f937a81cb64c5b907eb94467e47d\",\"34560dab25bec82f48cee3105a230ab5899668ff2b843e8db5206df6c35329c7\",\"995e0698c752e5e0611d0ce9bfb71e0f3f6d5f68083d017b8d0423dfa034ec1e\",\"6fe2f568a8f4fad4e255fdacd97938453693b538973ebc0fa25a89234ef41d91\",\"8948f8395d94c8f9dfe3e957e564ece3d8c28a0a05ecde6da5f94ad3833ace16\",\"82fe2711a6b3cb26955b34b37c464326affb2577c66a2608f140be575f8075aa\",\"1eeeda34bc9dff91705c49105dc4b71bbbc4aed883090ea90fbbce87bb7dec6d\",\"456d2a6c88e7a5f8760b4b143d2f895f440381c5561463d4679dcf7603a8cedc\",\"6178498b021a6624b633efa164378c85e8b219727af978ac7683ad1cc066eb1a\",\"92ea95d8be0467cb685c91891e245ee0b811f6f7c354c5544af63f3cab6756c5\",\"edf35aeb0004b6b9d826eee0e1b3fedd8916f0f579cefc92e16ee808c8907158\",\"904ba14bb7f2d2cf6ee9bc054281ab88dd5399223a0c879de4f7589e7f2caf11\",\"37c7af4487ca92fc370c58679820c0356d93091c9ea049dbdffb0357ebe67111\",\"01a8d2beb80d00d1386acbf5eb67cbfbc0dbae67aced8c6a9d19ccba3c78d1fe\",\"5b442ec25bde92a27934b3b0ed89cb538dfae92f0be51787a2486c93dd6e9a51\",\"179543fadddaad8a665054335d143bb0f8a893c73a16e02d3fa6f72fd39023ae\",\"9429834372d6b69646f6e176ffd66360a0895b2b8b274fe3d39c48cbc7f1df62\",\"c70624e8bfe5e65a3b5692d78f15e899ebea49b92a3299cd0c188487c5a41639\",\"d75e68dc13208df8a81fe0c183e4acb9be29ca92841015daef280f443c7c845e\",\"d397d6c0d72af194faa934d82157774d4cf26d8249c9d41bb5667248cf5d7f3e\",\"be14502d20bd587420306ad004f4ead57039d40ec34d6636fb45bf5ca95efae9\",\"f622f731a1bc7bcddad62f9ea74f2b8cfcdbd75ba1bb910f536816ae1c6326e5\",\"d392d393956c0621c31f999a5f6e10d729e78176dc4a24b11b1b7ce904135b37\",\"ebd725a60b77e7d0db3a67f1dfdff92791c6167cb8d98f402525268972c72193\",\"de2142cc5f5d586a50791f5c8e68a1cbdb34a14a965200afc298012c4d83aaf8\",\"5ec2a9cacbe5de10dda9b91253258d8af83a2d81b4b03ab316bdff2c7bd9a782\",\"25df6c300bfbb9076c13ae79dccf475e050289dcefadf03da21307b8166db5ac\",\"c0667da6bc8f8751de40f39ee421d345ae189caa0ee79d3fa4c4a93adcd74603\",\"a17c50917a493770ef16b7bf2ef29939b82c60e17df454dc3687d4de4ae0caeb\",\"b1a8f88f440021bf983c4ea2bd3533222eb7d46828e48bdb38328b1292dece54\",\"56c2db8b7d5cb0a2560bc270b37fd5c2690dc7daeff3c5fa09200f4bd8fed6ca\",\"1d569b2e48e25d7b0256ec64a98c724913a9b52db5b7f3c1eb2bac9937f8a503\",\"b5ea8c1cedf95ca0be4add13c5858a7b046d68ab9ac7a017e7c0ee9c97520214\",\"3276036993c57520f14587ebbe49446d3814e9a135cdb89bc6bd6111c184cfe6\",\"b3772d75adbbba8baeccd1adc6169ead2d98493f1b09936458752fe16d8eb224\",\"bafbd6b683ce47ab1b3f7532575a20ca99d3a189b564a4ebf4af196b342d4dac\",\"cdb91aebc2237470210ca1b6b980f1667a88a0d9847dd8ed095cf1a6bdbd727b\",\"9d4789d1a285d7190cf12f31f9a3a76dec33514224e74f3673057cf2642e7358\",\"705a57228a25aa6c384e7e3a115e39c2995200bb7821bfdf4eb206962d3a69a5\",\"06ab8cb2a08131a32abc0f1e5af7a74cf0942fd73315cf43d4fdf64b134d6b34\",\"bc04c1170685789255a0efcb5b7356b6257f7dbe5d7d98263b199c8297035e94\",\"2c79fb07bb340ca8eeac2457444238de9cb982cdeaba1059a868479cdbd8bd27\",\"01f1c2dcc4743e91e1f639336c42a6907b5784d07e5c6de385c3dc50a91984bc\",\"8f050b34d652b85fdcbc28c6c5359b21abf77d7dac1ee4be42b27d82a7737a14\",\"70fca2d9f188e0afade940801fad7032d0be647e5e0c4a3086fe3528202de44a\",\"ed452ccb0bc25ffada5ff1c92130c5ef15b6d4858226bcec6c8d164552e549ee\",\"0d3df95f71350ad5bd18d092965f8a4bb0996243b780abdb9d5c427c25109c85\",\"893fb8204420e2ed57547efd8cf1699baef5bc2b2c0e7f771b80bb1c4f2846bb\",\"20543a404a94b896b6b5d1c768311bb1d62c29faa7b5fe5001b4cd17b36f063f\",\"97861d75c9c7da9dd63a653edea4866dd50e45e7b31331a13d98dfdbde3b648a\",\"ee1d91432d3957145740495902c56e83c2b12467b930fb1483cf4f4c4de90b5d\",\"92eae5582c1f638f86748b8a387c60387e537a249ca20e3c76d8dd98a2154dc4\",\"4b8defd930141146bc1e860d7e6ab7cab72b9bddb6770c9a7e10e71b1c6df3b8\",\"8cd4d61fd07d190fa0578871f5b0d24cf5341133de0917560a1e1e713e531345\",\"ef6f857537b0eff97c9e335c8fd65cb3f5343e9fd466782dde6332b4b9e3c891\",\"61e19b4e5147dd0ddaaf15456fed83169ade2fc5dde6a439662b9d6e2f57aa1d\",\"5a52669785b511cc0a7fe11cd8c0ad89b569ab2f1eceff8f2948dea0b60936b7\",\"487dc53d6b4a80ab27928eaec5c5627e8f1da0ad08beda78ccb24963acc98f5c\",\"7788da30df6c691c7a9b1fd5658e959fe3d95c59446d3ca13bff23e3afa15a88\",\"0ae9d0e3e160d43433736964e329e062cabe7ddf68f9f2757c8bfa2c16458586\",\"b783d94e48af13e332913d0ac21cc43821bbe6089eb1ac10323e41a0ddebb7f2\",\"78966291c4f7547b60a4c2943d7be690aa48d10010c363219e800882f2e2013e\",\"0eb32fcf82b9af43b7edaa172a8998b0b7615600648b0b36cb22509f2ae40ac2\",\"ed1179c2f760214dd63e8876d647224b7063e39e65bf8a6ce474a4a501ee3775\",\"8cb43208204c8d7e8557bff8b121746ecb591a2a4f2745af4ba53aac321478c1\",\"d768c15fc447cbcb349dc75cff5f97c35d53ffa50affa4e8744d036f32a60a44\",\"e74a7576478f89c4d1d375073571363e6640470d03643c68694804b523c1ef46\",\"b83c095b9706217a5c2bc5d8e519d7ba784d0522593e14153e804d0cb5f14a41\",\"83acb7bf2e653f5d96ef237d7f2c16f0c5c1d73560f53536585d766786f045e2\",\"4713572902162060e3e5c6caa34d68891ed8d884db3e2366f59b09399e3c94c9\",\"0a922d3a6557538a759dd7d7263e17069bbd2d6cf23032c72e5e6f5ea198f51a\",\"a479653e72160f9843a262b081c2a0d4ea0898a87eb733f28df84812650c98d2\",\"96141557ca2737534ae371cbd09b8521579561cc51611ff990276084f5fe7cf9\",\"af0c153e33631959990dd19c3d27c6f499a103de6f7845e46297a5201b86e8fc\",\"1b0547746f3bf7521f4d0c8967288ac59a2b16f35a21cceeccc49ac44d0de2bc\",\"8a996850368bc2f5f9ad727fda41fc14fd60d134eab5f1777fe5e7af622217e3\",\"4803a586f3f15c770203892f3cb4befd654dbca15df7cae6c840e354aad136e4\",\"ce7277da78d09d50e2313ca82a05cd4826b6a4f1e19a632ad45fd99add72ad78\",\"8560f3a8815dfde0566cd7aabb0a3baa313e872cc9d3583500ba1d032c598974\",\"df009624256424f9e47b506bed4b65774c0e6a4fb6bf34f3d05471e150d80e2f\",\"80d5e14b5fe1ce77285adfd715e995252a7e29a6af654c9168059ba14dcfaabb\",\"0fa08ecedecd1accfe507f4702f6ccaf337a72b2e44ef34778219cad189f8111\",\"0406c7bb6b7bc40adedd6760e6ada0578e35134ad097d89f4e941baa017e0a87\",\"c381ee1a84f443700490f97c8c97f7cc4c022978cdc7d9f9dfed9eaba669282a\",\"4675a35194ee70364dc579088d377a5cbbe05a001dd4d12322bbe63a694b9c52\",\"18f6ce48e3bed765db2748bb0f56266d2e20214b89f4bcc84e49041b2f853b74\",\"83ab7f2ddaf88d8f13f03457b2b4de89ebf6863c03c0a2bf640fe7fd00b19815\",\"423f8cdaf76ebb875ea998c06f16509fb9c7eb8d6a97c7c598336abd3cc8c79a\",\"685b5738d1f2b60a347e764bb41bf9d27da2a022e13d038ca78b46f6a7e43f09\",\"5045d84a45fe82e6fef87c8759b5f8ab6e616888514521d71f9a1f29855a336c\",\"2508bbd5b8d5b594df7ce4c7066f14bb9abb2bb1a0587d19d37bd22082ec232c\",\"1b7d368e385897f7abcc02c12ba516b775cb2b3bbd97d88969ec9d38b00659a7\",\"db732e80d4d63e1e432f715f48c17d73436e4ee8f1c913913a1869e3c8923efe\",\"4596277b711a2da0268e894f1a15e763d4f88e6de57afb4283684971aa62684d\",\"ea8c825a5081b3d9a4bc70a6e285bfcf873fd02d21f88b205e8e3521c4480554\",\"732e074d22be5403cd92c92c33afebfc2cbf381697441b6c44b49559298db455\",\"feef3ac9e7cd5409ec366d163f6d003ad7990ee74ae23870ea411588dbc52671\",\"c24ea67cc0a1f5a997377420d538d49a273921f688f48ad4cae7ffc2df3a0ac0\",\"e1ce7a651f0c325b1d6d8939b54b8541f9f726b2082c7c2d904c8d1b1a179a07\",\"dbed4f9b34744fee35ddc147e28a6b710432851e30268bdb9556b6248018e9c5\",\"24777d0951d5471a46c756134850de774daf62c681806aa6205eeb4795612cc7\",\"eb304fbbc88192b71a6c7082726b080c840f1438e8efa030720bd435f9c6dc96\",\"c635bccd29e43ad4045751adb12b535fed54f409d6d1e00950ad4327f665aa21\",\"2b70bfdd770d9056258cd89fbad691c73499eb90418976c8d68eb79821e4cffc\",\"cb6f43133ca68918c4dd3badda6b46df755f9a99339b2bffcef966ea39d4496b\",\"8c258694a76ad9b41cc1b1494239a36f32c44f272fa8cea68ade7f23e81afeb1\",\"1c9fea92dfada2cdca10baed2b8da6901b88fcfee91f8cfd77bee845952714bf\",\"2e60aa4ff277479aff39a44a818f7e9428873fe91b9ba5854453b44ebfb8ba40\",\"3ec3f6cc69f2ba270a600d1996a8aa0578e9f5d98045ec356a29aab716b489f4\",\"e7083ebf2bd9da4e02f7db2c7fc5d34e09c3c8bef92759b6060449ed35d8d598\",\"f046666ecd5982ee76f027f18231c6083693e28afeb4462eb2f5791de9c621a0\",\"59f01d61d22bb4d86f9ca78944473e6e3f7cc77bec83d201343f0dbfa9e3fc2c\",\"d2681ac8802931001e5be30daf2889e68370b357482c7fd9b01bda0daf9cf1da\",\"c54c9a9c0e419f69f9c93dd15c8cb66f8db322f6efba4acbf172839590c2f4a0\",\"757d224d0b3746ff5b67b13b67c173f2cb24b31f45d9912ed7b17789106b1f60\",\"dc144228f70f53943061141fc8bd21ce1e9c349ce6e7b58a993c8eaa0c62a718\",\"faee7663e3b18d75c523ce0a7115f4cfcf532b8ac1154ff2d422b08643fb2bef\",\"f6e180061718c92729c9b5a90ec75350d8645ad0bbbee0d8555814c39ba62f16\",\"cb10a1d04c3e70cac5f38c61874dd311ab7776f7cbc79c4002427fa826ce9962\",\"473113b1cba8598692e20b6437c700105e95d3c36c7600939e0f75c68ef6c20a\",\"c7ddb5bafbfdc5db71e2016aff706ddbc9a183fb7cd8e88a4e178ec4b1654387\",\"2e689e5a396ccbd3ea8a6c41b5e84bd73d44ed5c7db4881e42f73be88fd546bd\",\"ff64735bcd005812e8db5ace61c865acbad94362f98a2beb98d15b23c94b291b\",\"7cb6bfed1ec95f299aab286a9905984c8e2ffb3d552f59b1371c0e005a7ebb3f\",\"92ec7caa8968350c89ae90187248c55b4de110f01f47d551887a6d9146c5704b\",\"3aa16eb6dfa9bd71dbbecb2801afde9f7275631241f34726eba57f2f1dc0474d\",\"3f78ec958f8cf030bfa5dbf302af02c310296cdf82a0a8ee4dc06b51b76fb197\",\"eaa66ffa6c5833242d23279dd8928f10ec1c51144c40a7f2d6ea53f6503852af\",\"b2bb7046defae86bd044a894c2366c002da8576a0d09d5e397d12a668e0da9a6\",\"87e303b22a4b8f63e93492758fbd7f8e3a193ed4a4a115d98eb814cc0ec23a1c\",\"4e7e369905a67fd1766d1481c13bbfc9400be986c53cbf28b22b79290450f401\",\"f81ce89185fe888d9812c66accfca336e839696899a07beb9a16ffa9236be31e\",\"5d575b4e15a34b6400ac371653606b0a8873d9ba1717c68545982866b5291ee1\",\"91f98cce019b19df4bfea2e0088a1733a7300291224a62f2b1e39282d58306c6\",\"ed89c7647daa0748b9bd58db89731323bfe1c7e243b4e3d9e332c65115d6291b\",\"3f62e4d170d468876fedc11c8e804b966b135586a0a77c8241019a03f6a7a866\",\"4e3b3fe4dd72d14c42a49fe015cd033a5a187d80a34cdc42e5f8f6364f3a33cf\",\"c35d80e270e9b9265267524299faf1bcbeb42b713e2db229561dadce3f46385f\",\"dcef8a945464ff1c3824c95a68a984d2b65f3d20d93a28284e52f172239f4bf5\",\"2b433361c2b17134bd7c26cd9549f23d68bc76c9f44885b5df1c751ce426c70a\",\"98d6625411368daba92aae3090f4d6f68d6c099aedf8eb1611fdac3284bfd69d\",\"97956ed619fd20d7663ca31aac05722c7fe42d0181690c157850a4fcda65f4ce\",\"d4c45088f72fbb60a0d4ae2e0ada403be1028ec94d838cd264aac3d9ac4cf8d0\",\"7e394764e310b0c183e380e45f708d16b84df699ed439219d85dafe6cfb16eac\",\"091520ba0ddba5148da68f1e39610e306f0cd09677e6ec3064408a0dfccc0248\",\"0581a7da66763ae45558e6708c113ae4e60e8bcdf06ab2bb23f7ace3ebc42994\",\"e676784f2455976b35dd69ac4c5df0652f660576b4c19f7827c6600dbb309d6e\",\"c01dc65c53027209beda7aff824d07a0911bfbf36f0dc34f2887f89589cfcc44\",\"b0c43e2467961e547c2746001ac81f12c5438d40a0436531a66dff9b424ba77e\",\"4bbe76a3d63fd9a4bb12d4f3cab99512696df0cd6a9fa1a52c94921ed71537df\",\"24ff245bd603cacd24667debafa8ce005b236c550f8003187299e0299ea979fa\",\"9b7b2a2cf2a2f920add69b24a489519b3051a9a9d854e097b93ce638d6882fbf\",\"e28852d0f55040435ecc9a920ffc14add6f2088bed2df34d88d6ed22679577f6\",\"f2a4c4e5e64c77e61f1adb133befb2257de263696e38e4f04be3ee7c94e08067\",\"3a257b0eae49f9d2e484fcdd41cdde219a13b07f1578b4deae510cb1e449251a\",\"7c610aa942b0f4f24204a615735791725861aa5835bfedcd427157b9fc7a9508\",\"eb14a496ca1e1ec0597e0bb9def65e2e305e84d55e35208daf0b80dbac1114a8\",\"553f869fd78ba5845d5ca1a887142d8e6603658415d7029ac58f86818dff1aa7\",\"0589191c4d00ba82e13b3e7c94c3965c1111458a879e86b1b6ffb86be96d2a69\",\"3ba23f3377e8725cc007c3de89d68b83b169e26115d6baf4f91226b2d5357af2\",\"cbe82312cf3a3ba6d8416a88c7cfc9b78da0492352aaa23adb50bb8062aeed30\",\"0e068191a2df62fae181cfcd53e6708d25e2657ab769e6208637e41906ea1855\",\"eb59ca1a8ecfbc820d552e1888edfa67e66bd99dbc6fc361ee3023ab6c334b71\",\"22e87c3420ac14a9677fde75bda7069d2342f4cba63a0f83af8bd3bc8912d7b0\",\"6d26d352bec299e7dfd99fe561cb68ca0aed111e1192999143f0993d24287f2c\",\"bdaa79841ae842f6550053848c7a223608246f3b029a5145877d339a8e9cf3e1\",\"27dd097f29a540ba5cb708403c27346e9dab2da60e2c4012c6d7f1bc26eedaa8\",\"a71a2e16b80bfdc9c12e933f36cecf19ce3e284d8d5a6b7ecd65c45a484c105b\",\"c78c73aff564f090c9674e466296bdceba77ce7e12a0267792c6581c6cb21dc9\",\"8d06724a56182bc156b4d145232b15dd18c31d86b9df7e940da537393ff0a07f\",\"25a57bb23be2e30baee6d71bf11d76e164d08784d7e01c32376646f2282cebc4\",\"e255be99c6abd8eb852d03bfb137f8af3faf873ea8afa91c0786b147c65cb846\",\"af53f97e516fae0db6c0d7dc037494a228b64eb90a21d861739e85d42c50a27a\",\"124ca5005a10e6dfcc2c60eceb03db8e4ba1eeb667633c9da3f04be619fa5b62\",\"6409fb2e79353d37b2e6d6b26f8b23167e9bbb556e21baa57abd7739777b04e5\",\"cb7d3498a6c57955c07a85276e4dc55842db990e1d289d4490b1366aade41d86\",\"9811f86b249d4d994f170ae3f235afbe283ff24c29dddb983126514e9fb21eb5\",\"8780871183981c6371ce030e215d5e388b7c36ab54ef03a434b8e456afe263d7\",\"72ddca3bcde85950700be2dea79fc56fb16ec41fbb08a19f81335379c7ca60ca\",\"859d5e2b866d99d93485cc37c2eb46444b7be04672c02659ba9f5158aa47aa1d\",\"0e113f3224f38d5f0de23d5411135baca8460c64c3fd35f8c2fa04e19e480347\",\"8f4a21eb16c7f37c6fd162f6ab26779f530422ab43690648b934666736057a88\",\"c442827ba46cd0fabf60aa1e03fc6041970ea2675e47c66f2eb7eb1b3ecc6cdf\",\"f4d071d7253c0c741b4560c95fed1780c9b80688c28314dfa8686658acaafbb5\",\"679b6c3049373a954bac02df6efb4604672b411f527d06432f0a639d03775b25\",\"9b52acd9b48df5251be3dc226aa2b55f08873daa15ed4900f6c1375c22ac9f27\",\"f8f0faad55d8092f2cf530525e60229ec0667cf3c5ae83152b8f705bc17248bd\",\"8cc06180ab05bbf5b6f98f24e5624cd7cca2854f3cd96091380c7f95a2744684\",\"bbac1545d981be070011bb6510db766c0e8d75f05c14deabbfc34e45d4f8b687\",\"59acccafb13c166d49bbe5af0230d85e30fd0dfc21e65689e9e1b6b961025f9d\",\"1dcba2d2ac599e60beee2e67a855beee65e50f6a918e00c5ddeabea77b15a925\",\"fe1f639fa91b8012c1b5d9676c3ff7c332c563cfda09dbce102b399dc47afddc\",\"a50ea6d68fb3965f3d3cf642aa4fb49a17ff48d8a31602c84a88b0336806fda4\",\"637ae7f5e2e00498b5e0e0bcd886862cd43548dab7dcc726984cf75e91b1d439\",\"af9736cc2901c182edafa8f8454b76e7ba6af3a7281da9c8c448c8a58e31d80a\",\"8a1329f0172ea7ecd76077c8c8602c1379af64c60f4696b32f3b19560abe59bc\",\"ac3c8413f3e23623679fe3c21a410f84488c4185a4d173336232966a4437b92b\",\"79b4c97e33835f54043a0d0cd39b9009f96b9528bceed38f17922ed888ea8d27\",\"f727b287874b25a6085cb50261c7f1adcfba4d6d4f4dcc689afb8e25fb15b203\",\"36d144367474f99420d93ce2526076a04d7d672a09c4c42962d5f1a3f2dfbc20\",\"78edcd4dc199d26a9986f0aea98431dce9a045a9f9d06cad25be3a6483ae74f3\",\"2f993bdda7e6bae9f56c1a5c0b81ca81ddfdc3432f3075556df8732723c4bddd\",\"78c8b6ccadb46b47dd789c7f1fdbfec1802da21bc7ae8e1f781a027315fd1d9a\",\"11c613696bf6990dad0e25c751c069851726d8d5ec55eb56c6e83ec1dafefe4e\",\"5e67e6903d9f7ae4c07b40b36b861bb6bd164c92366db4f11c84a7488eee13f7\",\"f760199b7acdaabcfbcbed60cfa249c3b38e8eed4eb0cbf78cd08dcdf805ddac\",\"266b6b339559e01ab8c1d2403d17e54079026182283bacb8a178c66cc11bb089\",\"baf447753f56ef7032949b3e407b06378ebf0c660c19fc97c35a4d5c46a39816\",\"309fa471cfb42aa8504f6209728773baf291696f5c12e6d3a1f2d07a5e2f7419\",\"71dc8c94965c25ddb08ce5543db3e2dbb938cb30a10139cf9f4ad39395c0b66b\",\"8fb4690ae80974dbbbb24655aa89fc4ce0a5fc075c018d57981f403f73814ad9\",\"e998c46b651994cb5ee8ac56a805270dd41426b1e865c09ab4ac3686dcb6096c\",\"c1b479465181854eeb5c46014055d8f8c568529cd2046f48d55d1846000803cd\",\"1b771c93f85fbb47e92807458e6446722075ee6820bd45cbd62efe1e1cafb407\",\"52ad7eeafc520c5278969c78a1f9d15d5a49fb5718df220e98867ebf9869a2c5\",\"9e882374ec6a58974ca479997bed7deec08575301bacfc4b3f87ce6124324f89\",\"d976b0597e8ea0f76dd6d138efaabe2324d7a7db2e4f41bfadf24a97897e8af7\",\"80ad9d5bbf4c35612457639b45acf3fbaf68494c8f16d83e7c9901074a86ecf9\",\"09706d8df8480085eeaf85cce3e4450f7427bb87dff413be2a5a520082a4b004\",\"dc0f5e8b46b9739dacfdc8c134ec4b4449ed24c1c5b0374dc691fce861b99cb9\",\"b77708f9eaf36b0d289fabd98e84e44bc456637e4cbb6593538f10fe05c8ec7a\",\"7e70883cedfa4f6da6451aa1f6e576c092fa748293479a14b7a4f4d9d19c0cba\",\"4ec6f1fc783021cba76ce25708d411498bc0e9c36cfe5337f51b03d94d14279c\",\"e2602a07e30afc75b2ddd5129136d837023591c9d7ec2d34c5629afe779cc94c\",\"904ace32e5306f982ec961b89640d7156c83b7a41b183d27f50b8295260b0c2d\",\"c52af7bfdd0e945b6c96003d720f35f06790d58a74e0646c508c4e46ec733fe5\",\"3b180bac7b01c3eb62df64b31e1a0226872236f4e3c20c46cd343b0337803b11\",\"1ac589e02307a602287354cd7a7bdcb7a3903cf5dfc9e56ec67a931b9e4a4136\",\"983ccb8c5222919bd5e50a4b3f5f8b0d60b0ded0187b20dc5252884f989904c6\",\"1d3ea0c03f41b7141e0c3789e8c3c24345f96f528de307e77b021c4116da8102\",\"6fca5881a260870f73ccf5e04b24df7ac5c08d8e5be4c495a2e4cb902cbd5083\",\"00d98249c86eb07542dceb52f5aba06051ee1f4408f999ad31de6bdc8e347c5c\",\"2e4d77c0c69e7098d4cd83aa2d23de89e7e3bc6174cc1bb7332718d9d11b3eb6\",\"7af821ef51e9c4ec7a728692fbafd522100d168d4d321763696b3b13b70f4dba\",\"048a8f6e53fc40ca4b901116bc4df7248068d31a9ced8eccc431a21b8a6c614d\",\"342d2bd7d06e4f0e91f821473b7388b2a7569e3069c55ab44e41ff0576b2668d\",\"ed66db046669c7c2118d92c3d733177855650a1f66960ea37b3ef47fd65ee316\",\"245dd3714d506528fe73d44f2c9695424e5de074d551cdd88453126ab2b60ce2\",\"1595fa170d1beb16072f8ad3af751f149758f70b356fec322112cab1461b8b99\",\"fa244138ba69bb568322f37224d5de7c883e0272561033fdf1b803c31c25f8dc\",\"3fbc028dd992e81c3e55c1a0a8b43809ca749c67b7d2fd95b300b6988e331ff0\",\"e03ad717666a238d890138693b4040d3e79075793dfb518199c3fd16a02ce795\",\"00b38eb0e91290afa5085651d8ab9efb857f7bb092dcbce7afbaefb29a710bc1\",\"95fffb4a0f690bb32e203057e2bc4d373e93a9a531b44e40e73dbcd8b4e0fb2a\",\"b08a2c84bbed6fd635de120bfafc7a2803e673481c4cba07a43baef65b53f4da\",\"6f56349ad6e7444b7e8b0b10b71fe460a757985158147e81c4ac5c2921855e48\",\"4662ecae80e461ca031a3419d979de8a22176e2763363ba7c2442ef5a360f362\",\"9267283870b7444e6aa07b3b9013204fce03e50fd8043f202ed106e3214f8191\",\"88e8f30d1676a36e099325e7c5f19fc21dd83bc3ed615c302bd0bf23ae125ee3\",\"5a8f128760f7810c862cc939d269683dd082973652adb879ab1e225bd0e14358\",\"cc2bf626ab48f4fefcc171cb12d27414dd18c1585bece36005c2b3ef2354e9c3\",\"b7597d77f9f6b0cac1b9a58c453d7b66517490766a7d1eef2eedacab4f41ccda\",\"83b6af3375eff430a63ae7399c1b47e9d1d05476b6e43ed681d4e7ab0f985a6a\",\"ae7d27cf97993d485cb45c02fc03013ece0b9f94516ad4b04fa6bf563513675b\",\"c9874c38cd9843729dbbe70079d8cf959a92b3a5db1f20112e94752c2cdbba0a\",\"9e7e2ead4fdd47eebd14dad277b4e6ce8bd3c4d465d74930a714037f144a35e7\",\"132ff73e2aa00a4151dd37e3169bbcfdc8e19b55dd8f9224461fda714930f281\",\"a2a573522f4eb590058cded9954747189323551ce8b8dc59cd9b19d34c092f84\",\"83bae93a613660794b68e61d6967b1bdfb98cbdf0fed60431a1cdf3ba234e198\",\"f616be78b03d133431b8f88760d9c28df7435ba63ac29157ad444014f09d018f\",\"aaf9fcb1b6c1cc825b4635e9ed6ad8c9798e35bb3e5f4a5454ff528375d89081\",\"212c701b1750c86a70002514b319855fa1b5302c12f5b6afe6cbe439654ba98f\",\"b9896d95ffb986d8c8db51133a1ef8fba6f9bb45d58b2ab7f04c894fbc1c4041\",\"b3b5abe4eea32ca701f9b49e22f06a3f854453a85e36d590ea70d6f921240970\",\"5dadea9abc8cadb115f60ee1e28599d725cc266864b52a4f2cb36962d52a9b11\",\"b5d53ea62a67ac87c77b91028b5643113868d85d0c53a17d2b3097def797c36f\",\"3927ae29b0cba85cee0e3ab85758749e87c8f641b22232e909b85a809c04da09\",\"d24ea7b258c273679c687cf7dadca547f8214a82ad959d51e914cb25fd4bc1e8\",\"012cebf1b327925b00f8bded53a714802d5f8ca522ac47df22b11824eb895438\",\"a512ef4f8a48dcd697c5e41f9813bd5171dbc216df2b748c914e3f3f09bff66f\",\"820d226db3aab0231d73c6091c2052c23e28e73fd899ad54a36243bcae4bf0c2\",\"5692e0eb3c833eaf2fc98e3de70b2bca09099d07bd8dd018325d18a76964ab66\",\"1a0c698e4c37afc4a86d13ec7ca4acbd7a45c69d8bff3a9b55bb60af466501dd\",\"5a0262df41b9c5e7a8f9b2f0f265aa1bb82b8afecb4fd226520dd6e83802a734\",\"2c0f5638ea36accfe5afcadc6f7c105d3fffc67c849daeb9044141eff475d2ca\",\"df2d1574e7210b70d1f57accda65130277f232d4686e8ef4f284b2b4e2fb3684\",\"d39ffdab8775c858eebf3187f0421da66b48bad0749433a1609d6808cda4b361\",\"d716edac6e5e2656e30b98b3ddacd136e4dc6b8bc5fb9c2a4730713e021d0a37\",\"616c06c1cae1cadd391adcc5d54c29c3b1b5d3b6824798795cbb91f2b459bf2c\",\"e8bdb6cf852700e3dcf59d20220d876951a1b8a0f6c84d3a391b43c083c48981\",\"157bd0166a49ec766bffe466dfbca031c84683f9ac512ea4ae6c52110680cc3f\",\"85e06f87aa4498159b2d6feb192200beaf54b14801a6900a4499fe7805c8cb57\",\"777f6b86ff13ed601a1d7c6747dd0aa9fc9bac30948ba0963a00b14fe869acd5\",\"d2342df98d85a1d583e9ade39790ef620d7f77e8684ba18f53155c6f0f914cb1\",\"2ebd3e1de720a4e811337ec74af1561b17bda2c3ad9acf560c690d511ee73a96\",\"5154d9ffbe90d079fa4db8810c69cbf1b2fe24cabb192c34dc8c4af33ed297ad\",\"a0e54ff55b720b0abeb12343c6358e604789eb222d742ecf4d794c8ddef64cb1\",\"e821b5f5fdb4442acd7848d6ed813c78d927888922e69a78c5c0c75c0cdca046\",\"7f81a41b4fb8f871ed20ee3330560c7b941f8dc7f61e9ce1d0515ae6c762bbec\",\"a94c4e99a22038c988b8b50047f3faf9fe4d8d0688715eccf74ccc585f69471c\",\"48efa732283923ff142cabb5c31d5f737a8b2fee86e2fcc7c24232b941a2b4ad\",\"8a3dfaa44a318dfe3602b75054a95f9bf6df5af050725381d206f42c71465228\",\"7694d0196a1f30f2ee0f05e268eff31cf8c84fa2f75f5bea1b1093601928abfa\",\"cbedf06a8a561ca5ae7d9052a15bafde91fae81a600904899cf9ad5f5de9cb5f\",\"d58d5c93eeaa7d456e8029ef588085b0553615b19f158c409959a7eacc5b34a9\",\"c18858dab2d29a0a1ae94032323001ad5e53aac527b6611cfde4b853d7ef458d\",\"06385fb8d5c67710ed17858c117829196dd01031a13d8f9056ef59047733998d\",\"b2a435bcbdea4ef12cda1b9365712f77137b13481e353472ef2aff96c202ad24\",\"d13efd2249ca312a8a00567967d5424172964d6ab81b8b82f1aa42ca91bfc5aa\",\"3e5016978d63dd3eeae4c5837e31c62d6225d4311ff34ffd4c14d43aae7825b6\",\"39fdd7d623623d77d26e5580eb4870ffec494004f07b8e445e1289b88bf3b3d5\",\"41f2e9a695a764f77bd80078c15dfbc051ca8a683845937edd34ad4c18b9f929\",\"0ea253cc46559f8041dca4b59d9f673a00a0c7c7ef5fb2e223921846c081824d\",\"3747b5a6f0d16913e89189bb90e54477ac272e2ae37e056d97c846ab824d98c2\",\"161fd697d85e9f6ec310aa1a64910c237d657e2a70725f701620d470264f1983\",\"5d9465f31c600bf58d93a46333f8ce8bb84a47882ee6e29b433db4b607a8bcde\",\"3a6a5780ea450f12704037023dfe5b3d3b429c83d3a79b89fb25a60c22e7602b\",\"00f0047b756e478d34ea2fe8b0628092ad55cb2cbf4bd1b42aafc8c9b3180e7a\",\"b3221af414c2ba83320e45d0eb85ab635d6b64fea1362462901328d6bcad997b\",\"5263e929bce810d72f43461e2436074ab894a91c8308fed5d718d8389cd94b43\",\"179cf58d2c4ffb71375451191cf322cc2f46455e758a2aeefe44c4f9fef2598c\",\"6104b16a5594658f5e954cb16b4d1a890fd7889096a47865744f626cc474ae02\",\"ed9e7923a0da18f97af57f545e4fe4cebe0d592e4cee9f9b11fbe3e630aaed63\",\"5b7cf9789716d6d508212d7a9dfe1674121633d8985da8038afdd0e232ab9d16\",\"a7629e815a9f5ba130bcd80295b7b4346208ea7e5117c76705310506e0bbcad5\",\"d7594a719211431882f0175ad27700209cc6e4b9489462c5aba3177df39d2050\",\"1f8d1dd0d46b691e8561e5de420f59dde298d368bf6dcebf1114f26441bb7aa5\",\"0b7ed0aefb1b8783972a1cf557159de8c55ffb6e5d72b8c530b1de0afa2abadc\",\"55fff8ccbfe16a3fd80770bb5a37dfeba9ad28376309b77b5cb1f470653497d0\",\"6937396621882c86f60862ec87a30318dc4a457973b5962024dbe52b2c62c3aa\",\"e68b476528fcc24012d72be5d030e4ac2ee14905a4bebc0943b411ebc9441aef\",\"8a663eb89e55475a033d8af4e1af862f9e1a7c84b73eec816743c6c68e70b644\",\"e442450901909b5353655058d375fa58e72a3706a137700443fb73e474e245d7\",\"62bd546f3ce98e67d0a25265bfed53d7e810a0f0c77630c61adac4a95a9c3558\",\"699b9f6055228c6be39cd025c5c3fc27d46a0c6ee83725e47df58ac8d3e3be94\",\"ea955eb5c66e6ca9b9486a61160ac4e3e18b1ba7e4cc7ba0d7ef8164c69a3e4a\",\"26f3bbdcc8f7d21677462a442632c01129a8d58bb3ef8fdb75a37dc98a59f564\",\"6872e95f52e056166005fc3787fae6fd9869f64a33122446290a44f846da9876\",\"c444c52c30c5f908696d26b84969dc5855d47aa6ea1e223c7212cccdbbaf52a9\",\"ed8fe801755478d1a93c0ae744602ad39c05a808f649e9694f76e35e55dbb61a\",\"58dacf2027db94280afa2518fea1a54695f1e029d5a7010b2748e8b009ee33b0\",\"2657bca81253db6f049a5d188a0305ed24a487ea161c5e42883ee1448b2ebfcd\",\"ebdd8718bd535d94ed216fe703b3ada0c975c2d90ca11a95c0bd424f99d8ab73\",\"eb0e4cb963fa710b86a0fb30d88910b007aba395da676e2538277b490fa1c3f4\",\"c16b5249fc52d6e8fd7883f3da0a7b95d89e204ddc680115dc201c1ee20be531\",\"7cfbc2d6505f32c85d80ce5baec0b8d5432e5ed3da57615ad51a892d72469984\",\"e902712435e5ba8203eca1f2522c693fd7e5d6219476d50bb2365496d8422f8b\",\"b2edb2bd0146ece4a6b063d71972d6625f08bd53b2cd16a22d57452547164738\",\"5bf978204eaa5a2a974a0f19cd066c2fb5b2e07cf07e02604bef800b101408b3\",\"1422bd213007f218c5aab7bae80a8c69cd2d174f3b176c573a4d020261078f1d\",\"f5c8a543faf6a0d5b584c5e6b0fc81abc7d801775c0a795dcf9a42e3ab5bf0eb\",\"4fc6c5fdb5e5d46231d90e548592e22a6def18aed378822f1705b137a1b13f5b\",\"67348775ec721fe8dbd3f02fb39eb68d848a702b184478deda9fdd29197b7bdf\",\"7cb89b546a683bc2bbfe8c4f88e3634aafa25de861be1eb0af78c738e201f1a6\",\"86e0d20bcc0c07796e8d71ac4a1f0973a20b8643aeee53dcc5b6694c9d8ea8be\",\"e727ba5f1f5e29826a860667179afa5636b8047da76db27f4e8b6c47ec0b9b6f\",\"47bdb3501545598dfdbb41d6808398631e3a71f03fe57e961f711c3c02f29391\",\"900a3119d9706bd3ed2374fba64dff52f1d81e1f749a6720026e6003df901ab4\",\"bd9d0c06796534bd56333cc9e7f575d553615ad4b55555259bc0b1faeb9a87f9\",\"2c278fe57dfc174607e3a479f7463223fc2528905b8f93ae9e6712f6dd6b5bf6\",\"d4ca1018c3d7fe4de85bdb059cb260bffe50d1a649e8064dd9d8d679fdbb1a4b\",\"0aa370be69ace55edffd22cd5eaad02a43d9e3ce62491594bcefdbc88b93c983\",\"e54f6b5dd7dcfe3087c03035dc009c1d2742bd314881cf5d161ff26b3c85dd36\",\"303c71b2ebb8401b396a3a8f6ccf4630e6c1ae2a79e66f7c33e2a463db97cf9b\",\"a9439bc2b0a8e6e248441de840164ef3a63b32b542a8c7a463de012346e3e0b9\",\"ffc365238852c8d2a6c392e1151bd0d644f80b83b349456de19c3396a2427653\",\"2a9b8dcb11e1e460118b9d9b02c2d9f1af7731c93632b25ff4b820e164b6f759\",\"372363967c5604f016dacbf23c8a2a5160a80b39db880ac74fe59f8c14d29df0\",\"9a78fbf85dc43f019bca09e5d419b2159e49c7726a3d659616b2285a4e4145de\",\"62adcf9b76deaf59038dcbd7ed8efec8b52acdb1ef98681d85400371b2e8fddf\",\"ea7fa0c8ad8ede7b6b8a4c694cff7947be218b6589cc23fdb0a2a60dea722b42\",\"368885bfb7f5688a5a1b69d9d95eb365108328af3ded542d30df3e55231a517f\",\"d74db3f0c84f79aa2599d17754eaaa34f19b2cc2d16448bef8db6283ce4e6b88\",\"d23cd7dfd6336b7b0eb176ca8f63978fd4d943ca56345cad48185eb4c217c377\",\"ad35d9a9b7b33f130cb67ff098c1cc4138c523108416ed1bd703c8e64717c72a\",\"240f8e56a56dff759951fccf39b6ebd2199d07a16a81686e228f66641d0d3d83\",\"7979319afc7ca432029357e0f44c2e8093d696a9c1d5ea9dec3d19549ce590d3\",\"5506379885ac9ae172043ad9620cde840309ab2e3f466a86458d98fc9b2916c3\",\"e2408221144069ba96bdac0140724079f1fed09b74a9b1af2bf52750fd3b18a3\",\"8f5191a3a800c0c29030bd014d869d3b4f0489b929fcfea7773c0b3e609f9056\",\"819a662c270811e380056086ebd88f744c38e4ae17e70138b3121e62586af474\",\"30a534550ab64c7e7b696e80e33a25c93cc5198b6cae3277192b055bcad221a5\",\"cdaa332fe2dbf92b20138390135084a903d3aa317f9c57ba63fdd561c78ddb69\",\"2aa2b15397e0f23f8c7c29d819caa3a941ef7b843f851cb7e14133dfccd3c63a\",\"4c932aeb91292830c9db87c84ca78faaaf9fa5a0d374f019aca0d14294e16660\",\"3bb3d6b9463dcc20bf4e54160468be4a95cd2333888a99a2830b2afb743420a5\",\"ba6f9491adf9929b7ab1e819f67b53e8d0d1b8381ec07bedea79f104c3f5596e\",\"06ecca217228d6d2d74ebd58ee3870720bc72cd53ebe44b8232a85f401082134\",\"5f962c6f90fbd76e68ca9d3aee84ed58473ce6b0768edd04dd24a74da7c47f33\",\"306353ac7214565baa3295ba27716fde85507492ac6b94776bbb793199ba5b13\",\"a077f50343db6b276701e41c062f84ed2d3343c2ed8ca2ea0c46ecfcc149761a\",\"7a7621dea45b06ce7c02d08f89af3d4d2ebc963d7b22085bff2b39f8190d8844\",\"55df48214e830ff6e1759fc4e4dbdb0755eb83ef5f8d8b1e711a7bfc5bb1aaeb\",\"8a9b4f18f448183ab80dd1356f9c360ec2b01afc46670f7594d83a3aa668ce65\",\"be97f7ee900927424a5ab128b723c34eb03820465811211628937d72e49126d1\",\"fb766bedeb309f10beba9d19ec455878e74a14fc54534ebdedd14589c57eb00d\",\"7c50db78d8437e7c1923b5517487523e104db7c2092b9665829d86b88c184c64\",\"270b71b15d4a99255ad1807f1bea27f5c9b41d2886723bd77f4e423ffe9a4e0a\",\"d333620f494a96aa36ceaa25b18a720764d9efd8b8febcf246635c3f81db6c3e\",\"30717399cdb8be4b15e5f23c60f26f12459322f672c45f99375de46b4d18f82a\",\"690f396685b314591316bdc4892570914b0bd9cfad83e6d9cb5245735c386ada\",\"f6126199cda734b348ba48506c2a3256bef7ccd1f6a2de007d3c570f0d6582c8\",\"bd1ebb21db7a09d37c55d41c2ee443519525aa28234d956d20fb70c2f935c64c\",\"032a1b9c3c778ecb3b77b6f487b6f842007f22284bb22736c865ec88bba4b394\",\"93031f9994d5fb2a2b5686b88a80eb3e1eba49ed6c822903a013263b57ef0bba\",\"a527971958055370fe303f423b4a13ddd59496349301bf6f58a67276a93e6fea\",\"24eb24d33ea8bade991fec3cfc37c6b2d23e416d7e5a2162fa176ce2d75a363d\",\"ca2e16c627c010ec6400cc6c96641551f68629c8cc4e7114a1e51a19fb933473\",\"8a65c6be2b08d6b122ee6dca209f6fcabff2fd6677e2bb22dd11ae4adf9b07e7\",\"cb5d8a31a2c44125d5605e6a002807efc5a93e09b075ed4e4fea9a736cdbba13\",\"245189ad2f4759d243f430bd50029f2e18f33870c7072ba5c9e991c28acddc2b\",\"fc086965d0af6203a48cb197950d570601e8994329fcf6062f63c96e5ec5f91e\",\"7f489b19f2597947e489c80e4e0a802454555d7ae48aef0b518f45ee6326e28b\",\"30946101e59d9b871c7a99200ca1a3c5d93841b5f0713c2d0248021143cabebd\",\"eaeaa3094def7899c5d25edbfdd829deb700e34d6131f36f142004c2efe4925c\",\"5febed2c743c6f93d534a15db95ff9b4dbf15fdfa743a7298145dcffffa26cb2\",\"4088f5aa6bfb54e8501c39227d8096a80292799d99b59d98783a4faa95aeaeab\",\"e3fc368334feba295288cc352fbf21ccf25415d6d43f9f3fc7ea6835ec6c2a1a\",\"b8fd7b679d394add616f85cd4bebe702d09640b77b5da8cc10bf27e75b7f3339\",\"52719a7017be480773d8ae6dc0af63dda9508e786966a233fd563eb81d9d1091\",\"b4570926648c3dcf5793766ecb357ec9ba37b0e3eeb9340d25f2a49f99200ae0\",\"343dddcefca343a172c6df049b2f36328f711d927f1e9d5dd1e35fdb13578ec3\",\"3253dd85cf8403588672bdd892ddae5296115e9738d31fe872a101dc19760456\",\"c074d6d40ce8f2cd36c74e6e29f2f9360e40145a015b4283d10ecd9f38a73324\",\"e12cee589ee8b8ca8b557b794a32972d587c96da98ac7e150ab00e0c92400183\",\"daf1129e13a084543640c1517ad540cfa802835ee1b4ef1565678f617d071b0a\",\"dd522d458607d0d1577a942e3db38f865d4e91eedccec2e5ae7b2c7de19c6b0a\",\"273fbc6962ca1980e8572dab6c90e80213637d6df493c2b93cf04d2252121002\",\"eb3828262f4ed82942d2f3fb75e030cda9588a2889b8854a10da512f81dad38e\",\"507d454b68335d959109a6942f5c1cb8b0b8ffcdf7bcad3a606605cddbf7764c\",\"6b534d77f4ffe4629624d9a2c971659f475a4db5e0757db5f8818ea0d91a2bce\",\"c147277dbfd42e25d89225597733f3750974b690d2c0aa01ce4582ed64e3b31f\",\"9993bada237b46b8d1b49e69167909b1b3c9afa792a118d4cff58f7f929753a5\",\"b7b673d94f71082e3a1fa6a88f9e7cf8fa9260ab29625ec7de132cba13454bd0\",\"43570f1fd92d912f6a5f063aa320dcaa8f1746751cc2d1ffcc784fdc8c520af1\",\"b4b3170368f74c2733cc5a01ea2230fd39912dd111d1165a43da4e681fecaf23\",\"7ef89a30d3f4268e0047154f709c73d183bd0c2006f25896ed69632e56fa13e8\",\"e38c2279bdfd4618535136d5e260fbda4e9cacfda2a9b269df68d373065be4ec\",\"5eab7c2ec5b9caa268708cda39e39d07b66157eb04e38b60b7f61dee1c6d1828\",\"a88f291b0c1f7566566bf432aa528ac988065393108b57e9e91d9e5885cc6fc7\",\"de12a4b0cbd87bc12e0732ee16ca4bb45dd86208fac504c74b4d3e8dd419a179\",\"3c5be6b9de70cc1b037613ee1b66f5c3afab36176303c16afbbbc3c0aa2e9721\",\"5c72ae6d604d54015ba429550bd434b5468bcc523ce8f07927ffec8de3f00924\",\"8dfdfea4d6e784e3f726a431a1b91d92e8728ba8339dae59807102539df98feb\",\"dd2e4bedcd6362c6fef62e5e83f800a0e1f0ebf613b46f35b11ed48021c277db\",\"101d0c28f405b799de9382c73075ba5907992f3dde13c92ad12f5ff22697f41c\",\"5e655dc710dc138ddc9342308a89e16f7d1287935a99fee75cd51ffb4cd700bb\",\"16c5c863386707f74897f460dc75888225a5ee9188fae44b3bdfddb3e895f8e3\",\"63bad08fab7971b21a51bf2458f6d7dc2c4994f4d28027ffee7a8bbbc50a8155\",\"aa48c6ac506cda5c5f2226a6e44850aa9ba24b5fcf507e93a2a1d6dd9d312c22\",\"d638c7656d012b15c3ca2a3a7a83ea60d60fefe870c88d921b6e739cc5dcabf8\",\"a82e58f8541e8a2db8c714d0276003d5da68946e3b8961fa5841a84c516d5c13\",\"9475248362c14c7ef684906b69950f5d1efc754e345d361a2226cc664ae3bd5d\",\"1b30092a55b520ef3733985324fd7cc58e61942ee174a8702d944f8a7b550ec3\",\"0d2a48c7018d593687c8c34f452dc7257c9949f01419a33bf4f9ad218a8f6976\",\"474c24e3d4f612241273d79793776ea7a7dcb0af82440396228a7daaf19b0daf\",\"27f1c07541f5c2b2472e5f22ca7e724ea7ee8cb8a4b33b7a788de6078a1695cd\",\"64196998d1f6f4745514ab4777688901061c86cf90f8b29ef55acde744209672\",\"ad45768b524c570bae768264be70c3511fdcd72efe5a95f3bed2c9044e378de1\",\"d480f676875980acfefd95205ba39124663fbe2a348b837851f084e681c1f2d4\",\"66d1c80992ad5c348f95f75c9ad5c9fcee5b8339359d62577926fc0e26082431\",\"c898410737c017c1c74065d47aff3c398f11299d7eeab626293df845c5edca16\",\"1623e1e6bbcd07f979046c1836fd2820207ea685f55dbf5053af72e91c4d76bc\",\"d8cc9823280260644c2722ecb8afaed00c75a6699bd7f557b7a5f8600ed5774b\",\"c92d30ffc309856f3006ac69b65be674d99bce12a2ab719c3f53e410e3c5fa75\",\"72ab40ad2be4fe470335407b7ecfd7d84a9c8891ed6b4c3305ba000805c3c442\",\"e467c60a02cc035ad21455af37fe564238554f1dbb93e5dcbbad2045c4acf462\",\"05ecdefe52955d0fcbcadecdcd9697c4cc96aaf76976f5eb50785be85439dd92\",\"c46a52c3d945e373a0ce8cf591e8766415dccc8c9d933d8a6f85a3a5d283751d\",\"8647f3c36ac7c98511ad4320d392cbbcdd203b81a5c95b049cbae9114f365f21\",\"4931651fd67ab53eb0f161665df44f660e15b52d463ddde4813a2de735a31430\",\"3728da4ecaee4539d8c1ec112a981614a8bbc14446344c27f6fb4bacf59839b1\",\"5c5ef3cdd022955d63a17970fefb2770c284a015d3806d2d77532fa5deaead0b\",\"c83d0b1b8617dce1d4ee7ed772167b618cc8fa5bc336076df43bc080d03d2de4\",\"df8369bd90540abaf36fbc8dc4b36e66e554e2bf8fd3a9d485819417ac9047c2\",\"6b7c60a05b294869c94de5a84d70bed448853f0f81da9afc46cf64cf217b5fb9\",\"36616371a13a4c8d29c9ed80f318f3f531bc0e7e5c56424a0101cbd9b0e13a5a\",\"e5662c41919af37de5b8256cafd1150b50ecabd389bb07c3852686d9cf0f2ca1\",\"1905eea22bb4108e8473fc5b6d1962f968ec090d54759465413a3d516abf30fd\",\"d228102b2b99486984285a15b931153937f7273bd0d2ea13506763a5116ce647\",\"58cdd44847fcb8e760987c468336a210638dabbaf97b1b60a77d08afb25b0fe2\",\"c590974b164edc7ac6efc49d7256d18d77cb9c5fda47bb4d5ea6d77e64328e7b\",\"5f2f6726d0c01065f6b7a69c6d5125fbc7c7ed3c228533afdfafbe617828956b\",\"81a1592dcb0901256dcc9c5c154a9d7b19bb63e984afebf9ef47c396ebcf9fb7\",\"b5a98a8a5bf53dce1ab6008a26a4e2b8feaa9c367708f3896f4ec4dad7ba1a55\",\"c5b1870a9767353230dc258e0e3f7bf0e5411e9f871cfab99a5d3da1fe73ec2d\",\"f19e6cadd60687014cd8e0685a673acfeca7ca8014728ec129bb6fd3f7684ccc\",\"317c9deb6d759571a78e8246117a5e4e82795b3c4f9a4b39b8f018e0d04f4264\",\"4b8c7869d1392a1580056c3e2741f4ff68c4c9e5da3cca986b44184ee5ed55a3\",\"88de05b76ce31512ba387f026099d51874af11989ff70a75a5fb5b0b4f9e7e12\",\"68ac8463d15d2844ec6911299055b3a4e6ff8c1ec66cb060233d6adff9c39e73\",\"669b38f6a23dc42fd210d014c388ea23f8fdea0f973bfe995521372b3f722561\",\"362c410c33dcdcf0ea9e92f11c5a3a234d3ee1fe29df6603a864a03a50b79c15\",\"55c3c61cdcc80a0ecfa2ad23bf85c675fd677f286ad0f23f7115f256b289baed\",\"08ed641e299a5fe1653c669bd7fc4323ba48fd3a235c35b311cb8abd4b14b984\",\"75adf634e1a33e01ccde87309b4e4d59b7576cd194cb41ea24b8b98dd2501d0c\",\"de1613b87ca5e00f3663cc32697587044fffdec7b6e0bc703bc833e7b627410d\",\"f270339544c9acd6494b866f398bce398757ffa9d0f73767d080a3805acf4a0d\",\"ca83135934bb33d9bed2eee8cd4327cee24006cbede2dae52da93f54cfa98f40\",\"7ea145127b553ac478f1da48d103ddbcf3726770f9db0e7d8ae55cc386dadc99\",\"d1a90ac9d7539c4de9bee05eb57fac22fa2545bb2d711cb4edc4debb05da7e7c\",\"773e74ca75b12e235d427f4ae8e1ad4c043eedc3576cb0bf5c0d99e2a710c061\",\"3b56ea0b7492a0b25aa3ad1702041ce69850f7974dae8bdb0f42cb3a2788543e\",\"7191f8f6db0f4acfdfa416c383f15a583c00cba850ea311777959040a91fb4d0\",\"b68a484e6233e9b5b32049ab0e12ed2242990c4f8ab61970e3d3ca193545e6cf\",\"79ead2c43e3d9c54b7af885229a3f8a91ef386624f7d18a88f0e726c8c4ff9e8\",\"6292d313517e51ca465de38e02998213793cdd00e7d6c946a0ef0ebf9ddd2f98\",\"154584c8661c1db82bba8e4a3093dd89637ef7c916514dceaac04ae7bdb25327\",\"c6da5bba40a94c80313563385956d69a8ebbf882a93d93a8d6c89eac2d625076\",\"a2028a7f95d3e5ea0e741b22a09e04f4262e36961165468324cb970f155e23de\",\"affda5e4d95747120f8f1b3bac28bde55d32c487007a8248b2e1dd05b7a92167\",\"bbcefc52b34bb4c15589bed8049ae32c057da7370ad00e2a89483ffed27f7f07\",\"2f847ccfef3478c613d5e2c0375211823f72e0d289ce19ee0e1e76dcfa7e7b91\",\"083e2c1d1b37653875da073989d206585b22d553024afc6d529382ebb0eacc23\",\"1f0e1f841c468d0312b936c317265bc3d907e751a649857cca9251fd318da703\",\"a7acd80543f8aa2ca903a7f68fb1dd981982d1d2951748a88b0fb1e2935bca64\",\"e521a650144cb862c239c7ab3e996fa248dadee209d04d1f317a86b4f8deae7a\",\"3d1b40d6d530a1e04f169e2b76a15b439b4350763fee2f4d9084f8fd445240bd\",\"df902821d29c01cc3e5267da57a841cdd236be248d4df3e8ed8a0747b98945d3\",\"303d5da231adb2b08ef99d22f266397bf6d8f406443bacd50c7b4c4b400303cc\",\"bdf66fada49b79b1d9672a891fd903c8d1a66b37db092ed7b62651fb20972b51\",\"396ccf6ca1dd7f184cc0c4803466ec0b9435402d34082556e082b167126d53cb\",\"7a2feae56b54499bd14c4f710436e3e1417f74b860c174032c7ba5a777e78444\",\"7589380fd6c470bf9608578fec7cc143caa37054c3fb8f900afadcbee9459efd\",\"6a157b4732bfc6e1bcc83f8b4d8b015b4093c628ae69bd9adae71d6270050b4e\",\"d3b011dcd1481fc30beccfee14f206a849987b3b269926d0917a19ca77f126a7\",\"9a49916212e8b298e46b40a67857cf4fb877216302e7bc2ea02ccaaff18ba4b7\",\"a72ce677263cca06939abc0f21672c883586e8d1841024682d825eacaa038cf2\",\"023307d953f03c2f75d78bd9927a87309142fa179c6c966bd77c1f59ed7635c7\",\"4f917e4898ea89286475f6cba0c6d6d718b120bae5e82b755dec9b5c7de3cefb\",\"e939664e707a76cfea53a7487f041d8eb39e53f417af993068e978bd971462dd\",\"17702aa2522eed3d62770569468e82af656732539c74c1423dbe4ffd873cf5b5\",\"027686345ce044eb9c88e0df6db1530cb1569971132b5da1b93b279a85b6d07a\",\"6800e50487fc5cb579947c7cee56c69e9fcbb2451bedd9499ae190878b2d1cf5\",\"f35d213e701116c27eb2be112843ea65d85652d6b883d557a0de05ad0d46c8cb\",\"f8f2c552cbc6bc3b27c121e6fce3e3122d0b4a2b03478128d3b53758c1abd09f\",\"6f79007eaf012e689ea07edcfa8340f0d010b3b029b67b953b08fb6bbf71d20c\",\"e31339d4d2bf59688069556aca3b55a1fb8b208f410fcdd2cc0dab7849ab2134\",\"24a5978c1d77b5b0916ba900356388595804f7d132904f287720b26bd3c85d75\",\"05bdb5ab14e42965e4c224cb45e5fe58d5c6aa0a713db37efd6dae64069fe702\",\"52af02c532bae3bb9347f42c752427fc655f86e78352f74ac41553994260350a\",\"c0d7248b35c5951f78c5407cee25135fdfcaf43ea643b264baebc44eafdea91a\",\"946c03fb69905db95b7e79ed4e11ecce69f4f9ff6cb5c4609570dc27af84be80\",\"b285517983b6ce24fdfb147bed139f9a6a05a01a19fbfa510d41477ccede286a\",\"5685b6ea5ae072398acce895d93060a2d30de0e45a8e171165a44179827a40d2\",\"79519f557ff15422221321d426209acdd025325d60ca632f361fff245ba2a041\",\"3ea88492abdf788f25c151e6dcbd1ac9f22cb8966ab2648e2f9c629505a77c28\",\"e5e343c483dc0909b7ae9c4b41020a4a5b1b779ce4bf9c21be5734277287af06\",\"dc3f7cfbf5be7b9efc31bc5d1bcd4c5dd2c408185452afcbace11326919fc5cb\",\"addfe7b2a37403aa651b4c020af6906e65727470a134e5862fd3ef479fb6cc7e\",\"952729387f3a00494bebac589f0932cbfb0a28afe7980078b8e0243bff8b6a39\",\"607f8be0133ad647954350bbfead599206a0a750bd886e35d6bfcab262aff6b9\",\"943c32e9b1bb058863a623a7cf7ed8cff2cc6671020c12caccbd7c1823e422a1\",\"bd7e493a52de1b562b0f1fdf0d6f898a3d437104d9f96f2c750772c21e987ed9\",\"b9a441ef14670a12597d892f388440d5d6c80a374c860fe7f0adf84d506b5c6a\",\"6e327b484e5e0c09beb1c10b062d0490ebe690b8eb4ba34f9dc8ce779c35ce84\",\"575d1aeded3eb717c84754e0399d679775f7c22a2a63c89c1feff73a32fecca0\",\"7b68b6e4b4255e08c688d7d7801743af2107d71185c997d40e8a139fd5ea589f\",\"cd05fa7bafb04adaccbdd6e594a618843c643e1140716b1870f1dd63d6f618e6\",\"80e254f583a54fc10f968fab8b7925eb35f30b4d804db53265d5363d85345b88\",\"85dc9cf86654247ecfbc28d0788361e0aa0788e27310fae9b2d60c00215a164d\",\"aab659978f18be4df65cb8e1f09596827aafc9deb2e690b1a0996146b6a34761\",\"49b332917a553c4ffde6c93eed626dd76c77380faf39d6b30305ecb0592427ac\",\"fb493a8522439280125da0189f0e8eb84bc8bd5434b2f25a7be069cb6c4d9f47\",\"1910db51dc6959529606acefaf133baebf30325910c2f1aaa04f0c7a69d4939c\",\"fc8e1bfbc57d29c1d5e5d850f8a104771f54926bff77f738c8e0932a0e92e13e\",\"2aeb129e22aaecf532f9bc287ff9373a637cddf9e45e33b3398b9648d0fcc088\",\"470e2d77389f710082a01b589c868d238cc258bfcd9728c615d255803e0a8f85\",\"d6a9c7f6801523d64435be3633b823af30d27d723e24d279a6be92819870f5aa\",\"1b71b784ed0b3685198b4821c8b6fc136f2416c5358cee9d2f62d54992b13587\",\"b67696a73a45fd6c91ca69c27f2af547ccabd5fd5a7ab906a198bd23d5d7ca1a\",\"066a8c7f43c5c302eab70ba4df3e4d8363c7fd8a02e85cece3b32e7de589b67c\",\"5515e0ed60952cc15c77d6f016e300a4d43ef08129d4f8942f489a93e83d7ef0\",\"d6cb67a2484d51fe2accd0c53165fba374b9d321fe2da5a07a34a83876092f2e\",\"c01cef9e27b19eebe09d52024a19aa30ea1a42859cab7604d757c5854c5722f2\",\"bd413b0e868a60178cb5f3bcf5f62b08ee9e40f5acf64201cbf9029e42d7a312\",\"1f14291c56cca493f740e336162c6af7299e09522ad9048e1cbb30e40b8ecbc6\",\"1660ced783041f13a6e75578a0e7d7e28f84fa04cd91817ab1eab21f9e631152\",\"5f10a7f29408be9532c9f1308237c9106cd4d68f847aa6e9181f87c7169715a8\",\"c34c6930a5b84e7a57845da0825d541afdee45c15c9e37aa763ebc585189db54\",\"d1750b862b95afab59852efdab15f1719cbdbcd459497de1f234de0b4345f989\",\"17330f9b1f91f94b01b3a89311e0e1d43180493aca33842ac78e60d8838153e6\",\"585d877c5a1a6609c9b07352355d04156e458f65664bad9d2ce209d56585ac67\",\"f7ae4d77206f41f38c09d09ab111930c0e111101bff08a376e3c1007fb8de648\",\"78cf227e5294304ef24e9aa0bdd6f97b1940ff4829cd3885c6bcc3ecc0c3bd82\",\"7ecc1006c605ed09855955e263b6f3419b23b6d579ab6f8085f0d872e465dcba\",\"0b2594368f925461a82d9cec56b282012ee8ab2a9bc8af03d388b0c239bc15c8\",\"aff8f46c15eda9e1d5cb573573268e19cae53edbc6283ba3309d6d72ec2c5b30\",\"c448de13281bdf4b5daced48c5ca61afc5bdd35a25b53815741c2bd760c21664\",\"0779c23631edcb8f72a3052637dee73dba944991aaab0286a6740cf040961984\",\"815a619c349cf0f884744b19910111f1f855731a52a9a2d9577db546c117d70a\",\"032be1704385ee9eca3add86d58d6f2d60567aa80bc75ad52fd67cd7e643d4c4\",\"096e2a54352b85444788b688ab7e242a010cefc7ec6c7f64e84033932efafe41\",\"0c11baea83be123cfcf5c0a83f8be90e8d4482037d4764a80a3241d9f11b8a4e\",\"9adc1be0e41138fb369bbe1a9844e97d32ab8c90cc15c617d2cb58bb4af231c8\",\"48fb70f78ecc8c26c7b16581d7b4f51f3461e1880548dcf1920ae1ee7d7ac086\",\"4f964fc32da40640b78c7cf91870dc312e0dd3c0ff44cd0bb5ff538567764d9d\",\"27fd0feaa1f3164cc189909f2912ae69b3319daf64a15415b1695c08c5ee01f4\",\"1a4858435550fdab8a298480ce2ef8e7dcef0a6e4d83e7e6282997c6f293df3e\",\"66d696b9ac574fa82292d78c1a201021576e77522c4ba003698da21ac7036953\",\"7ec9566b12ffdd678a91d25f0b63666752808f622bfa7e0198ed20f8e3556d88\",\"5f828bf1925620466dbd4c2eb0fdb499e002a4567e869eeafc25211c682654c9\",\"9818ee467a1aa810c85c47d17ef36abca6227c793ae1074cc4924db0ca92d665\",\"52b7779ed1a300aad79463c6a5ed075bdd9d3c0e198fd4a6f273d6063e1b90ea\",\"9b0bf74d13d8ca3b9a68dbaa673d329c452640e67f9203693ed9f779ada8a132\",\"5c9b3444cd2dbd3d48e4c271f37a12286fad6167fc9a7ea2af089c6ce5162e91\",\"c0b5c12152967b4cb9436ee462cfc7d2bbba3c3fe3e3afe0b38c27e96ccce3ad\",\"137e1090897d46b5067313c2c095b77bd110c39298bd2febe038da5320e24083\",\"7f111583e4b27c8f9203fc8c24fe867d545b5727499ecb28632c677afb8b5b79\",\"62db9af6a78dbc949075c6ed0ef8a6185fea516d19e785ad61c7acb1d350f7f8\",\"315752fab5d5c1d0d3aeac56add63870aa4f76ae7704f91126cfd357303203df\",\"c85f21f617ac4e312c264fcbe3ff7a4dd4f4eb2fae9a574dca5ad22995c6d45f\",\"966814450c5cdd75c924654543ef4aefc9b9e5ffe18126fbddd3e43ab85de640\",\"2b2aaf78ec0e3ae26e8cb449e9a36c4bababa5b9820352bf44630f0b30ee58bd\",\"8dc47208dbfbbe3974f295588b4ab90a2fdbec343c2663bc3a36e7b79160e731\",\"2eb56eff09d273fecf90b97410dbf08a055fbab3326a8dd149c34a59ca2f6040\",\"8999e5b609c6916fa8938fe70e49da943d4aa527dce39ec5c56f019d70df13b6\",\"ea9359a48a2e0ff8dbbfa281d32890e20e3acc6604dcbf608d9d6fb5f367d34e\",\"d0544d7cc8de5d04c5f9add2677823ac60d88e4bfc940a827785f47f9426277e\",\"67d5e7f2c4a70cb2b3756b6fb3d725104baa8f4a6fb3ce8dc66fe7788ab3fffe\",\"19b3d00a6010136beae4d90c5800120aa22a2a6f71c00b08a791098b025e595d\",\"a8c69eedc49e4fd5b4bb82a450aafca5dd9e01bf2396466c4780229dcf0ef5db\",\"34f521c4585d6b749bb285c6f96ecee725afbad5484ca7bfaaaa6670ac0d898a\",\"65936203536938794c6892304af1a4d7f2ef14105ce50eb20d017fb83dd83527\",\"f621d6f08f296d3a45d57c46790c3869a301325797ff56d70258f549b3e75a0d\",\"3671b621361f6b34fe79c5a45d8261fa706457fee712ed78de34d74bbc62b60a\",\"2c136e40adf6dd502e2dd12738dfa56881f4fc414d57d76c67fabdccc65e30e0\",\"040686b456110d2451b8571c8236c03dc22ab58cc7cf26184c3ccd0e474a7d7c\",\"f56afa24218835cf72ab272b1bd4242f93ac1ad1ae2e240daa25a8f9fcf029ce\",\"47e395bc92a433bb39f0d91ec268f4909ebdbf77e28643b984faa132cf00c6cb\",\"2a9c22cdfe835e5b1391b02a92d1e76581a76eefd4c2629b64d5392283d2b044\",\"004cb9b67dff8558e00c949d4bb8787055a70a7b4b7082307254331de8908f38\",\"16b41464e7bddca6145e7e9a90d1c8ca3b7d94979e749768dcb5e4e951ab291c\",\"26ffb9c01b9e6f0a7f48da5d383e35feaa025de98304ff8ae4c3a03249874078\",\"5aeaf5391868ec5fb0bfc5c42fc3f454dd8b4d04afc9e451bd33c986284581e2\",\"00a40cbfa855c607a294ce4184a0fb80bb47cabecf84b2d4d919f034567dfbe1\",\"5973ce8abe1ee0eecdbd48a61a53cb6357fada8b84608a2955835b1955409519\",\"048a75c93f675b27444e2fd484df8aa09e61cb500a27f822f9310173ac6af77b\",\"7b000e7a0aa17fe725544d3493849b9c0e61f09f70d63da32412481c6192f7fd\",\"580d6cad26737731b296ebf0492cf9bc64b3eefb77b1ebd8b611f6e26ddea25f\",\"1d05f36b3df035c513f2be3902593cfdb3e28901e18dc667e39c2750a0117cdd\",\"6544f57d39d7d4a4802ee6946f09592f7ad3ff9eae5eb87fff8c532faf1bb994\",\"ae135ef5559ff61d3fc0f92e0cdfd7aad829374c8b50db38a2249c5889e6c0f4\",\"a002972f85437b06ec9d68aea5298eba3ef7a325219494b6e03c21ffced427df\",\"cd6d14467379203c872f995d889096dcf5aa3b67d7b187961741516ca79c234e\",\"051abe525b8d06459ef24dea22ebc4369207e60932ae4ac00300eaeab9466a7f\",\"d05e081e4371e0042f153d5b62be811d24835ad5a87275d093aed174d8244837\",\"a3176d256236ecfcb094a7f1a8be1556f0fcfb16387614c4e03ff1ffc34d3aec\",\"8df837984830732fad6616f0eafab29494c2786dfc6e80a817bb26f4656cd61d\",\"35b207ed01fd1f17992b990bc920a6cfa8413b4a1eb15f0871020d38b9feab3e\",\"fe52cf616853efc1e5f5af9094b26d3f1fdd04a3a888c46b06e54764365bf335\",\"3fea642ea72833e33d66f58a0cf9f38afa488810a2144da9967657f1e2eabd98\",\"2e4a0375dd0c3118e1ab69b55a52e9e052e150ece129a800dd73df54e7f224a6\",\"12d0b634f22b4426dc4ece7bd8410bd7d4f884a0bf4acf04977426456bd643a6\",\"560f6f7e39c08d93d455fe3cd21893e4fb9a7207bb30af00ce4c47b2a8bcb4f7\",\"e8078400e6dd24b15fcdade37286a82abfa9637e45e7f8a381ef4c196de22fc7\",\"3db12896f155e0897d9ccac3fe62ff12cc93b5b42d4abbe48d66e5c0f2ed3b6a\",\"7a1d913a94601399eb6f3c6f11b778d45b7e6cd7a86e6e91b2e6833564cd8784\",\"9105555ad9686e7e05cbdc4d44ed933519ae9d6f5f645dfe7b418415a6d70ee6\",\"6678313bf90e904e8b6c6e238d38edc511bbd61922574d015b6b1650cbde4541\",\"98cfdf2a4ad9ed06712497836bdc3881cc78796ae3508577a943fd7d231cb16d\",\"bcb9d38ea17cd82975041980d33ccffb0a776193c61fbe7051290a2796e1627d\",\"d9141ef6636ff9b1fdabe2382f60460fcb87287c7e33c1e2c70420ce8c026547\",\"309027109ba2e0b68da988b0e9150e4b5fabdc73ed73359e018353f6eb951dd6\",\"d2033684a5647b1f601237a00d084d3360cec56b25ef6666bb1d0e142d01b08c\",\"abf0b3400c554153207fce09a94c532218bd39a43dfc5f32a51739ee7451c8d8\",\"079da722b413a9cdada0b5013dbf7bdb26dd8f94ba5e6eee73323a8672377883\",\"3b34feae13e8cc32e865ea562fb8e5cbaedaaad7c8f99af01ae5937836f709a9\",\"1d4b9dec43353f2f7bee320b1a0e9fc35fcdbcc17f8787795993189ef73befca\",\"f97ebb21a2808658db670e69e6477f189a2b35b00f381aea07269d663eea7eca\",\"bf1119ae3b253c39356d4be467e59158093a0a74fac4528352e11d742a87afc5\",\"b705f77d8f4bfde698966041af4603916abea884f6f9b3314bff709c4e99998a\",\"4da0b22a866a5a99e3a1b0df220d7c4551ea9adbe51b32c9639c735c64893156\",\"ebb2e86ced7eb280d47c055ef6b28131ad662c0d7a902d6f257d75d93c2e5556\",\"b2ea53d01dbae70639a5d6b3d688bc026ba384b384c3930ea886433a6c185155\",\"37aee6c2e90c571f4b81e9ce3f3b8a14c3e9fecd0f57d2046d833e9e656a533c\",\"92a379a5b4903a83711293cfd644697a950a063c8ecbab8899a4ec6342ad763e\",\"38d5b232beef8c2a821e012de757c5be07a7ede6af8bde924f4655d21c922631\",\"1d5e727d7dbc4079719828c0d213e9f178e06a327e237fcb8aba072cc2ac8965\",\"d2598d5d712e64223a24b1794fb8d05424c9d76b8f9af68fe7b9b7aba76d7a15\",\"97a67e0a219331529f6cc06d843d57a5e9e645269d960b11bd224bfbce411839\",\"d3507748340695e4079994acdd8411493ccc0c888ea28be008957f5265ab7a78\",\"eb0cd29c28cb9f0a679e3b723b1098a03bfb19003d3fdd8d8e0998a65667e063\",\"8a134a8623bef3398a162dd862f0c44a3963d30f98e6fc119c0116d102277078\",\"6ada513dfabf214c2c7ad3c0094918abf2818d7c1c772642d508fcb9a3610198\",\"c612c29a3f1d4801b4d45005ea91ac8bd91aca7799d42e68a92d95a3978253ff\",\"3319834c3621beb43a9446d67e4364b8e5afcbf8f4ecaf12cb11007a76cc32fc\",\"040de1917d025ad86645ef76a815dbbdcbd7c940cb4ea5cab941cbb52bf0aef4\",\"485037223ee5806066fefb2806c8ef597a908d93a78cd1f2ab2b81bc79695444\",\"d5cd54afd6b75c8abbd9ea73bcd46e58a916536b61816cbaaf4371a9579b5961\",\"a4666dd922b3e995fd38d1957cbb53d4033cea11a72be1aa5e29c97fa7a2f430\",\"d1410cfe1ff698114ad4fec278b6d98a51f30de38dc310453ae60e3d72c63581\",\"f72026f1a107b616f0716c4b47f9d5367f31c442ba69d1828b74e06fe76a208f\",\"54fe72ffa3dae4345a4f8cd28d2f21d379daf79d4303378186d80aae809c49b4\",\"7bfe5cd2163d425cc98f9009a2fcfe38c99e83c8fa3cdfe366c17e20d55e51b5\",\"7cc4207e7e1415ff9bcbc7042ec3bcb2acf47b67c3db7b3a6a29df1cc40b98fc\",\"878fbe75b4397f20f1fd99a60864ff02fc386288d928d7f7301669b9111f2358\",\"c5b77097668f50e7334123c948ec408cc4c3e803d1b45c39e66f89a870573e01\",\"16b5039e7a027d1f260be291c70081b8dd5f3277b20918ce97ac825ef7542dad\",\"7ded2c051903de74a98821d597027c4fd679abc05bcacf02bf0230f252a682ca\",\"db0b7921959f04e992a672a36437865f96f5f7e73ab0aacd9a19e8c2524fa8e7\",\"bcfc41e870074072177e104a8a34172e024920ac6dec29384850e958e4fd7020\",\"854aa23d6320b807e53ab8e85ea24083903c0a6a49de78e17fffb37bc81c098b\",\"bedf7d899449c52d9fd2df4fab641e24c5e5a1252a0de1dfca286b615a450c3b\",\"6ee65242a6de5358c5eff35f8270fc387c66ef68b4742f61a70ae1221f8e4bca\",\"f0b4160c384b493da51db6cb44334a9634cc2d9e040d284cae57a358f5b93adc\",\"01fe1c80f3d617a199123b098bc1341f6c22083aa4f139816a04c362299f7a34\",\"59726a85b7c746c4e6b2f6cd5ef4584f1b74555f73aec46e08a9d88a8018c352\",\"40380890fcec5674e44211772d8927b236499db5e05f3692e3bef5965c2ff369\",\"803b9072bd4cb51c4fc838ae83bc28ef0c743a99bc6280f1acc01317ad8b17ee\",\"e39cb28e8bb8cfefef74c98594596c1782873820f7578c8fae487074b6d4e5e0\",\"ce885ebeb959b939b20d9be26310c51e98a7511b39d9884ab7d121e9ff3834bc\",\"6820a8a37569c8702d12537cb09bb59a62745b82e647e54709ef1142057d9ff1\",\"19ca6b524b2802c64b7afe1ab89cfe8062390e7b633a7e823e7c1cd2c74e8740\",\"dd3c92f6ee27330fbb35670487e380ea585e3d40460d9b7407de07cfdbefd480\",\"175622fcb677e435a5ce34e8fddccde503cc9eb5b471f184077f7fe4206fba65\",\"ea0f39847188424e30d8c4a63c6b2fa5b7172341a25e6691c5bf7d04a8ad1e5a\",\"29b8cd839efa2b96f7333c21b9b1cc048610252ca0bae9638f16cb8ea74f00dd\",\"5914b388805d1c8e34dbc69fc04cb2c43db9b234f4f61481d3f80c68276660ad\",\"8a0cc05b3c331f369beb4f681273e6f798a603ac78e2897f6f5f32c21f431f67\",\"b70d98e9312608108c581e0303c75ffd85cf7ff6405084277e704497b7111763\",\"edecdb111c16cf14c39242df889821a16d209eb533d0650af317a402bab0b27f\",\"ac2d2a905c06c65fc1ef0f3fbf130aec19f652a4d2fe630352268fb5129b39a7\",\"e8a12f1960d2d9e8d3beeaad55368683de67a2b0dd398da68bc8be1c0fa79685\",\"0167a319ac8b46b1a8e76c27672145816982d35c9241bc7393a18b980b207a38\",\"c699fd890252541f469d489342952b4f91f3f3853128096bcb69a110a7505a5c\",\"17f2ea7c7d45340a5918cbb21a283073aaf1f0f1d859d24629defd77db752177\",\"418115bdb4d7d75ffbe207cc676f673af96986613ee57f83efa6f3d231125283\",\"f0876c1126b4a7a15025a5fe30cd96922d69d4bb4301896f17412038f0015dd6\",\"a2517f7b1dfb5152b5ccaa54ed39cc15fee0373810211d62b68c742b5d1aa69f\",\"78fdf4c1434d5fb3c27f71a7800c43530c7a853cc017286ff8ec13e01f5494cc\",\"ec66ada220d5ad92a89f1cf680664bfa776c52fc350c1bc6346cc36e4541be3c\",\"9068c763e4141d6ae375dd603ab73f39cf607b588ec2b5d757f5d0ea6ddc2da7\",\"80c92b447d79639280789caa3da6f03fab5c3f43f0edfb953c7fa548db571a59\",\"c3daeda2efb14279d498ec14b1e5799784a5d3d1f8f7ef651964ef75fb718889\",\"35050a60837c0ce3e91db4980f5c83e815e11b482206e44ca38dc818aebe8b0a\",\"033a5741ae5c0a8c17d30e181d78cbc23dae87bd5aed55aad4faed070393f998\",\"4ae04bb75f42e14f6f007a750dc85b986685eb843c65912bc5a5f53872f721b8\",\"09dc60c21af723213c6918d959697c32c6bc53a4aa3e2b28d1e70f6f529a9437\",\"182ea48edb08a16844e28538cb532defbf24f8f65d4b88b20243b96a9e4f88ab\",\"b340d3463967e256c3919a9914004a26934add8bbef05cb91c57419288078236\",\"d868e87dcda020db1c38f1127abe3c2a9e458d24dbf2db944144b02a713a28af\",\"9d36effeff0a0dbc602ecb5a2b01bd4e216665455db6af5330ab85f919ff84c0\",\"02d24f8c07815e6450180bd4c47e923482e5708717b44768bc54459e1e8bdd2c\",\"08b8fec4d06e68d13291f360d62e2251a79eae50282f3270152f36857fa1ccb1\",\"7a441211f7d65bacd99c469a85cfcdfceabb30f333ce1671fc7e0e96e7cc18e3\",\"1893e8e0b06109beae8af2a03ccc62b0488a74c623e12187e1798e567aa98482\",\"7789bb29d8183ad4c714d1de997bc31801ab9d10db67d5ecaa31de4cdf85da60\",\"7b08c48b78972e7d069dabd5844c0bea61dbae519a6f722f6de608018d4b899b\",\"a1323afc275aca44206400c1b4c34eef84d95a8027623d4ac3d1fa3aa5fbbecf\",\"3d176733cefbf1e1197d12755f7bd55807db9c0cc5222d54916e953849c11d0d\",\"448375b7397ab75e3f0c0c9b3dabd1e0afbb55c41326e2d3a46f95b7fcba0924\",\"08f532e48401432be97161b7c6bc15d12654ee8fc49bbd1f6638b23e5b31437e\",\"67fa504d89e1fcfb965ce242468254fc01ddcddeba22b2bbf0411dee523f47e0\",\"e3e7611c18be7b35a57e04348100e5a55856e87b28cbe3c3ed8252c66fc57712\",\"8748298f4c421d8d0937783cc65d8aff67e3b34754fabe0ff6787cb97e66fe9b\",\"14175e4796667bb28d4b8ab87d8624618d9e28a19b21ee2ce0fb8333fb4e6dfc\",\"d0493fad9b9cc4f56a62a1e4d59f845282cebaeeb97e576ab3d5ccae7d8066bf\",\"74850893855d6b6b16c8e5be0905907a76584c542e8ebdf2236f4f1770c2c107\",\"7eb2e70ac794f5168cbaef1bf5762d0c928898d84a3cc86c3af548fda7ffaa36\",\"57b00cf1d4c6962a3b78d383a8bf4928b8c9539eb622ffd557c4113a85f5fc96\",\"8295588865a24ae189692dd604587f4dd43cdca6a0b65810e0f25386ebded3ff\",\"cc2e3b891e35da1f2216a4dee95d3bf2461734c3e8ad136ecaf15d960e5212f0\",\"13810bfd4d290face576279f3e0643ac34a5c932d53f46576c0da71195bb8be8\",\"9ace9290cae152e6e3e6060e202742e2e2cbd45816f24afb44d0342f611e1819\",\"391a154f195b83a7329f356446a01c1018c76904a059ea27adcd10408bfb1275\",\"14178a354970139701ea6e4c5e1eb9ddad570b095331209fe902e6b3c213dc55\",\"f205c655a3621279189ebdeef55a5eac53c1ba28e7041011f4eb0c947eadad94\",\"23a71be821d9f1c8c107719426cf87a4e04e2d49035d0522e320413750b4e72c\",\"c056b4c10735be70cba970037ab340e0772addaf6b6d450b070ef719e4ea8a14\",\"ad416d65755cc52f467cea3566858fa8de5f48370a7abf10dfdeca4dfed8cff2\",\"760357aa863486ebe61f46fc2e147a3688d4602ca6bbb9f1a0edb943229d2e86\",\"b7b7b0e9236009f9eed2863626a35133db5068faae31d4decfd20704e60ddbf8\",\"2069c1623a249b08b0f93021bbb0355e912135e0f9aba87f73a03f2a0443ecd8\",\"6440d23071986d5186b68f388bf876e472aedb88b2b3c4ae6fb70c53e6053d92\",\"42d070312673d6e0cb2d21ab8e26bccd3bab6ed23979dcc60c75338d06c861c9\",\"9e43c314d80f4b059c827a4431e4b7569e98953a38a56e13db366665494ab66b\",\"15114a27f2dfa3df3d3c0fd5023d1ad28f835c0776ff5b3080d63e1b59e8e8e4\",\"59675c4b74bd14567befa8ce93f932f21bbe53b91b78f2c05a9fad7e4ab38ed7\",\"02a6d6f86706c1d1c69968136eefc9e6bff9ee8b3386edd5f113dea6f006b2cb\",\"56d0599fc641942fa4dcecdf759901830bf59dd7d113c84db66626d4f84a9e5a\",\"bcec637604e13a6b2ac3d868feabb610adae2fe1d38b70bfe3aa18f37e7f9d82\",\"adf4a09c37f5f9a957931cfe88c65035e0a2a61575a98dbb1baa6aee5e42ce32\",\"4103fb7cf96ec21ceb9fc9bd21e794b543a9ab1354d4f44eb7c82c7bf8164db0\",\"dcd141beeb5cb41149d5c47cec6da5f202a16d36ef89a6396a7d82d151af712a\",\"7f43c0943b1d2314fabbdfa70750f7932fac06d55a19525213879ee21555ea85\",\"7f9451ad79cfd84b1dc6582ea08352e1f6a0a97461ad575a3fac0526a99a4469\",\"1ae5c1c300889585cd7030a751ef87aed67c11a86a5393ef96405048ea1744da\",\"f6740c37e474885cb9b634e738ee5dba300081faac1b72191f93c61bce313bfb\",\"7ec4197379ae50907e7fed78c7d9cd574cf30c026fd4c9b6cfd6c42c614cc8cc\",\"791ae63e7cc7336d7959d8a463566eefe4c2434a1a18408a1dc3a4723f0fb3ca\",\"8e53e3f19a6781905d391d0a18a99bb5bc5cf5327f2549db6db34f9a774fab49\",\"68d6a52a3ed039a9a8e5ed8a696b591aabe77f2a577152317ab70d3c23e1c40a\",\"b44769a5745c1b3a0d98b3f1bd46e55fc9044c912401453ed009367903f1c870\",\"a3ca48f26d63640d459d2f119df5a24db66e3deb7254349d138538882749a387\",\"0c6fa49ee860581728d15a4ea1a785092ecd5915081b6b9028138069aed32aba\",\"5f9c2e1c3359fd263136abad136592688dbb7f559e58c89a56e85c33e44c0b14\",\"510d3816edb7f458ffbae76f3ffe805cfcddf813aa68a5ebe46e576ad17dd560\",\"cb4e3e01abe1893cfb4fbcceee9fa0c4278fa37ad291ff03cb600b4ebca4f35a\",\"29f950bb17764550ba287727a8713af0ccc1828c55e3153ca14115304711c031\",\"05d684b8bd6770ff88613623e1198841bb440935d56e872d0e535da8c4274123\",\"91b643a8f46921b6b8cc2095a7f808bea1530fe78195a3bc48d5549d3eecf0f7\",\"07fd1f3cba92a1152debd8d03a0dbba6ddf06e774db2f5e2402a8dabcacc1b9e\",\"d97f76db85d4b0507bab478ae1246986aef9097a83f326f8e313948f64972d3b\",\"3fc8a213520b155adaee3bef130c22d5d9f1154cb257666bb6034c0fc85c57c8\",\"d08fd9613798150eae420d821997465ae09b543e2c6c4329c106f3b3dbc15b0d\",\"bf14368e4d51fb38bd675e8e0eb010bc8fcac852f01f3daf9bf15bb464f3b641\",\"b1aed9bc235c55cc203a70ea14953d9db6f620c7e5fc47bbfd5f8ffd4fc30868\",\"a6a759c1f676f17de7ead73caf7d890b8c57c5b53353729d992e9b2f96a6e1cd\",\"2e8c66eebe68d1288bdaf8684636d3a78a40ef20af7b4abd8906a16253a2e5f9\",\"4eca51e746bac9f0dab6b6aac44b65653c84536c346ed1e73f4d5af91a764fb2\",\"84ba16fb28abbeb7e2806f71478b7a1c96d882279f3ab184e3abb08e5acb7b75\",\"f59bd2c059cec179991f113e4cde5392541a381ece30678e3a32d9b02978eb88\",\"10c9c901f1c53551ade53a7fd2e437e05d26c787638d6a374bfb8d3a3736620e\",\"9aeef81d8db25633b7f6803e560c1db2318cd7f4fe1adbf4f1134a3a8b83aaac\",\"de54358fb1b27304c2c67e497d85484447b56a1c9c7b231e8e19c7e492e7d338\",\"6b0327dcc8ef7cb6eb7f06f6d64c4d668aae3bd7b0f80ab9a7620cec37cc23fa\",\"2c377f9ebfb43ba3adc3583f3b6c15a8c0dfa58853a3fdc2e613f32168157c8b\",\"29c90107c6c7fcb5f1d7f4dbe236ff0074fd0cb3131435927ce41803c6b7cf10\",\"526f7baf9f9a5d38333b8caf03b4f65f512556c494b8da1d0b7df90f1bf670f8\",\"957e420b714370d3849ff556543dd8f28c230af994203119c3e233b869ea4e32\",\"26d2e2053d65b030b957ce3eabab96567e0babff1dab2840f5fe4ab7dc184815\",\"2b440d9b813e1614ec49bff540278cd9ca6b94eaed072cb7dc8b51f439ff8b5b\",\"ecdf671ef03602c7be65a25211056f2e26a80edce006725cf6ada97b21b44fef\",\"543a6270473f218884c39d14f5e619e06f6f9e7e9c2b8457a2deb49b67b02df6\",\"31ec3cb2beaf5a2bcd091e88b6f01160b847e58c1bd14ca11e1984dae2c83570\",\"a200a25d5cefd50d0a1d51db365eadeb6efb8b21c6d5a5d036e44cbe7c462f09\",\"d7cbe5bb73378ed6f717b11813f8abad4d726038473df6ce9af5250b907b5f3b\",\"44ff867b9006987738115279d46b8ec273fd3826a9bd8d13614826568b9af644\",\"549a4c23afdaaea7dd8742688d8de180cbeea7a40034549aa0f94ab010defbc3\",\"0465227a35d2eebe7651dd2120b01e1ec66d0b46587c14f3a463b8ae024fd6f8\",\"0c92e8d474319c33d14228c2a822d3d15524c43beb85bced55d751b3f6005228\",\"26297acbe10938c8040d6d3c26957a1bc8cbb898fef7e257ed37ce7143bbfa10\",\"74cca29b6a644719811fdbd6ba5c029e0f3cb76906ee4af918db5a7d25ad82b5\",\"545e223cd2a8df335479e6639eb7b0f2a18bfeec91215ca32d1faab350bf1c3f\",\"54974279e8834a80cd74bafbff9d73d170e8cd62c5712a313e6d3bfa7227fd1b\",\"079926d98db103e5f85441eb24191a87a454660b71fdc5269247aeff8f2701f9\",\"b74faf5a1424c8b9e18d1985aeeac595b9a23b032241c6648e9b4260db897f21\",\"2dfde3f73826b719eb365d6cee89306340ecc428352f994238f6b5998efceda3\",\"65c5bc06ca9ca55147a39ec3302115267ee36622df15d81ce6a9ef967779d0b6\",\"88d6f5865fbe889f690f8dab4266642c6c6a92556d26c483581a99862d3c4bfb\",\"d3ee363853c8de796cef265641413732ad473e3ca367e1e86fd71bfb637fb0e6\",\"e2ddcafe7210a8498fedb530e2e32814be4d11dd9807240d16c76c2c4485428f\",\"dfbd6e1607ba7818b702247a8e5b63d27295f6dccc34874839c4b8c225dd89a0\",\"9a492a3c88f74ec10e1fd87b93ddb1ce01165054a3e58e2428bb9200ea54be23\",\"c8a6dddfbe2d4433a73307044916b759c1737a9ea982dc7825eba711da94ffba\",\"bdc066c32f8a0dee05335322e409d640cc275ba63bbd95729e11e5a103c8119e\",\"8316436297d936bb787a2cddfd1f448163d43173cc18fd3277df2ac699c5a45d\",\"e1f3098db35e738cc6906aaa96e5627e9194e3b94b678ab704c718eb1a479e37\",\"33be57c5c3be5d226ec1b90547f5b0d2fcc14cb695192e694c87e0fc08e643f6\",\"f85b7aeb0752aefb13bba3402eca84c334d3ef2478d714bb8fbc0e5fc6112751\",\"716d2a97b3ce20bfaed8090b47e61cdbcec1e842717edc81ce97cd4268b4d746\",\"02b7d2b1264edd9f9d54f292528c3ec3bdba431afd2e2809fa60dbd0ba7d7327\",\"228fc386c8f59f700968018dd99ae5c787c24d139dec27916fd288a8238073ac\",\"f3752a234cda7f82a785bb53e2fd2679a638c7398b780f93b1c13dd0a8a4b763\",\"1423a50e235873a7e7e4edd0eaeb8b2c6ae1e62f159f3ab181f0369fe2af6f46\",\"76b173646a9365ec148c8adbe968b1acac4c019496bc3c8ba00fe47abcdfb77a\",\"7f3431ba780d5ffacf8632f3cc001a55d9b16bafbf88af2f1b1574058393df27\",\"d6c49aa114274f7a3150d33b9b6f4e8ad6f3cb1d3c0459fb3c9d178219f12995\",\"746806d7c781fc5fa3ce6a8f5e8796e2d7fc3ed8101774c8abeb1070d2bcf80c\",\"f33defe9cc6ed1da53b7c9459339c24ff26a91f0b5c1f3414539db6cdae46462\",\"3da059d35a1ca7138e70d548b4b5e9dfa831dcee753341937470b6d57c708e14\",\"4b78536339d766113dbb8174b3dc25f98ab4593b3966600a641ded4a8282591b\",\"1266ab552d503585b9c8472bef73b2ee1d2fd3ceeec275f53ddd0a6a9c902875\",\"500408e0092a2ba61f7356c531846ea914be2f63a44b0a599136d5360ee2cfd6\",\"0faf2469fd08876d6a98371fca03c56fe9743dd5a608ca386ab3326e268827f4\",\"c809d3e6a3a6060a769f3775ca946c26f48c1dbe09036f6aadbd73904761abee\",\"ea0e0981d9bf01c9cb9ce24c7daaa94b8eb1a0825be64c627a6f9f200c3e80a2\",\"ab72cccc90985273b2a1837606f60e52716243882b4ae90132053ff095235a79\",\"78b2c0e9c6767a845ab65c2f2b6677a41cffd84365db6f12454e79df2a62b7c7\",\"3ba083943336598d911e1e571a0b7bc5796552e0e694cd87c65b70f0f1ee11c1\",\"0743070ac21ff80889f1ec6d9ea915e8bd588492851cccc503425468b5b8fd46\",\"c301416b6531065d60a93481cb95d0c60769a5a6c485c80ac818dbb276d8e794\",\"c31450088de0a436da5e576d79659ecac4bf88c996fcaadc6e63676f0afdc332\",\"31d3e67de2ee772c98c63a00a29e1a9636b02bf1ff80dd69f2b61011f87a0e6a\",\"1235034a6cac7e08c72648010925d4e44f5f812aad6e1e3faba60244bb0da3c2\",\"2e7aa958146c2c8a01678d0e65a66574476cbb9ed6008ba3690d56db76e6de40\",\"0203eca3647b4f25fb658d4b26cbdf28b007d50f85a523d8ed322ced92bc059e\",\"c0ee0a494acfbf7cfb332c70505433fe88476d385ebcdb78b8f7f90724203f02\",\"a907089869507754d48333e815c283271db733fde61aa917aa67832e5bedb584\",\"7e23645c9ccb2220efdf6c6726b9b7028c861510e3e8e538f9df50cd07c9462d\",\"df39faab809364378768791082aa1129d0c2cec92837c5f496f7ca44f7b69569\",\"b0fdb8f25426ed13d0dd9f1f250b6f62d230dc4130a7954fc0c28a6eea833ac5\",\"f987a58a929ca654ea88f11fe0c2425ddf46dd03ec4072385926996335efed27\",\"fa241bd1b813957b9e28b73d1aa43dbd04d5b8442ea3d493a129fb99b1baf3fb\",\"fae095e8462d335c15618f7461e2311e3e1509faaff3bdc176bb8f1fee21aaa7\",\"e86389f00dc8e7b7c00e59d5722496cab232a337055a1f29913fba349e882b61\",\"2fe0e5b98ec732ee822e2a5cc3b766fecabeab813753427c2e831753d5fd9c53\",\"ae4f9682a195c029234902cb11e4a046bb8a51c10e297121d84544148839266b\",\"e2edbb37515694998f4a1ac7592203f1656f26163a15add7d7f464cc8260bf3a\",\"f668a3738239acdeca4b4bc8cd229f21288b0dfb3ead678004300449ff1e5781\",\"579ad25b3719bef7377a3f98a6e1b76159c1d05d0fa3a73b5b898c504e27fa0b\",\"26a9ac65dff0a54cd0b604fa22c6e2e04a47742e0a86bd02c40d1e51bb948480\",\"c21e56f6eeb9e50566dcbd8959958dc3aa94704a14c0a4b9478bc7ca6ccfaffb\",\"a182d2f4805cfec3cf885346d9de7c270804a6c7262b5de976b958b2ed363edc\",\"4a5db7c2ae16005c9d58248fa6704095367ceef8f78b0ba327d4fac397a87af5\",\"0a13c0a868451ebaa6256dd6bb5ae88dad1637e9d2d1885a7b30f0cde30ff83b\",\"614050bbed6b3056bb5b27779f512e4f6528d90927981d7503213bd04521866c\",\"f809fc5a9f18e026bcd3d1a80c35ac683806edd4aa002a22f7a5ce093590d14d\",\"b5ed454f5b89aab719767d9b10265bcc30843cc1b0d9c03c45dc614d597e61c5\",\"53d9639b5e08bf8432768e8a6ea48620164e8cbf6a04776175ae847752ed1685\",\"87913660c0d33516f03db76c408714f0f0a5f8779c8ee12cade1fbb27590abc3\",\"db69c66644f243855141429595be611e3f6c49f9b64338492ef8ee0d8fd40904\",\"fccaf0d065a6ab3bd77da13440bb066ac6341b88031f68d23ced45fce0ece8a9\",\"9e53ab249225c2c0c4f8efa59c844fb35f3c6ccdb5c04f5bcd63b2ef49f1bf3e\",\"f70cd1e62d22e04d58a871fa3f588d7bb5909a51962100fc2949e4819c5972e6\",\"06c4747a82be649e574602ce1bde2999261e4ad4c87581a45a3da162a350c414\",\"6a77e1988881a325af49fa8349acca5b9e91e0f5e028797ce036f7e987e6d324\",\"5e94c348ba99adafdd0f0163503c0022c6988abfd938f65e0af76b772a9943e5\",\"7dfb42e7e844289fcca2ceb14152ca505f04682f7900eb7a509aeb2793fae528\",\"1f9e7618999fac9cbdac2e927ee7a30d9084958fb1e389e725ce5628795f3a0b\",\"1356bc4be0d44f03bc6618a548571ff0df7258b035cb4aac7e38a6047cdbc0c7\",\"7b7ba3953c91762d71fbe94cd53b9ca792ce06272c703c35fa6c5f5d7c20bcb7\",\"b60889f0944214dda0ddb31754ad50ff1eeebbf974e82a588cca682d1792cd78\",\"5208c3ee4ea4b1b185f27f3813badcc77242a461221c554a182002ba18aed54b\",\"ea5b2a5fe9c724d2ddbb0e9c8a76da664c281c91dcb2aaf8cdc3fd7776944d4d\",\"c58eb5b74b12bccfa4092529dd98fb6cf349fd5d9742d57e37babcf4d881a92a\",\"f3d7b56bf59f1c7e233a427b47d931cdc62f15a8511e64b3de1dc70e8ad09722\",\"58afbdea41fb2d49369055f72bfedcd667c2af718259d5abde29deb158f29263\",\"12524fc68092110cea7746cc7abeb435e3fd69ea249273d6e9844060342ee5b7\",\"c7b59cb4097f2afd024d9a9210629d40240c3579f601d518dafb9532c74f325c\",\"fa6c6081ee113d07aecfc4b344da91652ed240f670c34ed847b5954ba34024bd\",\"70d78bb9e1d822a01817a97238ab606ccf50a0c675eeb7c403de703a6f5bf9b6\",\"e7397ac77481d1c7a88a537611867546881a5fa1fee8c200c69e385acff2d2b2\",\"31a46ee931997e83cd613546580064b2b4b9f9707f98ab92afb1ca8bdbe82f85\",\"52a215529f729c1c5cfc6cd22da049461d47fed08b16ee8184bab654def76556\",\"4b22e4d1e32385b55fb259882a85b6337f7e7202de3d1ccac89bea6cfaee4630\",\"2ba2d5b380a2c38c8b917d5e34ec4fe659ad66e6897a8b6737081f6039d65159\",\"8e6899c01fc4857ad05c011665e9d3bf516ea3dc78682c8ac4a03cf5462bb4bb\",\"97803884b01a389f1d389908e894bcf304e127d52aada54ea4acd6a8a541a489\",\"d09b06f62225e608f3c41eae17e6278324d5541741cb57da334828f7442374d8\",\"a8f33a619ede819958ed8c274a989d353067940331582a9a8cef3918c8239452\",\"3e285e4d0d3240c0100c0d9036568f356a73543fac13fd9efa1e385165d432b6\",\"d2113c6c5068fb91e98e137a2f0b6f0a64d1581c3329464e3fb0b73e0664791d\",\"0dbbc1da7c71893ae90fc6270597f7ca8893351e64d64a3a7948be2f271a67ff\",\"888ddf305440ff38755688f2a25d6975ddf698eb1aac6cd8e3bc5408bb4a763c\",\"51bc095c20f5c7376d5440c142983ef7f5a34b8ca129027d601ab81acacf202a\",\"b883655e039dc28cce767cbc75059c5c7fcdcdfd5ed285e873d3c1ec9e09062e\",\"ee51e68314104863c437c5201cfa6ab74520c55a061c3681e869250c60d2b7ea\",\"19a2538cc90ac55a183e145e05f63a7de362f32838d4e4cc30a2f062ca6fd3f0\",\"01bdec5bef6efaa0d13a0f272c49617f97b125f2205a3b11d290075c8d80fddc\",\"3e6ed7f6ac93cbaf8276eb841dfaaf01e6e1b4c6e111335f19d773b057665c42\",\"999cdf045c3d8d247111cfc55017f61ac4fb233d5a08e9b60bbc279648031049\",\"4256b52b585aaead46e699817d9f894ad4a936dce0014f872885c7af24af3417\",\"96943d97e3d81ae84b9166d583f283c3ee6b47e846cd26d4924ade5408bf3f89\",\"df8b19fadfb0e5231bfc8d5c343c20b77660985ad49eaa8e631a93254e89c484\",\"0335a0c37b16617d94de6f8cf355ad0220f66b8f1c4d91081a1033a093f73809\",\"16e74c270ebd12c1726ec8190d857e27feb0510ba8e15b38ee9f0364c55f220a\",\"39d82072c549f2a87e57964d24ea6a1d7a0aa71ed7042f527c36625c0cfa43a1\",\"8e238a61988416671d51bf405ae645a937bbf47e9e5d4c1b465a2fb50678d6fe\",\"09af9489f39e13f43bc68896903a1baf9c3d448ecb853b79f971470f95167835\",\"a4b47db099491315dbef1c4da2bfdad51e791223e1e409e6675517e6a32f6b61\",\"30cdec96fd4a6fe08655aa1e064b48e61118e00ced931a547df5d522fcea9be3\",\"470f92c401e827121c3cc52e3f71faae8312d40cb06d335bdcb1f04b1712d89e\",\"4f5edc2f2048cd33eb352a0c2b66634b8dbd1d9b50bcb05b887711b21a0b5144\",\"a3b640016fc420ffc53b719cbb3c2b0e28c35e791764c89904a505b2ee4a2ba1\",\"692614d57696578c2bd4c6babe993f1064052464231dfc69565e328166191b6c\",\"297f49ffa9d68df0a321ebca24ce5cf6e7620a11531d827a0c3d93d7f6699418\",\"da17d0b50bddfd72df057a8082c46e0093be67ce0c55480f0e9270f782af3052\",\"b1769d7b298bcd83d3bfcdae2ff3f55d701e31f6adb858e40ac96798292d7880\",\"3b2db56d98905450e9dbb6bd45cbea612ef76371f3fae0933d2c3f52a742c822\",\"ffe3dce09ae0d3e13ae26a587d58de5a6685027090fef0bd70e288b7e9fb2ea9\",\"bf886f75537ec97769eefff8732a43600ef4e5a96cb6fdbd49cef296cfceb3cb\",\"9311b3e3c6b1f0a69cc5505faf3ab3915e19e9c7f23eba404fcdaa95d0c13a66\",\"658894656fb09eb998f76c6e69a03e4003e0aa245f06327e90585f3cdc0036c2\",\"75c01ce4d95f62488f68caed676fa73550a9ba4a0b0331cd2991f829d4b13031\",\"5473bdbd51ed9bf8ae93ad8d27ea9ccb556d743f10dbbfea2b1940219b1f766a\",\"9a012573bb3dad00510ae955d98d7823c43f5e97832ecadd0c94e64feef76a2c\",\"9ec3ee27ef809aaed7edbacc9f66420d1b8a5b50e92f13535e10b8d7f3ce6240\",\"d3fdba78f140f2f6c52780a6410f0b300008081a25ce0094022edbb7772497d6\",\"f25cd1e235694b0245ada4e32da9f44466340e798ff7258833cd173106ad7586\",\"0ab143bcf7295f792b2b17a07329cee1ded43229e1a35908dac4935c24b24581\",\"cf4299b166aa6e3299f65cc4a7d378f88eea863929c7922f78a69ecc38aee13e\",\"a886c9acbefbd6e2f4f4701f70c4c71b302ed5bd8a7193623448f7edf3b07c5f\",\"bf5827435d2232242d4eeaa52ed77371d1b9b2a2c3980782c07568081b5b290d\",\"d28f2b5eaaf09729dfb0dbaada26aa892f8de8e55393bbb6fc8939b569ccb16a\",\"aa9face0e8972d06b8e950335a61ae89409feeef4c64287fdc5cf77bdfebb077\",\"b114de4d248e166e78135796908264d3025b3e392d0ae4277989c51e11e33435\",\"29080b45d907d5827136781118a8f91943dd60feda7effacd825469ccbf03a21\",\"96b3e04431a7e9518214213e2ac242278a8fa437b89ba6de06c8f5d5aad840d9\",\"c4168260b968f199f910e57575d5c8ecbe241bc3f3edfeec63171be6c8a18ebc\",\"07a3383bb62e13c7bed1e9150ab3b75af034fdaaf6269ad955e0b336b81fe0fa\",\"35b48ea67256bf0ff367a3d22bee5f0af3dc20288c4b826db12980cac3dd77e1\",\"da7bedc84b4ac256fbca582ebbfdb92b208ec0c72d796d5fba37a169293b8351\",\"3e36a7a63034921e0d7ccbcdbbfdd7535857e52d8cf2818038e183628107d054\",\"fb714d27b3f0ddfcf6a217b2250f872ad52e5d14cb18f9bac8dfa7b746ba96da\",\"2d46a9b27604bdf2103a143e006ea249836b355170fc5f91b038ed247c29e697\",\"a006736adbb3ed651f9865366ebb03afab44c3080c2672ee60fbf435e2c679d4\",\"46146ffd832b003e0a59ca7e78eb88d13c83d0a9946605f2c2ac4c2a4682d361\",\"56ce7d0af5ece769f09dc045f482c1bc57f3e00bb3321fc85faa8d5b9fe8dc5c\",\"bd0c7163767bb287466bd8b93527156f1b0c76c72c5b684214e2fb9b39b30bd2\",\"16244a93713f8871ed8dfed9eabeafefcc62a76298bda1a78dac2798d431edea\",\"404ba400e4b34804d0a6cdbc29a625d7c2fcfdaf5eddaf3da982b4941c549257\",\"dfc37606240065dfeb40d00d558b72aa71ee23e5ef43d24011cfc8d99b6aaf76\",\"f5adfe127002ab567c3fb8b2f916ff9b13583e42552d6612c6dc670330932422\",\"484ccede1038ce9b2d42121970507dbd4d2db40d053358523c8912843fa23350\",\"5efabfe3b9f04f63ca9126ec449717961f88ac7c0c08e33ad5a59052663d32be\",\"f781aab3cef47f843b09c22c38ed29839e1bc4a88424852037735640d0fbc28c\",\"382918d1f0dc54c6233d1ce60a3a601bdcfa4dc6a7c36956b1308a4ee8e6792e\",\"807897624a3937fbbda8d3c97cba2e21bcd42dbd2a7b1da942e6a271f2340b53\",\"6707d113281c1648d2be199acfa48a35af2ff6e2b3b8f98dcb0bca13569d9d22\",\"29b5140274440e16fcdae05296571f1c03600f53fc703ace4e5cbb63eebb33c0\",\"68669e3f1f0ccac44e8bd9cb12e3656b9807299894e6bcad418be5e9c886084d\",\"935e2ec95f50436166a343838ea982feebb94d222b9f3cb3e609891a028d474c\",\"31e1f042aa81d2bc46a92e1ed482956ac23f5a2737d857898ee2d7b8a041e494\",\"17fe9ee1d66215475a01eeecbf6c95a66ba92e4be24d9c0f485a95a139daf8cd\",\"38597d24470a8e893a60fdf4d6135cb77adeefbb9e9ca7b1e3fa5f78ea020783\",\"9a155241a9cf8952b8680f51f2c010e0de31c1b6267222ddf99f0b3c0a0aff3b\",\"e65cacc3b1b7571237c425e81323e20140d802dd5178fe22a7f1d783f8b0fd1e\",\"b3c4e5aa3c5fbcbb46171c165c17f682a97b7f322097c07a5f51cc6279edd1fa\",\"9ffafa41617a01abe1d58692df1486afd6fe1d997002d92d340f1939d34b8eb1\",\"cec0692432c59d2363e1ec4311fb5c99e424fc8aacdf3ad4ba5677d5762e1f11\",\"b1ea81f13baf2a4aee4214560b141144e25e4414bc05fe4f8353c5eea2ed2a99\",\"316615eb2e0486e76e3d6be2edc4173148d84d3b498b805e7cd289df399c5afe\",\"c262fd49aaa9e71480ba3b5071c8075a8bfef83c3b23e38bff162696bfc2821f\",\"3fff8237b30836bbed5dc97121af3da47668a3054edda47391ad38a2f2e822af\",\"134b6cc053c97c5509b45617cb7b7419b3e08b3e00c50026c90494c259feb75c\",\"52bc7c59acb08ebf39225b08f642923d233a20ed6ea227e2379f02c6ef86da15\",\"99110b4df1a3bbbd5595db2ce00991e65b1ae2ad9c90d1be6def34e76229690c\",\"c66cc69ade4fb0494845b989681d04d878f3806c7d3262a16255e39bd617781d\",\"571f80b618b364f25dd862e5655e55c34aa934e081cf34400d889779b5f4bd14\",\"e985ebbd998ffbcfb8ac86c9a09b26b2ad27435a6a1c5f2a2b27f03d538305f6\",\"f358a2372e5271ceb0ef9538967422ba3eeac4e327270a15331bff285dc3cc68\",\"5362f65d185fb38a0f410903f2a48abd09cf80e3b102faa9eec0d48ee8a8cf9f\",\"4901c377bf7e76ba34ebe942c805bd724b39725c4e667084d32f3db8aa4e9350\",\"8bee74f3fdaec3480867f8073fe1c1289f791bc0cf73f5986866596f4b9e885f\",\"5abfde375004ce593291df904d6ddae2644b3bae0cbb2fe0ed5512c45a170fec\",\"204b0c70fdb5fb7f181f4fc7d5e4d45186923e5501d49647ee3c73e56cf02f6b\",\"1223f9a7c522e54dbed9441e87850c47ba185a99e8bacbc8dad214f2a27be04d\",\"2026dc9bafeebd5441bb580aabf327e722771191ed45e8136c6cb6692065e1fa\",\"0b0c39de80e0bc84ca64bf4c8682309e5fec3c418f6f32b6bda1078810997ca5\",\"8b16f8b17575a306a36258790b55f506fb295da233e1945f22ea64ead969f22a\",\"d7c5a65512b234b61c374bc916b4927e27d8d98bc3b81960ab3d0a6ab18a275f\",\"f46014f15b63d30ab107a0e485dd5c9dc0a6baa4c17aef0111ba9c3a95d8a37d\",\"7d9ea47551f8455a5be6133be40c77e1cf33cd2ff6f0232298036b8d32b463ee\",\"c3b409cd86b1eec1b03bb61880f31924455adf8b9f2e9e1bfa858b5d14332f41\",\"6ee51663d6466e50dfebd6fdc6f7702cb9dc1aa242663b3889dc463ba4ce2264\",\"b01809defe808a24161d12e645ee9e8236c390df9be7a470b48c4aa407452111\",\"cd1f8f2fed59fd5b93e8e12edd22581742ddf5851b19d114268a640ae78c4cf6\",\"b54c8b126c37c7a9f8d37e05a2173bb7691d54227798599e4ebbf89f5398c62c\",\"e9a7050bab5bab7f248f0d0a7506d845a1f9d1c4ab69871bfae64aa204f6dac4\",\"9e556d311622fe0f68dad75d380aaf50a52100e2b39f814566592520ab2a6b7d\",\"69df1267d3d3ead6044a389c0ea91e8e8b978a9a82a9972b15eb1457622d1e74\",\"3a4e96d5146207cb207090541bd50ef9224ed24123a8f8e38be00026156b3e7c\",\"b8989a1895f8b70fb79c783605ff9b9190b76c65d7eb3652bcceac2891812a79\",\"b6b3f9c965d7a91f5c71faa2031e2cdedbba0fd1681f74b741364f40f9d97e9d\",\"3cad309bed190c449270d3a9514b65f0346f091be5e4e8321de139afcda98527\",\"b4358908ba8b7246382a7231e13c191ce78422f92ec976400030a5635eb97acd\",\"edb06bc9ced4f8509510203c20197a39b1cf87bbe2fc9d38fdc4d1f76b53651f\",\"1be01ede3fbc0ecc1a8f01b11cbd38a21dec34c404ef6903feb3d1aac3cd836c\",\"cfb45683c07a58d5e791ff795112847d23a5c422c747cad53e8d7d96b8288935\",\"10892ca4b33011ac7c824467e6aa0d35d37026a0380add93540886c3382799ba\",\"3f06e45c7bc4df21914aaffac968fc16457cb13b7d64b621d00ddd7da929c31d\",\"94536e3d031f3249ecd30b88f64dcca6498010fcbf05b30f35437c9c396a519f\",\"2dd89bcad8ab32645bd842ad9bff25f6020f7179194f146c6761e884ef798055\",\"2e1912b6be865a98c185c9a5e6875857095d360dd32114c66df794d3309bb587\",\"80ffd87d8574769d968994e9d3e638f5333affb367d50329496aeecdffef9b69\",\"8952c7b09277ff5da452250efad878f86bf0a0ecf91f075cdf39615134b69ca2\",\"c51af6f64f04b3c31b27e00e5863c76fe9c2514c8dcbe9d2bd05ae0b25840d25\",\"24c58f69ccc2cc7ed39c7431916777669213929c565aac62f0beb8974071491c\",\"d07e174560a7025b5e353cacb4e8d2d1dc5dec72cc936b8e3e71aad3b0b4bd59\",\"af02e277f6e6717a4d3795bf69025ce4bcebcf5a925fff1cb6dd02afeb02c38b\",\"684cd58a9e76857be4ac85ba63f8dfd9578d850f44a86165a4f731615ba76d96\",\"1e8b1ca2a6d3970ba9ba6177e17bae9302cff551983a34bd76d1ff5754fcc3bf\",\"de83a58e3171895d803cb88ad015d6718e70218ca0e45568917d0073f3ffb0a2\",\"113b282d03cc3142b24ba4eb4a010856ac7ed019fbd4a47d8a5f0a44413128a2\",\"528ecdcff87c67c30948cb98639671c0f913b6a0cec488f9389822e50380ac35\",\"10910288f8a7490f03eff4d1a067e4379bb6eed0987de56fa27e58488371df64\",\"b2c19285666d3f8e01e2c417b6a9b349fcf4b18501df82b3150b4d83cd6fca4c\",\"3f12724d0850addc15184b3b33e3b2173539a8a7ffa6c7e913e6fd979e0556bc\",\"cb07ad0f04383b740dd4d811df7b8314f6e309b07e6a371ddcde28a3c4519e0e\",\"eaed645f166746ad2d0ce1d5a39afa4e2ff48d5e9486b98187cf4c48003e191f\",\"ce1d487c29efa812645047b928ba451373b3cfdfb5d9724cb9e486b0bbb43a7a\",\"f8c983314b57194f908c2de009bf379ee4f710138dea3bfe8c1e5410e864103d\",\"1d503114405d34b1102fee6fb728688156964462b81c0a4afe0e90149cb75db1\",\"e89341fc639fba6f3f5c9e38b4fce8587950770d8e4ca63dc7c0670288fb80c6\",\"07185332149b69d8c28fe42a476b67ae400ffb20773de9788d60062c99430cc3\",\"cbd92d86753554f7da10d5973162734d3602d7d5134a72ee5005609ca69c6e67\",\"0167f4a65c2b675a897e8d4f1fe42c5005765c891078f84df01d0401f6918a01\",\"207cfb643a0e45c22e8239daf62d08bd281db6a752e3fb61cb44a763117d708b\",\"65db5664950002cb808ab1b7e7afa907dfa66edca19c7fdb229ca1e8a34258fc\",\"017b648cb417bb73d8b746ab98701c8916eec4c15941e26e00188e4e2dc41229\",\"9b429aa1b6b20b6a867b1f3475e652aac83634ef8704a20ff0cc069d5b0ca506\",\"6df65ea324b3d3d69e1661f7c896786ef697b3cd6340b46ecec268ff21aafdd1\",\"030dff9316d29aceb09569963f2622e25b2ad36d8017e4f6e3faa2a961a49cb4\",\"65dc0613ce981214228f4273a918d1bd9a9daa3cf6159182b83134710b38714e\",\"2d754544417c29847e742748d14814b2c37d285d9fbc8318b955261823bfd63c\",\"cf62e18d7a02160a650d0602667799d8c89006d7503ebe1d3525b382474b7e60\",\"62513ed1e979a78a9feb8392481308ba1766568de843202a4548047a9619abb9\",\"5beee16d24b1c38da75bd6584e05796b8dcf3e041ac0764329fc14032e8c1d7e\",\"9e6bf37f8d1e4821cc2e53d83b77d112ffd0b46bf34b86d547efad53dcf6232e\",\"94e34c186353db7c9d13a07fbe4448933ef429e80e74d512049b5e8fa5f3b32c\",\"9c3afd8bc4d4cb146490cc66942ed3446e984a7302c06fe4e0cc1608c624d8c0\",\"906afdd74453a41208a238a2cb29c9a6a359fca1e914bf1642050d318ef1b2e9\",\"3852a44af4039fe1d6e1fb48537a100bfbe8a9b7194edd61d273334066496d2b\",\"6628e480fd205919bb8088426a0908dea792559e515c2a5c03654cef42cc481e\",\"17450be35121624508cd16a1088d2962657cc81e6b07d91170b7b1225cb5dabe\",\"9a30be4adfb81e2fb09cff343a2eead3612e2060f1f53d80cd99a453c55ca134\",\"65aba223af625801df9adf8f5afdd87a1f560e45db5f242fd8b0ffe9384c2049\",\"99ba5a51d33eaee95051d2982b87a5cab7a519f3d29824222cfa27bd1e137a47\",\"91fbb7fb0485ede4cd6bdc6ed3cb5bca4ab47e8a8f3f06168fea804279815be5\",\"b42461c657304a73b85f4f12ff1eb72d17015dd04ba6d901ba3a20cadfe66894\",\"3f724388b9b44c46e211bb3db407edaf3875c2138603fef627efce0fae636e78\",\"e46e0e666e7b2301f6e5f10fe85098b9efa97e5a1cbbe143514a02c0adcb260a\",\"6b3727a6d280f4b238c179e19389e45cb97a776ce34874b3115baede956fde30\",\"f670d87cb7cfa4f39107b5c5808d34dedf5f2f5a965f3019655571183dbb43c0\",\"85f1b39e9104d0a9335b902e71cc289fa49422e1f98d1feb875dd00659d553da\",\"3a15d422daf390955f1837e6307f866b31800b94b28c287a6c195a8ed546522c\",\"218e95037e042e2f36b2276edb7e3cef5885c7447e6656c459d5e6e314c1cb2d\",\"7227bf533d98f1d612a604c022820108ffcd2e2f2bb1dd2bbe50d343383ac2d7\",\"0fb68a890c30a6d33d844c73a4fc2f18375e31113e30d7063b02604d06fc4b59\",\"e8b348ef10db39033aa6e4a9632d93c87c3e7b97037df627bef4e1dd9acbc908\",\"240d169c171bc2344839208ab3e0ad2d9c3cf9f9ee9227d456af96d6e1e7327c\",\"4900ed7662b4d2618d1db278ec9cdb2df5f6a2ff655250259b6f353199a222ed\",\"e3a3b946917e64b8c4ff8cc87a2d371cd1322e5f62678fe0973453a3cbe577d2\",\"ef76ea4a7f4605ff96aa3582804a1c4174fd8dabdddf3b2b34a5d2032514ff62\",\"40ea55c275fdcb6af3d114058cecbab5fe8b461042cca27908939b264f46c1a7\",\"3fb66d1a81d113ac57ec9aa140df2437c11f42fc132365980fcbcb8357c85322\",\"9fc2765f8e6c08804dc12d6c5137f78a72bab98e290607f6495042fb18643c10\",\"acb4a85ba0c0724fbf51c4e2d770965708ee5f8f5e0ecdada7dde1d98aab03ba\",\"b5a1054ae2a2be5b04b4d7230e4f550bb547fa2a8c7a3e709a6dbae9cd4bd168\",\"6b2cf830a8959b9aec43e0d89a7b7c380160aefb4374ad4d00f7ceccd6d81dc0\",\"e789638216eeca63bdeb469f5919f081b0021f56b966562283932f179eff8c2f\",\"9324fa5b87296ed2a192a9c1bf05d881e9f6c29fc3dce02a194e9da5873468f8\",\"69585c3f622845c02a31ecb23e3fe56074c15971ab669eb942d72f3f82eb4f74\",\"386dc787d40e98af6580378b774ee4a84b3d5a4d2ea2c68b4e298e48971fd1f6\",\"e232a4c4105c051ac5881f9a5dc620ca87eabdd1aa1be2a26c9de1bf96b7f28e\",\"6c355a9d4a48ed9e0e0ebc45eaf68b9c5c6e95c8d070de4213e2a0492bdeb7c3\",\"e41df4f49bc1878e14d7962eca54024193f8d782b03d9f6125a915dfefc5f325\",\"c5e827f73ec87fefc1e89e86a94cba73f0ce6b726a80f5a98242c3f9a39fe486\",\"f436597d1a1f286563169c6c0bad3f6c9f8aba2e7de72d38112d5675240d0a91\",\"87347f97e7c30e762afcad545a9ada8e4289a8abeb6e248299dd5ec331462cae\",\"183e90fb7c32703913e44c98bb670713785256208e7c41d67c9f562a53816c6f\",\"4de12103d33fe9a7d06696176bfbd81aee90eb2b33d8be7848330ed543c2cdad\",\"f638059b95b90489dc68a4c8c806d1301a70be16097ff5bbeeeb8b08790200d4\",\"db4110b8f38508850256ef6f6cb4e7224adc6114f6f55a28dbe3e38f2583fddd\",\"246870bb964fb50a71ccfd90b48c6875dcee1ac2a8b2a772c5d32f36e39604be\",\"554e23845e648ad3d327ba7d3745171bf5c7b4f1837b5e606bdd3849d1e84bd6\",\"44630494da215d1ef7ddf7077bb604e2266b727dcd538754a487b010e090cb1f\",\"72d43ca37d4b8fe8383950ab2b1c55fde742e0ad40f41a07f52086968fe512c0\",\"0b26ae3a851f7c4de7f3b9e1b800c19ba086c51acec6d11ce04c5eb504762edf\",\"de60cc31fe9fe06698703718a294dd330ed9cfd19edcfe2c53d413720809e250\",\"9c587f9bf65e314ca371ba8d19faa44fd8c17f7852236b230037f3c12e22623c\",\"d3d6b52dbf95d64e32c20003848138cb4bb1351a81517e0d15bcbb37f536952f\",\"7ac4f2861d7c1e381b6236cc754c445d524508323381fcaa575adce42e2291ec\",\"3ac8e59bc9e313ef53793ad6b96e974c709779faaeaa6ebe6fa18fe53f4818b2\",\"97ad1f41a258f417841ebc871b0340974b6eaed256526a537f1ac31ae742e07f\",\"5b4c890fed469783841e4c3b00dbd7c1217c214005a63dcb9ac4461edb07761b\",\"859b318a521d15cdbedaa28523202ea3f579ad791f1197b98514a2ef0b29eb65\",\"0afc30775dffe42c4da331181842c346ba61210b45a25c9062af6dd3e2a16da9\",\"a3587b80c6ee37be12bd6c3aa90301e2ff460e339cdd0fa52c33842bcbb5ad11\",\"3bd26f1ea9cbd1cbf0c52beac5a2a9b2aa18059b1c2d6b056c1c0d6f6f93adb1\",\"07b0ea74cfa65b761ed9c592637ebe5a9dcc302f6c8200ac7ea0a7ca0a13390f\",\"06016c908b3be3118cb66bfe887ab42aa1e01c9fa5148ab5b01a757b9556e3b9\",\"a33e2d1817dc1b58b08b15be30e9386d9116f34f8a202339d61e4df467fe2e03\",\"0bce8235e59637cb2994e4970440515156af0c977baa4de3981aab65df1e3a7f\",\"89b028d21a8d5673b4d9ba232bd718f068528febdc7c6889939a696d44650f6a\",\"994dab39fc402d23a70e47650b351f8d680e97b10d406b19503b406f9f040b96\",\"5d7436087ed8255e71677fbbfce127cfb0128b777d1255aee3f5c3fe4e848cdf\",\"7f9aeaca0706775a610fe0fc60ceaeab42e9223c154a5e8aa95961b7096f8a4f\",\"fd2d95ed02f1cc835b4b6370e5a782b23c967c00ac97dafef1dc024b24aecfa3\",\"f1d48c6dc25f0d9aaf84862050ab0bc3ef0f09baed520294b8329986d2ab147e\",\"130bd1dd906c7c506b42f7c54ab440a408d1e5c563cbbc1fea908b389e0953d0\",\"9974fa214e64a0f2c27976e534e43e79aff413115bb7fba0d119f2e249dcd352\",\"78560f25c194f8b67e57d7895e525324d02683edc129544c10af1b538fa6e8cb\",\"a788e7da5618a2fcb1539cba133b9fbef1476790b851948dc4a6e48fe0ea2df2\",\"34a02f0bc23f7532bd585c58d58f6074b7db5a6cdc2ccdd3912aaba088794514\",\"062f306e1ad32b9d560a871a235eecfe944ef83c2821e22155549b5f2aad8d6d\",\"616be7362b8878722cae7b6c4880935318cf224344d6dc5bdccf0988d9466d54\",\"26edaed3305f08ca71b84d29aa32ee81c7ebc85ec932c9ae5981f3a0fe8f0122\",\"5fe8c6be98e896da77854bd79968c511e97279d036e09fb350b847c0ee0aff46\",\"49233e2756ffa0d175118c4b3b69c993b7596555ac5b16d53c78f3a7566d10b3\",\"1dfda971ae707c5f72a7ad15f81fec1d04f5a94da70e8431dc3656d439b5b63a\",\"e25a89bf03c899ecc6b42246d44481ac8d3abdbce6ca5f437da667659236098c\",\"5d0384b593f5e7d5eccd56da827a5610101c81c7689708959eb0727c70213289\",\"054eb2d9887ce1bd3e0d990f8fb44fab8406ddbfb58bad8ffa7a5a269db02d69\",\"09678327fd41f1513bc92d87c6100b30af7c16dd4f212dddcd4d01d1f1bf108a\",\"f8122150a496e728fea4298c37632f7359061baf7abee11142083e3a40e4fe34\",\"7ee2fafac6d6d686e7fda717f6ce8f8ec684e80fb0e82bdd921de080390bc6b6\",\"8aa4f90fa584d2b5f0f24e22696810db7c5336203cef512f476ce73ad84945f7\",\"137dd320809291d511e0fc6f16a3ef713e4448f1e0304b73da1c9561d428e21b\",\"769aa5f5968254969b8018ea7a03c43414a4983fd537b3ead9831c1a35676245\",\"3ed91fe3196ca4a223bced359188234dae98c55ed48eb2a37a0e459198529827\",\"e0c69736b17c89f3717040f8ee8aca7c8b1a2e300e814af110e47f9db3c70412\",\"fed6157233f971e0abd8cd979085d5c7a48e09d7417764d5b23d0e69ceda5256\",\"741602ed64c73c308bfeccc33007b6c292326a792e48e6c9e5dbf8b43ad7c934\",\"5b0a2d4e111072205d62b79200762e440035e10ddd5799d8b4ece998cd5ab967\",\"e2d012fd0742ccbc10ee08900de078c777779673ce5aeb97d0c8b3095e10de82\",\"db5b1979b2f97df8ec70e89cdbeffd1dd374016e9a11004f2d0e3f5a31318ddb\",\"7c89ea6307db0a5f2ae21723ac59d78aac7804d641398d4ab8abbf36e3876da7\",\"677a3ca3618b7c97752fc16de62f96e7480a516c8c4650b1779e4d194853bb98\",\"c483fce55d8c36c76ac2f3e8f8a320498343663d23c910848fc04ccd7e5c27ab\",\"680d8858db03f69ec83084ed0385cfe77cb0a0c8b79d26c0d8a0f310af89d51c\",\"6a1d8cfd37a0e3e0addeede25a0cd66576c00afbebfbf41034fc7ac3770f0fd3\",\"49534e8fdedfb96a9e4cde33a15a8ff709a8c8090b96fb234e8ea3a54765ce58\",\"2b749e8ac7f42d42276b214c2e14c19f6ffaed415122970a8889f727f6b8a42f\",\"d73c584d8e63d59cf81047cf28e6cce92e850c51b0eaf92e162156976adfb4f7\",\"f555463bd80e08c61b2b678b2d5c8d4e3958c382476f301625f55750c84b20e0\",\"e26eeeb3788abe90af95344f9c579b1a8bf4303aa79b79a055c2bd70a3a4b334\",\"038bfd04298611d3a3697583930c49ee23b09cba51003c252d361cdd5a7682b5\",\"7ed4b40dad9528ec7ef95b939fa4bebbcd75269d4bd96df3fd3adb2695cd8b33\",\"c48e0a44c8cebaefe2207629ec3ead5f52720814bf7c4cfac740ddc33654c0ad\",\"195690eada836041016e7e76d93e498524e0b78043ae903267184bd288283684\",\"3e7d8e38e6407ba35bc19f88b823244787f5f33cf99cfe36104b04f7e99de3b6\",\"f6e5b93f816c6bc7a29dd4c0ac765184b7afcc26a41a194628230519774f4970\",\"82ecd00f0988cee3edbdfcc5afdd89106d7daed52b2bd024852eb44d9747e523\",\"466158b06571f1716dabdc9f983c7fa1863bad861ca33b66cdf9356f8981807f\",\"813de062c9ee1dc09737ad05da2ded297c7bcb73b611b9fea7153a90975d0756\",\"2a4924c089caebb6b4ba6ebd0a5728cca0d9f11b7019b7405cb86af0d12511cd\",\"f685940d78b0e0094777cc1ffe71914b99f96ff3bc8ee6aa381f0122b9cef149\",\"4cbb5f2c46f9cf8f8b7cc57cdb040dfc5a22cbc0a9fcef57d72d05c4d8b51396\",\"a046f8214d1953257e1588d95ed45633b39f00b95b42d24e6129825836b03435\",\"f9f3124e002ebd01e227256ea45b93a4258d515e0e83115b6d36b2a834a12075\",\"89ec7efeb79b3fe894b65788b70f188d0a28ec38041ab5762e67de70d5dd2914\",\"56d64e1705b20be2d20b82820aa2533979b3f340fa63ed75f96c27e7f66dc62e\",\"f22a0ecdc57f33e2ac6b6b52ffc136116c2d6a200225a400b1b479d8413d3131\",\"5040744c796af4a2f36bdce968af7d222af31749485054c31058ac7810754eb5\",\"86c17c004f30473d1a860df3426f8e56f15400b02abd96b3ce0d3b5d2b95db65\",\"830dcc3d39a94bf4448e1fc7993172664a45161873b24096b4cda3be5210236f\",\"07721db3e6cd0a60cdf7f3a7d79725fbfdd4e0f188bb895b6880f7104d32fa9b\",\"8190e8a6580ddfe0196e85176e45f46371821f3c56d6746622cc9c9be9e3d15f\",\"bf832093a755461bfc216b4f48ee0828165c3f4cf7d26ff6c9218737de672366\",\"ff0e665b7d10b5eb69df2d1123f4f3050d162a210a03b44a18e9a4faf141bd6d\",\"6265ec4e7057f573e3cf84953603422ee092dfe4674a0298c04b4e311e2cd8d2\",\"7020b2fb92902e2e539051361c7a148d08ab40e487e848114b5fb68bc9d2816a\",\"0780791f84511fbc4bb7c09fb504e2e9741f83be13f864445b07ef29080fbb55\",\"9129e5ff6c5dc2095316de0cad1fc5a62543abf9e462eaeba5e3c9dfe4f014ac\",\"07ee36e5a51d08d43bb763f77c4d601129d4521947fe8c1dce955b78d6bb250f\",\"bd26334a86578ab0c4ffcea81f748b4876e8488e49d1442ef5231e6d343947db\",\"588a976ff27d0be980050267b4605117b53b6d1718e510c810fa4b7d7206ad53\",\"4f34ca5b5be01f9727101a483a439ae9e342e57748d9ebfccf68df0acc5f190b\",\"eb9552540365e9ffc12570e4991543702a0241557fdf4d14959643e619fc8a91\",\"efd3e1ebd853766a2d5fdc6fc794e148bad8b9fb22202883033b2f9bbf708b2c\",\"c739177ae638de31d023a4e2c9b976aae3b5f357b549af06ab93dd34c02b73cc\",\"c4a8c7ff091b03c246977db2c47b60f70e51da5a48a0cf837553d29b7ac8c2ba\",\"2ca425c516bba735ae9eef8065ff47852c9af86bcbf7eb30e31fda5cb5fc7b77\",\"4884f306c2f9d26f90da16167d5b5f81c1618fd3906334ee47473613dd231d00\",\"31c31e8090848dea58168a29f82125864e271b76e77816942381be24c092329c\",\"da537f35a09aaf3c3e43f5faca030648d5516e5a11002c35f2f23e0d73152ec2\",\"f1bc9487b4af73472a8dce8430f52787675f44152dca258c0da177516907e00e\",\"3586ed5aaf82f3fc5ceac9fbd32c9cee8c1b6c1fe39bc9f8a5ab49de2aa32deb\",\"76fc1f6292d3d5625fc1ab6c9e3ece507eb43bc9510760ecc341d3995df86c16\",\"6bcc47b01a75e7628d672a5babb6974af45e60cb26d2a3e0184fdd2734f73b4d\",\"e3a674e67f7d03cf1d25971e756c8c1175716eb7450884c49af4d4399f3fcc22\",\"82ec8c66f96d35f505726c36da57efc1ec6349272e8c839568061f6f0a16da85\",\"6b98d965f481177e15415a8f949b80c3b6a7a21ed4529ddb1789415fa9a3f0fb\",\"9f7993818a3f56a2843470d6ecfbd39e66711ed3ceec86ca844644655bade17d\",\"0483e49b8edbb1b83c42b4af3304153317189d40d308d132b67b3eebb5587d09\",\"04cc376952df5cdf8c6df1a690be4d8162359951791e1c5b6df7fa65ecab3bcd\",\"5c6cc241e3278f15a3c8964e7d2a1faf61d1e2531c5515e491e00e9da6ad8ae8\",\"e2ec2bdc3b8765a86902af3a6158fc63ee6f805047615eee10cce529e1199500\",\"dffe1d9593b904e1e2f90b5fbbb125e9ff02e6a685ae837689caa38c49d96ffd\",\"c9f8b67ab9e28ea1ff66d73b282cc0e34df733d3e1368b9a8fdd1e92542f5948\",\"3f79749a99e434b956e7e3b46e0933f38cf9ad762aa3904eb07691d65c701091\",\"9955a9ede07d5c2fa8f2f695a185ff9f6350e6932250bcab7c5d35bf43ee7089\",\"68c1a18a75ac7f4a9b7079371e327cbc9c47e35e9d8634a2cb7caf2ecf1102d1\",\"1fc63bc6eb179ffe645a6e7763fede25e8e730f9b9171493a472dbd5ea5adcf2\",\"53a161450ebc90ecb0eafde13791fb7674a9ef1a194d8ac191d4783ba8c1fb04\",\"f3f7692a6e6ae4c26eb5f9012a64949988f60644d43cd37d9dc109ae67fd2681\",\"17925edd7407b7d9b03b70764e4f2a7788f9bfd25c80dc5444e9feab44f784ce\",\"4c1a623b0f0c587e0399e910b8bac1fe667daa8d30d120032ea2ffbd7fc7c3bf\",\"e628a8ee0df58e24dae456374d178dc18ddf59e81da19e70629c2626081c70f7\",\"fb347405a809bb6abbcf10e72db02ffaa8f633e822fa86c618cac5d4348f3ad4\",\"97c3ee0359a218480dc89067fd02dc76b5988ffc1870e4ebb481596b84fdd68a\",\"8b5d33c19a1d3303b29eee3bf2a15b6bdf76aedb65590b14979fbac6d125deee\",\"aee027146243972804f3f194666e84f528d440414b29ce3a56e160ac902e6fc7\",\"8c85eef74ab205e7f306042f89538c3fabfe7cc986862d51a58ead800a1b005b\",\"fc5aad4e4c2620211d56bb9ce8dbc7c0af860b97da5cc1063f08f6e0cfb22467\",\"ff9a6eb9cb0579ed5b523e0fd6ab79561ffff4f86c4ceaafc8ced1f98e1c59a0\",\"30d63fd85f71297ad3f67783ec1974e9f548466feb62f63fed8d6872f0778c38\",\"3a431e0a4158265664c476b4bc3d8de69eee7879e027bff7728d1a671215f398\",\"036c61a5167cc4db5c3da887b85dbaf53329f9bbb51d2753177e8da30eea4a88\",\"dafdc781e5b1d5d4aea268e8078ef987daec54c7469644930392ab3daf4941cb\",\"0894e27205b7f1bb08233878ec781febd60045e90daf34a7fda73ceaca8c22e4\",\"0a4dd4050ab1de731e9d0339550bd7346ea0f3ea088778c5f861934db4a1b498\",\"c3f6b432e802540077b55a27399ce701a3b7ce8db64a10bb71507b68dfe4d0bb\",\"97ab5295b373b4bd30dbf7b346c9b14ba05117c5376878017d286058ae3ba5a3\",\"1f66ad4f397a2e809a196a86d414fd99a97a0e2a6cfc82f5180704a7409cc1ca\",\"c22cc3f7f2fc1fa6d3541b400d68f2e538d358838bb12bdd17837d1c93f1df60\",\"ae9cd5a6a679daf21ac3439eff4958b0a8a1dd2ee29096ed06f5bdfa4c085b5f\",\"321e7d62bf50efe3f198143c155406e1eb95977d0451ab47f7e202d7b78580d1\",\"b40a5821aa238b33d60cbd85b492d1df5a49c6f419f6c02d2d3f4374d653aea8\",\"3cb585b3e84497e84704b19a963e516fe5cce3b17913d04dcda850614a157496\",\"8bfa1132e5795ffa52b434e4ce2842b6edcbd36717fe822c76ed9e694649bb8e\",\"4af6f6c13e457693bbc4657bfb6811457d20b47b50e32a5520c2607b11eab3c5\",\"15241126e6f3e8cfb60992b0a0a015a7aa518632ce9dcdeccd799654f5f85376\",\"908ebcb3aa7c6ded6934fed0bbe1fdedd1efc7d8a7dfb75947e329847d0df566\",\"b473ca5d8b1bda2f3d9c3817b683274650b94df34f23299cc2b9fd7529b08d26\",\"4ec7a8c99454d70052866767b79635813a15efdb1afcaabf1a903f088d2f530a\",\"c54efe1fad485f52c5a92acae7ab572a8d886d1fd4416313b06f7a5c9341c515\",\"02cd56cd3b55004fb75e430c89d879c1ad7f58642967f4cbca4e92bea4df0a51\",\"11ca2ba1265bf7716beee3e340c71098311154148d7a4aed995ecbb53891e9b4\",\"8c7cc4d60bea40eac719475f5ee59ea5c379cefaf2c70258b935013f563b3f6a\",\"a7288826df958dac81cc4024a4c9ae432e6838eba826d6db4efa5c4c11f19f59\",\"a93d0861ddcf0f3a089ff3e689b51f610d20026fe2192b80000e386f49e68ad3\",\"647673abd2e18e30ad7c387a6842fbe286e86642889fb62eeb3722e15de33dac\",\"05e126bffb7e6537020f4c6f97092f9c76446238934d2848f4ccec6c6417bdca\",\"dc7a2b803a5f7a1df2c59d89bbe6468ef1fe3b3dff25dd9f7f6cbe2b5837b6b4\",\"f6df4aab7cb1158427d3ee3e2eb36cedd63e89bbd7423ef208436369c6f8ff4c\",\"2c58ddbf2d1298404020ec99e2a39c9da9ce42aef4dc23824fba38d06e66c7cb\",\"2b0b6b3891051ebdcea800b83fb992383d8e06e444b4c2a8d29cabcc87c9a84b\",\"9b26304ff318b9f6f85f923705a77510ef5a4a34730a0e1be6232690fe994020\",\"1320aab53fb7b702a4f6b90a4b7a9ebcebac6a9bcf9b0026bda31428a3414774\",\"6362a47d598a8cad4c36e23f1819c604c64d7f892ff4b37faf4f73b501062cc1\",\"dc1c68e2187f8f35329df5d295742360dbb83808a051397f90a763e039f7d607\",\"6e016e3dd4fb609d9ea9fc3859b09c9ac3f1962dd2bc589a249e56481ac4e2a4\",\"0f1778aa01343befa1df28d4dd04b055edb2e7cb6e318e376b7c062c400742c3\",\"6b842130b96f0dc78db1a47623373c7e0e7ba386fa70125305e4d4b17025a2f1\",\"3648fbf3faacb9edc3f3aadccd098346b2c11b7fc55359d5c9f3b0d0c8422de2\",\"96fae2b60b55a39d2d383870a1934ee76c85d7c3934d6b5a5da7155fbec61e17\",\"4af4984709007b11effc47a9b2c4bbce57a7e8c9b10658a7b1bf14937093bb4f\",\"f904c863d8857a7d09c6c0ee84087fff79ed9e25ab5dac7e443519e015ad64c3\",\"044ccca6c85e599807be1c2407dafebed914faf0dde8b4b3e67fbc44e5871b47\",\"dea0669be2ca21596ece399fec2709e191ed016b5978114f296afbc0cdd618ce\",\"e31effa496e480c8b3b10d872b6c69dd19542dca6fdc3d78c2e903644eaa7936\",\"8f01ed4c1dd33f8eae8c50166f9472445cc83fe997743823d9a025125e913bd4\",\"2bafdf1ef8980a1c3278f2cf9c3da32e89acd95440a37a4f171faed104031da2\",\"7dd69396bf3355c22f914f7366b631c3bad4ed0bc0981236dc6aa6d542f1b1cf\",\"fe766e64cd3f9731aee91e97eb2f86acc6e3715a020ef1772a87822b93598dc6\",\"1ce59b31645ecee299d5516ef7c7160e7a679398d2a0772feb274e2fdbe33b72\",\"d69fa1adfe513d5e3e0ff39fac2c8a9598e0d7e93099c5fa3d8958b28841771b\",\"a90ce42a88d2e1ca9e920f2c5067a96d094d1e8c256083241897bfb1462445da\",\"3672495974e1b24fd921ef4a61c49f0433e89ae356eac99fc7c3bf2169212a0a\",\"4801659ce1ae15216cecce28017ae393d7c3bb6c733a69a912dfc06eaffc9690\",\"1bcd87261e0256f91d817272fe0139f234871b7219cd2c37f9fd5893c5eaa804\",\"048b79e362a5d33c96cf48feec79f6f1f0abf5ad52774c5c8fd7eeeae04cd8c1\",\"2927e1fc76a2556862eb33b3ffa871ac9c4b08ba79f8105ba506e53b19d66a0f\",\"9a7ff3f1faddfd528bd5f7f7d5f0cd001f02ca058fbba8948fc42bd8ddd787ac\",\"b7efb5de566df72eacf9b1c3e1a62b8fe0a13192addcf94fef35f8ea7b959810\",\"860c5ab03dc79b221a84ee5c47da2891b564b60507095d276ede3ff54dca9234\",\"6ee0877f075a135f73fa5ed6e868c64e80fc6da4438aa3e05c5709831f8dc5b3\",\"82ccf32d523f8e682e0391b603d09a392a647f59f07634c0d51a167e9ecb9bd5\",\"1b09587d83a51bd606a32f0fb60dc3a482261a84fa53053059bc57c8b2cde41a\",\"b650de6c1c08a4f8d9ab07f3727b73f4e230ff7fad32454b5d9e2bd20475ff95\",\"a50ce70a70341cd58044b8cd2fcbf82c2cd65b49b80c8f89867b0d7866d435c7\",\"9229767277155b2cda21c3ba34f323e40562db009e6c4aadf12b89df0050e861\",\"83a0a7ddb983a70879941e1c1a7151a50be98c41528853cc2649926da22785eb\",\"a7c1d99f58b515b1f89ecee2ea712b57df3bd58cbd3964b52f1c46e1f8cfd87f\",\"74f0412b02f3f707443e03d232603794aa0fbe46d4c0e071c214632a422a253f\",\"f16c87d702450f0f909ff5f0baa3f0a1fdbe65bbada8eeb82c0ba0bcf3dcb268\",\"291d3ed79c597cfdae63f04c5e0ff7b24d93cbbd30eb83adb450e7a88ea3fc6b\",\"1ba1da07219e848d76965fca69fae165f74b2e4ba312beb2202ee2a8845563a3\",\"a66b5b045deb98b6a2465002d71d4e6dd5f1d3217fbeb68c09b82af057ad96d9\",\"3d65a0f4fe6295a21833c5deff4c3c500f868b3a587cfb6d22f08c3ca496e4ef\",\"826807c93be2c13a7c6b75b2312627c532f6f940765c14591d906a4872fb4717\",\"8e6afb74f8f69d401db0363b29f4f111323f31b2006c226f0b83b05a702890b7\",\"8483c512b8e1b8d6a91d6ffdcc998f35398775adebcd3b8ed693dc7aaec22064\",\"72c17376dff1f2fa5a85c762db0a0f8bac969339dbd999f61736c9561c12f1f5\",\"3a0083758fedde32d63b3167989efe499c33ba061692003bf0483c8a74de69ad\",\"46df9efd2ee7c39319b9a5e4634a5346d3aedf6e302dd719205ed377cb1162b4\",\"8656c2988483d7601c5a4c0396b942a70812c94599f523c0fc2de3e876eef31e\",\"3f586672193551294e225ac196853e80b699c1adbbb9753345c995b7946ae52f\",\"ded3c67e6c38503b29f5eeac41d55a8ec19113f336d80845aa0d050e4b05bdd6\",\"677be7f621ce865e5343d944c97f3d1979bb94410c174d2bc323500050bed5cd\",\"048371f13c5300401a39085746cfbf4224b814092324c5a64c762002a287cb62\",\"00ce9340fb93c64c611af91bcb9f8091beac4573d571b1856644fbaf634f69b3\",\"40b709ccf5e84657aaa26a929c028e9041637ecee09c00d4246713b69593ec38\",\"4066c64975482b943c0d64d14f13c932e2f1b92ca5978726f30e413b8d3a1125\",\"87e3947e168486db8b94bf77c838698e15719a86255474d653cb4785ab89afc2\",\"d85fc192d236f58cfd7e77dec18615a8a14c91aece998f9df800c5cd7a992225\",\"186ece7db66db3ba568fddf6a32c023b93a01e8080e0fe33b71ca827793a7deb\",\"7a47d110f7b4b700181159ddf4a70e14a505ab87760d61ac1d2968ba51e82fa5\",\"988de384b0120cead5a2d58e8a27a375bd9700fa3b47fce631166de0d45f991a\",\"d8337bf63c2dfe2155d3d2d522cd4867f234df268438068878c6137955e741fa\",\"e82ddd103f9e8b8fecf4e98037293c28de2c248b3da252423cc8b53879bf4038\",\"9d76e024b6d4023c17f49d390eb734c51f6ed80aa02f337f08df46a882764348\",\"6ce82c62eb7582a7470a8f8326a086e519110dc8e0ec46ab3217d3f1968e10dc\",\"1028b10bceeec9f7ac02e8861a89e6c8d75a381e9ff86c96502b0064113f2155\",\"c88328cfa1112329a7429e65956a8c00379baf979d17ad3f7416ac85bf707654\",\"113b7278de1a8fc8d79b9d7845b761cd2c1cf20a440cd6cb8b33fa7e25b09e7d\",\"b322d8a3596efc3ee861e238eef8705efa66af82131fcb7ff3cabb56efff887d\",\"ddfc0ea4d12a4fcccdac0dbfb727330cedc52ccebbd9d741080e9cef103f2400\",\"0fc48be8efcfd11dd93b64d1e8182feb0c21dd23ef6b6390d7e15ea250216a43\",\"64740a483d620e7976c710158b76001b724cf451c8538864c11c551b0d803e01\",\"64d9896a223e165236ca0209bc3fb3875e6a320eba57de38b35d3621f0827bf0\",\"c29391dfdad4764ff43544956a8b8efa1498c06eb7fc32232ea85f2ba555df76\",\"84cc8168c3aabbcf7c36e37b149fd147c8da55a60ce8ef78dfe38b81abbacf14\",\"926fca1ddb7b95bffeda23783ee52fc272086c049fc84ed45a01e04192d900f9\",\"a7afa6de9372cbe2b9d1a62c01f27d53ca9cfe8d6c9e6bf98cac695dae3b1efc\",\"5bc65f1d8e8d0762194a9fb1305e4a546ed43a07f4784cafaaf686e67f2b2464\",\"148bae1e4dc3f8503f68fa058b62827f060f53916c20259de542d48ebff42d00\",\"5c1b55af6388e7061561015847628d08f5057d499489c27a674a6a424668f463\",\"57c3a9f585abac4d3df4bb1976fd8207d47489273dbd2628fa7156399619b4e8\",\"658c32775fa8ac84724d5f4cf56eb48f99630f3300732376fe4fb778998a504e\",\"2d1b10065ad722d8ac0765cb20b21dc8e11202d9da42423b2ee423546693eb1a\",\"b7735d5b60f5bd171a73eac7f8e8c18de43de079e04c23cd98b76d607292e36e\",\"7ad329d65e05e2f5953bad29c8c6e924a977e348e0119888fb0d670beb12aa4e\",\"69c5ee36106abb8cbc9299dadcce59fd6b25802d75731a81777b2d17a307d7ac\",\"76e7047a0b5600fafbb66e050066fd5f006dcdbc06c5b385c87ddbd2d6e95653\",\"cc050cb2f18b2d83bc420b1e361c51f2a3c30a0fee1732fd9e80f1dcb8cc840e\",\"7278b7f4313af8d459b5ebf6094ba0f7b3421dc763cbe42dc33144159141123a\",\"ecaf73a5d9efae17cd6e00c110acbd3d401aa944d2cd21145bfb617a491e1d34\",\"34092aa69f11d297d13a2a9a6d3d4bc7b8b7daaf07efc1fc6b5bb2a978fbe7ea\",\"e0a4dd4dadd06013138fb602cb425b5c318b8d14d3d5c2b5a1a4dbd4cf7180f9\",\"490469ebd132209148f4d3678b1895c994c215b82a2d2b125bbd03afc79e6f40\",\"c3f44b6c8d71f6557be84a5aef82a0d2b4271fa859997d92ef18849aa31a92b5\",\"5f3a569ca03841b0dd9f4975811b00fdc88db3d4f16df022eca6f4bcdbdbe1a1\",\"22b5d05a8532dcdc794c6f7357807dbcc7a6b8a7838a4433e4fa6a0a5cddf6b0\",\"3cbc5db4180a46d9ad3bed9e883eebc0aabae8021a25d9d117377a1c1ed19bfe\",\"12e06875762d0dd481613d6ddd20593abcadbc21d1ed6eb84e4a59593f64bca1\",\"a4ec44ca9be45f92f4623c1b78d410284a106bcd81db4287b8e9806df2651adb\",\"7c3e9be5ecdbe3a10637b378afbddc239cd1b6b1f40fa279884be3bb58c0a7ee\",\"30b8b17b151f071208f55a1595934cca44931e6220387e1a21896e410aad0023\",\"92e54704f33a811020075609eceeb5d59a839c8f03ed09f889454a616fa5b994\",\"8869ffe85707615d44222ee2510e6de07ea953b849553050c669281d77596482\",\"44e40a6617ec42c74e43a794804e2cd3a61d8f4f4aeea8f271f20bb6fbd93172\",\"b56b025d81cd100ef6e5c84bf24f92c3b8e798cde5c91384a20278338109ebf1\",\"9d8d3e9a38e670ac6fc4761281713028a30428ecc9f07988e80dac6c234dc323\",\"f67c0a0fc100792825127e9cb684d39c06a230b11b566a1d8d5f677d7562a614\",\"f62b53fb25f386d62471da8ff74d0c62507a1481104c9b129ad43a55f361a2e4\",\"4a0f6982d532f75d2d26689d71fa6666961a1ea4ea23856cd4a6d9a4620bbe8b\",\"50a10f2b293b8660a5622e6f7e24ec2d53ed6df082c241b787a996832028d03c\",\"e36dac89e211868b3b9885484ad73734695f071285a68ece15e872b352e4fcf4\",\"c1cf79604574f1cc6e5ed58ffd708329fb50c2bc3c7a62b44fe63574b3ea5694\",\"01ee827eb32ffca076be9da4f4245938b8939db965f59fdc05720d05c45421c4\",\"87da31452dbfe616f4e25e7fdad92b4d550fde09562bcca74616fce6a03f6937\",\"462a8d1b0bbbf0269310531324bacce540cd29ef1f828e9fa721cfde8fdb57f2\",\"58a9a77dc043bc45d5a3946248fe0f033249e1f1218b165314c1cccf0451d631\",\"a9185d52e7e420a780c360109a4337ba61b7c300667e6e7631eefb880e548bbf\",\"64cd58316a2960f965346ea3e76db06a583541e564e8e8ad4b085efd26cb1ffd\",\"c1c254ea325a401d9b4748c8556b01a18022c960ecc9df032e93e006bfd2adf9\",\"571d11002aeb0f1001f50804da002e37318598aad587da9aa1e9623716214495\",\"8eb6a5f447f3759df2fcbbc526e2f5589c8ebedea48746a0b30570cb6d0b8938\",\"4b2b88c8fbe2b4675ee8a11e05648cbe09f65975eaf4816f2be8b3fdcd333a2f\",\"e9a32f62653b56b463afa013e6d045d2d7f74a0a0b06587a21fbacd186db3fce\",\"fa3dbd5f06434195f0eb35da9912a7c45f894559491470e190ada968bef917a7\",\"2cddfe526becfddb4b9e1834e5aa505e8e7629deacf954a14f2c16a817e568ea\",\"4fc0f38c0e848ab1682d34a0d85e09a34efa4e81451f68697bf3c03607e79d24\",\"effb338f08e3a09a0c5ef7b1c28c0ebc239bc7ffbe9081619bff490d1b2fde08\",\"72b14362c8befdf884e054c81be7df3677d22748920d031210459417edba303b\",\"b567399f0beb90461643bdc9e28cd27054af288c6f9e23371b40d2ea4bda600c\",\"11fcc7d473e4fbb589276b87e67f09ece8c6ab556b6f7d87a70e5603e0c4baac\",\"e0b949c26b02dbfb08dc9adade5e04fe834c9edf1839a12155b6e51013ee3856\",\"93a91a0a94443cb6d96a4b9873f8a279fad5915d4be703c1fb127a568783545e\",\"39b94caa988eef50446943e0ba1a90e10892a69cf69dd78ae377140bb602ce5f\",\"01f2fab9ab645537a7d446db3263bad4cfb06664a1abdfa4e3f35a4fc16869b7\",\"321cb6f4c7404f036accdc3f52b6eabde7539eb9a5e9f68176fa91c08195812f\",\"faeac645d129970e7d7749031df8ae668cc84c12e341bf00f83a7b3a8c28af12\",\"24c79f38e578d3bd636a5f90da7c9ee70410bbd03b87db803ec1f5ada283cfad\",\"90e7de4e768464245264cfee12acdabc8d5e72b9bea0985f54fce1eb44b489a6\",\"4e599c1e1cec1b5fbfc29f30de2354b23ef13fe79e06e8d4d1b5afcd7b15f920\",\"2d92e7c07dff4fb7ad3459b5905f0140469afc7c1b5c828a2fa525c7d38eded7\",\"ea7f04219c5642698a96b51a95535bde9a28a895fd8ad01d29148e1442882f28\",\"5816bdb55e8b69456c9c4d02ed8a9bdc4be1a1e894fed63f197e6ced2e4d0e71\",\"8a6fd881ec4d0993842ddd2dfaecccfa325e81795a667110dad246b4996f0753\",\"504cc9d7d27e0bbd9a82ebacc534596191d66aeea9274e07e37a29ff78d98c7b\",\"850e9ff81b0db1473e9676e2a3a7144fd3b3e7ad650a2a9e3f167b6cb918b004\",\"f93f037575bd2a851f821f3d99dd818501c80ac31575b2b77a30fc3aabab920e\",\"66a5840817f6a8ad820e383f55f5222f592f3ecb4ed9d2515ab2eba27719d5fc\",\"31345c991ccfae17c6f5739be85552e2d0fb03a6256a933ee73a3c1da0f7d70c\",\"c14f6f7345e33680dad2de0d7c2f8ef9a68091011e924e44c4dfbf171b373962\",\"928f6808dc9f2dcca3f7af9a1fb49f48a86d130383cd14187db74d85a5043fda\",\"02a99495fa8bc6f266042eef782de943c55d8da50af12878c4e9c6384a1f8eb6\",\"a306ddbde946290fea8237c52a48dc4127872425a012e95dacaa7d0bc1da9b7f\",\"cbee52d08c56ea97a45adbcc927ca8660800c579803e9c3bcbcd48ac322e51bb\",\"0da1cd54c9f2724da203d81b0ece865e765754840edadd49ed146cca3d98f6a2\",\"7fa756f450b4979ffe0713298b4bdea8ea669503e17dda628dc97b5d9b296af7\",\"8dbe7ade70715dfeafc96aba58c741aaf8a49971553e5c2415017b67bfade9cb\",\"995e4b0857ba5a0361511cc3c93f3e1e39e41a466e4dc218387243283bf33498\",\"2bea8952d8a2d667661a689e60d74f4967e5692752915a3f53b875b2fbd4a7f5\",\"440dd53c55bb3509f52a50af6ad49e032a881939c679b049d1bd03923c033166\",\"147444e0c73f00d8c6932ad2b196e7c0b701c5fed65d8c615696fb9f6fbb1086\",\"26aa5e2a25f52ba817e7e0bb5485daa953dda88ba2f320a03a0bf9dd8ed35fd5\",\"719e7ed03bf7bef0e04b6a09a54243aef3d4530ef624c88ce32663ae748e6a23\",\"dc3304040418188ae59fdf5898d1c86735e2ed7bfd384cb2b003d0cfd1f1060c\",\"a14ef9084c0bd64dcdcbdf3d7d4cd43938944c064128f30accb891e6cd5d5a6e\",\"7e80664fccf8a1d5670d21ee838e468c6f874af90a0194d86a1e62f340041f84\",\"509b0891d0084f62475b229b4ef8b209fb300d9941ceab59f9bea2d78bcf8a6e\",\"83c7a93bed4e731864a12e342a75c3ecb2be6b312e2a6fe73098ef701f4f6135\",\"c140a48391074589a90c73ecd69b9925b5f4c0bf3944b3b3df577929aaffd5f1\",\"7fbfc350fca3f1ea2b1d03b1051890195d482821496c7ad328a4ffa4855f0de2\",\"fcc277d61d36246b61f3c5d78746916200d86d533fca63398ad579372b016556\",\"c23fc4f714a741720670d5bf7a22cb190f795dc8473728c0c7ffc82dfa395e60\",\"52f62f0aa12848797a0c335b5e9143a8d11f225928cd8c8708810846859a5479\",\"51408f0d6fc08b45c83aa4c76075c565e57a73eab98a0ce18ce9e6701bf36e27\",\"cdfe587c64dc7131dbd2fdfd302ec55886f9a2ecbf68d5e604144a558e65b3ae\",\"4ba3daf1f9ae70b61ed01df123763ef2542ad1cf528bc6d84f19cd7a800888cd\",\"30b8ea928d14af7713572b0a8f5c6ab257f94827cf65d654b85b93a5b5e1ab2a\",\"cf2cc060ba6e444529bd7de3a92de93e13fadf9d312237729527d5cc51429c39\",\"4a8c71ce57df506bcfb4ba8a3de6c36427885381934df7b9cb1756b4f1be01be\",\"5ea238ccc6274effb215b61489af84ab1f36e407e3cecb1b21be8be5823777b6\",\"5d9116b9989c17a7065693fe5b014d37e22824d588976cb8d8811f03bf3308a2\",\"65b860aa7f27eb5c644ffd9b6881a5b88c1d4fa9bce74910d9367337cc0db524\",\"c91844b0888754bbef9876e70427e352c4be9ae7f31a24439805b5b3763e27ae\",\"f7140ddd8e52882a128c957ca551d3db32863517143fe7d8a96eb50e7bed4fa9\",\"57c8730841c1389785c88a22bdc7bc6702dc991944f48f5d5c61eeef00e4b2e5\",\"848100fab8284eb6355bffed64db211d1dc9c73d90343bd0f3bd5992b280ce13\",\"9c2170c558cbb941066322fc71426927ee8d57aba75b552d7801b4c4f8a0711a\",\"32994509c9754c4157ab3b3e8d7fd702a2b904ba555d175305c4b6b996cc05b6\",\"55717124c6250279542aa2ff71ad9fbeafada412a7c631e40fcca2f27ee1f098\",\"73d0ea043d92e21bda94d6ba834f50d548eed64c11b293d5123043532e7dab43\",\"f7c1c487d61d97adb3e01b498b2b52ff9bbdc4f0cd9622af7a205db2ea4a555f\",\"ed6ad636707ba868cb9f3b9340adf28afb96d4384674c4b0bf9b78950ff563e7\",\"708fecc4fd22eafa51cb1e9d06c5f08073511cf89b39e71d689291c92418cfa5\",\"cd6232c9c2b831164be5f12df49e38ffe82a4ce55a15d5d3a91b8619abbfe90f\",\"4478404df0d7380d210ab043946de5d70cf7411db7a9f2c621143b7005c654c2\",\"68f5de97fda29ea5f039bb42ca292713dc86be1be31b871ce444458600f2cf31\",\"2d20b59c38fcedd42076ce9276d6dc80396112087f587ac7c42476deae1b939b\",\"fdd23a77901798507940fcb22b2b2e9efcc282d944f163d77fd1e5f295308500\",\"b07402f11d6232785fe08bfe995c65a226668e91b04f357d477341881d79698d\",\"7e2aa8e8d9a3cacd254870c09035734f78d5563e8d68d62d05265feaf7fef4d4\",\"c206520ab7f3eecf5d4f5a9feea50ee02aa2ff7e093b99d699b53033341a0989\",\"ba0486e540c5dc1440e4028b421c4164ac7f29b271ee8237994bac9b57d95d03\",\"b0bd829cc66f376692400bcbcfc851fba28bb34fdfc5b02eeb3320419563cf24\",\"40b9b08a4c25b9c4b7183171d9209477610eafb7f75f7a08d943ca9e3ee48fdf\",\"d29b989fae79d0247634d0f469baeed4935b99fa3b0b7ed93a245f216c802556\",\"17d155f7b0c13cb531d544db4ed37e7e0a56662b49b378885b3da1dc7d1bfa45\",\"c8c6e358d516031c6275ed168fe0508a351bcb323fc220ef158e08a277acc8e4\",\"8e53e50befe84f67fc10723f1ccea85d7a7f1e0e6ea82abb82404b4ccd98ee79\",\"ae85483cd3a44807574d3de1de78172afda969cea1eb651f0fcdd494694c8278\",\"4563e5affbf729354f6841068e02f14ee648bc6f693f84f4134ece9e7aaa8055\",\"542c63cd1356a2c96b77550085b77a48faab868f01885e7f39c6b2bfdd27c4a2\",\"af33362ad540e5a418261605561e2f2069e475682c6ef67fb34cd67226bb28c9\",\"2d8eef5345dc17046f79b46bb7ddf39367f6d3ad1f453bda1be153a45172d1cb\",\"6a37528e030921bb0fe5bba9de026108dfd1cbc4368e9fd266864c0c2bf06122\",\"e7ddaf5a31ca37943f55e7c8bd8b60af68f36b3681145405cf8cdf2670bffff2\",\"3ac5b6317f5a89581676c8df1eb2e92291b5025a3c441c574ad16551659e2179\",\"2b2285bd260724f3f159f69dfa51a573857df4ac143e738da24aa4b504472193\",\"3c497a7d316ce351d5a62e636097e54c72702163787e9515346c05df6dd11a02\",\"04bbb165f616b4baded5fbf29a46669a768a0c11f758dfe3a09eae22b9909db4\",\"14b02c0f1e1c254ee877cbd94290c49cd92cd8900a25766cce731dc10f495f5b\",\"eafb8438bfc7e731ccdcfbcb9d05e60d04db027baef18548778420bef24e55f9\",\"76d0be251095fe6c8d99bbdf53127257518ceedb93390ca785932343888e07aa\",\"ecb8057f54f8a9ec54e25ec8934716c627789704ffd7ccad35aa2abeb41744a2\",\"634a98ca53a03a5f2077c25a12975e6aa049d79d3e2415e4c4cd4d56f263ca03\",\"02906ac7a1e6de1fc09c1b19213e799850fe710751802e828365753e4c49f0be\",\"143334d75407d08807a4131fd87d3b3ce064c65b1223868973aa25a408bd5b00\",\"bf373440c9bcbfbcbf8c1778fb944ee97cfb033f8c7ff702c4e7e92b05c6d6e7\",\"5e6d815ea76bbfd04d8996fc3f040101c9c9524ec026cc7dc0a79d8c4881cdf8\",\"3f552622ee872cc8d9ea353fc024796723cd67c46d41396b101bc816716d2991\",\"39f96b8c51aaf53268d3ea677832d301c051bd848f842feb8927977beff2e08f\",\"d3f67fc346efef772bab1c4d9169bec173d45d6afac6633d20524cc95203ddfd\",\"76e94ac90c33ac63afb7309e0f8579c092f374ce2a8110a29d298848c1e37c85\",\"80b623a5927993d5ca9d174ed9100ef2910be58c9c94ef109959974a8d5b479e\",\"de41c9303e03bb4d296f25a1f15e1db7489668da3e5e2dfdcc6c4cc1d8bbb168\",\"362a5ef392aac3eac7a40c99d3d6bf3ddba8c0556afd02c7cd907f09a449cc3d\",\"5ecea0d544ffc28708c59b217abea6ee1b22b0e4a048215dca408dbd3463f754\",\"e4d52827a85fb2dfead284f33435866a9fab9ab6647ded293943134cee059f40\",\"a5a102acf2e8524c665713afcd434be4d89d3e426ae4949f59aa8b549b4c5940\",\"46bbe1ee22f2608d8d00f7374838a84e5577bc813e35c5f392e5c6ebd8aac471\",\"75d638438d9dbb0bf9e3587098d3276574693aec8c9905c09df71b78e88989e6\",\"37b69cb566f3c918359b4c94880f4c8fa579217f716546dcf2a93b636440ddf5\",\"5d5a6d56cf946aeef9fb71ab85624f5189e2cdd9b9b1aca8738d67e456fab274\",\"bc8a11fc6f5b5d3cf7bec96816852a425d6752037c7cff1ca04ba2f083d475a6\",\"e98cd1f98315b58eb6b1ff894d9da5a9ddeabb1f4d33331e172c4029f33a48d2\",\"69494217ec609cf1e2b12634e9f0cc52fba7c48cec1fe9e6e2ed1df591287945\",\"adc759922cc9750f61e0bae0c9c9f759f92c7bcf9afb04cbdddd751b5349c0c7\",\"47b05bdff4d34d1d1eeee2492076906acd6b49df0c747ac5a52a8f2062797825\",\"5a7e3d9ee53083009ab404678686af72a5845fc0fb7444dfac0d01d8846362a0\",\"ac21b93bc17e83b9f305a6a2e5101b2db325aa67fe152b652ce7722f48c95bb4\",\"61ac7579d6a22ca5a8b42aa575a4a299d0923e2bfc0223dc2dcc02c8d15937a0\",\"d684aff336e8eece787af4fa5956e08cfc2ce3f014d182a0cd1a88e24022267d\",\"36383ac0bd7f2e783293e1cdbcaf838707208388bacfde77e2f39ce498eeefc5\",\"8f1e315f21cf3557616725753dd3e51bcf24687a5b94218ffc05a55d9d629600\",\"24059c2f818545d7195259c273735a2dc00f9429b5d37ac9a694d0cdd63bc906\",\"93f0af2fa1947b23c7f31fab08608cce23affcad9d7f5c0d87b1ce2634674d55\",\"0f5ed01b8ca78aaf8ffc06802001b1869cddc94ddd92a619b5cc3397e8ce8ebe\",\"ec174d9c14be1ab05be31f20423868b1030401fcae3eedb2102a60fd424baaaf\",\"2b4633d349fad6256573847227cf874bd6375fb30ebc41951018df785fd94baf\",\"d070713420059111be417c14a4e8cbb30b9e09115349645c0b2629d94a92f351\",\"7d083e962db6a35504a1c45cdc864cac6a2f34f6ab2d34d26382394f41d17add\",\"9dbf7464d7b55a80d33241ec00d52dcaf326b9ea620caa8c3ed724a5d92e3961\",\"72a54309187b1cc177606e1953b78c09c2cd328c18d1987715e6257eefd81629\",\"16a0252f4c0ad23a1c30faa8dedc83f8814e91a07a059da312047118806b45e7\",\"5ebc8f67d66cb7c6d4f019a0baee63f94da9c31accab98c43b399ba55acf502d\",\"5885b19321b06768113766bfcdc2636e4585a94901aebc187e3ee100b8092a4f\",\"0148963f16b3e0be6d0d8a392c344919f24730866d9db2a53ef21fb0669775bd\",\"7e4682835515a6a799ba8ad12e88cc3180ab17cf5ee8f5b5c9ba4d791bb5de96\",\"10a4041aea06a9dde7e67e5b95a435dc4f3dcb68d601e09fc5777de0d3049d42\",\"76ed57e5edd08b3a6f3c1068e7c81dc07dd6c4c43acbfb2b2631876f7c01ebac\",\"ed90497af482e79f421bd0731f9d4c8bba3249fc10bd215ef3b07d4be4a16952\",\"9ccedef157992615d6829c42b5802bc894ec403ae779c727ae2adb9b03239a1e\",\"32b3a81b03b3698c774c7102494a0d1d94dd64f66b0fa6932da1c086dab968f0\",\"4739ca5202e0aee6e4af7ffb3fa6d2082e07911e99479ecdfa3426a24c63d185\",\"1f43b78d11c0c2e0fddcc72ee9b23aea8b244db2349f9ab3903dc896497e4a5d\",\"9065386dc012da3070fbbe4b919a47bfe313b268b9d7c0b6677165bff460aac0\",\"7eccf82b946907a2653e0f2fe8bb1765b4e1e3d471452646b7cb7ead42bca008\",\"48b4c0133234e30dbe51baa5ff7e18baaf557a8a89320af0e1b57b80db8e7f6f\",\"f7600ea8da65f9b4ed6e65df1ef23cb43b0b726ef03102864f4b645276bacdca\",\"86add56eb9f8a1de58ccf6e31e4694019c0c585b94f32387167aacc1a9286bf6\",\"182850c982be9afc13333f46b00a92f4094f9b1dafeaa80693d8a2b582631e4f\",\"d26754e970709ece41e1df9c1cf45d544f97b5c1f4cbe3339c27392c858ea88b\",\"08fcc25df2d7a6f8422fdf9290e852d0709f2cdaa6603a6dd31b91903032b974\",\"a9c16208f5b046d95818945737ce5731fa802de77c05a2872cfff9bc2324ebf4\",\"652105b3da79a4cf7adec0c91a5368c2b45b3eb13c29737ea9f5d12ac8f0609c\",\"a963d645ec8d4f662cb400ae10798eb28a46ee22bf39a2d548b50b28498f02f8\",\"8190153ab27e0d08c8153c551974ca23b8247a243facf8b06c2b6d5c21e6759a\",\"9f1410c036bd3e89b0daec4a5fcc80eb41104cfdef8f4b7e4d5f7f78cde596af\",\"0a69901dffae417e9e1b62863eee06c95ec277dcafa29d0cc93db8d88ed0524b\",\"6a353e6602bba8aeb36b05fa0c6edcb4a729673594d02250be40f20558cc7a47\",\"ec756034aae2f1144f4607e08fd8c4b99b46304434c98e6ff5feb35b0e2d33fe\",\"be98b379c8c54230cbad903b2636c3d707a4ba0741a7598e92b01fdae8623591\",\"471207865f5823b591ed15645201074341b9fc967ce63d7f1fb052acd79b5345\",\"cabadd0e8f7f7abb4285ca60630dc40409e7e24c9a560ed861ef8c6e78e28b4e\",\"875b448c82144654a819a66b2aab4e550261ab2d135533e67cfc4d55249c422d\",\"392c07b02c7bd063c34652e0efdfc5657089f103820104f8857c0fa778005c5f\",\"536e8ec09e527d75e17bcbda182ee4360d8fa74db4cf62cd1ca2f6e124fe9ea0\",\"a4c2cd83e41aeb3c07b4a55f2f56a4710b3c22b602a4c45c4063b3746ed7f450\",\"656d9ffb3c2c64fab817db9ce4006d6c4dd6474b070a102b4a089b35099b51ef\",\"97d4299281a8ee33c3422d198e0a4deb95d9c585e90926d98960feb00d64ac51\",\"13c0cb5378f16f6106f6395b08b06b9e054cf51cd90e84dd331db5ba209d03e2\",\"db8b43b1bd1580f984dc1110ddba99c0991686adc825fb4f2ed8dfae2c9f6958\",\"7d9a6fc2c7c132bf9e217767d30c07224f64af7ddcda4c063e3ad1cf04cba3b3\",\"73f9a38f705b207b8a2e66d3c75bdfbdfc2e8c8233f4e765b01a8baa8aca44d6\",\"4eee1500eda50fd54f3b1dd5603bee4a1d2295b1587f3d4e43fb1639efd117e4\",\"d4391d5703a2def97ab661ddc0af5214056b253272617287cda478f66988f09f\",\"abadaa7248edf5194e9b1c9eff8389bfa73158a3a3bb3527d5b9ab15536dda95\",\"8c7ac018cba6b332b7a56137bd4f5270e9eea3f6b7ce23f70f861eb7181279a3\",\"3be5d7a68cd3a9b3345df3a91b47ee022fc23949332148e98ea1f35ad43171c4\",\"420dc3f700eea24c0b9b987998af6c05cd29af054bc638c7b2f2c1f371d62f54\",\"c73bd20b7bda142b9349ad111de8577a3449434ab333ddac8e5efde8385a7cdb\",\"b629c082d84daf42c900e0cc3e3d97d3283be0883a59bfaa6692e351b96be3bb\",\"dc911b5373fbaaea5c11488d1c9303cc267d0bf1052c8b173e7c05fd1897223c\",\"48e1f1a5d4b3f8a2652b21a6dfc39a3cab6b73f26c8fcc72a07bb2ebf878ebb9\",\"1e8ed67747f35bf46325fa860fe15a8ed5938fe3820356f36f0aa11284d36abe\",\"46dcb18adf328ab36e4a5000fd55838a2dffcede558e1efa3b3f08d345850ca1\",\"5977fce8f4eaa5e0cbb4f0bc3280d14ab95c406a3c7eb2b3f77ac956e312342f\",\"bf66c7351d9c9e92d712ea2b095b6db3536f728e7aab5a40e4fb249d82ac0217\",\"93006ef1431a61fce25ab1c258c36418b3e4f42457b4abf13fe1f6e45511531e\",\"a53ef2eab2dac1bf2aa4f97e27e14015f39c52a2f69557d793ed65d250bd75dd\",\"0b280cab7fc27c5c140ae5b06a13a8338852bf943d98dea5e1ac4485f5337506\",\"ee8c14b6e2e8f7842bd86ef6ee8a15b8effdde0c310400b08547f40bf1af39b8\",\"252ea083342f750d6195a14093a2de66a6fa8fda85021dce37e0b95dbc926a03\",\"fcf4b17e307e4dddbfd29952e8fc7a6c7ffde5fa4f94dec03630b095695c5721\",\"fcace891765975d406d4fdf912a226d0af21059a2fb35320ec5285999a5953f7\",\"fa0c5c007debf5fcf46a36df6865800b8ce5e164226972da3408c81d85288eb5\",\"a788ad1132a70024b2a0020f5ca3d3538d9af45493976f74b6b0423cf878cd2b\",\"d0b7dc2cbad42a798b4d00502032a7e821011ddefc2d125c4b82d3da9e19701f\",\"be7144ebff87e6b1cda48f968037d2cb6c8295c0f6c1e6e404bc99e63a7a92fa\",\"453e29ec3ba197346ea88f3d1af2aad9ef0681922426c9afc5eed014620eaf31\",\"a6dd585247e018f3cf6a89dcf5e07e3ea506472f5583947608e2e4d44db0da09\",\"5e2d0f2d94bcab1c82d4bfab0df9c51b6015df26622b9aa41f4d987bf77b0b5f\",\"da1d50610fc73b104a64a50d571a3289d5fe19425c273434c6c5bbe2b9aacad2\",\"a2a5892ad2e745920e50d971bb0ba30c5b7dad7663d2b0c874417cc3c96bc0a3\",\"050422a02b5288baa674f8b39f0e821636eacb68dc6f3e24b402c7738dfaec4b\",\"cf34dbcad1d005f0e60d9efbf45b5c608b69dd992fb00601a33c3a42dea2bed1\",\"7a71ef517ddc530f806375db355ea6d837fc247b0f250d5b4b7428369d16c168\",\"45a82eb13766a1289dbb3ea45fc418dd4de84707b2ab90c7cc0c3d280bee2b38\",\"c108424fc4afe69f955514b9be8a8eae2f33c0ce287c02c848f58400ae99b918\",\"33f441c9b57b0bdc8c6ebc93f7f7c463ca6b0aae38bb7ce060eb4795a63fe48e\",\"3f5aaaa8e30fceffc4587fc928e4814ec1925a055cb5d9a8f6dc267054215903\",\"34d83bbc03166f3372f514564f12a3c3ea283482efc64226df652a214321ec20\",\"6941d955ac645288407319a045f6a49045409142a5689fc84c04c7ed919a4c83\",\"19380f33be0b1d13eb7cf38c5aff186a678ca412f13b46fa802baf00b7c8ed1d\",\"1bf41165e89d589cd63fa968cfdface7f966e97e099cac38047160d3a66f42e3\",\"569a8672061e955d421033118eebd88db6796d8cc927731df5583b1996d33f1e\",\"f1524e6a92882c2468e9b95a43cf00deb10e870e02bd99797cee2b709db9e7b3\",\"dc894c9473504d344d068f5c55dcdec9e29bd9b407899a88d07a1f89d61da09d\",\"80b3cd5954d2e0da1edbd019db529f8e14a22043ba406beb1b3a8e287f0ddea9\",\"de3486ee9d57da157379175d4c5952de3babf5cb4eedd278d772f7479be4e07d\",\"e42221becbd4dafd6e26a2f2b76add15ac0c55d9564aee16f245615dc0cd0ee3\",\"3b985de7369439ce2940d92d9c5f5075c829d42b3533a99c4b81f9c818d94672\",\"70d12b1e86c3d2a8a2e3a18deee2264c016773ee5c442025efbdfd3a521a1c1f\",\"5298f3062e34d89e1f0dd7e8506b92e9890dfab0e5f692ef8ef09f8db43d746e\",\"538f18b882d3a17a1e69178ea9f798fee70c5d5d3d5ef0d4a781a3387e647107\",\"cf4a9059f2282a9bffdb7f50d4dd3f1021cb5afdaa8d9d4e3894ed3fd272ac56\",\"6032342e387dbc24357283d7ff1851013e63ca4b7f535575bd29e4523afb2711\",\"5aab0218489adb66011d2111c81e83f6f6d8e435a2147cc960304aa4a1763e3a\",\"65e4d21086a4da1f85eec5c86d9db0bc56705ee987b436adfd0e73a57e19d052\",\"3d6cc682aa2c5959266ffd084ec1784bdcf203470793776281565ce3b5e15849\",\"b4cb5b0565a2ed24613d679ff4d419d83a2db26b81083ae44bbef69c0846e4a8\",\"a96d282727926306ee656ca2a8ba8cad9c189c1a9a312900b587ff9a35895d43\",\"c037819563c328a726bcd29b7019016815dc7b2e4b80823a5c5cfcef9d3205f0\",\"15824f469d82334efe571cc9a5581341aa76fedd1a65a46f0b24dba2be8782e3\",\"297fed5902665f691777bbbdeab50f5732721fd8f36360fca525e2039fcdba1a\",\"6fbc0178b2adc7e166e9db89455069e4222bd50dde4a54ead035bc1b8a7aeb01\",\"9388883686002fa8535b6862ab52b911fef44eac49fa205a6f194d90c428a2ec\",\"82610e2e6cbe55f3f4fedc52fda39e45ffe261b651f6ce1ecf0fc382b3446a7b\",\"6d632630d8d5c1d22b469ccc04bdde79628463c2b19c19d063211315e8f391f1\",\"b94a127d194aacfe4c3e1b3da69ec645e473b9e423f25a3d2b9f666057c4956a\",\"f6e0d90756bfafd5412636306ae6fb8c47dce0e35ded3444a2b228cbfcd74bad\",\"31bbcb708af51f7fc25251a4fa0dee7d64822b3876694e235cc91560c3013365\",\"6b2ffdde9beb1d06d133b906f412715aad98c7f6f6f4119cb2592293631c1795\",\"fab5286d585ce5fa3314e429b184e64a5f12dfcacf9c3cbdbc8ffb4851b4f51f\",\"b02340f79f8e162067ca157ed5d11466c9f42daec32ed7e6df6a65268b7eb6e6\",\"f7f84ac71356b9ddd5548c6fb5aebf399061dfdcb10eb051676b706d46e8334c\",\"56f05ce1517032f8cfc34d6f05ca01f6440b251118d930d82085a9685a977b67\",\"48a9f002268baf6b61b373f9b669b45c05d1a6894d061ec4f6425767038b5e70\",\"d8ee2b6c13f2ea93c5b7a0d72cd73e4b6cacbf0732c07787822aaa74d99916a5\",\"0ea1f80b2b9e425a01fa0268799f9b1694a33a34308e0fcbcb8dc5a3a227c84f\",\"17a3908e2991e076ab67dae969ba28d7894c253f7907b6ebe68b4c136cce2ee2\",\"bfa4086085eaf22e4c67ba9ac9380bcb836213ec6191d498d7f36ae68ed240da\",\"659e15c09076908da3cd7b7b2a1b99480aab504d371d3c90f41bbf4335cd9ab5\",\"50b91d5d5bf55d53a045f8b607c1a87080150ec06ef424d402dde1f3279321b9\",\"df27e79035a96b1559974ccbb66a54335e4c11144ee22fac08583e5aff335e93\",\"2a26956c94acfc3557abdd58da8327d88812f9070d87d7828cb58c128f62e444\",\"81e8efac924f6e0a2f64cc5ef4785fc0ea282dcb519cf585d38a653212c24ce6\",\"487d335b6e30ee0e4755535b6538d7711f3deeba86a5b5632d535d594fd3436f\",\"75d59ccdff568c389994e2871389517017f718198a408d4a13e15ac4d5ea4e09\",\"329b62f2ed23efcea58fb2ebd6165e132a734100887a90691e750c3232a94530\",\"bb5ed0dadeffaa4cb54ac8fe3c088c5f6bf5d34498f467ff46b10e03cdd1920d\",\"ac5724479a94c44b0398389582583458d7244e77dccd6d1a46eef3d610edcd86\",\"b7ce9721d82880f9eda02f96ba313732d5e13a4b7bfbfb75532d05fb37a8d941\",\"c6b1bad381549e23b0d7b0a0d805cb032c2723ce5399c3c216ca9e8e3cf4144f\",\"850deabd79a4f5b663a3cfacfd6bcf4a9bd4699a135fe913802ed156e42f2dcc\",\"64ab6cfbc00c7c4f7ad0575925afe4f89b85a871dca5b3c2f3b426e5c589414d\",\"57b22aff42b124a7ab4c2755bfdd16551a20aac3f1b7f35a1e7fc75f76c3e61d\",\"56cc7200faa4e8a55128a804adecc2f3371071891db50f2f133fa09621a0ccb4\",\"4c3d796fed6f3768612f865bd3086994caefcf405eee7efe3c63eb8a4413d810\",\"0e8a83d9c3b5df433e4e507ff96961eaf49044e2a88fdaa730132e548632b2c6\",\"c178f67953c8aba7bd5c050767633fd7feb024b703834e0e3958ec95368381d5\",\"ac8117b6af1be1c1b143afeb93ad135156358f578676fb667267af67a915a6e6\",\"ce60f41d371bb8d13d5a50da40a83de67a835340be73673ac5561271e901d654\",\"c3031746bc306d8a2a295cc156eb129451ec22025fc765cbc64f7c1d14904036\",\"bbf312ff0d0292d9aed37811fc11af70471e0f56f49deb638ec2f657d0c0d3f1\",\"81773e61b776db37b87263925d94e6b8f46c4355507c48115efc8ba1b657397e\",\"1e0ea04ad9d5069f5a6aad77301f3494ab613d9678bbf768f49f3a1c4b6db494\",\"badf2f2f0697968e7a06bd4c846f80071cb0eac5fb733f77b61309a7abacaf2a\",\"f2fca11f0c299255e24a7fc68122777bfce46df2066fe46088444f3c95676bf4\",\"96332dc228c68b90181e7b7d1fa5a307ea27237e6eddcd6f08ebd9dc32951fe3\",\"87f16f8f910b1077653a7d64fc195618b8ef91dbb46521736939aae2c3164479\",\"adafe500bae6366bea30aeb403311bbc7e3d73ed5ca165a9a1a3985ebc40591b\",\"a2ef7f45e3417671223c742d933d8d4fe9570ef73ea6b65433ae103694a5562d\",\"6a893eea3899b55801266c7cf5bc6fe823657abc89c65a57f0693c2a27f11478\",\"9a5448588c4b2ebd1a1d6d4f6f88cbfdac50af748b47aeb882e0765fca432e3e\",\"3781716f8d44729b91a24606055001045b407aa98780c09967f1e139bfce7431\",\"41a4dcc6a76522f4a336521a52f7339116161d59cf7d3923d3604eaf408e5484\",\"60215f030ee82a2e476d80a56ce8581e133192335e5416e86ab68803440a5305\",\"79d373f14c02faaa996b027c54e001236072846358381f18515f409918b051df\",\"868d3bf2b15863761000aa9b46728351100da9d26dc9750b097d123c499c747a\",\"3aea3d1abf42a8ec1e7b74bceac49afb14f25aa484879a2d89131660cc9574ac\",\"cbdbf28019df721bd617631186e61e8372a13f310860c01d34163df18bcd7f2f\",\"602d15e5d1ec1c96f3350721771bf1265a3d4d26c5f2765c8b86c3ec3749112f\",\"14ad7541741dd96b6681fea10a6f980f173c8486841936566d83cbbb9ca2ee41\",\"7bae9a7a121aacf0739c371d359719818becb6dbf7851a94893d77ab5a6a56ab\",\"c8cd9a3659ab9a882676e09a829bffbcab13d75755402aaeaa4bfcadd4e67d21\",\"5107f14bdb4935894177dec854554f4d1f6f0ff449fed437a6121eb2176c0490\",\"bb5f0da1cd225a05e1a82624cfff02e4f2bd6d47c203af683bc37bb0e3b74d13\",\"899d9b8a16b655ecb74c9149ed5ff528d5ca880171f622813321b2781bd93622\",\"57c967c3e22c048e1510c32b3f8e6caf8b040c9de974330d2bfc2bd92d7b31e4\",\"f32e2594a933742a31f97685871f2246a26636589605a6edbba778dba8d497fd\",\"ab1b8d1d5b7f3d7ad5b2647fc31867dca45bba2a53dea45c20eda8b4be692a44\",\"06fa4c68677c0c057de8a162fe56193a365463e5fffe5e64a77b761ab108f7b1\",\"0f658f16c956059caf7cf4643da53ccf63741747bee7a580013f99c59ba8e517\",\"6c2d98098a835662c8bee0e9df127112b5ad7556ae1fc274dc6ad73c13bb46fd\",\"1a726fc033dc9682095429dc2ff9e88a8f8e0de4299320ee39adb751b2f62fd3\",\"6f5d3a8d0a70ff183fa272281c764b2ac395883763973c12f86dbee6c309a1da\",\"614182826491e74dc5290346ed5579fa11d2e7ab653709b34631e2f5e7bfbe13\",\"03501844c7e7b0db75c4b19bfc3b96e569ee73c6d77ed99da750ce6f6c45087b\",\"97e2b2aa6dc231a374eae914355d49c02c10741c342e42616b9c4068191d6925\",\"fd8596df27e0d90f21550e2dd362cf65def9c189d6a633a548a028316f663835\",\"0315a6405c24a0a7fe9ef57590ead106b68df1be86f6a8eaeaf9e0baa6b84d01\",\"19927ea023b6e44ee2f1f4dfc986b5071518e27b801cc17dd0583900c6fb748a\",\"9035e24fd882c0cae2da27e58c7d63d4a6a0ad2d9b8da85d6d92a77420d036bf\",\"69535cc98724402da17298630bc38d7fdfab122f36d35ee65c14435bf6134b99\",\"a29345fa221a20ca7272253bc947f2f6ac002ac40bb83b58d1a87a6f8f660ecf\",\"4f95f9d780042168aa3055fa2b92a8f02aa2e37098a1532c5835755448057a33\",\"877be0de94d107d3556e65d811e1f96efa1aa6626362a12606263f802635d94d\",\"beba9e8ad77948d18c3b4212fb7e5e9c0fc5938012782bdaba29cc4a2569e011\",\"d6e6e464cd2f71c2ac3f3417580cce6d0965d933f85fa0ecfd9be408ef761da1\",\"df270b843e7146bfd409f945fe0859bec8924a4796cb3437887b7f4a2494a44e\",\"3ffcfda342ba588e7359649177fb6f6982936b63a09e9caa2e0688ff7dd99a33\",\"6f5799a778c40f71eccbd1358cf8b7929f882290065833f71e8bed5a31297eb4\",\"7673175a866967b906de8a6ac6c9132104200e193d656e1be4c7b0ee5b0a30cc\",\"9014a6b6f98c8b089d5ba45a7d1741033b99da7278adf97c7abffbd102202bea\",\"729393c097484ca84997e4f56645c7276b3630608a3a3928d594c0f01044f59a\",\"96d72f95a1a3b18c98d5fb802b966751711a5c60dc73d64ce7e115640f9db865\",\"7424d1339c7d81df19cd480855072944b063db439ce457fe24c9341a1a2ee72e\",\"e65e9509e443c8ab8113044bae73491ebeaab3fdb7156d3bf7e1fb2091c8726d\",\"058788f508446f513718a1f5136d33a098fd759e3c0b1d0ea8244b14a5aae785\",\"df1d0940d32907497fcd64c515a8c3030c5803b66fe32f9ba398fc83abe30db0\",\"775cd6e5dff0cbed01bc7ea8842ee83f0d4c35c555849bdd6b4ff0456fb2f16f\",\"9df0328210aed9e53eaf172353be51dd6fbadaa1a414171a829be2eda1c64c54\",\"eb931ad6dd0077a99173d8bdadb786a4b3cb0d9db661611679b740a3eae18e55\",\"83c17ad6e80b58af2ffda8e111c5ec61095640a464115ea1cd6f218ec4d9216a\",\"981578c3cefa438b531ebe723eb9bc50ec0def5418c79f2c7cef7a66722961c5\",\"d0ace27df6ec2317a8f4d00a1dc333bcdc3b568627cb79cf68b50acf1f612658\",\"1a25f10075154018663810bd14b195d31730fc89cf23c94e3894fef02f2c272f\",\"fd4d287ee777437548fb33e9d4dd4e806b328931be25035e19ca8fc190f72a60\",\"95e002ab822fc34a8d656cc8ac1908ace5980588c31308ef728cd151d465eb99\",\"01c6a22bcd8741554227461cc13c397b507d971d4b968928855c0cb8672a785c\",\"876800ba951e950ecb438500228df7f61bd11142d867c8cc98d01c45f672f3a5\",\"ca4172fce6b8eaf1a7d21935b2dcd71852e2612bd463c334950cfc34dec80c13\",\"f769aee88e78d662ad29757a53c16797e5993d4b4f6626518b5a7d2edc5c7ee4\",\"4a810d85388586050e6c70ee6f1596386fe0c626450fbf1b52d7dab30221aaad\",\"0a8b2087b8568cfa8ba1e2c1cab251638dd1e2fdbf14be418edd863c79e12056\",\"4c9b8f237d9bf66d140bf52a9496cadb1fd1c263c993dba36396551729a061fa\",\"b0c65f501648393307a30a1fd7414a79519189f6fd95b626ffc241a4ddac0e2a\",\"093b0eeaf0f63b7a6af9e4e63cbba6d6dd896ecfddaba83aa16bf3330c145ce2\",\"cb956b3e576547c28b2735bb7345dbb1c5410204d505b0f635091ca15a205b9b\",\"847c4be090551c2c6e81c091b282b80ba503b992410c41fbf0d2a23ebe583fb0\",\"6c68f15f7a88e99aa0ac3f7161dbc3052c950bafd700bac59a49e013480ba005\",\"2496411c2663abf7472ed92593320086885f12987429f6ef5df00f2ba7716f46\",\"ffd47d09b0de0d0ce2782188f8e0c570280f56025c56b8b1ca94848df9bb7990\",\"e3746ff2be96a5459b74967a909a865b66148bba10912141b8cdbdc4d89b5727\",\"2e809b842c76d9d352fcdbd3730ae250436781db2f0858703a8b6c56d32920a5\",\"191f2b2e6b47b1b92e88955c3ff0a53c8870020da330f159ac62d352ef8c5cb5\",\"10c29c584b6be6f87a8d69dba4b1b5a087be3ab350e699c3f9adda91ede2e55c\",\"7ffc8295a44ad2957b789d237244e8650c13c566ca32c4cf604953273bc80c6f\",\"27b51f6612af5d4752d303976d37e41a3a82d6938cbb46a18685df43ae0eb090\",\"3a00aadfa41447332799aacbf60ee624a8cfcc5c6ba97894c1a2cf2fda5528b4\",\"34eccfe269d128f5808f0635fc8bcc6fcf826dd86c9071e8d54e1aef5090ec44\",\"41a7c252f0cd24b5de55f750009ea445d80b0613cd98d1a87acdd9d25d796a1c\",\"4fd467684ef786c9556a3fa79be054711ce4ee098bd5235235ee9ea43bf694c1\",\"dace1e86309ba297918aafe57fbb8234648224b12a1b910628f87f227a987214\",\"bf3a6a915623208bdbbea8213a5f09c5bd0ae8b9549b6314e1eccc0d1cacc29d\",\"17c15c0cb0101223b59a360d2fd5ac71b20f3e5f582179f7a1aebf86fe2b9bc7\",\"1a9611040f6999a4c63932079452a48607d17c06a5d810ac1f4b7e5adc82040c\",\"2edaa2c0febf380f8ae66d11c103086cb5f525f0176f9b2f6385a80c8468354e\",\"2d3896e19145978fd7cd63f227098e20d1e5f8bdb0007ab464377edb2f14702b\",\"dd9d5dddf079b0d836de82978f3892a0169085ab455ce9c88d99a3ed20348d52\",\"08f3fff5984109a8bed1356872ec49e4ad520069cf8424d7d92d27d4cc760da2\",\"ba745248d219b9963a5471fbd27faf0efc9b44af0902571df079d0314b46fe40\",\"65aa9c2634531a504b6e3d40c4cccd150d264d82ea065b2b4cde8487cbce3d67\",\"c71fabc1fec3a7344a10bbb0bf36c95ea2b41d954f6e6a6353beedd7a64c978c\",\"533b33205abe6e07de748e4e107c4a468b1a597ad01cf90f3955b4299ee08483\",\"f5da2bcb27178dcccb68c789ee822ea5dcbfde5731b3078923b1c2931e4b3590\",\"f812c3b1fae09f165acc516d8a1e4b1a940ea469793fb32ab4596367f4aa9cb4\",\"d524b4466f83d7a817b3543587cb4ad3252c5e5e0626cf6b1d95336c5d9d2291\",\"f615a9fabaf34cdcb8eaba32a40e1114b6ec520f43d6947cf83dac8c51c93234\",\"ddbe236907a648710c4de34de15235448ac59f6bc6e0c6940591a4d8a30cace0\",\"1dbb976b37bb754468573fe75813ae12696a83543f9402543f52b7673af4182e\",\"8940a0617dd0a3d3315f2b14ae7017e15119318b6ea13aae9644cade92c2c679\",\"5ff523ae54d00b73bb63e21eed87cc89db74cb72fc5f73507383b52238c43941\",\"ec7867b85ca8facf134959ebe40e3aeab240da3ed692450c6acedf226cc0b1ad\",\"712c52b0e8732f805bac6b7355dee156202113ae76824f754d54b537eb5eb92f\",\"f649d2a192c34f719c1b72ceb0fdcc7833c7cbaa4cd1600eef106fe97d289c49\",\"352b5af4d54be9d8ed45dd99ae3ad84962f92eed005964d6ca12e96788b543be\",\"283572f1668229e78bade92cd5c4aad84528e79280c5a3a58695e74a0d5574f0\",\"ee701ada65e97487f6c71e516e337f4739fe5e060b37263607482703fa779797\",\"16bbd4f8155e02692f222614e452f09e00eb9e3a6f586795a5baf8eea63164c5\",\"930efa09a4fa03f0a1bb6442193c38ff4af83f4d0b0d91869a28b7e6fd6767c5\",\"f0513d794d1eeda5dd849030ec1fe36ed6f54fc71c9c087d85c9ab8163b687db\",\"150f9761829842d962fc9435ef68c5711af1703dc143ad76d7c2b441f4b8f738\",\"611d17d8f28c13db8888e438364bf0dde90b45fdec4608026c019cf4f0e14653\",\"63495027991035b77029642565daa9f0e3a3bc24ad99f8e2d9da2c17c7bdecf4\",\"2dc5dfb141cdc85fa41302403cf49a7828d87aa19263fb51d7c3023a8739dbd6\",\"c50277e9210f5147559e09f4a5ca04ea41c248c6d4a1e2f5edbb6b6d7b560111\",\"2fddaab29c0966a85c658539ca2c60ff1178081cae6b392e5140c48442d13cf6\",\"4d68230db7692f2be77fec57da443154f8bed5ee53f280efc8f65324ab207eb5\",\"df95b2522eaace29ad49705b90aaa7e31ee88117d302ab9fe13b0b7135a948f5\",\"5a47b36fb05f00d3e87e1216dad25c24a380b2438e503f274bc40f8c7f905f94\",\"6ff0f49b61f8dd489365142c84f4c96758751d6496f5b1737127666f10d43be3\",\"f77b9edd82b3677f20c286b72286e621e697df3fb8c027217d55c5987f0c0a0b\",\"a45acdb953698c6134fbea0937a745983f50cca5b0a3c3579816b071db405a27\",\"a27e925cab057a39c67b6786e101d9892363939c6a60f140a911d3f7f25dbe0b\",\"dd1bfa0f7220e856ad233e6e37e254e087aa54c65e9573a87664d0a9fb28fb4b\",\"21ee4e66c1d85a276c353d72ab51c467ccec74517e7edacfe27a05fd11e6f8f1\",\"52a84446925f09bdef49dfa90be1d27366e57f30761833c363c9fe07bb1e8b36\",\"b4d10ced7fba31e801df9b507fc49aa46e137b623cea4dd8859f8f0becf202e8\",\"556620360e96b651e3e7b40c30d44128ecc4bcec4ee63d29188ce51100bbf7d9\",\"5dbad89afddc8d2b6fdea71b32caae22bd84e503298cf2fd40b97c577cfeb33c\",\"8593e84e92fa471ed802748bd7e707f52c2d630e8cb2a051c210bc9ec0e8fc28\",\"7a705a0e48fe79d3871a760f86de843685a57de6e9b5aa93be1b4d6f365243f2\",\"c8976123e50e582749ae7f78ad816057460184c25235349efbc6c82475eefdf7\",\"8d36af551cbef20d866ebf6b4631cab05d28107daefbfb6f6279a3f5e3a1fa5f\",\"b40043031455b17c6d07a51deeef238e4cbed55e4747594e51e9986c9ed33458\",\"30db9d34b7e0011b8d6d90e46a363567fb16dc44737bb4ba96a51f39cc7784c9\",\"4bbe04703f93404723a2aa28b8f9acd505edb0f530dc491dd3f5235e52702aec\",\"c4583fca9caecd32d797dfc0bbda698c25bfb8ffd5590c226630d70ce5694c57\",\"a302e26c384d84180889470515c669efb26940073679c1914d5d4770fd09c024\",\"3edc515e126d643c86c9802a5be6c15597d35876f6cbacb6ed9d519c664a24ff\",\"ba1c715093c59ea675a00e448664028594a10303ab4c93322cf8b0d6f388257f\",\"0386a28107f650decfd0c3c322f75f9f3c593461cee893d0bbcf6c2ca0ee8cde\",\"aa97558d6a0a7504379677a54b433614a0aab1c9c8c9fb00b118d4790922c6af\",\"5fcce41fd5aef0b2488a281343f58de7a6d1bd9dcee013603fe002e9479363a4\",\"249d0eda3bf3c2a8463a35243396494962bd6ade1d823194b682418044e280bd\",\"7fbc618390b494061be41d294dd10bee27e2c1447647c0f25ad684f11b0cd961\",\"c7ef827181c447a87f796884ba54b256f80bd406bb899fddebb466484855892e\",\"622ddf88696b8d34e92466f82f6939c1e35f3903f102e5f70a082d8f34badee0\",\"23eb16fc00e7bc5302c3126ffc76603fbee67e14dc0f583ef0ea6995389a75c5\",\"29657977abe7c0de8f97545f58a237a51ef182c62708d7e90425d05f55d77d6c\",\"9870e02a51531c8dca9567379070e1be8aa7f3da934c577590feb6f78a05e482\",\"d798db05f74e9dfad46cd4cb23804c9be0e909429392e243eeb98891e0582f21\",\"252d2918334c152fcb55bcaaecb220e41081e76c52a36b1b95ac807c79c4a1b2\",\"fab314c259e65dd867a1eb4da7841cd59a474c318f53cc6e9ad1dd10a18aa74b\",\"50349d021522423de9e68be1f041ca6c93c1a09a38b90dacee62ff7a91be23ee\",\"bbc36e8b348feb8b6b0487b8391d8aa9a3322e86c12d1c4b5156ce2aa7c70bbd\",\"afa95a4d86bbbc9bb274bf4827115f589d3644b3d04b5d08376e9f2e17e6053f\",\"91ce8aeb8cdb328f3bbd39e9fc32dc0672e012230a5b17c4d3fb102b68d6dae9\",\"d4f8e13215f2f324985571fed81358be1d9a4d96d399533032894ef84e59d7c2\",\"1431b0de540966b922df667d3510d9ab88cd0003cf73648a16f96aaed61d4677\",\"30d36e8b39907c53feb4a0310668a9a8b24c3fe13dbe66c7c38310fc91d06aef\",\"0d1481b1022cc34a7c4a03ef4aa0ed81b077071876fb8d40f6dd3bf54d1d714a\",\"a3bd47d683f5c14c2cf853febcf953bd30755ccd16bf7e68c812769e60c240e3\",\"89b527f5d6bb7f912eb4871b9f9753b0020e397aba3b9b9b721b8c7161c40f27\",\"e2f1b87f2da5b49001750b209ea8dc071f20b5826b7ebfcc0134f6d194861c06\",\"70d6b8570068aa54fdc1106213a6035e9cfa127edfa3203c4a658e1ce9bb68b4\",\"e650d0562457eabe9bfd0064b04a6fd266bafe4df9a07274561e61c7feb1c397\",\"4789d6ad3592eda7c73a2e855c02997dab9e5811e836f7af88aaa637da005d61\",\"8ac0b851a4b608d8b1410b1b61b685dcb5a8fd8dec9a27d81135b824258e14eb\",\"227fd7f43470aae94b47f8f6d038b27e21c92c7d46dc49a541cc4f76cf06a0b0\",\"ed928fb2213a06faef12d222ea0f1c5eccdcf595cc9bd0c12b4070de5a80272b\",\"8d223f8c5504c2b20f6a408b999edb0126a9181b72f4991657cd6053fabe60a5\",\"de323d2e46f9ea81510067ae191912ab12ba3c03f0bef1c3bee0929745ab4529\",\"ecc62876e841f242b8630e9cb8c476e9fc195f1492af31f1548235a4c0b3d8de\",\"42a7e0ae8adcaee65d52dcee54fd4314606da8c6c5493bc6221225a232213f56\",\"2c8aeb81eb59e044444a1367a1ae83a4879a674e9f2e804565efd57c3f64751d\",\"35c475563d4adb1d37f27160304ff8d107b4ed9345139fc14a2700de9aa142e5\",\"ed775b7d7209ad2ac29759227347c5693b9bb564f066360d7e14efd0eb222aad\",\"1338e7068bf4abf7b75986c4da50c3a8ece6c2d142ae30489584d936b9a2852b\",\"140524ed9ad830cc4561f6e385138cd9883f2ac495ce587d63a10ec7dcdb9011\",\"65fcf8086c657d4f70a5590c362afe0721e545c92d0b027d2b2b8ddc2995567e\",\"b0adffab182cae048e18875d04fca877b7241dac02ba20a6d6f548a3952ac27e\",\"47798702becb513a811acf77431318307d6375d0c8b429c1c65c325731960c94\",\"6ab7b518436ccca1485de0c0d55936e370365702e2852a6dc23b96bc02a0875d\",\"cbffb1f7819669d12e549a751504393ce83fb5c78f711dccef7fddb7a96e9b41\",\"dffea5f609ccaf2005345f3bc6f0b29653d61462306f98d46921207cfe3beeda\",\"3ee5825586e0edc4a4a1a63445e9ab22bac21993e639589ae16c56dfbdf8de86\",\"23be73b7bb1b46242e0206130f8b9eecdb03f4305f13dcad012566bb598a799a\",\"7a3d64b2797cae7223212baefe3629680f4840e6dd73fae80170b2511fe40f0f\",\"5e96ed94130ad402558c6f2f0238a4f9d2b04420b9be374e6884f37ba2f4aa9e\",\"4d99d5d52eaf919fa5e419a63ea6aba2d89c8d9bda81f58b1111318fb7664396\",\"d46a1013e6e0ed7b6967ca0cb6158ef50832e0ab3eda8ccff3170601c9c2ff4a\",\"133e1b2be1e5eca9b4024167b1f0fddd2d10ad332e0b58c33d97538d177374e4\",\"f8adc2bedebe13c0acca7f976242daa12c6887fc18cdf179f9123396be69fef5\",\"bdeef93a087a25c351abbb763490210447341b60692447ac71d6f13d166bcaa9\",\"47e47b358090727dc00bb9251dd3fb919e261a3c88c49a9aecf20e62905e714b\",\"e95dbdcf4bae0f18081adbb57a32d541165d8a99cea270b6027a6042df72528c\",\"79abd102820af1fdad043c1a7080ed7689741ee7d9022e2b8b159eebf6fc42ac\",\"f2239c80440b4b0ce677be3136d95e5f8dbd19323efc51058cb82b8aec40df1d\",\"4097d94ecb66ae40a3ebd7e068a7fca0ba138611e210da970058f503585ecb20\",\"03ff8cb40c1fd2feb93609a6aef775268f83318f32b16d5866a763144d5309e3\",\"ecdf2cf6788d35d4e22bf4f8d91598a9d8206ae8dbd98b7a796c2212f77be7dc\",\"e616081139d75daefb064c8c99772b001ed74ab430df2e6a30a2ddabcc363fc5\",\"fe938d9b55f809ba3e5dc57a6e211b18f20666de5baaccda469ac0855e53fa6a\",\"d5425f72969c60f9268731f59b78c664817a5efd58bcc5c773888014e344f31a\",\"988197caa0b2c08e3af811a3e3c9c9487ef6cb16eaeec94cf3616435ba965e53\",\"e383ba230e9c4f39a18055cf06021e524e7369464bfa90f671ca7706e8c90972\",\"37558a60db31b8defdd9050cc30e4eed7fb4e4f52e0095d6608f60a7bb1ba415\",\"a61288a7eb05d632f60a5bac3ede80a013398339c0c70b3d1de2a561c0338fbb\",\"597a322cb391091a75754654cc5c6ea88c7b558872fbeb26d774e94cb132a719\",\"7febabdd8aab7ebf0f012420f5c23e797051d096087a0a26045b4a2260865993\",\"1c8da5f946f54a7cd56a1f87bbf688e93cb227400c0f8ec6708104772c2adf3e\",\"b4d5034332a253a06008b0fa9abd79dc9d3f6924dcc34b7bfdd5e7526d40051a\",\"b7a5dd90eac14cd8026d1f85fe8c933e93312cb400b9e91b484e0a2cc203ed44\",\"66504a7a09bc2471b840c950deb64ef0f2da98b6328b7fd8de33fcca4b9a5f72\",\"aaf2985f83824bf32cc7734d47ce879ac71922f9581cdf0ff581b12868ade474\",\"766db2d9ddf18a22463d420de8e49a50da713402b6d7f2351024b4927ca3dfb3\",\"6f3d0cabcac3df47f404a12d83d18902bcc8df9e0c2c7aaa9f09d8497743f0e8\",\"113bb1263c16938e03ef9956fd28ec975045fe6b265f7da986919d3242191d42\",\"bc8a44bbb01978311ca3575767e2a946b5997d287943efb70740add53c2a05e6\",\"7b60d19cc391b991e07a7b660c5273ccf6f2ed3e386c43b466bacd771ae6bcb7\",\"0c7462734ecbc6290fd6e9dbbe38d9331eba4d43f322ee19150e6cd36463ac9e\",\"0f08366b0e619574c0eebdf201076340825ec0d3ac22091abe54714ee0c3b154\",\"a724aff13681eda59d78099c7f12aa51a416dc8781e0c364aa6c4d57827ea07c\",\"8983fad3aca14c2760f0ea652a4c03d1095a0af2d511068305c553b58c7b0fc4\",\"bbd997b58c500d33738fff4192270b3b0378b765dcdcc488e9cd75ab4996a441\",\"08440b11665b57d069941764f35a5a057bbd30e8e193bc502cbc9c19b6aeabc3\",\"300026676eaf60f0203cf15eee5f92e48b827645ba0382c9ed23e3832d3125b3\",\"fe99db398df382b49d5977bc7250c912563bd2cc3d3940fb7330634134d34cf7\",\"60cbb48669d5fe0335af63c06ed96fa5be72e272ed5425da708850bbe2b170a9\",\"d6171b605e35be72989a0fe30011badb2c6bb8f259cd47714df85f41bcc2ec43\",\"746b70edb01e46ff27b3c70b50c0eb271207de5fa47e7071d4c23557507b9d81\",\"a58d27d09c7312575e0a408c53d47e62a662e237eaa6e57c98d2e398fe2b33b0\",\"cd1cfe8c73a5a63fee21981936345d0ea1f8060d9a393be33612c09c0f7d654c\",\"6408c60185684b61ac7664886d662cc88160f5e0292283c76d6cc0232fe4deb0\",\"4285c59c8037044bb816453120217ca71b0ebd5378989948522caede23f349a1\",\"454724f08057afe1b83548a819e5b5d8584e8b2c8349c70eba0d1738e945db23\",\"132d0feada35eed7378baab3a2fb4751f5cdff157b853f8d252506123387a080\",\"b1f2f6fa8562266881dae9dc787e80cd80c08fa8b42996d1c25e0250cbdfce45\",\"3f89e627ca51c103d89f774c6696a11872db2c013a3a1cc6b7fcdb60dc9381cc\",\"a3f37c65197f1d07d33a5a05f9ae5ee00dce42e592e942c75c927da94ad29d16\",\"444230747630340017e1133e7ffce8b9204d4cbfeae0aaddefc67912f25366e6\",\"fa2585d08d82161aa16d7a9506306b7ce7d436fa16699bc32e8dc8545ce6ff85\",\"5df62030d84c775027d68bc7675472308dcea236d7da8fe6b102cbc5eb25c83a\",\"b4d91b0279ad37386d2724af741263b35bad1d918456c67e998b74f33e0bdd66\",\"0df0475a947d44f6fff8d439f7fdca35b565cef252b29e2a21d2557184ce80bb\",\"741f4c542c87c43782b09570c64566dae2fc9a60828f9a57dc4d24f5f20b22fe\",\"942d2a35e862d86cb107d36eb1e887c4087a157beed85b49337261ada68dbdf7\",\"4e938200b75d6a0e9cffb084dcdb839af1cef430d43f405d897450ec51f6a3cd\",\"c5765e01d958f8d4296713186429358656f5f41641e9732f99368abf770dce0e\",\"d7ba34f5d9985dab197feccc7d377dc8e1b12477d2982db856eaf9a844ea4eff\",\"78dfe74432f2ab6437e222e967ef5917c7c56ad6ab2513a4dc39a64a9f7b9f5e\",\"d75079e1110be7a745ed60a508358c58a6708d08549a84b74561db086f6b0d74\",\"15a1f67425a98a8c1ac3586ed0f4251fe2e615763b4eea663bb5ce120a39c98d\",\"0cabc8cff90ed0fba7f89d5860442f256145c2dde9a6158093b8aabc5f956c3a\",\"ad174462488c42caf0bd2681dcf6eecc4538baf06f22378e720d8de8830e3e09\",\"09b628ef373aac1d9a3f0a352e608b88dec119a17d01b584cab65b3dbc393707\",\"f83ecd7d07110228947b95149d2ea852baf8d03ef6f9a025c8bc3aa5d0e388a3\",\"df70587fe711602bcb7c32338b4cf0cf7548ba68cbf81e1b71831b35a48a1f5d\",\"d658ece796581f5770e1faf966798f5ab52dff0beefae23818fe1c55810eb37b\",\"cd4cf8b87deca4bd83134832a4db15e7245c0c325f6cd3629da4359daf3bc916\",\"7ad4757e85cf6ded2a825d45eaa894bed4f2c698b3263566e8727aa42696ff15\",\"2e1c846a2c0ece01eab97d92817c7e9aad137d19aa5e47fb66b0889dbd281dfe\",\"6c40812e387bd098be8574eb09af6540b87dff7c4b9755a7387b3cfb690f082c\",\"b7e7514e068754e1a7533e30f3fa15faba090a27d4bfa2d92dbe0cfc60c5f3fb\",\"28fe66cd1f8b1bad37187e50447745729f0217bf3833d064ad72de70e7929639\",\"0ee215d000e529a31dea796e80921ddff9944d823658cbc07be4f547735492cc\",\"31b88a15806236b3151802b7b8302bdcfc6f211a4712fbd63d44d15db3c06a73\",\"82b913679a4a7d9937ad9950b70cae6ad0029d0df7ec5a2038f23d763710c3e4\",\"ffec4b557b793114c0c889e46010547f20c0de86e1ff93a9f61d97cc15c41250\",\"1d276a28715cff883e7ab9947b9d98d7c3f689f1983ac9088e2b53060d1dd3d0\",\"5a8292484cdbf1edfb3c02653bf5782bc174e42ea25fa0d355ec31c20a140c4a\",\"34ae44fbcdfea12379fa3f11c5b90575e171166c1a7851f31b3586913940e8ca\",\"811acbe684f96cdf0be80fd5dc74c68a9eb76a1edd8a102ae0abbc6dacc4bf89\",\"9c8783907a3d5f32a077c0fd75f996472da938bf2ec0dc856fdcf7da61fae36e\",\"65363a8ff1945225ec714074c0d5fece4cdc4b5326b57d1d46d83b69a193e54b\",\"1c629215b36a47dfdb0f8ef62f790a5cbb8ee1a446e88fb3ea726beaa2a6ae23\",\"644f57e7b2a735fdfe1ae10330f739feffa888d189143e53dcb03e4d9f600530\",\"2d357456f21e88dee52d3fb8a69102e6293149ebef25185e878f7cf13d5d7100\",\"6e7429f11bba404f44c3a46978af44c340646dbcd3a7ea6c06f411fc56ddbf44\",\"350e1e3bf3bd2b8926655d1c40c3a225a17d999dfb2f06b81989e52eda314986\",\"10e1479c33dbf8e2e3e740eb778dc0653f4c115de01a2824f82a939891ada6f5\",\"4a2d32ccae9a003139deab70187ce61c24e73633a40747d96a54b526ebe8f1ca\",\"e450c525052294221b13f6ad99f44647a86927ffa1092a85fc1906c102ba9ebc\",\"c768a950829a081aa818510f8ceb0875087b6cd57532ca3cb740c42e7b756290\",\"77ff36b1a3a5320f94249b0429479bc196741802cc64dab68f571baf7ca88642\",\"ab3501a229a2fc9b333d2c61c49a17349df5c3e081ede855519abef659f142ae\",\"61e3ee821c2ace5602501c297d9a39b0f68b92e43e4e0c0a179540d324f43274\",\"f822f922fb60adf32d7a5919783c8b54c2baa19ee9f3acfcc10105baa63ae8dc\",\"58fa4cc82619a282344d7e248e9a8636ea160c6fcd783dd890b124c9084b9aac\",\"fd7fff980a969ee1b85eb237b83a5111835102ef9301a7ae96ec1d872555d943\",\"235b4f4054ef4b531f6b51289134e0d93cbedcdbccbd30dab44622025e22649d\",\"eeee1a37cbc06e79befd9adc13366b19983ef8d4164e3f839579f141ea6e1713\",\"3377547c96938beaa2c87c7eaa9b14377dab8180d0d064ce3aa336de6e061a0a\",\"7a7e180e045e89f435619ef19b4669627d0e7955cb2048e2d8e7dc69462e8a91\",\"f5a8dbbc06adadf652db41993527d17af7668f083c6415505694dd2f2c68c95c\",\"4d504c2d766cc4c67d376873085bc064fdfcbaf2679d7134bb68693fcead9888\",\"41abecb7896ae46beab4199408e3d0c5bd39f948f8e5de87981edefbd8f6095f\",\"62d9a8301ba37ad093593227fd113b5f33b4b66b238f8bd22680f124e1d8a858\",\"bd41c61359ac3f78fcb705d9cbe958e026ca1fb4a281cddf8ebfd2bda5ce9eb8\",\"ff26c35af931eeb8008c4604f1e033db4c9a2e2af8b24fc9ddd85b21d99ff9d8\",\"fcd4ba04a287b4adfc3873dbecca35704df6bfd70cb549c108f6c7d728aecbf7\",\"24fbd1875588d2bc071e910efa801980c0778f997d880199108cea4afe952fb4\",\"d35870c246bd1996e1f782159b6fb5db25eb6605218f64b0a2e62aa9cda25edb\",\"aa0101efa0bfeefb7b0a34d4e3755e9b7bcbe99be290495b06e39d792ac8bd86\",\"92d9fa546415d6471b6c6f2c40b1a2a9f159e203df1aefdd97c55b39703688cf\",\"9bd0b6c74de4a1ccced46b7d90fc147a6a9f22a19c337a4258584c26c9819ae0\",\"3e3f349fe8faeca242f788946775c52b265bcdfc97b5f6163efb92b158cded44\",\"307ab52e4ca02a296c125acc7e8ee9aac4cf887cbf61e924ac70dd7119ba5199\",\"753c26d4f7fbea0102bee3ced06bbb65dfd869a2bca2d331f988d176d6a90158\",\"65a02342f7a178c96a922c982b4c5eb5dab2bfe250cbe52b0507bf3760621ac6\",\"79422e3112d7be4399408d0d286512594e546737bd610cabe3c0e4166519e629\",\"451f84560cb1dcacf45bd2c01383184894483aaeb15dd5370145e857eb010dda\",\"abf775ee58b5beff46cebaf8d2363bfb7e62465c7b8dc8044e91034f31461136\",\"e12b449a2ee657e8142a251ac4bb4b9b9da694d9f0fcb7f4da7673634e70e8ee\",\"d0af456da116e306292fcb5ff5041e45088be10a6b1f4751c14a67050621a073\",\"e6f65584141de4328ebda3f86404a5f99b26cea02b9c113532271e68a8126feb\",\"5b9779df24190fa6ae9345216b70e512f51efdc39591eeb203a73b6579fd420e\",\"0e18a5dc20c60904346a5ded0be4604065dec797a97ab5a7a580e5e790fcf75f\",\"543ad7ad74d212e824f2ce661fbbae8897c94b56d091c68678d204d01b4a3810\",\"c8b302eb6eb7b26a191fc3bac57d3a71172c6136ea887ff323275f563ca82158\",\"ec78f0221dbffa011eb566a3d12785f94846602308ebc3c3a41cffc6422a42be\",\"251c868e7836b07bd2953bd2638fa864f871a9d4786de01d5d14ce911890b6ad\",\"8b81623668a8d435110ec9a94f3b7dbf72670045bc429c55bc035dc9100e5fe9\",\"ddf8c38a3fee96233345d754bd5254339a9e9388ab6721694f073b9da2b6bbc0\",\"4e7fa05be34abb9a5caca91de69876b5787dbd33145b3d23c94c612c12c5a2d8\",\"1429c66527d3e52fa935407c7a41bc3eb9bbbdf01b33ddede0bbe6ce62be909b\",\"c8e09a70b18801023cd640f47ccaafb6de3fb61786063a4fcef97597677a8f8b\",\"fae99a5dec3494abc0c3fd33c6b7264b0f46a60fbda28601fea34bdefa7b774d\",\"01fab0593385dede5ef06f06c25204ddae669cb00c337a1e4764ed2909f1f39c\",\"61c89160cb6e83027d4ab77704c32ac32aa440c5a7cf0cf3cd2b72814a56e6eb\",\"ba9b0dcc87ec46e935ede924cce9406c73a256a50d1a8795d04b17f95a7446a1\",\"f8bca79df586db5041aafd87c1238cf4b8b6cab2299ea03725b686108ae7836a\",\"36a019b373502c384c8a7586e4c6d3e84a4e0d20dbc4d1f55a51e9c2517414d3\",\"4fa448268ca34f0017b607ed544e1d854a93d9e1d012be78942613eab9b6f234\",\"b9b8591b9087e834e1c48ea701a20700086fed954b3636a7b5c4aab29bb057e1\",\"b2dad8c916212e4bc735924cf0b3c0db54b835044bec29441a698adf7ab719f1\",\"1d4928f6652568a95be3cae8dce1adb3e344221731ec92fdf6ffc037f9feaab3\",\"93217d9eddec8e397c76288c682a6d5e1f1523d6c1e1a7a65784b1917d61ad21\",\"da1ee921322f5433fbd8183fd5d1c40c39e65c1da2390e0e03696da9e7a2652a\",\"9ab9ee399995cf256e42ab3cbc4be3d15421611356f3ba4302f7992c3b4141c4\",\"8d292ba76770e4bee6eb10150e8cfe984f8d3b144ab0e2d1315956705d656331\",\"81f90673904fc782348e752887ecc41e260cc6c3be57fa0291a0d2074b77bd60\",\"84791cefd18efa42b1e9d248d51924c7326ee47eb64689baeca9a11621b74f57\",\"c7e08d0c6244d6ebb7ec97ea21bf2dea7237ef565a96ff29cf0951376efb7f44\",\"b74c82ed48ade0b68fead18422b2a1ec8d60602fdb4b49804222ad1d9500fbfb\",\"d7d4042083e4ea9f065b82fc93b03b2b67d9b7c113f8cefb3e4a001b7fa1df32\",\"20de4464fc4467f660259e4ba15d00c727783d98be0da4a02ef92ef162cfd393\",\"ba7e6d6cc5583c20a78582dd97b5c2a43839b07011fc1c9eb70d4f32db3846c3\",\"1453a1288cc87125bac28b487825a9e504b17cc7052a232ce916cf1812c2a686\",\"ccbcb30d31e313fa530569f2f0c189c34a3fb65c78070eee6bdefa99acc1e9b6\",\"607e708a08466da8d13110b764fa7de39fa5c1a2a82a624caf37a2494a95e1d7\",\"984c3dc545616867300cdfc437a047f390eedef4a44d327696260c287972a130\",\"0595c67c775bf034ae09b33c56ab55bb350b6a8f7a554ee88d438b2fcb7cfb40\",\"536c839b13373578bfc5dadc45eb79b01e5364ec94c94165e279092e257e1393\",\"e804cdab724cd2a3c949cd6030d1421cadb5235fae5bd9f17ed48fb54ce1b64e\",\"fac549a312f7453c14ddabbbb9be36cf89dcb4678f07b0fc58b1b0a1159c06d2\",\"e9d5382313738d5bec96d0f9448c85db8eff81d9710d7cdd91a4cab6f403e1f7\",\"de7fc00a00d3159669919f36a656ad81805823d35326e1737cb067f26d766154\",\"ddcc8590fad954c73fbcdf1ef0a425a5731b1709f2c31c510092c0e6f9fbcab9\",\"88ca29c758bc1062eaa15fe0da70c5b8f54f26305fea042b5b43e46438a4845d\",\"c522c06f06fa3ef17be43bcd720db9fed7944463178110afc1a86862b5f1629e\",\"8f9f260b3c3d1b2f23b9dd3699a2f2b5b6d8c32a9159398734cf8d01f261d53e\",\"93fbd534db66689f75f092dfc18e33cc0f8295d804c0170e448c3e5b162f7f5b\",\"6d133c8af37c1a5c45cf6b86a5a628c0e291ba499bc134b383e78f51d53301c9\",\"a73f1c4e30b07daafcc4ef90235ddc79ec241b4e88bf51c34d7881b966e80f3d\",\"cd2c3006ae5335d36bb42a4ae6201f20d771916e37d403f1930fa83f54db7cff\",\"80b901e23f655fb2cd9c0fdcfefc1541e02436ff1191c6bded6f84980da6032c\",\"7f637589b31fb758256bc838dfbcb46dac1a2affa40b25f70c004a23bac3dd2c\",\"994439988282e61e5ea029ecf42891eaf99a625cc44c28294562d76651ec2daa\",\"7e06e69aea9ed1fda73aeefae058761a3ac9f04cfa0eef0c727e86443520184e\",\"8f5b811d4c2179e541d50b0e5deb74f2f37d190fe39acdcd47706be75b4d7211\",\"c7be2ada90ff350db094e07780740515948855f4c3a5db1023637bb550dcc00d\",\"f89ca53610f056db643503c0c730a52841f03acef4bf37519a2326dc1db01a11\",\"089d98f63cea4884ec71518a7409d0da9282d35ffcaf73287156521daf92aa7e\",\"9bf8fe124bc8c2f836e426195d95ec909068d69bdc31c09b1e25aa008c7af83c\",\"a15831299663e93459ce1a14685fdc3ebe60f2c75b5436ad587a741df161b38e\",\"230728f55af827c2e6c51636b938cdf7b74674ae5bcf775bbb4fb3ed7cdea25d\",\"c8afc2c64123de9f413077b4c8fad9b3dd969d793f1a1d0981879db56cd7bb9a\",\"0f5b6e068177e4675881fa31b38e1f295b6b7caa7f5334bf599f77f044a3d07b\",\"7bfbdf7914ac15a78454bd35ef27c8994e51bc7dc1966f55cfd834376b4ed854\",\"1f62db9981631455b596b6bf3ae9148b6a95bcdae4fadec855eb353a3eef71ad\",\"08d6da9d4f8ce6d39fd9b1ea9475b7be4aa7f3050e2b61b108adb13f5fe89d5a\",\"0b922902a0f284a0218e8dd653d745af23b6c38096915bfa40d49f21ae86c69e\",\"3813edbf09916438b2f841ce9281afc96700ca031986fe323b13749f48c0e28d\",\"0bcbd242052e226c212574a07a90fb56f60e487b00f3acb277696f6860ece0ef\",\"0e35a13cf261af641ca316f78cfecf9ab5ce610f3c3fb0f687602ee211fd1216\",\"d306685ada8f176c8b62c18bdbb129db0fc61b88a3df12a3a8247f06f23daa2d\",\"72e15a3c87ab4b81ca5453ae91bcacc50a23c1690775c60a73d542bbdc90f6c4\",\"2fb22b7d7f5acfd9cc422377f0dc27667f72877cdaeac925a15bc8a6ff4aedff\",\"83f2b7aa8d785b11786ff0670918c660abf0d4eb48e46136f852ab6773ece7ec\",\"03851999a1c43ea1b1bb948820552ccde9bf6979e445697a8b65c7e2d9048a91\",\"83d1224474532630dd8a55033a150fe8f4ec9f364a33e0c91a6cd94b40eea5b5\",\"f6d26c1a8a9f29de43dbe08f418ef817a41bb26414c4e0af83de01cee3ddb61b\",\"50cda730ea70a323b9e1a032a0b5c4a435bbee8057ee442dcdeb925544c0a530\",\"8eb975545e2633b60c9f6ac1e79c46407655e837c264c699396ce0eedf346ac8\",\"7ab4152b4352c111c3f5324681c5785258e85b30ae01a7bb261168a9b3938ea1\",\"a29b8821def1d1006ec9258ec75a2c31615f5bae44e5b392ce2ed5d8e0c5d407\",\"f10bd72d617c3a803da8389ff2c88cabe78165dcc2fe7da0abebbe95172bdb39\",\"b2f1aa2c79beaf5a2ad723ca8ab6b514b93cfe18b602c9d9f34c14012e956697\",\"903856c5aaef44ca66a687a4ab3dbaf3ed64f1d835f83d4cfd02860f54c01f72\",\"ebebca5865a4daced171630547b6e25150a14fd51f770f44cdf4118610fcfa75\",\"a0c75b135a2c9ae15782bff12c0d0503d8f483cc73174685f457773c62781866\",\"a5cfdd9b3b4d05525e2eb2e389b72b79669a66aed7df9f84e1d7612713f7fa1e\",\"fb19e3ae2e54da8e7cd49b2975e5409089e8b30b5f13fba4371816f4a049692c\",\"297203da2a384e4588d7a6d18bef548474a75f40085684644dfce8360a258b79\",\"24d459386ec5ffdcefe4a706a77c9319db04462353a54a96730561a6a7e804aa\",\"a4785ba2988423d105c554c026c6a33907fd935a74898ad1be877da50651e27c\",\"59d3681c0a94cf2877aff807b63321544ae6b09264ca044dc81046c0d118fbec\",\"2dc9d3d67e2c3cb87d3773fd0553ccea7fdcb8dee0ba8cf15ec873082e76d137\",\"b0c7d66e76e54bdf75ba019a1afe2487826cd2d42d9a70a80a30c32d550bbbc7\",\"4bfd4d01f7575bf6b6217052b14b29011a898e1d8c682bc2a018fcfd93f7f005\",\"0275aa2d5ba0d1925049f7b6d65aae22dbe0bb4f3ad7412a77880c6b9830be2c\",\"dd8e183e6fbfc9b6c6f85b3f7a56b65856401f9f07dbbb0d383435931dc82e5f\",\"cb2fdc8ca978ae21c7e8d3071387f7812a7bc6e06ee32f678e005e9c5e83fff3\",\"16d1c593d5755fe0947ae11369d651a2fb7f6f74264e7e06ef43afe695fc48b4\",\"46ef464701487d327f4c49d943680bbc4a4d5354b18aefb55d756811618a71d1\",\"598fbff952b3704fb344364d9faed7ce184bbc2abc26a23f05ae9cb1b038beff\",\"31c2c36858c6b0223370f496d5a17533e755d44b4e5cf84870e7adf14592455e\",\"29f162ad7b8be50df8dd4d3d5919bbd31aa7d8b2f2b2fb386789ecf055b9735c\",\"16b93ac684c4ac35cf8eead9fdaa19548bc9f895ac2b981700ff85825da0a552\",\"1da4a74da89bbbcdb8bae464039a593d3afb22fac54632a2df91c1ce18531d97\",\"752b9907cd0e0883d28bfd69c1c10cbc19cd43929508f99584a3735595c2fb03\",\"bb6976b3d77d67b56657121333f8c4a97cef37d8fd168205f5de3601c0c8059e\",\"636d75f850d7d99081f7aee6f17e31cf3f8ab039816bd25c050039d1a6a65eb1\",\"da4152f5b904d4ef41a0dc083010f8039fb147b5c8d1d833ed6d39cfa8d882d4\",\"775e9660d60ed7c26f8546334f21fff338f93fa3889cfa9591100b24fb8f5e97\",\"627396ca53051649914ce64d8a2a9d8ecc8f1e57d10220b23d10d1c32c26db41\",\"19d75f1061955f7069590c8501a72aca23ea3a733b5efa493512361aaac68863\",\"95d56bb9c7fbb66ee863deb0a473176e80f4157ce7ead08c98c4ffb369836a6d\",\"f6ab34fe0c6a313c85bca13a80747a08c4fedafd24ab882aa46deaac705dc8d3\",\"d95385d22b6af7b8886503be14f679c545640b5b10663d4971710c8411d11735\",\"fcc5f4fe72c8cd486c36474650657ae934e5b345e371f816725081ad25c6ef04\",\"763e53ff0ace6614d21b9085d1a3b26d7198738c463bef720880628be96da6e0\",\"5f0996e05855b472199b42c91a51d39ed6f46abab5e8f94582c294eaac2a841b\",\"650068d80a07f0f950a3a77d06927a8e5a73eeabbfc40dfe2b07078b414399c3\",\"c90002edc154bb35540c69c20283bca92a5ffe7e259ee7944d4426c8ca265731\",\"5a179e1a72081bb59fa03fec6b81ee833311e06872d1ecf0c11080bd82113cc5\",\"ef1924f698d4be6ce9d54d5d587c256d1f5b7a811f883572dc7ea8a6f8cb4897\",\"5c87bab4b686b7138144b019abcaf77933836800df9e9de99a295ef658e0df5b\",\"7e8803c7f02324ce99b99c35b72467fe9ce3d8d432a6eadf7e7654bee83238d9\",\"bb310b7c88ef54c9a743c3bd6d29df3b8971092fab129d3f8c878f55d29ec4f4\",\"eaee3b73a45336c659e90ac8a491ac9c38e6addef6fd2d171ec0b40193d21747\",\"a9b618c3c3909ae0a338206e69a7fc380a64799565a823ebfa133a64f3ea19e9\",\"abe07c7121fd41d53e2f69310eaab1a50eb40b6455f822a1ed63ed3aa8e40a29\",\"1bd4e1a71a38d5adbc0a97a78b04204fa52f135ebaf0c8c0453c8a14cef57d3b\",\"44deeca74b8811a1c5af36884eb2f9a57270c4a1a5332736b2055df0784e9af6\",\"d9eeb970777d3b254be7ce6eac6a2bf7a0e6b59daf9de421f817e56eaf3f8efc\",\"cbd3f388e6d0f17955b95f89af22b60fd330fd213205cc8cbe7745ffc403c800\",\"b8fb63a8afb69c62acfb8760beb2d620066f989c25a2e4f771738e35bfda89d6\",\"3640f8e63257ac3446aca17dbf018ffb886a926eef5e1433cf9b6b2552f2266a\",\"0891c8b4470b2162167f08f881aee2aafa25386b4c75a73f42a3e2bc5135d440\",\"a94396b2df8b9c9f6adb7cccd351ca9f57d1cd29b46204ac7402a2cd6a6a869e\",\"39ea78f20c8310bcb25cf8431dc8a95d57800a2846f48f5bc37e1b4ceb2ddbda\",\"c7e70476bebbf79a3ca4771830514a6a46f3dccaa6a78045c5eade4e2540c270\",\"c7da1cb9c8f5bf28362a0fe58477cf82e89cba127fabc4c5dfe36f0aa4e5b3a7\",\"47e9261e5a5a4eb409d862a987b8864bf33689d43e363a3a0e9e1f8a936d2088\",\"60ae4c2ca5bce3ca68e505612365d1d17a75f330474dc834923d3d3e9e091e70\",\"14fea88783acee14d2f9d23ce214d079a013541813e4582d16b2ff517452d248\",\"09ee000a066f3bcfbffb264abfcaa169e3b10e337e46854d1d56324372a63214\",\"43f3090975aaedd046647b6e79e2acfcfde8f2cb686ed5ce81b7b4e6acf03664\",\"93192f5ba80b9277ca263ca3709263c60407c20c5f35554b09bed4bf417c9a3d\",\"9d18126bf8f856011fd44411eadeccdb0ddeed2771cd4f1e8185a43c2d2031f4\",\"af14179cef004056117a474bd8ceab3a28777a184f21cb9f79ca80f72d264f63\",\"c0cf2bfdb3d16ef8f5bc5c0ca512d4740bb86c048515189afb50c97c184dcd5a\",\"0638d184195164ee53e085d268d20549c44acf2319d19e7acf0c7034c5cc804e\",\"3f49961f82c4b9e78457f6cbd7d66dec106b1f545bdc3843684d84f58051d930\",\"546a7102492dc0f0d253285674078c78b59b932426d1d219aa81e9e0398879e9\",\"ac3140952f62a08acb5f062c1e9ed3fbcf9c25f7c535b777512b35c75427ca66\",\"ed555480ec5baffac8a19fccb88e649472972c0c0c55c95dcf73a19406c03394\",\"46257b0cb4c0788105a0f7180f528ce685814c99bbf0fbbc246beafa71063090\",\"04ed9cf25cf425ec785983fc4fecaf2a6f54699e4048a78d76b81ec1caccd711\",\"3693aaa614ce998e0549ea17b5635231b1292e2cae2aa8574a39354cdc749cea\",\"5c4e6c13ca94315eba3e4aa58e3c5640209eefbd54d15d6fa99a6a26b4cab380\",\"13fafc29edbb952156bca408f6e57d394eb836cc1a95d5bd82cfdc376ebed0a1\",\"dc5dca56ba7efb4e7d1e2a884c29259c6e8ef0a4d248fd4486f7f777a8dbc5df\",\"5dd30b3b3c46e2fb58457930c62a676e1e28e82c847aae9d50c5655fc5e1f6b2\",\"1d3d6bd40df6a668889f4bb905c54037e8c16e0fb414bdb503a1ad5f204fea5e\",\"8a90093aaefc5b8abde56a1905cb57726ad2bd1101dc2c8400a482826d8ed23d\",\"cf8a28789329c7266358c7cc39502833d70053c5b52f91f07dad7d94a40d95bb\",\"d6fa8848660d1a27c45484cf06c4a513ff41dcb8973d19c6f97d5f3187da9e5e\",\"cafc11fd61a8f36cfd526d6f112faafdd285c8720ef79ed0e1fd2f828abc70e9\",\"b4841a1cc6c1a576d357df594cdacc1ea5fec59dabf6c75f7ec03fe90377a3fc\",\"abeb6499f04ec2b8a06a018b7afe69947b1a86366bc9150b5825a4537bf04abb\",\"b7e9f9d370390c5fb4a10c57d34282cd1b0d805f04cb5547129a4bee781b397a\",\"09082236484091bede64c58e9676d59fffc7cf740ca1aa83f1d8aec832bdebf1\",\"fa208a01f5af72fc24524770b4f5da6f6a79cb572817e16b13de44a336b40728\",\"a158d2de225bc9658966c83a1b62486c353b0c663178f52dcbc899c322199296\",\"35f33c0d47435e084d05d08795eb5209e65cf7f1cca142546ddc6e22761b4e35\",\"626811122c1c13bfa34e3b9e36f0f59aa3cf4c9448c0930d45c1d67bac6ae4da\",\"f6cfd40b1061e57997562bd764340edd6ab530342aadf8c0f9c9691490cedadb\",\"6b408e8c5718d304bd22870a0fd822a0252af9555ab14375cca50ec082a7401d\",\"90bd2d3e8836328309cf57f41bc081715d5f537c7457ba712b25ae7b9c13b8a7\",\"98b0b03587894a8e41ed5b010fbfb6b6dfdb50440379795f915c6fd4b7e09a90\",\"f69d3ec27f6083e8d56b7c4a26b69a2ade21093fc48fea030c1589d72c5da0a4\",\"752153e5d57e9059f49eec32d58b49624573f08da05a7f7a83e84fb19aceb94c\",\"2504a52db64654affc4071263512fa49068ee1e1a3933388722d5f2a3e94ab28\",\"e202e22d33ad38372e97198b6ed549b0646b7d36762b1ed9a67724b1806a11f2\",\"54915bcc931d419ee87b90ce88ee1be92d93a44a4574748f2e584658b7b5c03b\",\"317f281aa156d1be6746549e8f2a1548a52709707947b53e4fd97ba635b386e0\",\"79afb9f6475e09491b91e8f2c9013ddbb415cbebe5940a1111923475e8401c09\",\"a43460529d1613f6cf7a302312449d1cdf638266ec55e860d930985a0c241378\",\"2cc8adcfef3aeacf1c9bbdd5a7b580f00352796620d9a5521851ca3697f0f982\",\"fb3cfc7b9a306bfa52ab4942886a11d82121a6d8ff43f124ae7c581f3e3bdb15\",\"7f0bb38520c9b81d4c285f1c9153a4892ee478c91e6bdff212f187b6e7b79714\",\"3262328eb44d138cf61823ad5d641bb5b937ee5bcdd16e8f59e6c58b227db54a\",\"44e652dec8911778dbf51f93622fef131a44295cbb9937e54978b49d5919d139\",\"2f07fd4ad7222b507916f605f05dbb648bbdf8390babd2a0d9f994b3adc00662\",\"ecafd05ff296c998cbd45a2a5a40e74ac8b93999de008eaf9f32f6818a2fce28\",\"28bf940b505fcdbcc3bd5ebd634f2cd1f2dde741212d52f5da72005915bf279b\",\"da0ef0f53bc3b9361d6cc8ceefcfa1e63be845abf68c489089ef158c9f0211f4\",\"fec773c292fb1c1a451b8ebc390ea27cd2b324b01c582cb5cdce84b0a4b8671d\",\"d78a6e41a8edb1b45b854cc2b9866cdefea7eed9945b1ea15b6284bafaaba479\",\"0433122db7e36379b0d5af874b4e6c25140de5229e0c688fb25210c5c1947b93\",\"3ef457374233afccc1173fda7dfa82e78f5f8e5d6108b3030d15cddff354a299\",\"b68d8aae8beb35fc75f2def9c0570164755f21800ed9c0ecc7fd13c014bceb5d\",\"7d51f47be064a3fa8fd4d43ff2df6a95414d3c068a33c373e51124bb92c6dc21\",\"06374489dd17b5e7ee2fb0e94ec80214750f61cf5eff7913ec9aba6a73c98295\",\"498032ec72d26550301c8f7a5afb1697e7fceb1cc86e73ca1c6e31ec017c9c94\",\"e0f2a8ead862a8190418718eefacc50707b962082e4124d81ebf06df2c4edb50\",\"294212e9c663cc5c991d304f50505455bffab1b37c0b7a867ce410e674411d86\",\"e799ebe18757d15a0e7071ae3c54cf9966f46c5e4d7033b9a7e73526e64931fc\",\"403a9d8e3ac918e4db7a84d4bbf5050e1922e208c1405542a4a3781c8d911570\",\"938646eafa5a52e33ba25d44351d3322e4ca5e563d45ce984dc45803ec2474b4\",\"4985167e42e67c92d2fa2771bb0c95a1834abd6a49170c4de16ef7f3009d8c1d\",\"487d5899ea03e30d23a5e91ddb5151d0acb92e59a93734ef529a0024304d5e32\",\"7d99064436a3ec8b9105b008d8ab9b989b388695805b7d07e3ab9acdc0c6e1a1\",\"e1e5b2aa4812632da4af62a9ec9caca5cc8ce56fc3d222ae49f821b5fbd15ae1\",\"1ee0f2a4486b63b3dc9d75d1405cf739e9ac9f0ac12dc0c683a4677ed1cb1e17\",\"8b92a8fd977458478f02945f5e4bb062ee9dba613450ebfe393ab1e883349582\",\"b0bd34582c7c90bc79d212efdcd882d16840d063ae43c3e092784573796104f7\",\"7c86e9942713a2b9d463c3ec2b14f671db512582ca91c1b445bbb9ec41532c50\",\"a04db4afde67697d450f5bb34712c5dd6a27507a62d3388018e3e5a6a4c01e17\",\"e2dd657e8df21a7d795a5ebaf161d81cd90e87d56ae44e9d9c7052b065f6b00d\",\"ecc9237c51602ce4febc27d5d55a9aa97a089815ba7d228ed791128672e23d30\",\"40d101244bf1c6dc1a9d8b40bd536d1f5e86a91b4bc3aa6dee99eba42790a9d9\",\"dbaf60d35b0cbb2cacf3d46a9a60cae5543e597f57f94d6c5b6813763134a49f\",\"49c614e21f04e49275dd2268ce946df56d00c0f5f16a11093fa7c8f9f821d0ac\",\"0ea4c30df91f7244ca170c93be7d047b1a3f9296c887861745e6523f0c53f8c9\",\"0ac0c0c82a4351fb6724835c176cc0587eabee9b8486c57a7fd60e113c71899c\",\"ae987d44ff0af437440513bc279a9d66857ea1fde7741fa16d5679ff2a5a1eca\",\"15a7b0a6386e74c3e03a136941156e009ab3dd301680691541e7dea9b2164b56\",\"0f29e7373e879f209ecb92caf213c6651308d86bf8abac3d8e364384d43fbd38\",\"a7324295e8383c11cbeba28310bdf609364ed4cbd80af95c8cc96cae90a3057d\",\"7465b154a4ab0cd46e4519a46658272307e9c14a9bb9f0624c5fdf2495118f69\",\"f490a589a2d395cf14a9a4e9f9960f752e1e96459f9108cf051e73a46c3a0552\",\"29c51cc41e242cbd51a0b73d780c8e9cc0cea8f87264a04f509311c764de6cfd\",\"bed67aba7adb8e0e1dfa470b7b882af864f830ea480805eb50323b801f8cd76f\",\"65bf2ba73d2033598dc8f60f57d41797aa8a72b2eb2d8e5f5137a387987cdba6\",\"11b1217fe41eb635ded4b778bf9d792e9fbefb6d6734289d98d62fbbba65ce64\",\"5e0fd90fa9939f5236121aa2b8be4a0a127cb802d54cef71ba5dbc5cb5465a73\",\"f212c94058832fbab009466380ceac64a3acbc68e79256ec61096f60849cc43d\",\"34b7f96cc443dd2912d075d4b9661d02379da488db76c4c81ecafb6720118c89\",\"0adfb0cccd7611726ab81b23d18d1d55ed6c17bf9fb6a6ebab0773e6f5078fbe\",\"a6d62ba564e46ddedc200af074aa5597e703236496056c27bb4648c6c189b9a3\",\"b166dcfd3a89b9526a88b2953cdd605a9cabbedbd6ced2c72e5b046e2b903515\",\"1f2890e0f2fcd996f96b25cd84037f2ca95c0dbed175655bd0105ecdf5c89259\",\"6ceb3478d72e8e662fd0f49d44e82eb92adb3930cc5bb306728d785e774e0984\",\"6bbfe247681f89119cede990e6948799ff9b8686d8f964c4937c59421695a8df\",\"44cd4dad7639cee6b066de1ae0416060a355b30c280fb8fe2d7d3be5a9f3275f\",\"1028e03c6659dff235f8cf557fecf9b78fc2087f4ede2efbf4e7c29f1ed01206\",\"ce13a42af3b2be930c66edfabe5538cec3eaf2db0929fa4cd229deee6435b1e4\",\"3cc2c81710863d6ddf693fb2727d832aa75a344a5256c5584b5b22ad6defadde\",\"b70d8297fca8950d4e8adddc94e05b416760b597229d102838fc300cb79c91e9\",\"bad4f5a93d94c03e4f08244fa5cf064ba4387f92d2aa60cd9ecaad79af0a43bc\",\"cde6971c0979c46c0569bd71927fac21b3612fc3b4c0cc99f3d536956c4ec2b1\",\"0c02bdaa92f7a3b1abbf8b75dbf10d7e471ee27673d918a6fa78e3892912206b\",\"938176100b4f8a8e150fa480eda123d8808b6109ce7575f37ca64b4db0e5ce80\",\"185e9fd2e77e63fa1d2b971c57e5ab76e2a65a8a153373178f81b4ee2a04a370\",\"82f381ab3af60beca6eac91d510af754d87f664b5c4d5ac0a3f2a29eb789cc44\",\"795ec9270e7b2932c731cfb5e5b7dbba7e3404f12e7a343f6bf2022743aa3d8e\",\"d1266f48954ae0ad00feb1c9e1cb5fd5c1d76a34d0c4c1e799eb08e942234530\",\"3ae95d1f1ffd9645bd51b387b39b97180953ad8a0fc3828f82fa5da6e9206545\",\"de04399e3e3fe8bdd0bfbc0e51d1c2ff2c9493dc6b5cb030b1a82a445a9d064e\",\"102ead8c4fef0b5e8ad35667b8b9b1a79851c123aa9a2a55b27e6913c91ca26e\",\"fd22c1f61134ab56933d72dd0f7a4bd15143229f6629c4d7bde61a2a34414fd9\",\"95a3d2a1429d6943d5b5f97fb14c1084a80b559a17cca7ef2bfcd6aa077c8a7b\",\"e888ff8d104fbcd4180c9e7cb3eed52a845dac4aad5cf800e903bab7cc38724f\",\"3bf1d25c3489495fe80b4873915d2ab82d9d6384e15cc95c4edb5e11c97f12de\",\"6e5cfa83a35832a261fa0d284552f999c5aba8d7c51347b9818882d59b3df43f\",\"156aebd4e3c6902779fe509265bd9f031071d0642f58e3e500e58985c7593adc\",\"4918c3c7e26a1a6cda32fcc5f2dd964c3a5c39dd33a09907c1bcd9e50ccecb7d\",\"7770fcbaee11f2bd86eaad745738ce817c4f2ebf2ae12d0deb0e9456a52903ee\",\"98ce2086fa132067dd20e8087d4f89f3f72b7322b1a777b7314ee28fd5a8d093\",\"3c988217d7d1b6bba3d405efd20571e78c835de49bf9f253d54a03fd7c7ad5a2\",\"de05a85a9280534e73d7193a779c3897fa7badbf3431ca649673ec17d8417f6a\",\"fb2d33fb245111dc942ca6100fb805216c1505e2370868225de02e58f80d95c9\",\"e4c24614175e0932817ca43383b304803fe01f999db5744978a04056e11a2ff1\",\"4fcebf30220d55b00be6c13f4c1812d54ba7be16bca96ce70235c25aca402fc7\",\"90ae17b0b704215693b070a4e00a82138a7816a868f621b42e877b6ade9f538c\",\"5beb6f7fd0c8e61aa8912383db7c1a34b98372c2b8539a0c45d859cb6e3f0257\",\"d2c51d177c1268f256ac6dc94c21c78ce664de35427b6b1968b8799621762f0a\",\"2f801b15f3289eaadc7479fad1b396afab2f07785ef8db8ecde1ae7a93c94712\",\"7e0639177236feeeb2aa7642308aef491d5acde465880d0f72901c2d932244ab\",\"5bea0241f418b33b1ec01368da986fff14030967d3e43ffec948848f2e917ba0\",\"5c820673cd9ba3b0e30e13b5072ed2cc54b73985f90ec9dab20ea5ba7ee23529\",\"ccc01fc582c7ab2a53cd7547fa550cb1ede9f90aef6d720f6cc0e083712ca22d\",\"90d79bc08089684468599c509bd1b188cbc33abde29b5b60e464c8422134af33\",\"63bcc3b28979e9aba1cb701ec34af72893ec8d8be3d3cc07f8870d398c946e8e\",\"99d71a6cea8af48f7e53428512e7eb047e1255b1fcf59904d67d6f11ad9efa6c\",\"ec6fac02104b0a8e74dadf364eeccf4488a8f43b415b9ce9c59381645c8917e3\",\"740cc71292e3fcc37e998bd2b407042b2cd2ce38995b504664f7788b58ee34d9\",\"d0fb2680faa3ed2a4c65dc1304146c7cc86308990fbcc1885f0f6f5e91ad52f7\",\"e607b9b610c8ea0a9dc9685d9f28312d0b4e5c2f732da32374b4e41aab8eb5bf\",\"dbc4e5589f6aa76f00ff76771675feaa077217d7c7bc871d63788dd8ac573d4c\",\"46ae94f13bf3ca4ba8c0b6bed1f31618eb1f3fd7d12d6aafb5636e0ead294754\",\"7613ec85f002187a8714fda17306454c73431e3c8474bcfe47ab87341d57cf4a\",\"1ad43cf328140c97f532a8c67dbfd58a7761912f41505a3091606cc643da1a8f\",\"7d905511a655b0f2164324c338d645cbcb8edb63e4c066d38bf763949f405389\",\"8ed9fffe0b0ca7a8459cf7793cf5cc0a7d497c84ec71c7b1fce1e2b73529a1a4\",\"bbce7072f3760ad3ab453bb4298160737cb0640e06b8d14b7d5b8065e749dc2c\",\"ca8d967ec966d2679ad5e0d39dcbc56efbf62bc999fe78384100ae7aec6538ae\",\"27ef518dd9973031f6405d1b0a440a5657755c1faa306b153ed140675925c156\",\"db7e12423b756fcea693e064cfb4ce13df78650e7bcdaf6dc4eb93bead3aa4b3\",\"d7b70df84d63b7066a17d4d6b8a113450c7dc9e41bb2527e0f4883687833f77a\",\"29130ebbfe86c48cb4cac86d87ea169c1d24b30dcaf5e228d4c3e282b2ff3e2d\",\"bb290785a9483c5a0020f144b580fbe2ea4c4ff1802d1f824a9de89f621cd015\",\"9542451d8f7c0d9bd93c98bc71d0270df817d689262b925dea68a2c498be9290\",\"62044d04bca386f718257ec4541b44d12ecf1d849a953b649894e838df390fae\",\"9d6d5ee2be31ce2fcdb7bd3eadf5250a5d59117b59fe529486c215348bfa0beb\",\"4a85940b1aa00c31216fc8ec28174075c7fcfc75b67825f2c1546a90c8d1d2b0\",\"3b1f32a0904b2bf1441854c58695727ce80db05e0e9018a6be2400ddca7bd704\",\"1103b34af89cecab5c3758bfe766d7750549d03b19de8f086800c0f59e722477\",\"6c837f4890cfb1fd291112f29025a0b1b57ded4c6542b782c888fe5c96791738\",\"f06c4be6acc121e06f5e763cd0504d06e55d72c0fa699a87ce9870b9c03b67fe\",\"92a2e693d53d3c5d84b55ff32d331694d33f3fb6859af0f1efbdc857636adbaf\",\"1ef0fdad0fe8ed8b6c5b156c0617893a6df1da7ee4d11b8a45b3c2048268ffe2\",\"f95c03ca5854c5bfdee4b5f3895c27f1c230e5f87e9b2cb16b1a13bb9deb5f0f\",\"2b07a391fed4bcc3a002d1c7df19946753dfe1715b80d6243efa72501ba9c9c7\",\"0fe53277f648b089bb481f4a0bb2cb04cf00f139593d50403c7ecc5d41cbd29c\",\"1d40c5efc7b148a86bf38e0bb55cd3ed77ce034b390e8f1e8286265c38d1b891\",\"54cf031451680199a00c1c32bede45c63ed8ef389f914f30f42612e30a1a4f34\",\"f3ea9c1edb039a00135070a0806875397b81c8ed4676185c56d843acf88d8a37\",\"e3e2796ddbfcf38adc932a613f85d3a43707e8be2961125b81d103866031d55a\",\"9628a1ec94a0c753dacb08bd4d1db38fc7f5bcba56c953dbc910cee08b038f3f\",\"01457e689ec4f48388390cf576e0f1e91b673cb06d006c4d9c11bb1d6013db92\",\"978508264c25b9d249026f545e49a55f384282bdd98025a97764fc62f9d0dac9\",\"cd4a7c5581c5f3a934c4c8e6b073444378e9fa5c7647b909c35763779345f62a\",\"550c2a0f84d9320be82b638b265c53c0c7dc83e6c2806e37be195cae7c759057\",\"8a464cfe7a9adc0d2c744d04e6e89c3b04c19573e5f5a5140d925824b8f3bb82\",\"45c65e8bc810028175474dbfff0df9a5300b318027932fa121fb29163e6adb35\",\"bb44f311a511096b54be69aacf7e00d9f23bea8b00267a52163904fd11438783\",\"90d6f1d57a5444dcbbea6ed3500d51939e5c2be5db1f93a5090a1ca68c14a7df\",\"8fe042726e9b377fef667f2f04ba8bd6097743c5b3d5ffe3ef4d271e1292c8d8\",\"d3f9931dee33dfe3a58f567f423c5b8a81daae1659075ce22244b17072e6fa9d\",\"f44b598be79a071ce7cf29682a7e35d1fdbaa53719a9b3add50a8a3ac24401bf\",\"4ca7de8d619b4cac820f34f53d1bb2437e21f56000ecb4cd22439af21a3bd9e5\",\"bdf14dcdcc00f5b96d5f3adbfb094ba113ba7522eab96984563e7cc230f0dc24\",\"3c7390e7174ea60412c21208ae736887d680aeb0c3a787da19256319503882f2\",\"5a2d99fbb0722468b0d4e3014b82de9573e0c3e8fc6682cd18e7dfc5d6fc439a\",\"690aed5753238983036e19e343309126b38a1ee33abbf97ced8b75077e6d8b19\",\"0840c1ec3596979cc280215e46bda87660d5c8f7413d603e37aa1ab5e9d158c5\",\"bdca2fdfff652e3d7f73b43dc2b4d17e7ddbfea21ce54c5e6054781e63e59413\",\"358be3768f429680eb56b2b6903d60288dcc7c517f730ffb9d2aba30b40ec777\",\"b2dd66c869492abb014e93e09244d31309a63b5b17f896e665b403bcb9ae4ce8\",\"dcdc52e66ca84a3515dd2a5da6828f7ec6699e6d73feed0416a9539ee79c2376\",\"c649e3cc990c63bb81ef029d9041ff544555db7ec58163766fab2911f60f9e90\",\"36cb5487e033cb08860f0704cc395ec3f1ce14404941a99b4d25b8fbb1b005b8\",\"b3285858097c56927d87716d4842b19857394280f7c0b01a07658d99062d88ba\",\"c68b099ea407a6470ea57824d8c2cf98bf2dea56b4d1db790862de05e1235a5a\",\"c4ce2e683822123affdd8f2e51afac8e18fc8116f22bda6c14103b7d8d226a4e\",\"2d87580aee859dcd7732a5fff0d108cdaa547ba58f757bf4baf1b797b986a1d4\",\"8a45480935020d4b45cb32d01bf96fd090a2552bc51cb71a751dcc7c1323d07d\",\"ed2aae4060d0242bf3288a2a7748c035555e40c1ec610b61ee263acaa11163f2\",\"2f0d9ef6bd74fc867c7bde33f5bb91c7acda3e95849048f5de0e56a5a57071d7\",\"64cf9de3ae6e39d73b6909186b6ba9aec1990d09c4a119ca0ecd509a0be9a908\",\"8fa088e0e92e4edfac8af013de9fe775ef934f022a75f21347727fcee8c2acd4\",\"83cfb51d8b45134318ef2b7062643a6819d8d3c4bbc9272f74ac315aa28eb962\",\"63ef36493ae066487d87e7dd1796979f9e7719ec8f1d06774a68f2effcfd3809\",\"44fecd0f13ae6ee573976037c1cb2be8ff1eecaf9556e25008438862f0e00c91\",\"9020837840ba358a809740151c2a16c95e68bf510607d74d769518464c228289\",\"b369ebc0fa699af2e40d52b67f26fad7fe5f87e22f82ffe99ac05221054f5a1f\",\"24d85e0cdc6477b7ae691e8cde0e7d7e575af9b678b044fa6f9e8b8d1a41a2e1\",\"e0cb1920f60f3d3055459a0269c99068746aa339f530593b72cab63f01f9cc52\",\"5c8d3ea1741137613af483c70770eb9cfb6bb876eb1722f73eb6da44e70ce18f\",\"5b1126cd3da01cf335836a3b042790bca3156c6b8a9025e3f4eadce4fb7bab51\",\"706851c4f3721da7e7a04f1f4acb2aed8eaeb6021dbff41003b84568fba8cdcc\",\"8c808fc840e07428a35a1113c84d01241ac78c70c8303d24265e07b06e5b7ed9\",\"26c365b76e1cca1d653cf4bfb5d8011f51d12e6a0e775d412755a3d977326b22\",\"0718ea157e8d34379a0934218f0be518efa5a345cd4da5448e3ff069d5bcbea9\",\"237af37807248123d1c78ed18eb5167dd64748fc4e4d79bf0e2b3ca2c305a2a2\",\"cc8a02bd6ef496a6f826e0afe67f37a78fbd239a52bdf8db5ae027e3c1504d90\",\"99d3ca08b6cca506fb7a0d88ce3bf100acafe3dcb882195eaf62534476414c20\",\"3e06f23b97405bc2a7976b8912b2a5c5cead7ac865837fce23a82de5abcb1c73\",\"24e3925d8b56d98cf2cc9cd6bcd643307ec2363de82f11f8e11ef66cb098dfe9\",\"0d727c6b40716226593c6907cdc221a8f85af51cb13862281a44aeb663f20a30\",\"d78e351f765a55d71f49dc8e404817ae8663c85a8b975ba1e506e7e46e7f2a04\",\"604c53f0e7219ca02fc4632cf4b76d99804d1014a94907a4e3ba559327760575\",\"9a5d715039b30efa77cd9ee92cfb339bbe5abab4aad6c100756164d2a7e6023a\",\"84f3285d24d01cca8be08f7633493396bf15251d8811bed83310056023d8d641\",\"db9256f6bcb716c09e0e285ba0638bf8946d96fe5147543f6bb83038e16b12ac\",\"026bee9e3c3a1e0784cf01260870e465f8f9342e08bdb993f6a9dc68e46fc57e\",\"60ac746beb491bc224db8a46ffa53052103b4c963f5ce8a638bdfceaec859d67\",\"9ba7ab80d0f0c13f0c1daea099403c0bb22449165c005f43acd9c4c817b2fe49\",\"0548d5c9d1c692094bd7a52e68f0deaece0ba473bd82c8bd250eea44dbe84a78\",\"390191cd30e35fe2157f4bbfc243192a7342cfdb59299aa2231904d9fd36168c\",\"0b82bd3dceb8399f5fe0f2a9e813b713f7fa6513a850484fa3a79d2e72767c67\",\"65273acf7b4802f5db9fc0e7a615e0fbf531ae8880082c8ba0995240f9b324d5\",\"2ecbb21ef60df34949527435339d72b781025469d3463108b4c97af1ed2f88d6\",\"7a23fc2f3765c3fa4c1c3574988fe39943c4faef9766d75ac3636a63537f8fce\",\"6208e975f243663a9592a8365b5b22cfc30aeb269981ca6fa441f1c662b7ddd4\",\"8ad2cb6b7761516fbabd3ff9aed4bcdced9872ce97093e9c031723494b55aca8\",\"991b59283e9fc6843f7695a72bd7f021b33cfc90677b4c2004687d1b72f77508\",\"7bd9992673be04fe086df3bcc8e54dffa0832303a19c0346e1711971b1c72a28\",\"fef2c30f096f1383a28d6fab45b26f224c80136f66dd8c12236daf1a6bb97eaa\",\"0484181cc74167f858a08e054547c93824a94c5df8d4059b0d851b6be214e3b3\",\"fa3ea52d0699cd6f11d274bc4d17a188bf6b6ebd48c835941d696ca619287965\",\"73332137589d58695038331b134ddfc26ee8c01487c9c2191790cf3e32731a53\",\"416790b24b32ca4174cc864d0a4a029ba1f384ab03e47c870c27a9f0c65e0463\",\"267cc9e2e26684cda38242c727fc913c82050f2157fac18e238cda0b21b8323d\",\"2fb6ca8b229bfeac9c11eec999002b1e8f8e7023a657659b4eb27028d18908fb\",\"4e23a773bee3e01fb38bfddfc3f8237f7675c5df37aaae8837cda4e46c325eea\",\"f3572f332230aead6b712c10d4be8b284278c3f7547c975879433e4593d5794e\",\"1155a260d084fde5645231b619a4d83389d7c13eb6b5f8eea032846e2d84b02e\",\"df602fe4529ce3e4ada57d237278a7d76ff93aa84116940b90cec3bd9d4fabd6\",\"d3b31c5e43c1f9ed09f4aefc16081a2f6aacc2131f9d45a224bc162a338fc133\",\"de8280eed3e1070000c1622f5badd17fbcfc63229888ba363018fb5b4d678e32\",\"2f00cc0a6441b6fdd75901ddd24686433698ce937003359bb4db4406ecb9fca8\",\"082ebd481f53bec881a001a7ceff9bb241fecb565b7d33e0759ec805a69418ea\",\"3817019be754c77831cefa5170cd2002d4dfddd018d42f10d6ef6c4aa90b387d\",\"d50d55b77610954ebeae25c67d0516482b1c8c6271bf01d4011896a06f3f2d70\",\"0ff7c3e4976744e5b260fb7768990077dc90804ea2c9868c6e81045b405638ae\",\"623a36a2e38c4e86be169f0e52d91d31e628099096208e46ce2ae99a2346f7ae\",\"0c2c84ce963b7ac3dece04316e71aa20ab5c701986ac17f024ededd6478df60e\",\"2c91dd3501e9933d2c0190d0fe0ed40f8159f91ca8935302f92fe23f0059ac2a\",\"562b7ca9f4b774a035d89914d19c50d09753d008808af083aa74e75207b50c8b\",\"32ac64b8d040df388f2a99df2c47c48103a158505440d57777794eac5e451382\",\"00c69235d4bebda20e5afad2a718f94b5f72a10ee294e916217aa18c36fa7cd1\",\"7ae61c918703885877c0ed0e741b8f0cb0a574c524f03db66fd7f5756578543a\",\"43ae68d10f98994bdbb689754227ac7d1bfc6cd7f0805b2a843bfd1dd92a64a8\",\"7f96f5e57dc33449dc87f808e188143cc0800de2811149c60720ac610ad2fec0\",\"defb3298de8dab5dce855059c7bc384a3dd8f2115dcc3ddf75d084c606f74763\",\"e20a628fbfb5f07014b42d5aac6cab36463117441682b4f9f47c91ac28b26ec5\",\"523585893a1d7920a0e9d71bbea61f8b9e0ea8eca2914a5e9a8cd0021dc74aac\",\"18f6ed651f6a8a35cce11e893374d1693df33ec59783dfa8e717b64d6234576a\",\"6248432ce832c49c4885eae192163f9b57fd499a03650c54864159b216a12b70\",\"33efb11dd166be72b1baf3b73c1ec73c034abaca24f727a18e460949018829cf\",\"4c1983f20df6a580cbf2139c5918c8b3c5d226504f51b3cdb8bef7678097d75a\",\"12895b88b2be9c6685b2d84e906624afd764c03f7285eb738108d824aff09699\",\"a562c556655f20138d75e6ecd5e088de26626ef6c5a26424d5c92ae962ed8793\",\"b100c5e8172832124cd36cffa5b79cfa020f1a46d922ef47382f23849b72fc35\",\"9ee1c9b820ef8405d34fcaad1e64f84d92cb892d107577d078c4f112bec6df63\",\"4624599de0bcfda46607b62fd430da10c12d89d14f45fc706478bc41cdaaa1a2\",\"613b223ef03a33673f3a7d1e6f8fe50084268e485373e592514298ea9a6415ee\",\"e1c14752e24018ee4de3b37e79dfe0a277ccbeee9f269bbeb78947f9ae9175fd\",\"6a10f5facc040998ac50189b326715875f04ae6e26d301c63cd3583dfbdd64a8\",\"fce8ef40137acbafdf8db40f48bbb72adc0d31fc8c2d3b391d632c9e7721fe3b\",\"ec5f4c709c698b1c65118fd8ca0ce1a6c5726f9f4d3bb63fcff09cc41b83a994\",\"6a97b1ae0edd2d61197e84ae8457ea9cc5bb3aa31a08651eb5bfcfe28bd8279c\",\"944eaabcc6ee8af6975e237cc93a5f48fd93c86fa2f1115177def071d68f3c8b\",\"3cc7d012faa2921b97c60d57ef6d5a06032d7512d42ec501778afdab11e397c3\",\"4059d6e605b309528c14cb889617b1e648e91e14d04d1b87a8a23d852ae03f47\",\"eb1a91dfe5ccb431b8956f48ec7000d0cd42c07a9fabf6f159bc2660e1b741da\",\"6bc9ffe39be0e1f5e2f94c9c776e4d92597d9f115913fc7571c927325d13fc97\",\"fce9ac343cd346e111f713e02f65901f697dda7786b6d69adf66dc0f85236488\",\"2c8a8ee92673807d663c88642fbb00b0bf1178e178ad388f8c8cb1e118dbcfba\",\"4aeb0d673045fc322846199b1dac40d13d1a9f4e0de40236ed4e113a3695f64a\",\"4319317dece2ac01a842b4eee2321b3b71b55358b9daacf4fae946a217d5ca97\",\"4fc26ccf936cfc42446e64b0a054c750fc6273c4c63c7d6910a2f2395a317e6a\",\"1e8918f858b2e9e4e28bdf3326bf49f0ade21ecf8458ce0ec2c4510297f5ddce\",\"d5f1a1a4ec2ad6addcbc6b334fba2cdbf5351f7dafdb86d5b25703d2688d9dad\",\"063ef0f00e8f25ac741899b94c1e69ce1555fe7c4b10fbba6f21440709dd3d2c\",\"ab87e172e1bb588849292bfa78a8dae79c902a3153b8cb0ee974545f6e701617\",\"940cf72d252260ec7d618d0d46105c6419a362b5a4489ab309c76e326260f6ce\",\"9f52e81d54f7ef576370a210bdcfbe9472d9872ee4d8f7a9e6c7f5b123a97c74\",\"5db6337dfc1285a781d430ac6558d0ef3e886bb4258ae9fc680f3a075b0d0755\",\"73768a9b5293bdeb718eca5d3a033d7c702f6b36986f166ce90b704580558ddb\",\"4f20aea8179f43fb361bcf67ba824b0ba3ed105961c7593e05d35abe2d0dd99b\",\"7a26b87383e33977708021407718d508d81d3ad26b06a03d28d922a8e07d8c27\",\"5b71c3d80b049ac3f95cc41c27c9090cf7aecedb6a991cfb96ff09cbebac6ed9\",\"faf8969726d04791b0da9296276513f4c548de116fd7b2d60de9cf13cd07c833\",\"9f587af583148dd5ed441ce9b85feaecf16c2e35e67467cceef6163763f03d47\",\"8e598a0615bc8087d607c22da5f29b460745a80777ef51dedfcc7c8d8c4274e0\",\"e9e8afa9d6044516a785771089554ad17f5e1b882fa8dc17a32306a92c6c6545\",\"017497d8cc91a5d2685d6e9dd03bf25e4354bfd81e2872e5c57fae897cb20947\",\"afeeadd1339d3c435726944687135f19ee98ada903d928dc56bc73a48cb1df68\",\"0cd3c18e6e74b5d65c9d845e22e9e7aba07c9986ca8894949e01bced9747eba4\",\"9ff8c37d2ae3ee11d604154f9671501282d8f85e68ee0a13f79631f8fd109ce2\",\"ed36b95743bfd57e02ceaa467c702dabfce0f4cdbf98b91d97c3842028e24c5f\",\"67b83f70986eb2bb8c38a5f78e994861ee1391ad2128695fe2818b85eec5ae58\",\"f7d6e304aee48e59fa59fdcca2ae6ed0c771f048fe3296e207fd8f22b3cd0864\",\"53618c026b8fb3e09a3c448bcfaca550dfa2d679a2483a0cfcdc5ca139805a39\",\"042a8d0dbaab5d01aeacde31171133ad966c480ab2617afea5b81cadb8aa095e\",\"7aa5dc5150a799ca003ce1d4402d4a1ae7b8136d9c2e5009a4386db49b9e6328\",\"04178f2229eaaf98acc3174737c7eed93b277f3a820cc8483c4575c82ece6c18\",\"164d10d3d7b1011f310f3b21e5ee6c8e35c4fb4552e12babe07a8dcfa14aea61\",\"0b956ce27dbef51276357d6b81553036b1d2c45517fcdd30dcb31eab4b245691\",\"5730dbbca6b8ef9221fc06230dd2d50e2dc7418c59b1fcd3cc8ec10259337775\",\"0231a4fede5d69fd8b6717d57174ca339bfb976746685b3eb5aec110a72531fc\",\"9920c47777a7e7de37cf1bef3aa4d07c6e41414e7bbb32648bfe8f6b8596842e\",\"707834d63dba9fc6dc6aa313e8341ab4b553b27c146b7fb5ecb747d5b6462a6b\",\"040ccd2f78eb3fb56a18252cb8ac0f57e5a26284744212ef216408ebfe49bf42\",\"0fe0cc98ec09d22953534c7bd6c644e24cb8237186bd1cc88382891d6d6e9e86\",\"be4055cc6f83411afa51ffa2f6e03639f136ab4abc32ec331940eb4484f1d0e5\",\"6ab23448ec8fb5ac209b82c87f05d1d7008ccdccfcc22b0c327172ebfd425f29\",\"502a365f2431a0d52fa4920201b0d15b259effce4f618717e92768117c84c65e\",\"5715990bc304db4b5c6aae9ef69328574b930af50bcbd241d9409ee6817bfa9a\",\"e5c4cdf558dfd35bf9d19fc4b513becf93a3399bb0fb2466996f2fde117b2ef9\",\"8b4c62c465e04f914db1e4810d9aab00fce60f32cad94ce1f65a2d8cc2f1664b\",\"1302d514a5357bc3a3fd7e41a43d1445ffc13366e60c49a2a6adecb5354f5262\",\"4b7fdf0c0c051b179bd6b4aae9f743df83b01e6ef13ce2c692c0cb3b1c46b049\",\"4cb5e2f83fc1b35fb4769dba4c01be99d98de3f41b03e8d4031c9f334f7e5294\",\"551b5c474602602d84ae732b77ca43fa22fd08b42017b301a38e0f732788794e\",\"8c69e1e7c0cb2fdd8ce0d0d1627cc2a7d08868c3e49d1bce268dbd0c798b99b3\",\"93ce869088382ba15cceb9813242abfee99712bbef66e22b2ec17a22498e9018\",\"2d6931a6a9d2d386404bdd87faa9dcb7cc09573db699138d5387e9fe714686ac\",\"8d95915c1e19d7ed2f542603860c3504c61c63a25bb27275fb5cdaca315e8a11\",\"1d1d68e64a1078f0106e1f0bbccaa6e20cb95f58e9556964cf9e88b019cf97c3\",\"b7efedabdc12715e415d874aef36d85af893dd934fad4df61c2170097c304e0f\",\"0df47995f7fadc00cd3d82d8f799e94bfd1066c8ad58a876fd3af27197101e41\",\"a94135f4462994687c0c7f26f0d6ca4eec15d0a41d15660c9a680a37e673041a\",\"8aefff277a916d9e3dcb68a911dcd472c137fb61d764fd389e67f3b87af3b3b4\",\"e7b9105f57571fef7522f783d7e366c087c07ec041c73e0552ddecb19ba5eb83\",\"9b80f5742bedebda158fd59a3924c1fa72727c043126e84f6e8ca522708407eb\",\"2323565ce1492dde25cc9901d443090a82cefafeccde533a4d66f830da853975\",\"ee7922dac65968b455502bd3617ee8b58ea1d1af2143b5f67d5701b561bf9b05\",\"d069b11d8f820ddc5c5ab229e11d13a564695e584db371ccfeab9b8d9dec20e6\",\"19ece811f9d559d82df278f3fc5bee9d3f51a42e62d5825b2182198051fd2907\",\"253356adf1e0d7956b438e8deb2c86847001b2110da237daecd58527dc237e4e\",\"92c0ec671507d954c12b253a8df9bcaf3893023241a5f00948c17892324b3c29\",\"2665a0fb7544bfad12f542e653b6c731a858165685063b95075352b6c1b9af0d\",\"e61b0a56bc7e3ce8619b2f02eeb1847c619984b2824319579835f83590dfd34d\",\"d88f1a281f4f95e6bd0ea244d7f65ab13082f88287b9ea31f25f240490bf321b\",\"032dc44b241b0147bc664fd188421b27a003a3b38510011ecc086693e19fd8bf\",\"a7eece61f147bf03664ba11670e3ed35d6c4d23511fcbc0c0be662de4605e997\",\"3160c570c5abaef51c0b3459094be0fe43707f0cc8ece8257df5c18a7cd3c732\",\"84b2b88337f8c5f3be4e24510ad375b568eebcbcf96f5261f8a0f39cbc7c6659\",\"00beac0950090bb1b573470bdad2505e99a43e2221352a6a4c2709a0e22385eb\",\"0921592ad2ae39c0b43bf25b23145ae327f07a7a4543551514ec9e57d731e7af\",\"f6aa6d2dafa6e94c31c2a1d2a97d907f2044e1820b06256de76ecd9f697b0814\",\"4aacd88cf5dfbaebfe681e7528a36354a5bcd90177d6afae2efe7be07b5d0875\",\"1f07116daf0d84b537f6b8bae1df1481d48c2a28e003c69ab439a86a2c41602b\",\"57b6b8ab81813c35b6629aca630f092154522165bd76a3c0a921c14222621c21\",\"d6cb5519fe901b22b0ec6939f71924ec4c7ef5c2cd2d8f311934d0d5407a3203\",\"f904a23aa6105d144e1bc67e32ef40c96d19fc2ad84315c3b542b30c0a602e2f\",\"19b6d244e47b003008e19d56b6b85deea1af4aa13266d0470da5326987052620\",\"f53a18db6aaf1a789f314edf3d7bc7099ecc9a22b857ad69bf9cf3e56c36f928\",\"2945760d16b1f47b74f880c8d752ec8e4d5dc5e24fb6975bddb32eda99c4bbc5\",\"5e17d969b1f644f7db5b4e6309e00f6e1a326f353c7c35e2317f5b664c853bec\",\"97e348c3b0789dc644583e9a66ec13e996a35299d635fc346fa38eed16a8a036\",\"ad6662d4b05e7fcc83c8c2159b2be6956c95c3023881dab2d010d3313313e4cd\",\"7f41f7d10b355942abde2ec0630e988593857d6bcd2a869c155d3cbfc2b36804\",\"f24a5b27af764ac5d8e59b886b8b5320ca1f4174d3007b3a9f54fbe931c54ffa\",\"8b62c819123f1812761bad21dc897e1c4bd307fcf48975b0c60f8bc20a5d1798\",\"474927373605ea0c9933c552008d4dc29a42d3b8ba52b0fa5cd4c4d4775fab49\",\"0e970be0d1a1a5e9d7c4ef19c3fb5f292160d4be3ac9fac0a5eda0fa9d63e8a6\",\"f3dde47e572fdf51c1040aa3f8cb8adbc622fce9f3fe8f50c9d875286afd567a\",\"e51d7b2a3c8f2258a6c45ea25f58df5df22d9dd9bfcd3770f1de0da31cece2fe\",\"0bd6e881cdc0739e0c4c6d9b54b7ff77535d55830a03991c83e7077b3b0a6c8e\",\"b18f43c8fbd9746eb443362645bf0f2a4dfdc62537ed8bd8101f1a22656bca72\",\"49e2b57122c3fd244fb49069665eea4686a6a022505818fe7adf746670138a53\",\"004cdfff8a87af052b9d478e0122257a86a65285e3949aaa6ae6cccbaf9c4038\",\"56f6ae7fec86b9c9405645006356f52ba64725d2326a08a1a3bfa23353e538a8\",\"d4826353701c6af238e781040238d421656e73de8a9f84dca87a7468c71524c6\",\"57c9ce18f6321542e83f9e2f37f38cc9e13f3fffcd1b687a72373d27c7c7e248\",\"d6957a9230834482c498535faf9d49165dbd0d9293651c7fb3058bc49ee57972\",\"de3f7207a1ee83c0d94711b4951040c8f109efea1f2b75b1bc6481030de4a87b\",\"82cb34709bf9b6f5b75c9c56e701872e302780a059df162e984317e837f3caea\",\"67ed961596ff54c138ef7c59c5488a1fa86924127f27d0271eea43381db632a1\",\"cad758fca8a365d2491ac04ec86856d5790d4505904088daba579b72d62d692f\",\"711713d23d9570103293652773ae21620f16b0f699eef133b5d3d09c6a5b20b6\",\"86ba8ca586a88dfba028eb3af5f40f69c5401601854e21c09a457e69d2e01eb0\",\"a15087b32f2c050585637144dffa916165c2e4bd396d849be14a8e378d453961\",\"8be8b807d79b1935af137b15c86685be9d6d403041214534472accb0f3e6c50f\",\"9794e2d64d11c9956967cc6d6b3f7a67906c8a1578001243e2a1bec04b6b8ddb\",\"ed035b6c426412bcd181c5ca475ddd2c839b4bc1f8642c0a85dddd322d336985\",\"6c805f514c1f054ee5f8e60380f2367f15cf0ab166a1911835c5b53fdcf99e60\",\"3446a77af1793f4aac202fe3833ecce8e12406d79471f1921e0ce40d2fa34fb0\",\"3e89ce984544478848811e45192e023253683ff23e140924a083bd64986c7ca4\",\"723826b7da907b4bbbcbcbe3fcb964beed02ca15bbd4d57386c959885c133f1a\",\"e346c3d06f628d6b3f4fa455b4a2dc2e7001d4764e54c43cfacf84ab5dd62c8a\",\"404445aa832c0744de0bea0a5218b68365bf2b531ab0290b90c2bd6d4082d8f4\",\"e1f9d538ee1fbb21b275cf251dd2e4a957dff1391674208b73bc4f17b3d4a286\",\"9dbc570a899f08e9875a203623c0aee1b901523726578f0e268a084898b53cb2\",\"e2c7e77c178631502077407f1f07dd22d55694f7c023c79faa091a0e3a7116e6\",\"6c35ce3fa39defde3748dba448afdb079ac60ee1e08ea68fa559a21d99cbda5f\",\"ee43d92813fa9aed84e0159330b7be19ed1a21e717d836f0896b0f1d0600302a\",\"25432d8fce723716a85947825579f1f6ca43e39c492afc5c2e9355fa2c29e753\",\"121c4792a556b435cf859a017e0a3f464ef0ccc0991a0608dfe16e8a21082e5e\",\"ff661a20eb0885f7da2df97558c963d460cd2ec9f34e71cf4fb0b9193bfbf100\",\"b6325bc03674ef3ad08cf3d23f5b42d8b8539fda172b920be081bc61d22373b3\",\"d3ba04985770c160962bfa40702b56515bfc31048fc45e256ff1ea572d0aaa2b\",\"ba30523ad77ed7d0b9a70581d4c12d40006a215c2aa951f3ff0a09022bd75a77\",\"0e535b4d919398c8b01798305b80597824501ef565532a1a8c57d5982d81d3db\",\"90200280627d01ac8223c47cdd9dc965d6e0233d85b93ed6196e527f52d3d927\",\"b3d23bf95ce5101fe4ca5e5827cca53a80aeadbfefee31496f23cefdbfe95564\",\"cbacb569b82980a24305566239f66923488ff628991734a3c15f5a27a4aca7c8\",\"e7f58cacc10a1afaab25cd7e0abd0131f6d06cab22f3692c327fbe76e97d0b61\",\"48ef272b55ee7401fa5b244bcbe02bbe293afad7e79e862ed161c512ad61637e\",\"14be71b2081ab85468c9a4f6225778c452d5b949a205cf093802466d45536b7b\",\"779c65a49c86d898bd06e82685a40bd46960bb87bd2a32892c6605229dee75bd\",\"5d3455e86dbf8320fb814928a7c5219393f3330a6424c49b84fc1e4b18808335\",\"28d07f85516e73add635478c9fe4c441f372fef4fbcc897382c6274928b69650\",\"39d450d35aad8d113986782a104074e044b8f2230f79942e8b0e16e4a7291110\",\"8b7273c22468c2e6cf8b6ecb084317f0c0b49895ae1028089724dcae28471944\",\"124fab8483c36a182a9be572654415dd5d5cec811376819db0c302f1c5f28caf\",\"91396815ad837311739a2668cf45ceb1f34ee920e7453c0de6be9ef3ece71688\",\"23dbd447b6b875a4c166f249b7c83c23dc911997cdba6f527b6e7368b3b48cd1\",\"25c013d63418277c363fc0471323909ce909b2bf1caab9ac6f51e39c16c16673\",\"e969dc4cc9b33178fbd6283f85acb5926106815658ad1c4d36a65bd131d427c9\",\"dabc03d1516d75c370b41e6ed54d7fecc804895449658b1c17c6045c597c2851\",\"1896bff0f75a8617844b9e18f0206aabd3736a3f4300c35e88707876443b7567\",\"d75b4ed2966756effce8f860c3802b22d51e22651d3db43b56a3a64e5351c313\",\"7ee8a7fbf0a5ef18f5d9c89f2eda2c14b1c27feae953205549b2cced92b21e88\",\"1542116110ce45513bc8fc7e442e2ff4cb83885fb525b29e6bb947572e41479a\",\"866608c98d5946f20f61d6d0c085cefe80c6da2271ce800e8c48e4f6ce530eae\",\"4d1aaa979f48fd267475840a2192da33dd492ca3794c0f9e15ada46f368568bf\",\"5ca30b79c29d211cac50c21b75aa87499faf723f8d00303310fbadf1c3c58f96\",\"01895c63480b02ecfee095587ea2857e3ca3f5b60e813f6f935501a6d75653cf\",\"5fbd3d856398206960327789d0def5c55b6864442a098bd1f245785977e3ea60\",\"fadc9e2b76b8c449c3a9f6785fe2b1c6bd833248870f72af1f430e3aca281bc1\",\"71d27c2f949af976b2e408a6bdc377ef5c8f3d3ca746ab32d227e3ad3a8cec8e\",\"6a18923800a2f9643251874e5509c24a0317fde3dcdb22948e43295e46c7d704\",\"837ccbb8e5571328d5c6efa03f88b0f69e07d6168418806c23874b3042c59636\",\"d60942c46f4bcbf684939023b5ec4b1f9a1d206838be6594e16f621ac06f4465\",\"c7595e6f42ab105168dae50e090588a1be9f9ffc472fe92994fb46f0abcd7c24\",\"07c0076316116d1e3cdefbca048ee94f86135d23ff468691c7d272db50212a40\",\"703bca54196be40cb1add56cc25226b30d58f476576509b402fc7e769d909a36\",\"9fc75f2f192aadd35b0cab776ea81d1dae4f27db65bdc67b9f42b5a86aa42ebe\",\"6707922ee73fe3917e4283a1e4e9c363f95a9c418ff16e524e2d83cb512a63b7\",\"3b086c91ecf7d11cfa6e6e6ce2f7f709c87cbf101edf985a77b0ee82fd887d1a\",\"40222281dbf23bd18cc0ec783982f6ea71c42438fcc3ab37c73dda18329aa0e8\",\"54d47c7db9e03acd4abd306547876017b1284b36e8297f283749b32459d0947a\",\"2bc19e4ee99f2327eab2683243a8119970e7669d162614499e8180299288e97f\",\"debdd4b5ea9ddb8ddfcef32b640706c2d21c1f1736797108e3a175ae6a937bee\",\"fe8f041e094f1d17d4f6c70a79ac5ef1bc00c65ef388e363453c175bbb02e2ba\",\"c1e5e16c0ddbef13c1f7eb6fa2a1fd83f8fd7cbd18df492bed478996e6f77417\",\"c08b2dd1db27deb9099e8b0d0d61dd9136cde75961fbc6bc85b20edbc1512b05\",\"a492e27585cfa7f4e24853b3c4424c3f9666d596f2e3c487f47fa0106a3110ec\",\"2ce6f14ab02ad1d48911b3efc1cd56be9518dde71ecf616056bc256f1352ba70\",\"b22ae57ece810a53af799ec839b0d8db8fc1f20d5a037f8041fb128b3df582f6\",\"37e53cdb677e495c97537308049396484fc803367dc5a0b352feaf7a799c075e\",\"0dde66d54b9f77989035100a3b64aeb9a7cd755b46bc39bba3f317aeee85df1e\",\"b5549b8f86dc81713051890e4be9410aefcc52d6961dc927d907d6a649a3adc7\",\"a6e05e07686cc5d8a792e11d2e68d85e1cadc7ee2fbe3c5bf35b07765086803e\",\"a968679a6291e9a08375de27f1a1727afc76a0b9dd7f72f186bf3885ad056a6e\",\"a0da7a382bae281ad76c0cf1629b8bc58d8b1c6016e40ab71ab4a541fdd71231\",\"698770e22c395b307dff636fc46fdecdb4690b3fa0607a8379e9fdea1d9a4cf1\",\"e2ccea8441f78de1874a56acf023bf342a7ebf525a2fead7a7aa5f50104ef58f\",\"ce7c69b5dc4d6c513c85541af4e7052eacc4034e11235c91251a69aae6e77ebc\",\"7c3d1cc063b5ef5cc417252b11046340255a0b469ac5a53f7528d2ab3bdf2aca\",\"10420bfaf49b31cdcdf03752d9700711866ac32b460850e1894232a605520c29\",\"43e53de037e6bad80d799bc3461806b4163976c512ebbec0642b8b7fbee938a8\",\"645a78283fb157856167affa40df3f7a2df51167574139e1fb4afdfcc6a26c69\",\"ed59852babf7c44682f5452290f3c669c96d6be272ee52dec12b6a5e2d5033ca\",\"42eb27a3bbbbd5e104d8fc7e0e77ad1ff024e0e915d07fc08ece4de3bf6c2b5e\",\"efa33c3961b28f1001b74b5eabb99176bc4ae5cee0301a9008487cbed57ed9cd\",\"def690a11f04b158521a06a351426bfa2a74634246a59e8f29eebfd9c45ae68d\",\"39f289857c0443290ecd9494044ce67aed46f75cfbd7bcce84d04a8b847190cf\",\"0c9eb9994b3e453e1c499dbd4d5926eddf5cdea5d407fd6ca6d7f717078f522f\",\"7ab4e3a72fe56356db9242ea6439b3b66f0f3adce8c2cb5faf0eac85b0e205f9\",\"56f6673c66a7001ef1900eda24f31b6a30eb75ff90cb2b0896aa5803d8c46098\",\"2905bf03fcec6b4bd0a4741ece91fa726f00110e38a0872b44223dde421955b6\",\"0aec026c442061b194fb0d97c1919f62b17935752c69074a967de803ce8ccb4d\",\"1e4a250580f5be71972c4ce442bcf3e50c5fa9bd505e83f951e100556e7f800d\",\"f1714eb9c7bb749033864a63a399a8bdc0928fe7320cdb5e08c23d174435da2e\",\"3f5e73040219efe353453dc5ef53b2fa2919f8d3abd04a24017d01b557fac988\",\"1f8bc795875d23716de0651c3e765d6dd4122acc7ac2b758f11c1ff7fe462530\",\"4902bafb55cbfa4ee5cf5259fdd62cddca7b2033454b25d00932496d01b02fc7\",\"e1640014020e9ea6faa9c8a1f325882b7dd1a563b62ecd75eaec59ec386d5fcc\",\"cca311649c237338d79eac86496cb270007a3eb62528ccf235fe555ff7d25eb0\",\"ea28679cda435668eac642f1dd2954d35b958fd489c7dae71a52c39559413546\",\"6f9abf4420118a5b01e3874a5613d34b9848eca3cfbfb578e83d08f55de34736\",\"7eb0e8acc11d30f7d65e9513a238248192f5ed14841730ad09a1f0a5f9d194f0\",\"e517d6c23218b391d04c53f00f3f654b197c94b8890c73fc6e27fb4baa115ea7\",\"42e826db39597e2468ad5899a9c4781857fbf2302d22314a1e56701ca76738a9\",\"6e5da49242680968b3a7f0595918024bbb8a29002592f0a66599964595c6ee09\",\"8f04235efe2ee7223446a2cf97666bae7f63f5b6371fdd319278aa1ccc14b480\",\"ad4ca443ddb8bacceee8fd6600d29d4c318b01e5c5e2542fb5b26e75dec93cca\",\"a1e580a705bf3a41ee9fbbbda4485cf45895640e9c07e2ae2230fcce62e30556\",\"1ef22bd3f017d7716fe77ca96d820cb34b5e59e57b722faf542b1a993ed26218\",\"d372cb650a5f515b102eaadb7212a4419bb4d872b9c30b1e39a649a4b39ab50c\",\"b35f6fc843c0224f47d26b3268f068591b3303096f82d38599025423b69c8bcd\",\"618918050808e06de64f3f3279ad9fd90f4727e95be305f66960bd496037985e\",\"eec0bed49a87e317db5f1e56df2d47962f2efdc904109b10a40ef2031c00f4b9\",\"de40eb6a3f0d6ad3e2c62e581f2db857f0c13d0bd92f0782246f2bf7c536119f\",\"17eaeb0749335cad2e1ac8486b3e2aed7df6043427774d84910f95c934f2bcbc\",\"52731c69011c3e29a55a77c53ebfe8c0bdde8411064a3a263d15080616e41e11\",\"ee3f99a999877dbb9d994a5ce52b17cffafa7b31c4582d21186838ffccf2810c\",\"5ed14998da5c627001efdb984fd9926a4482791076d201f3277c5eb4ae7d33d8\",\"b880f1225568c7a5aa1a78c7dc1b41a527c5e8e22aaa0bb4289af8b7b641a688\",\"f0338d682f55e03d9e38e859784269aac2bb38ab96672e2e811636897f26a7c5\",\"38ad77a24688cfd2d434c6aa9f54ff146e2fc8c66505af3e2f4e02b1f0bb650a\",\"4a6f50f0d88b7e8e1cfd693e35ba38717697347c491fa5bcd1926a16e793c304\",\"8db74114e6b54f875bd5706128545dbf3679e019c9b4d1cb346daa5a35fa7146\",\"bdc940d6abf4657220a481100d8b0ec19b937695c832f06da46d5b299935dffa\",\"5f51d2c28475678e658518b8734689027c1bc458039889e8fa15c15e17d78439\",\"1c71536a79fb7f64f0a84b9a8c0b8ef2ee94b756446a2b31e757cb323d453812\",\"6e96541badcb3bf12320383563d444ba5e371833627a7cb422f4424b8eb8deab\",\"e94c7ee671a1dfe8023f9c1b98b3654b4c4506d8e518832982012bec06177ccb\",\"5ee47e4fce55b4365cd16148efe51565b031ea4aacd9b057390990bd9d94323f\",\"bbb475060d3e6c113a48bf04bd9078df0f4685637de4383ee6bd5c18f40ff8b5\",\"2f9165af1d687c6de6555ba922797e3d9d84cd56102524adef9cc9cea8c83261\",\"84652a1dc576f354a517d1d643927564a253a284166b64fa023493aaba2870cb\",\"28ddf6f06d38c1be7044e95c740ae22a7bfe2853443b304c41fb2cbc14bed1e8\",\"ea7f339fe45a77c9b92de75356807b6deb1a9cb43a1073c1cf3ffdac2a136c70\",\"1b2c08d971110ced12bd71f8153a799135cf744ef9c01eb0843383a35d37b6df\",\"7134467a8b60c062f51c0c6f1cb7e42bbc3f55abe6bfed313cd35c47124b1117\",\"e4986c43a601c461bd58c213c02aac3f8d4ab31161d83837e06e10e1108b6a89\",\"2c4ef5cbc39ced199573c2833ec0a51a174522f9904a594073210cf9bcfcc2bf\",\"6b9b8f827ad2f3b4d0f7903ef920c4bd37ee422084df5cf14866bbce1e1c23b1\",\"37a50cfd0c061be374bb1b0a57910345e0ec08d13a853a4af60f3f07d21d6d09\",\"da01c36d4b50c169af563aeac869265a7ac0820df41a41585b7c4b639fb03579\",\"55a194a3d96298d23b17093e28ecec7ddf971a85dd196e15c2602b9b3d090098\",\"7f3c2bd1ffa4009d32eda1ad3b1b26a27e680feb33a0211f4f65cc9223ce3f54\",\"23aaae6a151b60b5d97aeee5f7636939a725a4e418e9c0e37841957b8ce31438\",\"34801da731646cb83f35d947b90bc515ce7f444f1c20231ee119588578fe20d9\",\"bf51b179dc48b9e6a1ea37d2c8239dc6243d1ba4bd1a8538ffc61dee7156e806\",\"b37431be99d12c4aa9e574663faa4870976541982495dff421a776e7b228a452\",\"c2bd66cf3c016155a2003f5f2b5d7ac41f95bb158e2f830983b16cda5ed055b6\",\"bc43b81b1fe7e5cf6e938571ae6a2fc953884cc65a26ed13433d3f35dd6fed19\",\"496c69af6a065fb71b9182ff5ccfee2ed75ed5cbd4a1de1efcd06c1fe27db401\",\"d0dff8c742991afd6f917c894974072dc662cf668f2449b0796f7ca3f3f10cc6\",\"51d6e530c291d1a3f19d27e47bfbed29e4d0a38ab29e9bcd172f02d19bb26140\",\"065ee64ece38deaa1667a4ce9bd71faaab5bfe00aa0b7a076f8d5cd0b2a7702e\",\"b724e5506fb0bdebe935d92bf7f8439207496236cd2453ff276acb4f4644e6a5\",\"2392de9e41c64fbd12033a461f882efdac270dec166f9c8b90258968935ed145\",\"e4d46c3b52cfdacfa9af5f428479a288e29477754d1ab60714ea038463f03288\",\"b834d0013a5c6ba9ec50a87c4b31e67d43b070bfd6e9a2c39ec72a90368eafe7\",\"48fc9719f2da86b974186bef68282ef61c55fb8c491ef0653bfac2203078484e\",\"08f5cbc0c418cc460c4d0fccdce36e4d65372c08ceb522c19135be02ca9ac216\",\"a335a1ebb2c4dad9e5e56410abccefc4caad95c4a6a650465b78933215a353b8\",\"d86b8f6bce7198381f96081ef838e83a6d31cde58a7276c53e073ce748e1f2e9\",\"94c4f78a23bc01e491225edb207605fc2704dc347e3220fd64e3b940260dad2d\",\"3d70901f878195026c5214ddc3ef411b67ebc5491a78923f356c9d25cb492199\",\"2f67d60ba76cfce2ba9af1837f48fdff41a105e622c2481d672b841816fe084a\",\"e697268f22a63742d6ecc00dbe385bf2edf27fc2ac70d901949b2958e89c0af0\",\"bd6147e2947fd76c025b066d38d63a2f159c0ba8e082e8339c2f249e3cc89590\",\"c02715d3710c3dc7ae88f1892ab5b2dcd61628b18e075d0ae0fb7095ef51e2fa\",\"86c979a2fbd39b1f0ce6baef8e56af4f674af3cf830ac5cbab4d6f21969b9a1c\",\"cf287ad037592d02b0fb7f8ac175335ba7d1fa8607681cae55f41cb1c2b065b8\",\"435ccc79d20fa8a3f640af7c9a4352ad07c3cbf22e195506114e37f10ff3bff1\",\"803b5ab2f9ee3c8cf8e81f1641d4bf872a078fa28b13e462ed6c28791906ed88\",\"187dc775384e210aa076ebc9f1844049bcc958b038a8e259e99ade25a657de0d\",\"e4a5f339ff81e2573e536d9ca38178bdfeb93e2a5413920b2db00bdaf689329d\",\"837383c48e2175599e6d0f6e4617e3f5224268ae088cebb1ca58a61223afe613\",\"532021413375e99b9852a867bb691c53b076608b2ab175bb06e9e54dc457337e\",\"4db4a140d21fa8969c743fc2e3fbf2e1520ffe6d93d5168768e2afea3df7a291\",\"2e8a48093816c5e1f7929bbc9061e151c1d43bf472dd2c9a9040d430f8c5d27a\",\"c15c09eea0586e74fb57c258a61d36a89161a9d99d02c7344208802e689cd32c\",\"2ec86a6e22a01e6cdb35170dfb1f027f5738a66ef1bccbe96b78d4826146388d\",\"44a1a7dc22e9b2633ac20123017d49c2016672f5e9ef726d6a91fae1304783ca\",\"623b6fac29a694c844f3b172092b88ba10827271dc5d515b26bec98fad012194\",\"d96e2ef4bf27982e3335cecd4e727c9e87d154c45349c9223044b6408baa3ca5\",\"d2226ee387b308db4cb070cb85c34d08290de0d095d0430d6e5c658b26b53802\",\"e3c003364db4a5c86f2a01a95a9720176c13e31d90712d38435e7e2914a59de4\",\"bd620b5f639cb908b0b08ab5ec645d0b1bbfe0f944fcd4a3783ce6401b585118\",\"979d1c3a35829af0e399f5562261b88f88b3d7ebf978bbf27d5005b0412118b9\",\"a4ae7253eef20a103d70c738aa051655c90c8a14233f2e779c278f6f6527369a\",\"2e90dd8b3c12f58c787a891bed8b5cb5f3d062b9555b150900ea00a2d912fb9f\",\"b1b149ca797c16e06779ad314c510dc9096289659ac350925b0debb87f5fb0b3\",\"70617fa847b771c3193ec67e05cdce31752a62f772e43f77255efe1beefdb73a\",\"2e7546cb9c25c1f6ac7a348f383dfce353ab188abcca73c2cc31e5ba07ac7f68\",\"11c4c5255019da8811408b3636602f2eed521322c84241de5c5a9f485cb4cae4\",\"6e0aaadc3c1a2e9f13e71817883f6f9f513afa2bd4529253be3f2aa28c9ec05c\",\"46318727513bec77e926e956b13f0502493847da954c3a98c2c054ce1a8731ff\",\"730751992703c24b02f82777f97eeae567fd2f824307b8ad1d7083c3ff5ba815\",\"be7bf3bdfabcd493ddd4b350c8871923e4754bf845fe553dc27adbaac871eca5\",\"d6a9a638c4e9ec727fc231a8b6edb1e6fefbcc68130596432fec30856cc3b6e4\",\"d4c95e1f27bf3fe84c7efea8ccae55e167d8c41a79fc4ddae3270d21012cfe03\",\"beea6e61069084d7276cbcfabe45e4af9449dcc72ceebb394d0ed0d843dbd596\",\"19a4120b9bd6684ddc76defb99f469982bee55666043d2450f3796ac53ffb18a\",\"715b4c3b0585c95b9df51e60068750c9c463fc9f68b2d20e8385ba08709751c3\",\"ccbcb84be0c6151dbfd375faee385866fa2a7703fa07abc1b576e990e4c04c12\",\"ef242e5e06433684d4f90725f94dfea1ca6aee328e85cccb582b225c99c5b9a7\",\"ebf1fcb136db1d9aae04c3efe5549048fcf1d5aa448e2188b966f53b8f03a605\",\"f73d5c6b01c980413d112a355f505a214a2d46524841fb8334ce4992f508e3d6\",\"84c2ffd9b80e63eafe28f19114fff87c526160154c407ed8acdce9346c59bf37\",\"7faaf2fae57d4e18bf4b85d10a06e04743c80433efa460918f246017f2d01436\",\"0479a45b06e047f0c2a977948352b254a6f9613c0c81eedee9278135e23fdd9d\",\"a2a570508733e87ffbb803bada82fffbbd1ac8e96c47bc00d16084c2cfd3fc8c\",\"a35f37ffc4698e45a8832a82f1c525ebda7438f8c8ae392557e4cc7393844160\",\"53ad552af4f0757f804b346b89b226dc89ae37d8df9776427fab089dab3a4f33\",\"0fe1572019ea00bbb83e3697003f673d86fe4996bfba2e78de33e1cd8b43912e\",\"1d5f7bae98d98f5c0951771421155a883a8d7c542e608fbecd1958f202bb2fb4\",\"1805ad1e3d69319daf06db55c04c5254d59e16cb155ab5aeea2b63cb0a1b1b49\",\"6f412880ded89bf86fb38d526401fb55ca00d73f4a33339dcb9f8ee4e8129bc1\",\"0f701ce151607c78000ca44bc18be133018a1c055d3e43393c21d4600c805b19\",\"2b59c618cdd6758734a8c2e1a3ba1a391f931cc02154cfb8200b4e5b94bd1c86\",\"68062285dccc41418caaf3688d9c12405a6bb2ce5e03e5e2fd5a72211fee6e14\",\"d52763ac3d308649e42f59c7f25fed83af8a96a1d104152ea087e88c955843a8\",\"c46b35c61449294fc788b043d6af12a6132f7ef891ea7d0d9d56c9f6a2596ed7\",\"eb2be740bea0de2bb0d85299f85745fb5dc9af745893499461e97371ceaeb931\",\"7d3f50f90bce6cacbd5fac7499c5e60107039ea1005c6fc0e8726f41a94e5f04\",\"6c1137e2956ada640ecbe83e71cd15c4a337ace384e3354d0381a8e580cbb3c4\",\"b9aff3922f4d228d9ba5b6fb12b81f079bb25a54134398fef5fbf67796720b4b\",\"6cb75b63e9bfe75322df7957ad1f20ef9865dba7ab0185741753b0b5a69af49f\",\"3e1f43d5fc09c9c2257dfde6d374733b92b617481c36f5f6a7ba1fda9e21e408\",\"5ead848a4c64ad5cc6db2fbc6d3be2e9c9b3dd611a98ebcb37ec27c850f406c1\",\"e9dc994c0d0c862726d81516bd9ae72b268fea2aff335b2f04dc84949f70afda\",\"a8cef911faae70a434a7c325bc9ff3d4129c7f009f8a5188244b5d626a4b56a6\",\"1fe77ba582629a21c87b0487be83fdaef04ce48e2b0bb72f16074c7c62361984\",\"ff1b6e173a861b0dae2ffa65e42115077664decab21e94038735419be4044c51\",\"28bb3c601443d953b3119d746b246c24b825489f40daa8828816ab9ee8838b2c\",\"b24d9e65433041c16670dbee3e075e1d0cfb15aa7a8bbb90ee561694cf41c7f4\",\"ba0358d8e75a4e1a50291e5d4a186662a06577fe638ea160669a04b8ac2a1e31\",\"ca0a056a29752dcc19fad36e6e517115602367cd1a57ec3dbeb74e30553bc1d1\",\"543f5c988e261694b0f938f7a1155593f9bd64c53ed2da4fcbec19b7c3f9e6ef\",\"90c368a042f8a44a6a8ae977b5271200b26af8520e71565cb5c7b100c5ec8742\",\"5e9432e406cbffa763375a1e665aad638018c3b0fd3014ac127dcd796a8ffb69\",\"71960ce52a63e4d8a8593f77f3df43a340ed4537043461cfb213e67b8c1874dc\",\"493cd63da3e3afeecd02d44a2b6c6724341e4c37439b1b1fb18eff27c024b560\",\"442f9833feb5eafefce847c3a5cb9b025d7d194819f27cfad379378ff9399d79\",\"eb430b0d2017422efaee4da307730338ce35ce63af70c924272bd0a803971656\",\"1391318453ea0b55260a3dcbdfeb1913df8ecbb340e4af4a69978dbe1046ad5c\",\"1fc6b7835945c3cd829ef1ba738839b0654e478d00294a0587ecff5ee4f1c520\",\"a6d4bbbd290056a8997434230c3b9307127a8ce009345f64b3921115b2b15ffb\",\"5d8261409360d27fbf6ae3a95fe8773bebc2975952d4844cd989ddb9f0eccda5\",\"88bdde6de1016de65dccc599f74305e3461a0075c62b5935b84b0dcb839a6df9\",\"533ab7ec0623447e9c2e942127150c48638d098fb7179f79b109c789a8fb667d\",\"087667a59109a7877360f6337289024e18b5704abd1a9bfe154dab8b8a364829\",\"2b217480ade3c959444a228cc00907a346d005dbe14fcb65c359a453c2da3a95\",\"dd7c7bffc3116b7a71f45e63d0fca68bdc9bab11bbde2d29f43c5cb2054b84c8\",\"8593b9d70cff187c63176534dc505076f7a3c9421311118c57f2820b6bc8ca85\",\"62288e18c5387568c4ad65897f17315ef1dd4a50e86e45b540130d163a55f877\",\"394b198d355f9d43742976a8240b399d06c557832eda549921196e69765d23c2\",\"bcc5769c28636b176da5e25bacb2ca74aac26b8218eb23f75834b4dd1daaceda\",\"ca892f22dfd6fbf43d81d0e4723119c9ba123224e0ef08d94acae5d6bb43e38f\",\"741e7d8d68212703380801a08e1f8b70e73523959b5770c1cb7979a2a49d0de7\",\"bf7e8edeb9ee309a0778c3e0cec2c5cded16fccf06cfb3b91f1d5805d0e014bd\",\"9ceae561ea5ea4fd44d42833942c6eb88b46a85ba07f9506bb51622fecb595cf\",\"3b2e2d42ab6ad59d4a45dae8a0a4bc549550384a855c4ff5ec5f65a859100e01\",\"ec99c64c9ae6ebc116707562d8f6e884bbbae9087c0473b9496c8796a05a0906\",\"4ceccb91c43f85a4bbf9d48fa73f326765cda4c77a983e6dee8ef375920f9a83\",\"f00f8b210f7c8ef83b61ec68e02d455980af0245352f2a80d362c7d6528c32f9\",\"eac5393a1f6db7c7c75eae6a9357d0d77d15f3657b394e2ec9024f7f686bf59d\",\"3cb86d465b907e8d556892386854a7e74d2837c13e20ff0ea8847ffae67b0b4f\",\"d24bfe25dd7700161dbcdff90361b1f247715661d3426061eb2e85d5bfd6d485\",\"d5a4bb5bde290636873b7f71df1b092d74ebe59ab401947d39536607d35a7f2b\",\"bd9a4f1e8e0f3e1f0ec347e5acf0a8700ffa5ff70dbfa30a05b1a171efbbe1e8\",\"44d45a91475ecf64de68581eb06e1dd856e2e2c65ff0c63c7b5fe8e7d2f41837\",\"f241d9d9658ae41aebce805d904acfd5fd75387f5df9952f039ab7fae522d071\",\"094330eaeeac03ef29493fb5c6e97d2bac56ac66040b75ae3084b4e32bbd3653\",\"fb7ddc26052365fd69769dda480f63e9bc79c7eb4f93e3356d35386208e32fdd\",\"7a9ff4438dddcd08ada47f98bb11bdce86bd43f430971c70a884ee829e690247\",\"072ca52b1fdd4f184bcc05916e5bd2f361861db347f1f60bd1b1dc9a58a587c7\",\"d4b55b5c908a15b5fb9f9d04dae81f8a7cda3f6798c3f4e228ca21a3aa5c3b7b\",\"e80d8132b28410d9b4f5552a8ac56f0a9aa31802f1bba3592549ab1377ee2dab\",\"03df7af006e6863d315b6455e56dfe9b9cd6a66b38759abe05036786ec49a4e2\",\"b520877e03bb4dc892264337f40fae17bee907bc27841461798aaf3df19af714\",\"493fb7b51093c47366c555421d44349ebc50ea530cf34c54c6d1edb5ab4dd2f4\",\"81e8dbdab3c37cdaf054a909ae0470b74f44f75a4db68495ba7ffc80e01cccae\",\"ebff91d215a245c4e46d2da02a2bd5e7524e0eea9fceb6ebe2493f6ee824ca77\",\"27bd945bff6bc54835f24afce5bafb8e8c2f45746e3f7fe27b611dcffe320d54\",\"1907f2a244e5d18a3f9a1b5a5c78234c0f92d4aabe1fd246bb9022ef5d1b6f9b\",\"75823bb905a77662dd24519a31a65117120b8419a4c8ba149dc405ad166f6765\",\"c108a970b156f6525b54e712b3759c8034e5595c9f825d91abc77ae70a4b2cd5\",\"afa628f0286a84d6846113a959c916038dbc8a3631c787531820b19c728e4a2b\",\"96dc0f196ec9855763a4385990d0d4cf09bdd2d9dec9295a135487c8571b1950\",\"50631d21aa8b968fc589ffd23e62e6fae86f33c33846516017a8eb13fa643844\",\"8f864e2d26ae857552ad0d1258ad797ff64eb07698ae349ded060ee72ab9976d\",\"3cf0f0ebbaa5096aabdb5e9737aa3e45c642df5b4ae97184db80b3ec2fa677f3\",\"09f0ffdd3128948741e64431486c0a9dd9d42df087984202ea4a2cfc47bced2a\",\"e609133662774966b801b1ae9f2914228af15766c7015f4ce011741db9d62e35\",\"75497fa0abcb74157db207779e1be130a7d3c58c153f8f3c3666a8b75b860617\",\"82135d25ff5a96f00bf5569124bcdc2970a4b433129048d8a480087664a0bd51\",\"3bd172fe4de954051581c5f4d4dd35105ed8c94f9d4dfce13c5fb99e7027b441\",\"cf2a2a282ca8250b7682746af205b915af6564275c176e4e4e1322e9a3f4f985\",\"002cb966815601c8e756bcdcfae2f4d97016d6af30c76212d7f5f0a224d7a808\",\"ca1ec4b360a29b6f9043491c9ab785eb780dba2fd845706c544f978392922a58\",\"f30e9a5454ce9d6782a874784d7b41c1eeb345e1e2da9fc01143baf8cd82471f\",\"5f1913c9ad7c2f8334230615d17b412228b6ffbb1d26256f7ea2934362c6a76e\",\"c0094dbb6b1fdcefbff23a724b902c334ab6adcd8175e70cf26420c52b1b549f\",\"9ea0e6490422b59f4f96a550042321888e50bc0bfa19ef9f5af8bb15705f8320\",\"b3bbdde88905c69d220189b42a8764fc986f65d70bcf39b971f34bd30d4996c5\",\"200b03244d0da9652693bbba06ab0cd2ed71c9e54cbaa0aff7fb3a8b4a62c223\",\"f28dcdaab95d0896f79b4787ae509f8fd2e5d57cd60f832e41c26ee28ecdf5d2\",\"5c2091f39b8d87539d3438f6611f1ba21864e05791d428d33a651fd11d12d0da\",\"e2889983a3817d85144bc50019f8d2edee0a8e6f8d4a21072b8248509dc297f1\",\"69b1ab2944388ece5185a8781ea7fb3c23d924bb61ad0ddac61285fc49dd1c99\",\"a0066dcca047ef46f23ffa631a21be5fe74b5b85f99f581eff8445e81896de05\",\"a42c3b7217e20f6fd706133dca1ca628a33d2f3ff67c486099c1b1cdb93d219e\",\"9e3cdb0d0b2faafe1f0b49e7f7128a0ed2e42e682bb197e6308398030e3e0fe0\",\"a8fa954e05ab24e665501aae9bee40d2058c4170d7bc6f1a4e5f9717be1954e1\",\"8c6d51f4db62a5217901ec87f87ac6efb46b0b6c6e2527f9f29d33c90cd6219c\",\"d18eec15d2f10a635655df8cb688e286f683f9251c879ec326e4d5fd6164ce01\",\"b533ca1c808d331ec7f2cba29c146f05879990b944d466fdee5f4303cffa7399\",\"833508c7c1419e9450c7c993c46dcbe82ec2e67b3003060392c5ec7fa815834d\",\"20fc2a3b97872c176f28a4b2d18fe7ab69ffa6f9b8f4cf3fafa764c8c5f66f47\",\"2be07e095fd99193a4784fd6a6225d8b56de4b2e474325bf41e0e6e6da39dd02\",\"834fdae8b65e21900a09edd4d173702f39f162a4eb28a7ba462f923acc9a6de7\",\"fa49f50b5c95d4139e9abafc22005fb04c5137e01c41a9e32b391de9537edec5\",\"bf117817b10ad03199c128627b651c7e8ee6fbfb30e5626bb3e8b51c2cc5be1f\",\"722cd5de96109090ec6f924fc011ac8efd26e73b3f96af9e5f60724b8c81aadc\",\"94dcb3cf36166b0ef17bdb5926fb6d3b8d23c28f65221766f550336ba62d7c7d\",\"d021b0ea3d4ec9d025202ed3a5f3a82d18c6c4c351dc84c25ce2382fca8b5b42\",\"869dac229e91fca898f8ef9bf6468661be1127872643a13e3b11b3d6b1be1f22\",\"0d67c4a07c5930fbf336b5e4631a6b3689f8e2d139d3c8eb744c349b32ec8015\",\"3a76176776dd747539fd6a222afcee90af1adde9cdc7a704376614035d3cba57\",\"40c64fca6b63b389e6565c173de9ae4cfe773a4d32b651fac88492da10acca7d\",\"fffea76ddf665062ffa7cdedebf6ece109d11d19d546d630473213173cbd994c\",\"403b79f9a5af55c5fdd44d9ec56f848199a7da112211dd9bfc64d71ca5f9dfba\",\"214ffef365413e2bd66de5cee1ad3d5c88b227b41e023593d0beda058b8eccaa\",\"27cf1e7fe428a649be30a89176184c6be90f24b64d188147016641d725d060ae\",\"23686209094803e449cd581b30884600857a6a851f1c25515163b0c20369c82c\",\"708cfaee9f4588f2ad780efdf482c12e47ffb45a215fce242d910481a57b3fc1\",\"4b87c763fa4eeaaa25b022ce97ad94d53fbce4dbacb77e8b3ac5c4d4cc710da6\",\"5b01320acd0fd23d47ff4f43b42d8a5099bb6bb3be7ec30d179d93bb608118e8\",\"3e4e9716898fb14636b032c381f92fd6b11b74246096e01bd2b09c66c3d80873\",\"89dd1cc93b370a0bf46fcb127aac25025bde982a9e39334a042b71171dc3393b\",\"fc6f014b3198ba5c8a66797e96ac3536474a4ae18b9233337e069a87b449ea33\",\"d72f249a1071f3c5247cf025c8e81d36bf6a5dc5b7560e465919a48c87033991\",\"f2c9357d418e3fe66ffb954ced39ab59b98f62177c1e8d6e81e5fcadb68a30b5\",\"831a701ab8261e6aefa36f1acc55a86d1105aaf6d42cdaa0e0bb8aab5672aeff\",\"88acf10d6d533bf5fcff1e5c7154819e2fa371331c9d1df0c73294b30547d2d4\",\"cd4a7f9766e75b38ad8ba109f6e4ec760805b054071158890d8ebfa6a61618e3\",\"82e8269bb423e326dd765c198038377032d01eaef892cd72a882cf71bf605cca\",\"226fa7df31942e92d01c5df5109220fddb6ecc0de9e82d0121d45e119109c99f\",\"695d791f047dcad5062e939a0cf8c8f0a3be31bad259665d6eb9bd33f0df9567\",\"dea95485492eacf05767dc0258e342bffc1f1bdfd5fa38d229653c2aa6e933f6\",\"24d3ad54109e005848f00130a8224a856b798c88323999ff56a57d6a1236d052\",\"f21b5677b23b43c36e6d772bd2b1710a9cfdf2ca1de47e223a748d3ea116a6bc\",\"69aa27566dddfc2987b8b8984972dfa1087149113480b49750f23012f127216e\",\"bdc681b424b3fc927613bc8eebaae45b8295f1b238a1ae15362e42ef7a398b55\",\"6ca74dbf25b84672850dbd4407e1fd8fb5ffe4c16b6c74d30c8667ab89c7f617\",\"32e36fd43b69b9e835c3c9039d4b123c34ebe33c43f935a30f54084c084cd6d6\",\"ff6a96566202125c46923efa5d565d9fc182e1e27848961e1f5220c98438579a\",\"612e2e7d2d1de425b921ab14cdd8e1fb87c821e7eb29cb31f2b93f7557aa5601\",\"26cd6bb7c3f59138577a05a337266e192fe96c01ce2610fec9a8d60d4bf8c469\",\"cded4c9704852cd8a44cf4b5895953320c1061f204bf12aacc6ec2cc1be129c9\",\"ef79167065d2439c4b4f37bcb13c6b91ea98bfbf35ab1a3bdbf35e3c8e8dc457\",\"246cc0614a51e230d509e1b41318bb431969ffce16eef1f6d22d417a0ddc79cf\",\"9390126f0adde58b8aa77d36e322ed3e5406415d78529dfbfeddc57155d7bd4a\",\"92e46f3e5bfdd35906f735349b4d7bded050bdbd3464475b18a25c0d806164e2\",\"7337866461a4e64e23cc6312a99adda471cc2260bc66c312052395c4fa59a6dc\",\"232e7faf90f057f9811c67d1c4bb9bc77edb24560223b3913264bcf591e37cf6\",\"6b796b424bcacfa4c6ea10659bbc12fecb13b77b403fbc37e7b0455e6ffe1554\",\"aeb3efc999f3ef840e723649080ef19ffa02630322300a4cc52ca087e40091f0\",\"fa72865f46465bd5d6d7298da56320d1bc57522943e7a7b72c87421c2dfcd24e\",\"b2fd333bfa3c6b2f2b2868e094b350c7ccda6169665c72de03da21e40c8365ec\",\"ff0ceee2d31ae389d2c7f33fac8f7f21f3640be91c67f08d1433b2f716d09b30\",\"9d64329ed679ef6b7fea62f1f407f65f94e5c037108bf6d21ab4c965e669fbc8\",\"1f2ece006f7dc03380a9eeba77f473ff46506202e0025668e3a4e8811e7fa74f\",\"f855978fde801bb57f44f6f81b6fe1520cf6496b5a41d9e1cd71851c8a322d01\",\"df7289dd651e9a078bd2c4a929d8e6873e415b8d5b379cb081a8c5f12611ed04\",\"b80e79f72b65248f21b8bfcf8c0747d9573c808860f469d18a7f065c35d9f159\",\"21b4587456d611ac422ee9d84dfc7f98486ec502f5f327d454ba964b69b5784e\",\"316403d851d3b4ae9e34336c11d480194f65780a5e66c275d7407cac26be6726\",\"11f6726bae3b1c1012cd6561cfa25bcc8494b3d23057c62cbb3efa9e824aa230\",\"3daea8643d8bf8271713572269d90255242bce8eee42a7e4948273b6dc8ede4d\",\"c99e81befce8a3cd6b21fdb8bf94c7d85959899842723ca35a4bdff0e0443d19\",\"02b78c212a9ef93a1847e970622cce0e9c9bcbe46d6c7ef9152b24e89568d360\",\"c85f93e52e2a8a27232e8e1f5e390afd20f6cb95fbc8a4bc7b143e4d87ed3be7\",\"0118640ed7f284b7d95e0b5e565cdb9b54959bd2a9b11ab6bf41124bcb441989\",\"637f8bca2e2d3338818b0662fa301d4c9d25e19df76af81ae173a4cc0d83b108\",\"91dfdb2e4c30777c69a5481da1c421661c92f6e8b4aba38919d4e7fca9980076\",\"a295b1012b4bdc65c0f110e596fc932f0acf62adce4eed4e346087c41cc8cdd7\",\"02edf7faa56aa536b0222eff9187da45f6aa6e07a970e047bbec32d637b45c61\",\"59c803777ea6758305e4f6ff4533d69d1d5992be6fe2c993db64fb1c6248de8d\",\"80807a7f64d3030dbd98fafff1350eb13c84cbb75a0599c05941cb8d8ceb4cc0\",\"eeb7a1d76baeb290440d7f4a6aa99276af52e6218b1f1fb3c18a0776aea7879f\",\"6e88952f82af6cc74b1a9380979126c6b821fb466a03f22e773c02008b7b1bb2\",\"f4428a4e010bbcca6f128109f3ae4c131275a51d914a14008b39e6c856d0de4b\",\"724b38e72889536d79799499f8e85080059e4fbdd0690c8a0d7bcf1f29d1e6e6\",\"724a58c3025377b238ff6a86efd91ad54bc835459bb2330488ff8d9374c5377e\",\"37f5ac68640bd3d3ea53233d2e70b2019b9bfc1dadc1d955701fab8434c0bee3\",\"4c4f1e9f23daba238cd387733ebbb4ba66575c210423d9ac0732fb4c14074846\",\"2539039a586a32873bc3f49f0344b990b56e4f426c94e4b9792f5f5c29ac0331\",\"b8154858b0eb41839bb5a54f6aa3eba23725018ea8e37df23a1dd7cd1f279b44\",\"f34f12170e2668d275860dd9c60ac3b8ff4bc986df5b2507678ef4ae1e97e502\",\"78a23b6ce0cae68a994eb2d769f077e186dd78d2c577cefa2bd9b7f9177d695c\",\"b69aafdbd9028420fcde1f8b549cf8df44b575ce54540fa1608e2d7008308ac8\",\"465c9cf883ba3597ee5ff65c865fe06da1a8b4b5cfdc922d6c3dad760d5f4a44\",\"da1be733ee4f03bb3f312d53dbd74df934a8bea61b07d0ce36f0f2e6ef6d98d1\",\"8062f75b1f5964e4f2d59f5e0227767bf6eb9afad90694f2d03ff3adc4d6a992\",\"252c6a309261a02a50ea10a8b2d0ca36394f31a4fcb4263c22797fd917836bdd\",\"157ae5e9d4f52a1856d1f069cc8a21e632da791e81e8045240ff87066afba04f\",\"a70856f5c1ff858ec8e221b7868fcab125685800ef4d234e7ea0531d36431705\",\"4073d53b8a7b09519cc8d1eb207c4f9fc224fe39c5f1d490f94da466a12cdec5\",\"01db6f9fb2a2c69e74ea7d380fcae18ec57b3cb8fa2b9cc76277202039e1a7b1\",\"fb5b00478bb47ac16958ed7a37d5a51e1ce9f9f02db3e174e4f627ed113581d9\",\"4e279d4a444f2e4aba4bf86bc8558883fb8422b2f82c7ab96818ba66cc15b761\",\"80d7ce3340c990d1cd0194a3e2b262a36bbbd8b10621be97884fcd0ceeab25ac\",\"5bee02003eeafb74336514361a698da3259113a2719a6759016df0dfb6378a32\",\"9bc3521be2259ed444926cf91be8aed65898e1ab2ba8c690484fd934bfb2c193\",\"4ffb73803e23572b66cb8f17e3bf6c06da5e875e842b963a66cd06ed33909ab1\",\"48acdaed71400413ccb08757bde9d5b2c5abcf2f8ef3d6dd4d994d2615fbf11d\",\"ad736bac65d4a3074fba5af77418e91ac37e3fcfd414c874bfd5d5c14a970df5\",\"80fa686d0aad726bbc6b4141d40f44512915541356f4c9f118d1063e60ac2c4f\",\"a9ae7164336bb0a43d074821b68197e873b36bc46d0dee39cb4f640679f0786d\",\"7f034b570d2f377424de7efc6a3d1ab2615179f3c21dd2ee1bdc7c7f827ef73d\",\"5250aba569d7d82943842be590c69983389dc536b24a9e40b90f9e0e553c508a\",\"80c10037b943335142bed6ba63f8728796fcf260dde45d8e6db9bc4abe733382\",\"905893b2e4efeadd7e658d8ab4f13d95b5976b729ab6a1c0cd2bb0aacb5916b3\",\"6b66358309cac11d9d0477f9e1ecd4f6520a76464ff9929a23f21bf02fe9f5ee\",\"0515e40ad7f3aa30e6cb9f23392ae82f9e390116281fc3bd226383b7abb2f4f4\",\"357e3572abcbbc1b96818513ebd5ba96e037f74ebaa028a6ca11c1e3a2336c2f\",\"ef7b471c339f26bee0170ebcbbec178197951dd112b9b70d2c73702c76901d72\",\"9d86dd93817395806d6983141ccd1681d89d3ffd57865645213333ffa5d95271\",\"8f1f6f3b924fefe9998ae5bd2cc6c401515c8d71317871d882962d881deb05d6\",\"1023e37e8a91b69c54af606038cf2bfe5518a93947342c7b955467f4b5674f6b\",\"8e3236d699de36c712b770ae88fce01cba583b5688d0a5cdfd0edf2e9f45f6dc\",\"0097db43f1bec20803801938874802e89f224ea1676619815071fdd6e6ab32ef\",\"c122e2e15ce5f0d31eb63b55196e4d7b0b94200c5cf9bdb4d87379f53fd2d494\",\"78ebffd719b2d72d0c8c7faedb745a617903801c0e223172692ab081033b36ff\",\"7e1f9323e5413d8198d3fed0676d05cf07de083768c8564478a4fb435e79793a\",\"84d48520f264191782627b6a415f0434978abd1dbfdc945a27f5a98121b597b8\",\"7669f9e9ad16dc63849fbd1ee6e202efc641b3689954ecb5053a2ca3861149ec\",\"c9e08d30284de9a3f61a64d082a1c09c2218b7ceadb30037e0623baacea65891\",\"361a0d8229ea5aae24fc9107753e8adcc83c0472ed037ff527a5e6361876f7a7\",\"64871b8ba9150129beb942c51c533b9e2b27badc67774509b2d68fa2ff073dcb\",\"a005ea7d50d808165b5c306ebcaa682edb47e1718368a1bd9bc51bb76fec1935\",\"605e18e2a20a1654d22e7db94f7367072447c1928abd7f1942f3f27bd6ee4fef\",\"c7ccd9d2dc95488d568b9645fc8f7a70d754aedeff07674720063aece742b484\",\"23df3dd5c5f3058152ac42791f2e90cad7ac9794aba99258710b7ff07c446dd5\",\"e4e59c09d8a8bf85c2468cc20db9dfed2772f1385c2b6ab03fbc285c22ac4474\",\"501a7ac4e40e9b156a59f28a01286dbc940b6a5fac643775c5a77cbe916c4450\",\"e9085470aa21a960cca6c08299a9e343822a714c3dcb7c4fd9eafedbd95a1d2e\",\"561dd80a835de8f402f200b26dd046383669eaacd53e2028bad1d4b44dc2501d\",\"52e70fdfe9b43381137bf208a741721f5fad480e9ef5eb6cd702d2d48c2894ee\",\"236aa33ed3ce0542a6704d662d36f7423e1b40af6254a8a2e03d1aa5fe8f0002\",\"8c7e52afb91beca048e5bcf9271e5d2641171d32f4aa57d88b01506c30f79a28\",\"37c0889982c7c719a7a536a7ec60d60d03c9525b721a3e73fd0bd70a0a54a64a\",\"3389f58e3a1beaa7fe0772d9f27900edac12f12892fd5013eb3a375fdcba740b\",\"a7ca80c6b8334e401b07654ca8022c83f02c53a9e4953cdad2ce982a5507f9d3\",\"cae52c768169c6b01c64ced7265563a7c0401fbf2718eb73d56124186a5fef17\",\"fc9e4a975af5707894d2d113d3e775f98588577806149f7790fe56947b503089\",\"f27d689a6eb75228190b063022f38f0f135ba6547df67b37df69ff918080db8b\",\"46e1fc111bf8a8dae7908381aed93bef164fe29edc296573b48c8f601ba2a08a\",\"f6dc377b2763528add88e4f31ec56074930f02c64734135b9df83dabfac58f03\",\"b42669d7f517dac84f306ec2472831708719cbdb298448961e3ae7862b997ebb\",\"59ec6ad2dac4d191c15ebef784cf33a7d3a636c04cfac78ca7c11bd34a75da62\",\"ab250058437ef4c8926dd74138687f04d02b9b01fcca834b6ed9efc21fa0c73b\",\"37a147df0f350f34ffcebabcb17d4039e8d722bfc5cadd8c8b2f75544d7cae29\",\"d1331a3133a11742ea7bf400dddad2a9c890664dcfb4af6e5b1546e9d5531e60\",\"ec742dd515da7049a799c9b998a59665eac2177ddac169ff99881a797b3eff1d\",\"bcc3af530a3f03470bff9e1b02aba30943e5a5bd7bf52b127c72499d718f4d2b\",\"39c8eda0b1891c469863fa5c716e0bcff727ccf83a41d1bccc1654b6a0af5b76\",\"6988437304a0ff6244dcbaf7aebe5f94dcc615e2b84f4957bade45d225c17e1e\",\"8f438d98bf6141fccf3128bf813378ae8bfa108da7e8d47178cde9a684abf8c3\",\"ca976f3e9f7a217634bfb12443b156d989449bad2b42c4f497d10b2c020f5f59\",\"fbb29a23163e8f7daacfde5d1a88b5e23ccbfb44b985fdfa747aaad95edc8a3a\",\"d134396356b2eeef268d335e29056860a694bc960b71017b5a0e20a623e1584e\",\"ceb4c41c9a8945384f879df47be380f1efe833841710e621ebafe8fde9c3413e\",\"73ae65871c9bf6b80bea3acabbaed639e8b7ed28cab11d14483f4c1535e804ad\",\"ee13584498dca9ac0d7ab07c81e34e657fb53cbc09af0c4b6c00d6c13be84ccc\",\"5954e686e45efba92782da88a39b2a78a4f2cd397d5292a9b07d388ed1856f6f\",\"b23714a029701d8a6b397861d291bd3971a3425f395304913def7c4a97b57259\",\"1ac3c767e9dc4687919bd10774c1b4c584f91117fae41b206132bc55aa7bafad\",\"825392e801191aae32443143f233e9d122b3b95ae291ba70020d68c0c5bc2399\",\"309b66028e31b87a13ee928b0fcb880ea6f9204734177c3abe3dfdbf4783aa55\",\"8047cd5165b2c18d6e4a93aa6a412afed5631c205e6c9d3ec580368f1da3c5d2\",\"a39ecc3aa0b7359389a980687453a7a3615db02eb1fced1899ca569060a83d81\",\"ec22f0c56d9a54f46040f9b559f5bfc0117b2db8bf8eb818bc856c6cdf3910b2\",\"549e899dc87c50122eb2f074a2c90f59a80794f5d599969e072f48ba2da2b6ed\",\"17e44e01b06fdc111df58f6fc6bad31a4a9ac0887499d071ab8623de5efee6f3\",\"cbc2118b589a7836fa90688a4ed90b7f060ba9d5e99451580fe2f0d180e2bbc8\",\"6a76e4fdd745922287ee3e9409e78fe4d22ce3bfad2d10738d9ed798a93de1ac\",\"cb97f6f78e35f9f829a2c509bcb1d5114f0fcf8d51049d97304c42e41f9eee94\",\"8d4225f4161600e315037270c129b8f47d9e15f82bb406af3659d9388c14e564\",\"265f45da1bc9556b0dbfe4c7713919aff4a03afa390cb39b9972f7941e751b86\",\"e23affe87a4baeebab2236e63f45bc0587852d79146a86c699136477a4e8645e\",\"7d9dca4237922e43b5c14059f9e58300110b494972f91694b07a448eed0bcc53\",\"94f61b3740f3ccefca7451c5b7f934fc76fe6c0bd0f5911e17ca8ead626ec5d0\",\"b864a3d4b56ab4abb002731fd74f0839409e8780e29dab068b76e70bd032063d\",\"617848083a58968492689120f96906452dec39eeef0979aa57f225a0a9a07112\",\"021688d8758572d251960b86f4ce9dbf015caa9f4f563ebb8e252ce48c4de597\",\"88545388904f23a9f8d587a2696179e6e6c92c8d80bd8d25c5d63379c1ddc09b\",\"6904a7c0c875ad0d2f242cef8ab669b6297564291843c5dccd17fa5036f12c11\",\"03fe03786721571f5e7631afec0865f141e6f7767a17980a232050db640e64b8\",\"90f109d477d657ed2e63bc33e33fd9e555e096a4a31373ad0ee183bb0a90f15e\",\"be6fc07515a672f89dd8d7511451b694b34713c9ad8a6dbe427172629496d415\",\"eff324022de5ab97b460da5931eacdd53b2e91af47e51e6eee21d55deb83074b\",\"b52559a2afc917d4c041363244c83907398468ae111cdf4fad1c8067111453ff\",\"11be2244380c4c61773fe055c66b3c7ddf61501a9aeaf89e5a52b250893af7e2\",\"768eca88e9110f697835c9d28d9103d811c687a1e401deead40e5f8d96612f6a\",\"280e05a1b0ced6848f7d4e4cdcdb525f648d5bd9dcd265ab838d6f55f2ff3fbd\",\"eebf00c957584e7ee65c6920d7fb8d0a471c4b69fa3c3d78052fb2f4c45a56cd\",\"8121e7ccfc1ce2c5f3cf68b7ba8952c131c61468f7c8b2e77196f6a02757d5ba\",\"a30ddf72b23cf67a95cbd8063ee5a91d71caf79268ea34723e7044e03e1b2fe9\",\"fe4edb0dc10f46d3dcf79b1accad698ac9cf3022b0d4f3739b6b7c4346065187\",\"dee512839086f4ca177c1af2c19952073c2887ec5e863360f1a1ded03b8d4747\",\"cbd57b5a5a47251aa9ae0f0bc6b64ad3e2684c0662b16c748ad2699807acac7b\",\"6c1ba11a7a7beae2d8f3a0fee3d7c8986c791d44435c4bb9a1d2906103fa1daa\",\"c07b945cac4036cccb23aa83924f11e3727755a8ae1fe334e40e8ede131f2c65\",\"da324ca0ff9154e6641606b3e711f73241c4c8c4291f15a9941ad8b9ce91959c\",\"17c58e26465e6efd3eeb09cceff495bbeb4a4fe6fc91294f497e96f14127d897\",\"f64184291944ae6ad096fcebb1e057f1e3a3b8dfc1dd7188c7402cb8e6820f56\",\"cadd6c7426f0178cdeed8797747a4f2dd4be3c2eeebdd41db201e6fa5bb65320\",\"46a958c0c81bb382ec26be157d6eb8d6cf0beee9f7ffac3e754b74ca9be494b8\",\"f2fd5510f40d4f83869a8ee2594e675fc78910b1016d231c1eb1f43ad36e7c91\",\"16f7d9e6bbc94b364110839ebc5aa6aa37e71a27c0efd9d983bf95d9aada8660\",\"2a31a47d1dd511ce7c841fe93bf3dbc18c86c61c500953c7e72c638a35c054b9\",\"6bfd922ac7996d984c5ff44a09eff8347ba0975a38cde5168f8e4be54c87b9b7\",\"dfb46db554d85b024d1a0af873a99a6ae509366ffec648cdb00d7d6302e28198\",\"4f320ee10bd32fcb047be45a1d196c954201ec9c64a1d778aed799e39c7b585d\",\"d339631da2fa1552bacb4c1ef5818fdde65592e29a9b235d50e0f646a67ef75c\",\"cfeb633e123501ec9797542cbc46e8734f07567675036f811041bd86ee4c4bec\",\"3a371a983bf82e7629ab34bfb7a79b965c6668c8cce5393dca9cd836a45cd45b\",\"b0802fafe35d85719e6799af276d267ad3818b25d533fe607ae0b895d2faf865\",\"144ca19253fda50a229fd228a7346733db8270bfca53008dad6009126273661f\",\"5c8676d3b941a0a0866d68965e4a92dc02865686e4fe2b21a82ac3d31be64229\",\"c8397224da56e1036c2adc5a81155b2c7b3490e45acaac43e83a65b51d1a831d\",\"ae7f9db382092ea8de8d1cc30a67123b2b92a768a17b36360ef8b65a8f6cb704\",\"8ad6c0fd9083a7d71636f05f62cca640dec24849f50c86df115155cde9f003b9\",\"60820418b25e8f2b3c23e77a7ecadee0f84c5ee69f753e47176dab87b9eab462\",\"c8c8b758129b3319b41635eb6cabea7a94402c4ef1193cbebb6faee3b6ce0172\",\"d1cdf9c43b07f8ded91595fe17cea091dfe4eb858300678aff8cfed66070e5ee\",\"c3ceeaf092e7909fd6dc73bf574e95da893dadb531dfb2f54d07997e64be5ecb\",\"a0077e213bcddb389a4264e49db8121f4aed48b9863619f3441e009c08aacb84\",\"c3f1718a701b5d480694bf8255d35dc662ca23a1b7b2a627347695fd518cc71f\",\"23edb3db96400b1fa473fed925bc0cebcb5887a8478c2d38179c9d310c747909\",\"ffd7261a5fe4180ac9a90610321bf304c45ce023c83df9f32162989459dfd7a2\",\"84cd453a922d149b6650a08187f04a9d397c11c211e14d033d8324217db07c36\",\"e78dfd34416b74bd4a599e58d795db8e5ec629f2f7f311fea9c52d9ab6b1398e\",\"f249b7c91caa070d62d3bd9387edfe65f99f1b391a0292bd2e08df3f25ad0049\",\"691021e66f65e43932df021050efe2b8810b36f7ab5e5fafd0b08636222b7d3f\",\"563144b711bf0c8b33a5c9a4e45d3a7bb339f9b31b2436d806fc8eb14a790326\",\"68cf88962171ca6e017181940adf9d717d0fe6e0c4525b96f5035dc62890a1a7\",\"c700905b5db22e658328fdb65fb8e1437003f866a7c47e688120554782d48eb8\",\"9389918def7799855467d7d7fe36700ce0aa148182618e54cfe772cfccde24fb\",\"e1bb3783105a4077ac13ead1a6238f5a069562f410d4b947c2064c60ad8985c9\",\"0775003c143e27242394a68a8db034921187ce730fdd7fb10954f1b058b3c1ec\",\"195be0967a2b1a24d29500d4a44de2456aa900f0fdb92e50e3856acc59f2eb13\",\"95b4d3fe841f4fd74e7f46e2f70e6eb54c66ddc811412ee7e4b8d63194826b40\",\"df463f1ec0b71627803884d71d510df5fd76919eea097a2bf3b1303c0ae64254\",\"182e1e88444b4048afdf74aeb5562ebe87a4212387d26173c0fed9a9015fb895\",\"d13ed141f94c78651767c8a3a4d291fd4b7a0443b2767a7d0f676f8460f6103a\",\"d0e6730b81c737c4dc091f9a4432592a22f843290c8c0a5b71a2fae302c37f45\",\"b8a23bd8ef48d9438b83f2f8ac6cc2a925e36448ecefc5e462c61598f4e86d79\",\"e5c39810ebe72ff7eb1eed20f9b7d2b8eee330075b6109eea2c2955559956758\",\"aa0d1edf09f288270abde38d6b4f1a1ce65a3d89c439afb76f12644b8a59186f\",\"a993c79a278685609d50093ac0081a3e2f20dc2f9cf43eda2f96ded2bee8fcdb\",\"c267573ca260cbbb26ad20bff9da736c617a561e25802c8607bf586ee7aee0e1\",\"0dc6703d972fc1f92a148d8b74eee75b7c13d1127dfe8c2aa740f23f289b69dc\",\"a758c186554aab20ac47a830d2365a1e6a70a370cd061b331a1e7f47db8ec5b5\",\"59e9f0f5d848a1263e2bc6e08458e1b9726ba281c1ef4f5c716678e50f6b5f53\",\"beeb4a078cd5724335497f432c09e02a400a4d91ddeeab1622c253de5ff94629\",\"1267890d36a234dff7954551c7ccb0aad01bcbd674bfc5a73a8b190fe928ee7e\",\"ae470b3831db242529ae3503cec77ff26a81388e868df46b3731f8a01bc208f0\",\"111f192f9687c09058d228926b1c7c250c1f63c53b4ce20e2b6950f5289996fa\",\"923ef10edb88863b9aeb95fc1767ca87ecce26b394580c03170c71f990785dff\",\"5584289c2ce0e0ddec6daac8d025773c608b8be17aa68751bdfbe188f22fd968\",\"0cdae723ea3ebddddf03ddf05c9faf8135751667f28e775d085f590407ad21a3\",\"dfb4e5213697444ccca0b3360bbc9a66d12e5c6d9dbd8ed84738990026489874\",\"efb43e4459d88b40c2f5200dcb8ef3113a904859d0d39493aab3dd4274dab7ec\",\"8fd99296b8f53b1772b36ef340f05883bddea13f817f06b256fa551e51470e71\",\"403fafa4a6ff2b4ab6f124462e69671ffd66d3813ee4ff8c93173197c341ae1a\",\"b51f54af4ddefcf6bb1fe8803baafb5c7bc378812889ba8f75d7cdbd0d1101ca\",\"4f1682da53ce791e69d618341da92d17ae07e6d9a0f5278b01947518122ddee3\",\"e2e3d6c5b195696ae868b2494f15c61f6f0af153a41b5064e3c91dffca42609c\",\"f61051cc37f3f0855d1a0abb9cd0fea33f9ab6cf728921f241a07c45f8c80d21\",\"09549ea67d0bc7abd3a564b07889f3ab52fdda498c634ff064c31ccd23f2283c\",\"82578d0d708c229e245ac56de026484394d49fbc2a6b2c96839bf2cfcfd7daa8\",\"c2142974213b015b5eb2caa1c2057849cec5cdeb7e221d18ce3f75480422f1d2\",\"43568ad37584b2f85003d49eae3f8afab8db89e8ce2773094c34ceaa62460d8d\",\"c2910456a16faab71730258802a8fd790759600b02ed4cea613c7fffded67209\",\"8a0ee097cfb20fd19240af7c3e8cc048b32bf267c0a3a2f96ada9a1c6a01ca20\",\"4f80c337c87397b3977711f4a968c71ab40481b7c38deb7457ec929a9fc803c4\",\"e83f47284d036c37bd6a54cd94fc506db0b8abc502767f19c628b2fb5f46bfef\",\"d367e3b607fc71edc8f3348e86d18c66e8736909f993bcad9a559d2f9a910c0d\",\"2f62ba31dcc4c2b63524cf020b5ba3fa6fb0177a8e0d87dbb409c5c550a0d152\",\"a216a2871db69e9e6a311af7831bb98d72c2ca8cb308dfe4075e0f2e1e3bd959\",\"6132523ac3ae16cfc1957f0e9dfddf4bd0847808fc027d81dc093037dcd2513d\",\"20eca596b7545f2f80ab2666f40dd7b491449a811308832244629dd12eef8c9e\",\"afe464c2f1998a61ccb6d2dde7437f9571063820dc4137d5ce0716d05ff7d35f\",\"d531be91694a7480c999125f0deb0e7ff7176ea0305a0e2a6971a365eaf1fe24\",\"9c8ea26266bbc37cdca74d0c838e205cdd12d4e8e7f3b5059d13aad2daa8796e\",\"ed7709fc375419d2533278e51c8b19cd55b4e9cdf415421a79e5eb2674311da0\",\"5e3d9ffe70656e563133d416bb758ac1e84c1f23e18dcbdd6f27d7f1b822be8c\",\"f78a6d32660e05def4f943abc601a615d9528c7108696afd736bf75b1a3c9812\",\"ab61947a02572f49ca39559ee31c4b9cbf9fe35148a0fec4b3673a015acbc13b\",\"abda667eff6ea05c6da1823397fee653940fb79343c3434ba1b0defb42ab2b8f\",\"1b0c754e8247f0d99b921530cd2315dc5e3ef180b712cbdf79940606bc69bed5\",\"594621af4f78ddb2ac2de3a17416aceacc42f6765258593aae187f7006ab2339\",\"f9d94796210460d7519c7c0f16f48960816a0bcdcdebffe18313863a66d3ad83\",\"53756a574714c2f8bf6cf6265707f49cb779fb9db7b03fa0a2c74b63766fae65\",\"bd9fc52d2d68fe5a77ba0f6184fe2708a5dd2c564e8967d43dccf1bd52b842cf\",\"0fe14e9be2a2d16dc3aae5047b2e7ca29fec323a863b59c02eae3c9c69f8a927\",\"4c66d038db3d6f76596b919e8032490ac800584042d977370d12be85664a2a19\",\"2bded37f7b62a4cdc35503baefc39b9c7413211178486c0152e4788703f4b7a0\",\"a6c65a4bea88efe8b4e3bf6ded08ffb4b9a6ba42d86d10a901359d21155aa853\",\"874a14b4217d93e32c6b27ae1cc506b5139375a60b5b66dd7da52b77dc724858\",\"7f6e65ef14853776610fd5d0cf8694a01c66f4f633bf150d1b528d893d7837fe\",\"c3bbd6766043409f37240f98a8c2aba6892a4241b97c8327b6b362056131b6da\",\"1ce4b96ba6192868384b876139ad48b559314ccafafc9c7e9d8cce60e2638820\",\"b824b0455eb31c2514d0868734e5716674b7d1b293af121929711582717f6cb3\",\"112e55753c5aadfa149aa7248aab7ee39036dc172beb424621552265815f71e2\",\"ef9f0826e4ffb858c71bcb85ca6736a46388fc8ceb237349189a8c92134b4b64\",\"20421792272d05aa9f0519f5dadcfd8a964e9b71533cc1dc887613420fff170a\",\"6b23675afc7c896cd410e033b3436ed4ca251ee3d23f3eff1fc2480e1a3e8dee\",\"2cef4a91c64f754c4ca7de76e89e587751f2e3238f98cdb64637fc233c110bad\",\"b8f6c5428a20e1b824b137220ac3126f189767b960f40794a0257eb1561385dd\",\"37eabe4c10a73eb19fc67fca5b799829847525cc7d1983d00cd60248efc108c9\",\"813b4827276b42aef71a1c765fbaf8737b2cd0c5bade6bac7e5fda0e8217e612\",\"e1a94e12b4a6ff1e8619f0c77cdfe89bd7a7ba49f1975df7bc4db5b1231995bf\",\"36483a37977c260a418df136ea5add3a227226b431b5bb9c378bd03933493bda\",\"c73878c7ec4f43ea1f621f9a50de48d41d8a8c6209a0fe83779b4c91ddc65ca2\",\"4f24bf06b473ad792f38c1272e17f5e72b4d4549557188af7b0d83a417d601e2\",\"a2632d331478d52fe3158a8f5d38c580137141f85a20a081bcab4933316fc3d2\",\"1bb9d0ba3aa5cfe5731062006ddee50035f7e1218685b71fed02ad69a7e3c045\",\"6d9ce8813dad70d88c4813a54e3ccaaad6e788e11dfbbbbfb38790994ebf6fd2\",\"b254687d01e60a2aaafccd9f8036ecf350fb432110e59026bca1b63dbe2ba628\",\"2702af9853721e64407479ffdeac5619a58bb352906261642dbc2153f26dc855\",\"ffb1f468048fe166c762d5bed5872bb225fff013ed020a41e4b058abb1b61f2c\",\"9ae657ffbef3515bc6e56971e8279279161dd1676a647738a9769661f4f2329e\",\"f720d9ab6d5095263d96b6ae589fceb6beb20368bc3629912f1ef9245c99f5ac\",\"5343c61a98c3804ccef2d583bedd9b4d2569aa48704374aa1650986f54a7e89a\",\"9c6fcd0a39cf47552ab4e70fe2eb8d44ed507b7862c4a9595a135b087981aebf\",\"cd8bb3d11dae768651128fa6d8c16d765583fb6dacf503e8714173282cdd076f\",\"30dab96ee301c90654d5914c73d6b12d834df8f07da76bdd70bedb4003b1881e\",\"9504ca4ebc6427c1e55ebc1d5ea19a7f4017c3baff173512b55885ffeb45acd8\",\"920601d39a6864d34c3282929124d795da15cc0a2494bf3f611458b42b63586e\",\"a3b7270c4c1c3b466253720472bc2b2fa34aa01c26557db32c1eff05d41590e8\",\"214af091033d1f7d7a063bf10aabce839248786a594f21f00575a70dea610bed\",\"4ea573fa09753c394b3cd1eba0bf9296605d9a46281def173f41bfba55a1df99\",\"cd41f98790e2c4535a86108e846bab1eecfacfdf2248c80784174cc62e0f528f\",\"fc3376260c658d41f8c5d64966eda4d47370000f324d2b96cca9fdb2884ddfed\",\"39bff5d3ae2bf7de3a4010ed093983ab3f86e9b5b4f8c66d2c5c75e32c3920aa\",\"949549571efcb0e6513d5e37d55f85eec3d5c9bec33c3011b827c3aa8cb375a6\",\"0bba243a386a1dbedffec6d5f5c8f5f68b9322f28467c5ced19369ed2f3113b4\",\"9616e2bedf6582cf210e57a2e2d85f16caf1c72d6dc15cd7215919d01c600134\",\"784229543f6c19a60d4c4417c08b77e5c72eec993995f2e0812746038f7d2c70\",\"3dcbc239b7d6c88d7fb8453f8d2fb4d1a3e00c436afc6310ca5b30854f780e68\",\"7be56df62a92ea7da15b054637cefd6b4f8a648b451bc23c03dca4f8f7e67b9f\",\"4236cd1eb7e93beddd16f825e88f74d04121a49f2890b4bbcb8d1690fa13bc79\",\"0c399cf70847d974beed30b704b6b757109efb5c00032dbeaf28648dfc9a2f65\",\"64ccf4a440924f6ede7e529305cc47cadeafdcb2821b7bbec933b8a8a7cae1cc\",\"2e5188cd26c1409fd50bb343818a073fa3bfba04be2f9208ca0caea99ae75fa8\",\"f01f87da5b46871ea54b1ab4067c9ef08a02193b862c080be260ba0978399e3a\",\"1f7c424745abc55053903d4de3585533d7b72a97f4bdfcb15b83dec152b620e7\",\"1e8466ee044b8f6741c40db8700138e0010f5516ced2ee2b51994c712e42c639\",\"b937df96a07053b12aac501e22d2d17ce00500af8460255bce07ed769a1699ca\",\"f8be6cb34d672d8afb206e2b414aa92fe47a564ec9f240e0198ddaa0c11a6637\",\"3bea5bfba590fb05c3dfb09530275db0a67960d5c03cdf578145ff33497116bd\",\"08e2ac333fc2a6dbe55db86397d3b0b565bc77cf5e355b5018928b7f42de32a0\",\"81b38354c3bb763ea4a1ead3658e71d5001de87737a388b69dfbef2a3f292409\",\"9f355eff0097d40a1ef35ff9bca6b5d5733605f3a804c88825535a611fd0047b\",\"1b1c1379e4441c597575f688545f9eeaf3b3e06934f5442403e71c14dcf896bf\",\"a4a376c496296daa5dc459b72a406c14492d4b56cbd47cfbe7b64756b49bf577\",\"a5a4143109b1b65bde068ffc7c85c3a34ba261e4bd0219488d3ab7b723380e8c\",\"150112e89689dd4487b8887f27ad3736df5ae5b7984b61e21512910d713c7757\",\"95e81179d3addebddd3d6487f200d642fbbd48ee6380f638e500d19167a54eb2\",\"8fed7c19c615cd0a6fb76c91a326a92acbab71d2b1f51f33c716199341ad4bd6\",\"6f3ccb1453e8115f5c483a3d9e1773082b159df2c872ba017dc5a63186750e77\",\"dfbc737e133486d46f1aa728f8fd90cf72688ea8e2baeab07adfc803e6a1137a\",\"776a1047e49935eaae027b6050f387630c960b99e94bd85528732f226c198b28\",\"8968740ba6ba8fc4d40779f0f5908a048c9d7bed252715d0b9bdd5ca58fde1b1\",\"27e318dfac7eb993b633590823f97b1f439f481a1d44245a2f5d68cdddab0612\",\"f90b7d52baa00456f1718163d9f512fc1eb898e3644ebf52b9c786d04c2353d1\",\"11e1e795444adf5cce3f71371ff6c706a8b81e523a3da6cd643fd329240dd16e\",\"db78a102c053a031b7652e44e46ccaaca36a884e9bdddf50aaaa6aa2f9aa6729\",\"c391e35d7ab55623b151a898bb1a948a63564a34bf4d55740b96d1845e51a1be\",\"84e7fa88427075e7bc97f41b23a60b0e1d45e6c7c1f2362021c9bf9c606d1e69\",\"2e72520752a7af79b7671298bca67ba7f514cc7d8a10f84eae503440c81da4b9\",\"1ace16aa51c062d3c86848604ff0b86fd9ea3d138387339df88c177fafec02fd\",\"5e5b8c4b6184fbff10511a6c20865ed873b366568274bf6feb53bc69439dee20\",\"bc057c9f6ebba5be0e4c141e2b8dac2e955781f10b7fdfff35080af7c22a9109\",\"7533a545410e27cdce36f5dbbfa433faf909564c07b1fa757e998ecdd7abdf47\",\"d45d09415c427075ef1393ada1cad2c59a919b19e2618bba65d7fe7adde750db\",\"f6ff7ec7872c1c9a8425093dc87eb628def959e23ce3dc87f17ff648db3ca111\",\"b5d9f5a21fd6c8823c7dc6a6c5be41ca3c5a0fca4d02c9ff4089b2b2a17c4338\",\"fe69fd7732d01bdedd413c57fe06f211a14c67b26117c96a6e18d170355c89d3\",\"e758c9ec0060ed12172b6f3134ae26452459ff119f3d86de202200dc672cf6ba\",\"46eaf861ba056eaad11f694e5eb3bb815b1bf770135bd9e90fdd1499cb9f7294\",\"e2f071500d2f8ad5dbd6a74d13a68c5b3cfbce0de53d93b4f86cc60f17bb2159\",\"d953e36ccac77a4f61f9130277d8a160fe891e1ddbe44ec0c13301cc011a43dc\",\"f23cad49dd2a287dc6dbb1ebcb191c2f91ecc9244bef055679409ed3f6f55842\",\"b03010bc56f4eba0f572b5568ea27e327346448640e799cc7bdc89acec9061e8\",\"0a515d716ddf91cd32b3de7aeb62eccad64c460e6660f8b43568f6c1779c8b45\",\"1f2765bbf9e4aa1c7e2fa001f036d274363dc3e3b1b28d7a5e4f770da31c1013\",\"05d33b16c4272a9d9cd097feebdcb356696557d852f9cd7e941c2b97779a6dda\",\"80d6479753a0e11915295e6397901b5806e13a7f7eff572ad29d05b36e042db2\",\"f84ff7d2d82387850c886f69995b0e14f00c7486d85e2aacf16ba6e19725b955\",\"0ca4dc4f674b327eeff1e510bfe6df91d415b8a1932c733549221f8896ec5f4f\",\"d6733d51d2a05a16053d4d64ff2d4f62fb7f71ca175c7e353a2da8c94f7cba24\",\"cafda37ee0d01d4cdd3681bb869b860986c204c045b85a959e96e33e27dd353b\",\"270c1a8399c144ce3d9dbba0a50344efb61c74f38bdf1fbd688700a5a6fa15cb\",\"fc9eb04505f0469b8e0cbd82cfe2a3d48f2a85faa5fa639f6bdf8f362e4f5795\",\"2cbb17c4a3c8e00973bc3fbfcee888d8c95e58bd380ddc1145937615d7b00fff\",\"d30b37ea6f8511bbdb439127d820129dae6d2d3011e8530bb2be2311306f90fe\",\"1d609096289432e8daa7e6ca0ef70787283a990fdfd0d9f8c7825e37d3917290\",\"a04cfd67dfeb1b6e8530f5c0c6cf2c330ea044ad07a158f9f5c63a29e5163eac\",\"9b887269be79b7dc447f767f46a184b0c706a901160111bc6b00913e073c5e69\",\"37f3920500cb7460bba3ead9c66a086594c429f7d46114f16fb5b48310c5b4d4\",\"66eb67277444de7d342834ca2bfe5c679ec9de8e4ac66ecc1ef10d9a4feebf57\",\"0f9673cbb2e5b5d40f16d0d9684e1ed5c2ddc788dd4533a5de4e63530c5f13e0\",\"e6e2b4802c318bed7628d5f757367d6f0f11c18ce34771aede65671eaf62e8c8\",\"6d9b10921625750987e101ee424bad1d84e0bcbb563af8d53b05fc9e908e5d4c\",\"4de8aba10c1c8923082d23b8f5c10f38c331f65252721346408deec665226889\",\"a114291d855dff12c515cd0e83ebd3373f22e12d13b2200976d50d04b4a56b40\",\"5ecf7f9fb339aa954a51458cbeb06adcb64700d384d4537b7c7fbdb8ef98a6af\",\"7ec23585d403ecf8023031c652ce260c3d7c83eb681889f317b1f3509292895d\",\"fcacc6c4f5bcfa15c433c42cbafa0e8cfad87f6687ccb56a0016c8a5dc8314a9\",\"cc1d24c7774163415806e954760e308f3cd82171bb99642140b95ac40acc6d65\",\"d069bb43be362958f34b8ea095d83903605bfa0d2e88fca62d58cc3bbd751dfc\",\"ec51143b58fa04bfca9a2e6725d65ca34e9d764a3d32ba8c4c7275ce133c5886\",\"a06d68f48a9021cc7e8cf54ed100c9ec04fe445e8133db6e71a6cdf36d3ac731\",\"281e19b434d3b89e5e7b57f680c51e60bfcb05a379f1179bd00e7b36b02be4f5\",\"006cca28d9f579958dfa322fae4d88009e572679e6c5ebcd084d989c662dbd44\",\"0a0de95e09801089d2be41eeb399d6e6ec98c8b368ec69dc3843a01ab6377ee1\",\"d3f7b1ab0c54320803897d8f2a20f1e81a6785d303300c72142faabc8b5bb515\",\"f05a919e420dc97e0c0d3409597265631ddb79336a70d66ba9a98670ac64f66d\",\"59925ed04ac1710c350f1787ce27f7e2859d88e767a757cd576f623978fbb8ef\",\"2acee1ef4ffff476fe52d497cd7a153350574ed1a4bfe84e1feda29bc1d9f6ec\",\"94f072d9c45ea13dfc9ce4c8d8eef81bed04e77664ceccf2d1686ccd2ec14972\",\"1b52ec042980e5941c75e0a86ab0d42c719df4905a381cdcaebf5da49b2f9086\",\"da0f4b5764456f4f52c1df83df68c11569e3b3c4da95e0a2e1758b9f69730837\",\"3a7834625bebd1d2f717674bae2dcf01261a81e30699d2a08dc464a035a61b5d\",\"df489791c76f437b6093c79e118e2c31d01388502ebe26ac39b49a0bfb6b806d\",\"d28facc7cac1cd74105715229108c4a17a65c36ef42b8a0c7f4aa52874dc5301\",\"5b911c2939c66b90b07af9021782f92635f52c7a21aaf8563fd0ea5ffd827bc3\",\"a74a8d27efb29dd5dd7a41834f79c6b2d73f4395f70fa946b30e75b701f3d2a1\",\"04da239476763039b3eb94139529e532d639c5c3a61edecaef60d3374231a1e4\",\"10d9973a98a3950d19a3843a686f0e68f336eeef8f98fe9fc6d306dd3008927b\",\"15234f09ca9acd8af53c394ce25b135481e6d8f51067e28b56fe9a679e9013c9\",\"4fb52716c60fe135e52eaa7924e5d6f8569d59a2c421d590b08acee076d1bc82\",\"953efedce110d59b3b8e75a2f9e687b2f1106179448e9639f5a4806ac2585a85\",\"96eddded1de1e92b5b339164b57eadf422466bfbe1c0db87cc1f21b8fd474086\",\"129241a398cc8c767285cb3e0b536dba49f53d403eb07af758310422a16fbddf\",\"82cba33fa8d42f6ef5d6df3b3216a12f5669c8148cc755bf65b9485fbd13d983\",\"d1c2036cb8363ad70637d7c0bb8c3c41e15f9468da7d02a27d049d0a73ecbcd1\",\"9200ddd7c3c5135ed23f86b9f09c3a171d4e417c459e7a1c476719874b6d2c03\",\"7bed7762898456abfe4dcfc10c94a80389fe4c960ab9499b0cf2a2a132566209\",\"eaa8cd678a76edc2d7be77d2ba45e0f4244358a4c606494fcd88b26713be19bf\",\"b2430331d6dfccf1680966f9a8eafae44a1a7b055fed93293aef6b291e74a046\",\"222a2ebcf585ff14acf231b09dcce9d5b9d112708da84b6439e7bfb0287b038f\",\"4ee8017abdf69825f545345445a4008f8365ce72ea5b25018adab8f60ac6f520\",\"ae35c58eb1979100fafbaa7c989ab42d27d5b1bf83f9ceeb9020bc4ee4a6bf0e\",\"d3faf98f4fe78875316df321e12b7647388ca8fd14d04c3a903a3f21fc170dcf\",\"73c6836f19db3e4f79b7e0d30f549fb2aa28155348c9d5eef8a8d62dfb681a28\",\"9d128a4a90bf4ea6e211ad1811603eddefb24ee49b8a476a71513cad5d821731\",\"62c59768df5ab225f9fb618828e03a7b65a91fd5586b267d54bd47ad730d462f\",\"68e3503b8fd05e9c4df64cc534a56c3120428f1326b57c6eccb338bebed212e8\",\"9bcb18e40e212f01b539732c8603ff778abfbabc60a1284bbd04d7c99889bf39\",\"084359cf732d41bcd253c4b1645bf56759bf57397921b48134a98982787d1abb\",\"52c03ae49bd26f5532c518b0486fee05790a9914f9227225538595ba140c426f\",\"c8424ba8a08457c1b7b339531b7c9d300f424b5ff3db696354c604a011157ff1\",\"4720c814698d2fa77b0724fdcf15b25934058d18212140bef1936d477625a67a\",\"468f935ec46671795acc3649d34a835de113c6fd0e2bb5be56187df78ed9e1f7\",\"0541465a4fb7caa2547fae93573f66d0c51f6be2de1977cad6baca829c9f7528\",\"bc1bb72b2baa2ec71f086d98e82916180cd82d7b6d160c2980123a603f88a715\",\"a55f6335068c1899dc67153206b2c7fd381795b8141dadae97bb3257bbe44e66\",\"99e6d5dd2134e2e1868da66d28198b0aae89ba7d561088921caf5a7b1e9cf532\",\"c9bb0ad9ac4ca1331921a87235dd3db374471a808b74537497a559a7eee83e04\",\"29f89c58acbc7e465262033816dd1dee00c938ae942ca57a44c8dab3ad6e1239\",\"be29ac3931b900d53bf6ea886479dbfa20468d14c49d73892c8adb5307584b16\",\"48211876df3ef24cf8382e43b94841b6d4bf1a8189f9780402024f9c297993fd\",\"6e81da16236ea7b049564821c49b5229119fa8d456e61427af670d2a5d8899f8\",\"223c14f74c674cd0cd67cd45f1ce4238a1bf40a010ad8af44b8b317f995a5f46\",\"f9263cfbcb618191741941d1fae0b19078e77773fdc7bca203bee2ea8ac741aa\",\"06152a6a0441c4e59581b5559bfcc2ca392ff52b29952ce5241385a2cc0e1b96\",\"4e3ff5719f4e7e313013d32743b075ec64d3a57a0a22f9f7122b172a49392025\",\"cce0494eb5b81c0a3078b445cd7b3a36530b61cce9d380b98912614b571b045b\",\"4c94e150b3438f6c7b3c9203cd51764b9c63878f0d4399d222d8689cef50591c\",\"5b910af0d8a86184300f325978cc5aebb8eac1bc51916ac28db3b04d6111da78\",\"6cd040b2a1ab76e66ca1208e350b118461e3685e1967dd42b7fbe5fcc50940bd\",\"7223066b92a7a01a36205e6e3ae35c1c29f846b10a726ec6da6f2f5330ab8fae\",\"322953b9fc071b07e845f8434dfb428bf3ac4a6677218c3e8c02a0e7840f45b0\",\"ebf2fbef422fee74a934bc77931e1da00228194c56f3fae2de4e9c8992ecd417\",\"a99687bc48c91c71c02808ef727c8245cfa9f0adfa9bda36a71f08272c52058a\",\"7c0f1690cce55a20c346113c05686e1796053c22c987a97fe2e419ad4f8d8b96\",\"ec9d1c75e5ba4256137ae65027df16196d6c45469a08306314d6c63022026954\",\"e15a1bcd9e96e93cdf82781b76212cf7d6d054392bf24a585f2cb2cf11e3237d\",\"f3e3c3a517e04664355abcb43c8660c1fd96c7440fd05ae04e3c492288f8063b\",\"f20ee3f978b85f95c1f65fa9de660984b33721d392682399423b6942518a69c8\",\"9ec83b1a3da56d86b01ae10908d25ae7c3e2b1ffd09bc73cc1e0b2ff4d25e60d\",\"3f7b0f4b500385eaaa72d8eec10f614aaf1d51810b690d941d0931c14ce87eae\",\"fd64933526474eb6344141a32755a53c756b599d965fac23c269b880c6ff9206\",\"a66b15f304f5c1a059aa439d7c0a09b18b7cb30662b0169fb54215587575d819\",\"4909092b3f8ff2e0d2a850583b856b4f23ad40e31cf96f3f825aaf6423d41223\",\"0d8bcb98785f38583e59012b5a94989d39242eee496aa7bbc0acde1786b88f9d\",\"50d3ca57fcfb5a5ceca3cf949ad4c58139806c571baadbc6793498c367cc90e6\",\"8636b95ced2cd5ca049d1c42c007cfbe996df2dc8b8b12b60f4eec2e986bc654\",\"c0bf3c52a5b674ccbc16c2f73f8f849effaec57fb2c3b68aa6b1d8c4bfe410db\",\"46a20f28a21a1a8964be8c3052cc459aba4e5700bac8c521c938e5394c820475\",\"a3e7b2f65c68f1a34a908b5775dd7e12657389244787739b39365db7096d0237\",\"abd8283e0a8e6313480573d7eb4d6de377865702529c2fb39af925d62f5535bd\",\"ae66226aaf138c1fe8d63202a0496dab28bfd0222b31ec3116ea220c8084e02b\",\"00776f07605015b924d2ad2912c65feef05a8b7f71821de0a89a6afad7c76bad\",\"756d7af2eca2013cee8c9fd7d15f814e52d23458be849750a3405bffe6222639\",\"1c3782fac1689b14efd12d45c0afb1552e7638c1b8cbcbf7ea17fd2cd83e78a9\",\"bc59fb22e0bdf5f0d9f1f6540cfa8d986d729898237a1d6201ceae05d2c4a1c9\",\"2a2ed71b80b5feb7e6a93fc54e76a37f2ae83180e47c02e3d35e8673e09f0f25\",\"5742ae228a8fa33355ec861cb51df0b8f0e4cecc91825b5cb17c965d88c9bd5e\",\"b9e02464372ecfde16097b355395ca0a1d87e6b44934176aca49e4ea075921e2\",\"95035defec46f20e03609bc63b243c67e61769bdea2007cb250e5299a3cefb93\",\"2a6edd8427f35626919e447e70ad60ce1274ec48bc3ef0b4a9824ccab68b0c8c\",\"921613bcdb7d30298cbc42239d73725b72a989fbbc617881d7afcd1df0fa6ea0\",\"879255fea17ce7cfbda499f44800d42391a0d141899e1a142a364fdfe6be5270\",\"7ba219b9c6addd4e864d9958b3505c5f64d198fd41594157b75222949e64e2ea\",\"35dcbc73dfd142626fb33d1e17ccebc79b80c83e7e7effb4ca4877d1c392445e\",\"2676a0f45852acab7e4b7cfe74bd375f0460f5cb5fb44af43cf2564ff919e2cd\",\"fbb1486cbf52d87d867da232d1ebece96c82e596b93b22a5567af0b4513c0792\",\"169b80c2613b071259f3262a57164453afc5b866e2f33c5480de6945478027dd\",\"97e80351eae2158cfe7192dd877b3f94feb1ee26d7b43c4dcdbf8d8c2b7a6f7e\",\"de3e6622543bfdacde41a9f55affd934ab04440388141b591fcce7b2d07b6e69\",\"9aba61385d339a1acf706eb9cc41efe33c1dfd81dce32600bd0fbce7b95199a1\",\"5b7695a40713e54b22b2316a94543aa262f87d9dd0846eefd199bf6adbb4e9fb\",\"f097385d5f96660834bd98c06ba80638d789f0d1a05a5f8b5e7fb5d7cc39177f\",\"f8decb89505dca11a5654605afea67b58942f1905ecb721e5bbbfc52df55e76e\",\"6b4f9ec7c07dab6ce92547895a2e264028e724850dee8034b44bd7a9968cc4c0\",\"54ac3f5f2ab222135379fc78a0bd38c9263b79e7e38cb6d0f3536128f724d71e\",\"9511ba4401571171af8f5b6721e9fbb119f3c6bd764b02b4912602d6b8e9ae40\",\"7ccf55901c19fb91a7dcfbed82b85aad6fe87f53e4b151783ebc0e8d877550a0\",\"c1a45e30d4dfbb35cae516fb804481c81743aba07d7a6137a5375142085ed117\",\"9a813f39f9133bab4da8f73535f8fe3bf56f858ebd981e33fb91acb008e20f71\",\"6b1721998a56635efa2727441fa64cdc1c9ad5eb7883369b0d9a5139b0a7e17c\",\"4390fdf4d483b162fab19389ff756b676e82d1a2d872e7a6f383ecb191489983\",\"83edd34a7a92c459358c7f75ad6199e8677b726741d26628dbb1d2cc5007ecda\",\"72d3f8f2667a34b789acc4bbf041689ea66cb4ecb50199fe415d3c95db98ded4\",\"c0d9b640821e99a2e139554030b9f558606a341a84882f4b2e515efb149b2bc2\",\"cd1c59101426c136a16b85e94d1b6d17deb9aa2280d131225964fbb50f604e59\",\"744bafea404571cfc6c3507d21f438175411fb880c219bb1e4029e17a9b336f0\",\"b6f037c7fa0c4ea08e74e8d68525580667a33f46901ac83633b1c4c8aae41967\",\"d1283e26dec6f5fab36972267fca6a16b20d123c8fc79fab1426db30839449c8\",\"3a510b1d960333478a71fb996d32ac4fc2f4afca53a3a3a4e2266c22fcd58835\",\"583c4fcb3deb383d08bf7700c829e8cc6b63743f283c64d609a9b4930cad7b2f\",\"3130e0ad46ca878adfc8849b8c8d9c0b949ae88a96f39ad7a24fa81f850e1128\",\"b3efb41d75b9e9c95c4be5d6047fdfb3e9d3f5cf5ed2911595938bdb22a995da\",\"512f9f9cca827580375473db9cb9989b4285c5951530bafc5b5ea7191c6e1538\",\"6e2542a6b74ecfa104ee2d21255691856b73bfe34e8ec52cbaaf97d4c35c222c\",\"af22ee73f9fae5e5353b5cd747e3a691ea1f799fdc21bab851784ec5921dcd93\",\"340f70563288e79c24ffe48d99452ab8d18befd8358d8b11a34ceee355c95819\",\"2d3c9d9c9b030b6174ee0b9c42b4f19893ec9bcc7969f445fe9090305f3a3c79\",\"5ba4494d7a247ab34ceac71a3990b5823fe9f5ad1e809498f4bee2e09a4f4163\",\"d4ccad7cb91b4f4b5b68ad18132cfda2b421969a79c1f621737c6e02b8c26595\",\"f913c6421c552ac279bfd811a31678724d06d09ee2650b4749e7bcd35cffef25\",\"f588863182dd2c2e1bb0c9cd235a511a52b0a7e75a3448a673e729f2aff53305\",\"8e1d219fc11ab34797fbd9c09ab2b475e2e1ffd98ab06f08003cf03839a25b3b\",\"6a167c0f9f7c9f35fdeb691ea40b65152ce99f97f7aabfb0a736b93dd01ea308\",\"62508547d007c88f6371f9e4af9f221c57acd6c1e535f0b8dc87403ff3c730b7\",\"fe99338d88beb859602e2c5491721cbe3c23239b797184808697c1998350b4db\",\"fbda88a61938321ab7c5e1ad6ddb88dfd19e6d2a8d92efcad883c1849fdef331\",\"8516ac27f2c0ce736bb0f2e1b16970b64b0e99792bad8e5c2ccf57613617c840\",\"973da90b1d3998e03402f1612ea50cb07916ec4a3586a61d0a2fcd57a466190a\",\"ca3242db2f73b8c1ad7faaaf9e27a8a3559b702724eb084ea1b87adcea74f3f8\",\"9187c73c0684040367dd259ccd0908e098ea17b0d306f56d8148761b1ad1d053\",\"a0751402bf46d5ec00c2aa16988bec33829bcd5e897fc5e71aa3609db489d534\",\"89e66f44d4650149f42c9bb5db578f8e051ec877764e745bea81881646611d31\",\"fcde9b329cfdff9d9b9f451ea560f5995e764f7d6757e65cea0963e1d8cde490\",\"407133de61459bcbd79a32e28f734d0cfc2846b729ebdc16089f3eb078f0ee6f\",\"eec696e70272ad6f4cb96113c084af9eae1e2c5085b7f2784269fc1366be0ea4\",\"dabca27b869809d85d17730484d6e01b1dd265b9e946b032cd04048173ac7e04\",\"511e6a6babe04979ac2634472c125c9db4184038d701ef220ea4444ed35e4b58\",\"603d3e0787afa0eb005209d141472da31103b1b1d5baee1e61a666280a386576\",\"91a36ff57d1f8d424482419e9d661d7afdc91c127acc8b30b0a8f0edb9844b81\",\"12748696b039e3457d2f8d239b39fdeea23f67d2bd7a299b6720eee8fa619331\",\"e0c5507474807f8e35fcc5ba59a6de1ea05c1bebd09e9f05634ae4371f19bfb1\",\"7a2edabc63c6bdcc1e669725db21a6e81dd6c4ef6d7cfc8cda3fc760bd314b9a\",\"4870f308c70e0cde32b2f93196b14606d33c200f17b43a2a810b8174c99b42e5\",\"4b8f4c37be921d6fffb3ffe0361fbf45de1a8f39e24a74edb2ef50bb23a74994\",\"c990582671a9d250fbd56493fdfab0325bf6b8161eee449fc443739bc94ec3d2\",\"7ac613370ae96a8071ec9bd853be687c7a59e6dc1a42865d15a4da3cbbb06868\",\"76a91c3d7dac00c7ce10656a4fc3d3f08dca8cc86892294b34622d0ad3f9d2c9\",\"6ce020f6d2c999a86333ce36ffa2ca1baf24792cd2a85e242107f4aa61393661\",\"3aea62299575c21befada8661738a51adfd8ba3619f97c093f7db89d9f316a15\",\"6154ec120e94a1113a0093c949edc35b3032695d2a77de03b18c03b659682822\",\"e15d68b0c81ba3c501b9cbf35fc544049aef193180ab616edab31ecfe6556f4e\",\"200394756394109364c3433af06a154345d547b18d8527217ff88b270951f395\",\"95686fdd89fd947c826c07bb811d89d279845ca2afbcfca13e366f60b03d3574\",\"59faa80942bc16b3d916f659a2ba7b2e74f0af2d7b7b2d05570d648903ada315\",\"8d481e95358571100a0c5ebb064432dfb159597c5618a2f846780ea05c206609\",\"43c0709f2ec50e65581f48fb66f702de0ced1597bee7d7831ab069474491dd11\",\"e298041c03c9821d7c266785bf315e76571a0e71b9e2f2b657f4f8de5e678fa8\",\"80e1eaf5e9c6203414127dcd8b2d6c0040088f1397eb521ad75f16eab3e251ac\",\"c45f550d1efa03c0dae8e8dadcd1a163d563f700499b36331745fc4dd5703309\",\"a75077d084b86076648a1d537c96fe7de96eb8d9836bdf5aa2cb4a641922c72f\",\"6c56d34dd0d41695fe7218b42e3939345014453ecacdd9ef074c2d0258f2160c\",\"6b37dffa51d5d9d434183374c8594a2745e8ef883663bc2bc9b9e8436915d9b1\",\"0a56a41d984d19d0fecc3ac57fc65d86776811aa6fe6e35b4987ff19e5213003\",\"3d6b4aca82041af56d06a15c511aeb455d049137f67985fae8b8abb23f2e01b1\",\"bc0b7af7d8621608a4acfd2846a32f60279ef2f6c9a13246290f6aa0225e2fe2\",\"ada12e8c2c13338e214ee7c8e35f546079f10ac52d457251bc98b3a18e730de2\",\"a1ddef8787036c84a8ff4f7e46344a2e20cda9b02715489f9a7b7d482bc51afd\",\"abe02262f1994a2883edaa0dc86624b28b6e2fe5a25c1e9c00c6d09426712cbe\",\"381392dfd37fa968ed5a4ddb2dd307e769323e3b95e1ec42a1c2f3723a5d24c9\",\"6ac3dcc14fa65dd31b72e5f9b055473e83f5f35312bf7702d25c621b15802cb3\",\"6b73b8e2cfc4eea97e5bb6fda3136ecb9ed4e3b45ea9f4737612e5c7974c4c52\",\"2e5bc49329852d549dd5047ebc20f3bddfd315f65ded5bc33f13d72deed35673\",\"efec42d030c2d363ac3aeaff68f06188bdb3a7246cbc1454a71591629c2feab6\",\"0fcaa59b7e7847a030bc8a1675101256458b401c9238aad4843f0326e9409d08\",\"cb87a6cda363bcb6f6996d19a087fb3a3caff848b9b50d34f1930bb6caefa268\",\"355cbe8ef30223b60df9c604c52e9151418cd8aa0ba906e45502391786618612\",\"3f3b9ea9ef52521052063615da6277e2dc963f359afd29741c6b22bcf543129f\",\"a1fb6c85ca1d834dc766e706b82642ddbc3b7828fa6b8b6e738e8ec7b59f94e5\",\"5b99ff869bb563937f97fe955432818c92c23dbc8bc1baadbb037ce2539031e7\",\"17a718570bfd3107b2a7082f4a1d7eeb54bc9aa0ea4e28dbbb3e68d59833a91a\",\"0c0d924a80a8ce9c14966907011ab83673181ac9038118189c703fd840164df1\",\"b14b034597239be5fc3d436caa10387cc4ebd61414b575c02d42da472fca5eb1\",\"1688206cd4b875421d8eb698555f6fec67e71a92eb6704a74114502ac8c34d8d\",\"19968f10a3c0f2c41bc999bbc65e29b2d255df234df21d48d06375daee6baf8a\",\"3e25958a614a2dec127c7064d36d883971cf08c6e525e0021bcc4d17f5876681\",\"5f61a9e9c7d96ae89c39dc49dd65c5759553216d08495b06f5ffceeeb2c8e978\",\"bc35ec741373ed77519e3bd8f62c9cd8ee7871fce7b9cbe2c651ffe31ad23e73\",\"b7f995b77dfd2fa9292775019ba4dd9e3ae4eb305322a1a123830b193da87ff7\",\"7fd122dc5bf026c88497285269847cd91c697e98cd16130739f46111486be846\",\"0ce203b3d96a1327ab8fdac1c8e19bcd721f48e2c51bf85dbbf0bdd6cf059677\",\"35e59f8b0f2734b1f4a00f814e2275b895f87689ab9a7c9ad418b087f8d509dc\",\"9847ce3a23b9a42f5828726c27059c5ff01b4b11b348200e029afddad183c03e\",\"58e8b41655691cef5ed174a2fb3bec9779858d7928b327179854eede427c3a78\",\"20a9d14a3f3cd535fa6e19d317bc5c8f8515f0f8e4c4a3427294c71c15213749\",\"2ad2977b8322111e45867cfc684741fba24034617ba1ceaedb4e739112ed8acb\",\"53cf0ced698c24b161a831d417b7c6b3ea44b6253a1ac8cdcf4c78754f6f487f\",\"cfca6cfaa3035af7d487211c96407eb83bb30997cbcc11d561c418b9c0021f59\",\"3fb017805f577b063486af7eb0682082a23fbd7403e8bdccda9a0b84ac75240c\",\"7cf7dfd88c79e4e2edfb317ea2e2bb10efc4d8c2d70273624a4a87950d35b12a\",\"80ad130ca2f28e0dfcd9e48377cdc84fdf977a0762dbb0f9d392dc9df6170fe7\",\"c291f0f5ef5a967943a693c803871d96c263204ea096248ed1f890346602f65a\",\"bc2fa58b5528755ae81d2e24a0c01225f4e881e2eea99baa9b4242aa1e7d06f8\",\"5d911641595f08b5de3d3d81b382df093df796a301976bbbceb9d378c9694a5d\",\"750f6f48a19fd3391a4a58d80960919e3d8ae96189e9b01341252b7c26cffee7\",\"b1c2613dbb18e0c22dac64434ce4d17d1e3a27c95c2e7786a9390102ae471969\",\"92a393b6e1debd0a9db9f4c6ce65cc55823404c2f6ca40970bc2b01eebfb2612\",\"44a2fdf5a0ae487a73ac9c9dd38211e3c45e9e5df2bddc2709ca4f19293d6e51\",\"e41cc7340177bc614915ec1d5ffe60871677ecacae69dafd41afbcdbd067b0fd\",\"2a18dd670e6076be1bca4d73d8202bdff7693fd5675b64a0ad9e1c532e4c0879\",\"69f30fc473b71332117d93c6a8f3ccbb4723a6d8fcb17c40dcc6eccac79ef48c\",\"57fdae8fb754e65634db069fcc7f47b03a56099e36db7daf5c8e48b7a4e66b42\",\"b0c6d0ff2e3d61b546b77a060ba3797956fd4cb98bb26b84ea2eabb37c71d8e7\",\"12076143e3174e184b4f5afd9cefd0fc942b5aaeb599aadda7cfc5a6527724be\",\"0d09d5411d719f593c379099e4c1c02e1e2c57a6a54f1d3235e1672463b1c986\",\"a5fbb2e3ba99d20816e5c35c0a30fb4ce2a49c3d5d9c3c3c0e1baed79abd6193\",\"83558f680373dd71d75b01971accf79ffb270a3733738d9193dfc08a6d2042b7\",\"3e2a8f0b7fc1273893eb17071bf8831797ec54fe068abe123fe133939e0d419a\",\"a0bb94ec9ad1c1c907a6e56f66e91256278df188eea132dff93f8791b8f21054\",\"32502eb1f3fa5453425aecad6f6dd8a7f08d0b2ab1f80e86a1c10ff87572aa9d\",\"19c47bfeb442924edc4d6b9f0d37c6f31836b7d75d312d85480d51ff8fda01ad\",\"e8f37ff4acd28fe7d5090d1af42998c30ebb705758fffe1b5c7ae0c3936982b3\",\"d835c01fcc6ff3bc5efadec9fbbbee9ae76c83dcc1520c87ffda1790b8bfe50b\",\"4a06c52a9edf9a7e622b08e80af802b5eb626de43c0d9470579df09d864349c8\",\"4f2bea9ed6ff8d262df91f7dc29fe0e1f7816410dc2357a8aea92494ed95d15d\",\"20140a80ed06cf30534ebbd4318f11f2caffab01852ffba9943c92dbc7210db8\",\"f2e54987994b3b8296df9d00dbc97a3b326d604ca1fcb2a6ee14c03aa76ed165\",\"cffd264cb6447584e9959618bf1e921e68439469bff2c3fd37cbdb1d47468f56\",\"ef10a0fcad3e5a7bb68a1b6249dd27dbba224657acfe4f526ef5d1208f4fc794\",\"0304684d927b67da08dee836a85bffd1d90e7d2840b432a6bc6cb9c5fbbc3f8d\",\"9d29b8d3c2d817e4130074a83018e273462e00ae6fb3979444adafe27829b108\",\"81c6ac96f5a2b52ce0d24bb1a3f4c5a7266a0c404b7983586d7ebc5c2c631688\",\"81f3c296f66fcf9d73b2272d8900f9a43ce06100333bf4d302f72ea80e35a6b8\",\"36a62b597121ddbc037bb61cf29492725d362af799b71bef0644b79f03752d7f\",\"bf62d0abebe30f350b89f3371da0e98f71498954ec8402f3fa1d05df6d4f8ff0\",\"82bb4d3fa338567fcf99136d18e3d4e60ee90aef83f2975aa5373183aeed93eb\",\"bfe0f320b398e243add1ad56c142c5f6064c7bf011e9289b77c94af735398d99\",\"7be53faf02e8fda48442d0738047e485376361f8159ede31ee7cae58d4e7da71\",\"2d037dab9066af3bd180ff6d07c0f8f16e7c4952fd3aac5d4c944041990ee5fb\",\"7e6ff2ad8752edff1ebc3222cfd9cfdfbbefe256bb3a37ea045a517cccea6826\",\"1839cb1f4dd18c7388cd0bc2fc614268afc80dad409b64f7d782d785d4ec7173\",\"e70c5670429757fa53f532a6863dbae3621544719ee629cc8f69ddb734e39950\",\"5e8fce63ca33ffa5b444e317b20a0bf8205b81d01ffef0e18b14ee900b973811\",\"c56a8aa527a221e55fe4710be0472954acd0ae19e42dcfedc28fe8a35c1d24a3\",\"627bab7c1863d58269cdda8f33ef4ef9a161434a6f6828b9b294a4f68426f261\",\"dcff8bdf7f4ebe46ba32eee2e568bdbb03c954e272f4c450ce3515568216b814\",\"dab94c3d8fde50e6c8036da546847769b6eac9aa84372c727a2f00966d9e0123\",\"2597622528aa20e6e4afafcd51f9bab97733bafa3d4a02fcc54d8d9557f9e301\",\"07cf289cfa8476f744b44df63f8b6fce745f518b03dab211ae8e2890957dbb9d\",\"a587b7d3cf11f101d98bf13781a27e9efc528614e7db7efafaaa4a1126dcd470\",\"51e909e3c7d651cf70467c9f263a8225119b907d85a6eaac2c8aa601a88f4e99\",\"31df236aad411d2fc5484d79c5d2c0e9c85de3785af3e4ca59f5d196987ac066\",\"ce8bedab34b84e29313c9bec2109cb06acf42710c7f9136af18bb5b09e09e100\",\"b02105872b787934584f19e89d68f1227d5708d3472771aa6c96bc4ed4a56693\",\"d7aa2cd9a8b9428d5b568515a545568e75dbfda552f2231ac68779168d7edaa4\",\"74d12fc4404e277bdfd60bb4c5226699fd3045457bc1d03150490f40a9c8e3af\",\"07eb1ccfd32b0044301fba234bbe1a48cfd3c99744625291b7222c02beb0e540\",\"3001296d02e2e79c3c9859cdd6044848963b8d67667dd9465fcc3e6e4b2050ef\",\"be252a26febbac6d9bc0a4ae562ee7c2fbce1b8b9d8bb711705f2def86272c8a\",\"cce1eaba30fc7b218d25fdcfb1fa91ae0980a8abb012582dd94709d37b9d46ab\",\"356959cc73a408d8be51f40b149eeb210f757e7adb237d20caa9444211160dec\",\"8d725d198c91a811d9c6dfb8dc9f9c1f88084e51628a2dcfb7c2501f1583566c\",\"a5cf251e90435fa70573695f7670b8d425c5c2f8a1aba55d6ed5be6a2059e7ea\",\"ce9658faf85c35c2d02da7a9ccc5b6fba372b088d667d83ce87df78141a10bfd\",\"1f49326ccab1be4116832a747dd5a4bd16c3524b9ca7e594360e97bc20dc3a32\",\"f407c83a593158af587ae361cab40247261236524a73ed674b583b8a64bc7d6d\",\"c3e6e768bbb89fffca47bac188ebcb1d59cdce636f764c532c189b79ca0bc1ce\",\"529021f07a24628f034fd762256c01d59a571acc3345bf8f995bc8b1d2776734\",\"0f1ebd6bac2260ff1423ae4f76aa8be00901226627469aa7a066fbe618f63214\",\"528c8b3bfd3457ca13dd1c57b7bbc55d63200c69a4927d9c9b95ac3034ef70a7\",\"591192e8c5a08eccd20ece85b2ff9289b10bab701ea9cc7ae8a9f0f4ab4eb747\",\"8db7c2168a6e2b73de70f3ae80968b6d34a26725f06b050356dd11fb0c9556d8\",\"4ea3ec3dd0acbb114e2a9ca45e4ee1548a76ea273682cb0c38b984b5ef3ee7cd\",\"5e8bb406bfb8d0b3606fd5fd5a11b7218d810cef2c348c9d538f7ba4a910df14\",\"078914ad8cd9e3e981db296b81468e26f659921102fc26678da6e7fb87cde2c5\",\"cd431cd29639fc00a09b172fdf437bc611313b42b36c4b111772d06fcc19b5cb\",\"0c3776a93de0cd5569a7df9c109d9c9a7dc9fc965f1d1d27f91f3df1f39d7068\",\"b91c192a9d68b3f5af3b86ef37bf03b863324d5584ad7208fa558a7784802515\",\"864e99e82666e295c0be49554f1cc77592b37b291a48288a3f47686d08094a42\",\"3cf35fc0fc66416e1fb74275c20ec26278526979ebec955b755fb7adc577e0e0\",\"a805f246f7883d99701173ed47e521631029c97325fd811b43a35059cdc41241\",\"f353494cdd60397b3c4cdf6326439d7ecc2989904f9dfc05fc627bbe7516565c\",\"c9ca59329db8d00e02ca7ec5de4bce30599af6a177f2f25986fab218314fc7b6\",\"f2ed7d6b92e57c292ba977dae778136cc82558f11414b4e39bc17160806d8c15\",\"1fbf50508e4772096cd50c11daad7a07eae2915a6cf4f91bb59cdb2d4a1beaf4\",\"5508547e0616c1c58faa6eb165a0978d769360ebcb023d780d2a5f3aabfee924\",\"88d1d61fe87dcfd46f0af12ed126497fd5a4605c1a9821c2d5795977969cc874\",\"bea379c963ae0660c24944125ddd5d0d4436bb922cd61f0e922b6ba5484c6376\",\"3ce0d4a07542e62778cb9fceb531cb3bc2d24ddceb41f5574a99df685dbc5a13\",\"6a1fe7835a82e0166814ea293a5ea0717c1fef8ebfababf2953c866f70dfdb47\",\"dda6eeb65ae76268df7f953be85177855cda7627538401065b4363582fc358dd\",\"ade98ef78aa36303f3f86ee6181320f50951572bbfd5044764828ff1051329b5\",\"ecf167f3e23fe79ee532c3ad10efc2f8ad29286f584aacdd2e9dcf15d9872e3a\",\"4fd732d3108f4aedce84d3107de6a8c22512115a54b5d459ebdbfd8d0e1fd05d\",\"82f7592cac397ed6894be2e94bb7385d57b38f763225315b82c79d8c7528e407\",\"7de71a94bb59af8f97b9e9bbb7ca263d556b32981b8a498053a37772c73000f5\",\"52ddc705ac7ed3e990f56764a575efa143d310f248ba7a514cbf4e209884bc1e\",\"141d91a79ac8a6981cc287c8f9d391cb4df4d22c69cfcff2a0daa9d804954d83\",\"384156b9b5ca05ee6d7f33965772b685cdb6b8cd60fe751aea4878008c37e71f\",\"8c91360cfdf723993635797c26ecf5426112828eebb3a818bdf290c11a3f7c65\",\"c0a29d8201021494a699ed44017729cbe9724b89fdd5b06a95023b943b0ef689\",\"234d6640c63305f587cb979c55e0e96d49e2113ae478165f5eaccb72a3fc4e27\",\"8bbdbb8e85009960afb0460825ea2a995527e7568588d32a93f05a80ba0cc55f\",\"d41fffebcc90177841a2231aa95e950e10e6987360ff7f5fd148b449a35126d8\",\"bec52466946d7bc46bdc3df602e138a7ed6c05d9dd25509b63d582a226c45c7b\",\"4ac2699c814f2643c1fa06cae9c1d3a93bc6bffe3ea8bf9062228294f8a399ac\",\"7136b1be48c425c8a90d99139978cbea82e4912eb09fa50cf9b0b5e1f040fcb9\",\"6dde49dae7cb3e7a44d9963ecf2e6f307d41ced3172d5650886cb5a72cf9bd6f\",\"54e58e4a5d106b3002e01201b731cfb72d5284e1f9e29109c30516b8bac14757\",\"98dbe8439400f1ac06a3528f04a81fb12571f91441cbe7205888cb17452b9eb9\",\"8eb6c612ffe9a72cb38a109580b56008b7f2cc572f189674285a9a02cf14089d\",\"0689ad3f0f907d5180a9b4b94a793139abe9a5bda894fe05ebfab4b124dce1a7\",\"1463624624a9d50339e7a4462cb1858bd9c53cb187f28c6dbc1346450b0a48ed\",\"c7c6a003120911fa7766bd09fd996280bcd563ca35f57ae18f9f0c1cbbe61254\",\"baa0688af14b97a41342aa232e0a72bc3c97d7f55e2c59a1511112d57ab749ef\",\"6ade6828e0d54a25856552c6c4a74a0748c353f694bebfd8f6b80ad025193ee4\",\"4844a5a14a5163cba69f3ac347ddfc19c517edbbf06236ac57ba95ad853d5aed\",\"b08e36a7e120c08f1d9424f5c2d10be6f95c368560602e8d168ce59bf60fac85\",\"08d19ba10d74d073ab9790f3897e61d8be6acea3808a0daa2fd100020ca52b6a\",\"ddb2325949cb6c0338e399b6fdb01f454cfb96790b7ad98c40bcaf6fe3ec025e\",\"f4f42205f0be37e346f99d7ca03bd2c9cf5b0a4fdd7b7282278acb671bfeb772\",\"3648e2b5a1d8713d88ff53c7a888ce43d046621e2ba8f01db592d331dcade60f\",\"e65e9aef1ce829711014483d67b2f717aaf99878bbe1cf6b002b54dbc0893097\",\"9d334475f91a9cbb42eabd1628dc3fe033139048b11d4eaf046ca5f6188b5366\",\"dd974c2f6fb5adf52dd34c0c355411349f95ad9693fa5c81bde41d37c74d5762\",\"fbdf6d3181f7f6770ca7fcd30efc71f6903cd916cbe72742b332b9739655456a\",\"67a8be2cd94d0f7d6de15cdc166362c1bc0da9bd7bdc3ac9ad23d742563e7745\",\"48c24506d6addadc83dbceec8bc6fb9e9840793f692517a963ce62ce68220235\",\"04f011c575f69ff0d35cb099e01f1d356a1db5724d295dc0ffa4488ebb4f5aac\",\"fb1dc274336b27a9f5a2844e743b6e827e92dc4b880bbf94484bdd24550a50ee\",\"c283f59a70ecc451bdbc7ca2017fb81c45d46cf91f7f0bb0b88e00fbe3699fc9\",\"43e68ca42c9c3274bb931bfa5c348dbb502507e5d8862824f90f879bd66fe6c4\",\"86116010414de8f9c2447bc0587863f85897fd67a93849e7cbcaf3d60c3a6475\",\"c0577f65940876e220c5f8b517fdc67bdf52e09f1909a7167268a2c356460884\",\"a4dfbb25c65e0a631743d6df0d44364780dfde8a0f170156b24e9cc78bba29fc\",\"39e826850978f792a7c2b26a5736a23f0ccb409c38bb54f5462373cfccb7345e\",\"11885c332998ba616c4691da8dd7a925da737a1851616b70d3669778b6f6b358\",\"a021ce2fafc3a751483e4c44db2ec7c5ecd634dc370a54e6db26ea405c389d33\",\"fb1477f5c305331f30e125b201011849d09a4b56ef8208c41c4d1f863d2dc8d9\",\"69ff43edae329380c7364443f59f7816e24e9887a11a9bb0440efac9c7459b8b\",\"e0eb6b4422e7c7badbc18200699533dc2e2493e415b0fc47c82f288df758031e\",\"072b1ac983f02b1ea5c0516cbe7f971472aa67ae58d040ab4e79047231b1842a\",\"e2a5e590cb04bb9101be7b1a9c46de52afce72f75bc9a7d2476b46248f984919\",\"46953a3f21a98ea8fbb5c6e20e41286cec72dbfa04697625156147d77627fbd4\",\"6423b85ee454201fbb6e61c34626f55b3cc0250b0b4780e6186714a7407c93ad\",\"842b30cc0abb840836c724b3c1f531b0fb07cbef3606b6fe5e394d4a084f014a\",\"c82e6af795dfac83b65f793ddff060d7a3c50cbb93357b37fd7c8e9b9b1048d4\",\"0119ad4ca740db7e1207126f69dfef69a758c6b6195e268f5246a100aa965321\",\"8d076cce64e5104ce2eaf32ece860526d8147e66e052de739c8d1395a25df253\",\"07eaff7bcd94b9d3421b8bd3e8f13f9359e8e2b8dbb16fe14f222aff42b10ca4\",\"ad30bc3a551179212240eb4e02d36cccef07cd79c187c11e4b043450ee5f1f6f\",\"508f81542cda4402a6204bf37d12ed8b16daba9d278d0997a4c4fd2e2d23c041\",\"10e3f0a5e28aa6f747e6e9d3ae51c545c4923c1bce1c3a860e61fc2832acc13f\",\"89c0a045e87281b1c7e1e81369512d965519c861f1b0ab961507bf4ab22b2c4c\",\"775f070a51dbd016431fe19fa67e24d3b2131242aeca84945631ce8fe2875f7b\",\"a91d0fef3e9e4499ff556fd70e0dab39943b85c23c5d6a51f61eb339a3103304\",\"41fc0be629b02e59d8ef7beaec54ed39f11025005c988f41ff8dca9b305461da\",\"18e0c320194b854f55c29109e41ab99a0ce3b1dbbcc607a48d7e0f27283e2c6a\",\"f64ff7d0108c26ea38d4d0e64d50afd9f9ec25f95ce2d9f734fd0a66b00c1f0e\",\"6f4c30a69129abf09980da64e0a7aea0d7feb2db2b8690c4a55ef6991e4ea4b5\",\"0992777c5e87a605f3d283fcc282e9d8947c4fc93fef750e8aefd15d87c90ce8\",\"790dbdcef3939bd6b120ea1446ee1e6c9e57baf7c0ef3e1ea606d395f50c33ad\",\"d99e981955759241ab6602a0c0ad8897bf6ee463e7b393f0c01e2796f416aa2f\",\"debae5dc92ef53188180b333099d5f156cf8f1262b746f98464810045269918d\",\"392f9030abb15bc2b91724e20c0f044fab1914875977ca2c9ec5b29b2e53d506\",\"9dc878f6523ec8d5cc2330d752fd31cf925e9717d3d3d9333bd867192d659b95\",\"9a6e6c2c05114ba600a31bc5407c95e824f157ef90ca5990233311efbad7d3ec\",\"9c4d351067427ebef9c09f842b79853b15c3cab0676c4487a01f46ce2ffb1084\",\"38909f4ecabc71a7ea460defacffa2578d6d1bbe8db853d881af51e87f508394\",\"41f73123f09b5d463642b744a780c48d2fd89be1458d5666585871a00e61b840\",\"e3f4fcef6f98bdd642998fcb2cc03244655164af5155b040086d3a309a794453\",\"377d8fcbfd091f9712b2f27c309308b9c5eca4c18cc941bce084ad9797027352\",\"65efea41dd28486b017a284dab4fbcbb2454d538ec23b088d0dccc27c207f53d\",\"8bc9bb623a21731a973f2409168d918205f4aea1e03f145b432d067a81d375c8\",\"c20ce267f33b5b0832fb26b5a54b2d3be19eaaaaf82591bfabae6d53de13c246\",\"069c9b215af8e95522c2f29a97fd506824462020dcaf53d1c57f879bac39e1e8\",\"277d84297571ffa99a7a85ae4099000a035c8f7b8bf8586c1f690ab12c7e4d2d\",\"7cec3d098122720588358feb5ab66523a96e08dbdd84a8a70050a42aab7b6744\",\"a80aebd5c7e300f9a7e2da5019138d01bfe4e2c0ea23a4a41e0db704be89fbda\",\"92ce59120bde4c7c7f9cfa461beca2872657baaed7fe7bac1b757d41b5e04083\",\"2c0039f3b5ac025f953221e7fd610e9326f0026163142be905249b675a881972\",\"78c9a4482f7aa416ec6d587e405f2380a2ef8bece5d4f446650d51ed083cc960\",\"9ae7026b922c9aeeccfa63aa22d68e905c20a4741fb95061058254a23bd2eede\",\"9563c8833630809e2d448259af4b6573af585b98343fdcf5af9bb5cada41107e\",\"7615a142f36ee60211375e4f8ced6cfb437fdd3183b41a9567702e03b4174ef8\",\"c187d59507198a889ebdd153d62939f1b3ad5fc465d1622e6d8cf6ec54100d7d\",\"a8c57ea82090d328fc310be556024d3f1b97d1d5f056075dbe1a62324ecb1897\",\"9d83226ac6baa5b84b9eecb12c560a898c95e90b87b494828a8a376661b955c1\",\"f4708c86a6d4855914c0c8c68c859327eb480be4512e22a76b54ff9634f193ea\",\"86b4f7efd9f9bdf747a3510ff52d9c581851835a4114bf368702d48b78f17498\",\"7d8d5ba0d8d9fd24f36c41d0132b964da13182ce70e578e02fcde47e59706d4f\",\"b79020f428c1958dcb63886dd1da6a63e1ce47a95b742ed6992eb5bdcff4c122\",\"01af743b14aea4e9d345553229e1e75071c2486ee9cc5c0d644f7224378f1353\",\"f17234e5e79cf99039a8744259eb337c1b3efbc812ee93a7babeb019e477fc91\",\"aec585ff0ca2cb65ba8d13bc74ac78e63f6878ffc14abce6a73619b9c743fbe5\",\"49c0d072a257fbcc16e7d67d5b25994a55c527b222a889e940ccf4b283a7c22d\",\"1e6bf7f9429fe074f0b80f951a4efed93d0ba3ea8bbb59d872fca80c8111aec2\",\"da31f901aaa46fe0658f7c6f6a54576441ff475207d70c062275356207ec8df9\",\"833c3d36023c29cc3b351224e49ef276914a07e71e7438839fb7f5440ce65750\",\"ebb57f82f9768dc76083480820fc1e9f2d6208ee825d6e1313e6576f6e6dcdc3\",\"4c7a59d8826507ff91c21c8e2898bbfd2aaeb0ee0cef97374a614c2ea29b8832\",\"08c85d6710b27d67de052de2d717dfef575059d4badd42919b45791c627ea628\",\"e44ad4e3fa596c609aa21a935d262162e379f184358ded4d3273b0dd5de31653\",\"da5555ae91b80dfb86d60d9daf9ff569685f67fa9246de0c23fe29519026bf30\",\"63f81dfb0250bcb7483c9daeb09c40e172c378be170d02986273930b0f5ca18f\",\"c63f2c04f385560720f28032b1c9a506202193538463d03d52f96dc7fb8d66cb\",\"f563805c05d18d9be79046ee2ed31f1f6bc34f487640b929ecbf3f2769c057c7\",\"ac612ff9a158cebab2d1632ecf2b804b391b47de6e7a842b2a15f8543362a0ee\",\"790ddd0a1b78852dc832b312ce320fecbe7c8424f9270e6f0ccaf7b97c3a1ec6\",\"d8e2ccfdbd0305275b1ea695d360f49cbc88cb6f46690a710d36055d38a392fe\",\"ab586f7c4afdc834eb32ea802bbb44ca448999ba9ba17bba998ab2fc770556d9\",\"90109a93fe1c57a864959a4121eafaf607c583db7e10916cf3e47b742e1c41fb\",\"58307058cbc6c560094512d472161a2eafabee4040cb18dabb677b338ddbdc50\",\"912acfcda02a44fbe784c018a5c9eed31cb79482942e0dc7ce46f6abd8f9c5ad\",\"b826b239dfa4f0403b8373e2e2dbea4cfbe1f20bd515ebd4eab75f6b127377ec\",\"ad9884a1b8265371bcf0312829435213177f20d49edd1f8ac1167e872f8ce2ad\",\"8c7c6acf79bc1b9187d1fee3d99df8cd4ded577c5f2494933c866fd0ee9e7671\",\"e97c054bf44d93de69ad0d8bccab606c7e9afa9b2b955e607eb68a7d2659c88b\",\"af4e868f1b21ff87251414dd3cb6bf4986c4453f9bc9a64d56af9c465d77b6e7\",\"56bec36560334e2202bf5e88b361cbc8d0a9a40e4eafd1865b8f5bbffb130a32\",\"64bc34fa4b800e9ed81afeeb5a344df2a6e16c473a701b42aa81c1291c68b55c\",\"d78f02cde276cb3e171c5616cb4ba79039a61d9cff9ae0af0400f0d14cbe4f78\",\"3bba9816dd3a658d1f5957dae829f145d751e7d83a13f25d0a016b85f667819f\",\"803d8b83deb6f07291db3e058308496ffd52b2396732c9c1a932c37761b1b652\",\"c3404710b49c9f50a3c0743c4aa0907a47864ab34e0afaca9c2e8dfb3577ce7e\",\"e157b80a11fa7ca5a70dddb526941cc2738e9c1d6dfb7cec473f9aeb9b9dade4\",\"00d804fbf8a4df715ee3c82c915f15785b1f2c3e23cebc4f66674d3f549eb625\",\"a75c34f64cef907bd99e13b05cb403f21d3933b04ab7d6b071ed0e29ed1b54d1\",\"be63a5bd4364af1daa162c2453154c47337f4c3fe8bdbcf1f1fd8da1b2cb6b35\",\"7df4b578c8328f65c9d6c06827425b4cb959214c5d507fac1a25570fccf02dff\",\"305b043c35acfe9a02d7fa02cc4643a2a09869d8558c32258a933da833074e12\",\"82136f48333ba1dd7e261c0d44ab1ecc671370de4c9807b1e99d07637054986d\",\"78dfa67f1cf9708f33f9b1b0d6e6a71bfa194f614c2170befc6b57a1c32c09bc\",\"56b8d35b5e702b91bcd3c1af575c91b003c34cba1b816a045949781e57824221\",\"a176a4f1d98ec719f457898b7f7ea546a1c0e163b6615fcb60328b0d77711df5\",\"7cd3da32d5d32ff5dcaccf9efd810044a0b48a8f8f537c07ed709efb662cb9b4\",\"c1d7fc392444ef15817d15793a43711c6257f3b93f0fac2d2c297b376c6af216\",\"70cd019129dbf85c95127ceb5690c4d0f2346d27d10bde924ae0d6b721f3f637\",\"de297f4c789ad543badf8bf750b7dbcd69cc5ae58d71645c0191aa76db4e96cd\",\"3ad5316e954bda8aae976e70964eb8040e565827ca03f2276fd505a66ea4c169\",\"9393c3f7e879a6610d66fadeebaaedf94ea808b4381a09772a5d5380575095a0\",\"1037a27bde352f6199a1521b72f1f93a6444551758852524961211a8c653d3d2\",\"649f0b9d93009c32c6f09cd3c250a31216bc3372fc49c784b959fb93d4277b4b\",\"99c75e8a6f92954d2bc353a358d358e71bd0473c312260e285aea941d3b47669\",\"d4e8112cb32d6976ea935f478852fdf56663b63c162499fc557c8508cd971814\",\"864d6fb26b039bc37716b693ae6b7efb10ec2254f41310c407d8d340c083f6a8\",\"ec11fa39a15baef7a535ea7ab4bb490380d3f273b366149038a2188c09306732\",\"db09985d4d8fbce00d010a1bade25e546a4303218bf6b42f1086b870db9185b9\",\"3b8350609c46fedf707138c8b767833504d0a13fd89ae8fa99d476ba9957218c\",\"967944f937dcf26f6bc9a5de562e26071077825b3e81d39439feac674cd60be1\",\"0fc1491a768c201514e8b9d0669237a8cd944f7636660a4b7b439476cf638e76\",\"eb32bd00e0184af1006c1e172f36552fac7b750e80555f7660384323f06adfdf\",\"1ef6dd424963c17bd28f33737efa1bd50fa4a63361aa3d1eee8e1fedc12d0759\",\"9ea95c1b2cbca8f6f52726e8ac59be208fac5586a505f1cc50d55e59be7665fe\",\"a78c3d1f53df0ee88362669f95787d869b4027436d20b8778fa922b0bf636c3f\",\"5608631ff688e53acfba39edd61d16f38f26bb5de27622f7fe8a113b64ed278b\",\"dc61bf0a9d9ec082bea34cd81bfd4c1c28050036bfa3337a243a1720a7097090\",\"10a8d6b26e70b36f8002c84538dbe32286e98b3e77d24d0798ccfe4097458799\",\"ff8ce44b755a5817016406610b25999132c2b5feecf906ca04fce27455b13991\",\"c86a7d90ab0442df5ea277d47b2622398d4c1a515c9c35675e19ad0f9c86d20e\",\"271097249a74786c135ea3fe35c4ee48f3a3fca2cf4b5aae822c030653c79b69\",\"7c25f6eceb737962c56c099bf53a981ccbab88b9ed20f6015757268122fb6092\",\"6f19c8a4a1e0ba474b7eeb4a1c9b527425fd81eba84b6f32fa969e019119b525\",\"1bf006985b569c9b30483b709cd05899fef7e688be35529d682904221732ea47\",\"c5cb3854585ddaa0d32ca6bb9f283d8e29092064d2013b038909c07f516e021b\",\"580d622d30271a3c510e9b79b29869efb70b925edc990b687ab6533177bfe5c7\",\"cdb37fb576ca55bd580afc18405ae179b7bccf85814441d90c48151e74592e58\",\"f164de38f1b4922a2f39341813577d5356cd87a778fddb18e6611ce8b0bc45db\",\"ea0e557bee9b385a56a1b626b8a0c4c331551cbe97c8ab6c6aa25d44d0827fdf\",\"ff5da004919f6d10d1ac3606ccd3d009f22b10212e29c1925f73d2585c6c9ae6\",\"54b5a0a0187daf8746ad64f56175239418064eae3a0a1bc1e54fdc7ea7de0fb0\",\"061947c5c8f978936c94446e8dd279ea7fd7627df3e30a9a96051a59a424de13\",\"8748ddb6f707ca2bf060cc07d9ee733c18fc596da31aa323ae5ac548929071a9\",\"0406e8eb2f9561963ccc9d51471c450e708090d378d38817168afdb97016a2a7\",\"4b7fc7e894043aebdc0c3d2a61acdd28691c769d48a4fc75d19c09cde15aed44\",\"725ab45d9bb0a8b2c0919ae533201c21e01dd7fe264643a2d71c0db6f918626d\",\"0093b22ae75bf6811be4c8f5d9a7d3d0e8b1c5c565cfaeb35ca47d7f69c83ee3\",\"9dd3e2c95099c9e2e4884c8f7471ac6705db82dcbc7a7cf78805c256b5f3efa6\",\"99aae9970bd83f87c68b8d268b11888d3d5a6287f3da067756384f6feaf0925b\",\"0ae772132fdc2b0f9d0bffdd36e3724886698fa6ffc46dabd6848ed18e32b979\",\"557084e35ab5c66da7f2434c3ba48a3d8460423382610cb2227ff74177d9d4e0\",\"d6043316f84bf018d7ac3ea33824b3d8c01126720829ed7e45d85d76fa7945be\",\"7023e42ca8ef0776ff09e4ccafd880cfdfa46f2b7283c16ea81ad03698e6414a\",\"5572856f1af6725a02110b4c440eb6d8f8839e4b13fcad15f0b3d2374fdbd233\",\"8c786381d4116cec21b1a72de4e9ec48fe7fab80db0e73388e9cd88b93fa46a4\",\"ba43ca76c78810db78485bf9107cb96688b1dbfd8a8a141a4cde299eeffc9895\",\"7f949a96ab183cbd8926b7dad3e1a60b39ba02e18d57021ebb353eadab1fa8ac\",\"029901084f529c7645e3eceaacf564cbce18b340b7e6ccc46aade9647dcaaeef\",\"c7dd5da4d669fcfd3de0e77a26b0583c636a93e6835220f6ca7d4e03a9ce9986\",\"b1da480aca0fb104709e4b8d1bcc1b572db7d3327865046db707b088495fffaf\",\"e97c1650ac7dd2b9e752b72ef8815a29aab07f883782805c0cf8f5322c4ec8ed\",\"3aa530d4549ca9226d72937acde9762aa9a0885bf7e9a2af6d4b8e88c5830f1c\",\"2f0593ab7448756b14aaec2810bd37cf25c9a3729c239d11a92286f3cdfac63c\",\"c125fb9ed2f89cff5944d61a0f3b52caedb7def633272173b5187eac7fde6da9\",\"f5b66a237c36692ff0d4bd96a765b35242e76c1c2e02871bc3bf7f90655ff04c\",\"7558d689abfddfce7b3b9484f74db5da4a8dc6e393b51797a8788c9502a86016\",\"ad2539920c6d04c7ec30a5d8a041155be0fb001a3069a1a61306f0458f5b06a7\",\"7ea0dd1535d1ed0220014141b493171de32a340ab9ea8c0ff0fec3eb8eae1e08\",\"55e54b5c8348c77d98cf084dc759dabeefd46a357a0080affac55fff97068b67\",\"8d02a88303a6b554027dd1c4e30cfe4c8487ad8ef912faa666025e01064f30de\",\"53be76eedcf4b7c1347615ef109057155b68f38d3d62a297bb6c042cb4dc659a\",\"f0a684f823ad53167f73732d5a3759cab042bceeb3bc6e5bb79ae003d14e6831\",\"587a9996cedb39f552c923d1be574258aa248ab598ae7e251bc479cffd7a5e51\",\"f9c73d2c75c5965c5cc6717cea5184225d9adccf1cc53ca3af41e8445a0a9cf6\",\"e6fc7724410d50018b8d2b3507c96313e65b1a8dfe37533f999475f438f02860\",\"4a7c5a3b7660d3962dcb358c0f3c7ee4c00c8a44ca170f2a4502a4dd2e1c2522\",\"b12837d5765c058d3d8a3716fd61037c46d21e16e55321e5bea835a31e727bff\",\"2e8bd7a6734fdd6ce014ef158abf50447c62cff083492d76ff8e88e79cb65eb7\",\"ba4e35dff1099f159388594bc0c183235e3b938ba579c9b7de1ed80119463074\",\"b62a0b114fdba62cd749abc4281a590951d1123818d8d9a83c79363025846e54\",\"babad5edb34e4e5e6b74c3a076222220a6e7b920351074ab96f4162163662128\",\"8e67b62112160b4837fb6a513ba9fcc89ea88de5edbac1c06d3458bdb5b46800\",\"b9225787b183d98dd5a6760774564642b869c5af570cbd1573c980d438fb3b5c\",\"1f786fb4f94af1b1b434e38ff6a1f6206d2a3befe76df8ffb97be92d19b7b479\",\"85a70301830cc2267c26c16b2039033cb5b0699e76a37cf97529025999f7e0e6\",\"afccb2e9d3338c113f15eca70fe89f8cb2eb226705c3ba5e8bdb12e0939a01e8\",\"26b3145c714c160fd15641c37a318103d33185681a42273378fa66f85b2618b8\",\"6725f04192b20037eb1faf831c4accd76ce5a32ccc26ea3b863045afa8c18931\",\"a33ea21e707b3bfba6906cbf34c03088c9b3e98d2fc630c88a99f0f64295b7e1\",\"0268558dd7e09f942504c889625bba35f4a474990c401778a558989187f9db43\",\"24c639362fdb9a910ef76f5bd0fcbbef94ae1a8b0f8f5269e9680c56801c05b9\",\"09777449814af58b2a73e5e68c45b6d32b0df1de5fe2703d18b86ad84f89b426\",\"961fdd81d37c68c0b29569c8c7b21da43c956b1e77e73f367b550a5efdcfaa57\",\"42e84d2f5ab2fbe42383726e2f559bd3b2c166934315a53a62654210ecd28f1d\",\"eec691a06a95af82f774c57c5a7b0764878a19d925561cbf4a85725a702c83d1\",\"e91068d0138f3451025e80dac4ecda9cc1e38981330453d2e617022ec3a60848\",\"0915278b090c749670db37afbb8603a66d2f7a405ed073d97f4358fe26ff776c\",\"d7372e362552ea7694d573b34fa14975524a08397a99ecf74496025750fc86ad\",\"7887e5ef80546a567119acf016e1694588a58b9b82d4875a1d2acaf3e5c17a78\",\"f53d9bb27cc7d4a31483876bee7072c8941f97abd7c09b36ab67e846880fd8a0\",\"275fb00cda6fc36c2208f7e92ca7db50efe554e6291887d6336817e879079cbd\",\"b69729e374ca283cb196f31ae12ad38bbe033d2187d835801b846e297e242378\",\"a9e9985292a1897a63324829623c4a82acb85ce43af59451015dd1b04505ba64\",\"14fb5d78c49a5c42104600299aae9b17b0333ad196f4c8a4809beca93ae87b02\",\"202dd55c18a63caaa37d5bfa12f6b00e67c9d9c3f6d46d55580b91a566835b37\",\"dd8cc03035b21ccc54289fcbc0c59fc1695f07c0a0e606569ff2dc3a8de17485\",\"4ac7d68a722392d5d9f146a2cab85d92f2c11deea858d5bc9dd7b5af620239e1\",\"b8cac8127ee9df7d5cf51bf5a37266b4f5fbddea035a46d92032a1e99703ad73\",\"7299b169417142825346c96ff5c66946efe9dd072a428f8fcf82738cae743d62\",\"a472b086549843005fc6ea0683c314d41e7b42b9b0be6fd9b291478417b0f741\",\"efdda04f595f0d33337355de3937edbd851beaf5e724f8f6816d35918880cc9e\",\"bb39eaee84b998f90565bdf58089f47a4b5c57adf2f77c16c4349801d59e1806\",\"fdf16ab6cdf1821456adbd9d644855ae9c6efa54f410fac7d5c0365598c274fb\",\"35f17e31b725cf94d6fbf20207266b883edad7db12adff1debe23d3990ac7ca1\",\"d7d62c66a3589a1811adcd556eaf87d49b853f290b4fe6bfebe32022d1d865c2\",\"e04ab2d0fa79516d5bf3ac36ae9f223e8d79ad2387bad2684fa10245c0b394ce\",\"b8f450775754281e316c28482f8701853f7daebdc29e56b7144d0e4f19bcebe1\",\"063400beffe7cce14c4249951f29b526814404c92ca4b43d58a6eb866393b6c5\",\"13dc88092943c646e7a71a80bb2af869afa1738f008bc9526f70a6137c70f5dc\",\"1c797ba9cee9f76fb7b7e3b9b49e229d8ad111f57eb4405edab34d04bb2574c7\",\"f2a29f22ffa18deb9c3706c91c45099c448dc48ee91026b7e889cab595f5d7c7\",\"a120f662576d7761a74f06f8f2aa79408ad1941533e22e77c93fd4ba2ebfb348\",\"68d604ba80e8ddfff0d0d7ef846332895005d95a08f8fa5b0719fdf1157daf48\",\"20ac1c0af5c3dc38da79bd1076f62cebce4e10906f00a6417cc2df8b4aad34b2\",\"68c7dd98de4b6a3f9e646a58a0ea64a690e1f40695d66a3e5a1e1a8bda8d3334\",\"585b14d2d803e7914c958a32034f778e39c23b5789beb029c3e132bcbc9df6ce\",\"a68aaa0a5a6c1d4d1ce8fb8ecc02ef266a57975d10020e2a235d1049844fc140\",\"a7618496ad62e348efaa130307a3fb1803944a63dadde66e1139c042b7cfd7c0\",\"d3873c0d2b455b6ff8fb80e63206c69c5042997864ec5e34abaedc895eee1e72\",\"2e54dc0d98bdb2285b71ce49917dc325c57d7d7d2a1addb4bcdcde03da3d1551\",\"627672b9f780abecc65af4a80eaad6e0177d41b19de98a5c3451045619beee6d\",\"6f23e1fa021d1bbfa09a3a398325a77cc238b8e29bbfcbf6c10802e6ea8e2813\",\"e2804441ea4c1d7dbe336349e3ed7e287f7851213ac2c6c87bb7b792b36c8917\",\"ba2f86e4b4509acb5c9f865f7cd7642d2932e3b5697f2ad783943b3d31fc6ec0\",\"25e6d56c06ba4325579818416680ac2c8d144d2128963b6dd16cfa8f0b58bf5f\",\"d7ded2f66e9b7bf31282af643ec0487a57eb97dd0a8789f0baa4e73957fe65dc\",\"ccbb59a1ff12e53d22f3fd86e36c6ea5dbc7f1a2d1ce890932649a4571b5aedd\",\"23f9016f2b23e52b72449592a758bb8e2a528bb4c2936c0b10fef620f4baee67\",\"92bfd20d0397866af9235f0796a145384e89b70b2139f724e11b0fd94208b488\",\"18f9742c388e126e985cd328d78ea0987997de05ab9d524600c50e9b11645ed9\",\"4dd9611a25104d1df6430e7b79529445fdb31f302f13b81665c8fdc151fe4394\",\"2b88dc3522844b389e725102e208588879377969ab252144a787e5fdb8de2167\",\"f49b72446ef667ebce1d5beec65b8952738ffcf00ff6f9ad4adb8be7be80fd58\",\"7954ed7cbdf02654d59615a411c7f5ab924c768b3d5061b9d13d8a879062622e\",\"aa64c5ea7e6e5353b31b3d38335b12b6f486bb0d9f948509f57f9f9bf1088e38\",\"979e6fa41b3c671540fceba18cc5df08ea8c5e96ea5702c66f541a35facc842d\",\"e9227e4cfa463bb89e25cc76cfae411d765a8f41acc5060d78ba187ef79185e9\",\"a972d3a3c4605b6c230b98b2151b942becb0a0a55857951ca22a038198d012e3\",\"7bdedf0a4a17b3006c24acee507bc5fad96682ef1a9b3ae3bb40b58bbfb7a471\",\"684c57407e4edf67ca6de1ebfdb667a8292852994313ade657ec92c4b6b124b3\",\"aac75dcdcc1f597f1e37976887e175f13a636ed23bc8171227be8e8f3006a8e0\",\"795ba88131091219561e947db5f68bac5ac7f17b10d165c323c3e54179b19540\",\"e424703216d8364c5e8b712adb23c7af85cf5cb2973ed4dd7ee0546bff5b05c6\",\"b410064661386dd3159c92fdb46fa8473420ae4cde753639b4f2ac15f4c2f030\",\"096d1e6b2e73272595517e220280b96c4cfceaefb7e0a6356d7c1b16e478d6af\",\"b003a12aacc1bd9eabebe03788cb957598ca105a872002f6fc31d0db296a0c3e\",\"1562c7a3e740b507048ec90b0be10cfab2805e5307195105b982fe96617c822b\",\"6c85b965b4aa34cf29d7cb16014d753e62ec4b6c8d604ec4823233f37ce59546\",\"3412674093f1c0ce379582d460a0622c2ed148654e45c7f4099844c473e110bd\",\"59a130658a6734de080d0179bbd802f0e10231af90134102dc3d7ae5cf40efd1\",\"45b0b4285aec4e3e2e5e9507dac1d37a03c5bd1f12db2c0e6d6beff58b55e259\",\"75027847b0b8d611cca8493b1cd6a7de2c14d9d63182e6aea5bda0e040e296ec\",\"293fb0e32687771d6c183d1261db1f5932da809c507edfbf08e85f58d55bfc8e\",\"a351de9ffc0bc7b757d48c210a8c366b09fa90eaeee8710f25dab42a67dee390\",\"221f55daa3f86ff5299217a6656baedb8ef60d436ac8bc8926cea0aea85b1887\",\"ca38e0e5930701a2bdba52445e761d67d900bcd80dcb442594b23b59e9e23372\",\"6c3e9edd3ca9d2bdb2a13eae630ce98287af18a8f47ae4db289a5ed45385b58e\",\"72a692fb890e8e3864c73b0abee9a1ebbcbdb43a1bb761a905a5c1dfa04823fc\",\"6a331eb227998b8c12b886ea5dd5125b6709da869afd24c020d4c2b91e8e0fc1\",\"0dd606691cb51f84eebbf35352d62007699f45a4b4bb55472f2d8c2d70012d03\",\"b3cfc8eba4c7c6b208ddf3d686d283736e80f192079b1a9fade2c0f0e51f2eac\",\"0bc78e8b666d94766b97ddd815f6f09e9734bc511db9c13c7dbee79a499cb649\",\"be78a1a257ad903b3101236fb1160ea0a91a100f8f0e59e2d3f7e17bc5475496\",\"4214747923a09b897345b9026d6ae1789f8ad4efecd380e6337640451f66089b\",\"66f64072c15475ca7d3a2748f6e740d7080b7be11163d6c655618a0b4bf36f18\",\"e9ca2098e197a646ba14c51d2d1b69ca0af7eab1c3027ee028ffdc8c8681b208\",\"b5878ae7301a4edd1244125534b2921612d43dde75a975fb0e98c8d7d45e6794\",\"b3dc45d64c863dd622a403ade0f4408c84877957875bf07d0af62d908e7d0090\",\"106c8442aa17d105c6da3b8460694ba32d9303650686a40b260c20b4a3205acf\",\"07dbe1c9681d2de0c692d215738f52af79de591180c49de78aa7ad49666be26f\",\"862ee22ff495f89a406a236ca0d9397586925dd5abca1c2cbd5579d3b5326063\",\"557dfcebc329715766f6a94e240f2eaf0a4dd5f1819361f888813486f599ce9c\",\"bcfaa74cef876fbef93594dccb78f05e45a546b89faa9dc3cf6cf0ff201498c2\",\"d442e04cc15e8338027b7d28cf6d93db843dedf80016cbb4bae8757e39e8992b\",\"6769c03ab7c2bbe2216f7bb45e2450573190552593392939bc0c2192cc362953\",\"1d4425934d2e7a10452989fdce974a4b69ac388c05c7d21856045563b2c9d024\",\"42c8445dfbcf1850c68b99615a3962f190f8a51f80c05c498964ad3e767492ac\",\"bc05bf992407710be3315c8664010b7fe4e6282de941ce552a9d96b51411fa7e\",\"b4543149b623845ace163b12f78db13181e8f1fd46a950188154a63642ec1e5f\",\"eba1d1e5458a0431d60b928f797f38abc3ff0ff4fe1659656abc053d13258492\",\"431f1690b8ca31d278609ed95941bf9f0b669466632ce1b05f05485022bbced9\",\"b3ced08b17381d22a5173944e14ba58e2f42563bdefca00202087c6ffd1e5325\",\"d308007dd835719b735d05a9442c28241923d7cbfcd092c3af24190419df851a\",\"5bdcee03f9b712ec45fc9a7fa5e5166b3db3bd9781b5c4cdaab72ec11d95dced\",\"4b4b96dab099cadad3989d24a9d880c83515b9aa8734256fedf2cf44595ecb8f\",\"af0dcd71a8eb73e19c22cee2ea2c5eaa481fbe0d73bc6830bb5e9cc822840904\",\"e59d56abf988352d531f518b5e12802456dbf1f70ec48ce42e2e61f095c8471b\",\"9a7b74dcf7d7502bc6718931a90c537a5af2f8f5331d1043a3f1b3b827de2f32\",\"eff45ce683a3de90feb125e4267282d7cb55e8e8227b2a8b8950cadc331d1b5c\",\"834e6b880de32cd90afa260380c79e3d2a42852964badb9286529a1d2ac1c65b\",\"7d871e62bb66c8845b2bc7be2a08864726aaeb59b9900282d2aad6df0a5c7a14\",\"70445692a241e1c86391c5be33c6b30257d13c2a643ff48709b441552d42124a\",\"3e76d7a2eaf5730609c11a59ab458fcace0c90d3112b3cd03142783ef6cde77f\",\"2e2299fa454e5e64610ed84a68c04962310366b60e104f56b06113bc0cd6b83d\",\"31974cc4e567fea9ca6edd2b1c7c96de68d7135c867536173f543b911990e458\",\"a52f93a90a68ade1dffd69e9feb54bc565b395681ecdc14c0f06f90adf5eac96\",\"79b6c19b5070081af6fbd7a82c9156e6b5893db285ba623502f8806b123d4e73\",\"a2f2f930d1f9aee50bbb7ec8860bafa715a505b6a657832199308921251565ca\",\"8fdcae04328b6b4909466a2717b08e68ba0b75b6282eabe3b8daf0188fa3f652\",\"7ad5bb0337b94cd3094a1717a5e7ffcaf585dcac286cb197d2755a99dd178128\",\"796e581f345b5a0d402d5ebd3ec6aad2f6ed601deb4ff03dbf4fd31ea0bf97ad\",\"1c2c83869aa87036da8b9f85af3a5d43d1254d48f0e1a0623ae615c99ebc1d46\",\"672bd15fec6bddbb5482d5f8772724a8bdd6300fb11bedd101b49e1cfea777f0\",\"21337735dbb7147b1057813de6585ee3224d4b66cf3442e58cea16bece9dda58\",\"068ba9b0f3c2d29bca2fba5763b1fc1d18c47c29828e2c633331c98a42e06320\",\"364750d43733146f25d683a498fa84a114565e047a27f5097e2a532ca9c131ab\",\"633149b582e05375f5a78545124efd020f3880cc521274f57adc45900fb5e617\",\"8d3dc323e69ae89297833ae79114f9845c8234654c43e07a9aca2d01ab28a228\",\"9e44965fe7f58a3d0833d67837a637c017ffd7b75aabe9106940880929df5da6\",\"84b776fc675e6bebd1bb246ea936d5531385fb89ff47c3236db72292135717f0\",\"5bb8a756b29dfa6620598e51481abd1e3b805c2d90e6b1fa459e22d17821b694\",\"719ed61b14516c41f5002f4f647f3668bc106ffbbc6e2d5e85fd8731707bce77\",\"36f82902562ba9c71f9ddf74c29f7743bbb9f75669cf2a1ecc1ac9b9b976d0df\",\"b94c2c9391495ca46dd3aa4949c1b3f31cd8b6863a2b753e894c92a66c09b417\",\"17b78ec2243a313dfd05921c95fe6e68c6c1b9604d7984cfe36eb715d7fa7923\",\"bcab70789164a7bea4a13e87070d54808baf1aa1b2fee17b6f846d12c64e0b04\",\"030d796d94597e12a3ccb255da46c56bbbc374e2fdda970d9136c9adb51e214e\",\"48d9e561c3e280069928a0cfe88ddca98893c093647b452aae669ef4d2b0377e\",\"75a8d3c7fe94504d916c406b0aaa949127dcce389c445e164a5166eb0ed06165\",\"be62fa3b151c71e648fdf4be70857823c23af8b8d717d366d8f3f5ca9b642234\",\"80f82960b4d771736e3c43a3b928541416784013ea5ca28149aa18f3858c3bb9\",\"74e9b68d11bed054345d114674120ffd0949c0f949120e2ba2f09f5e10a256d8\",\"b8274a281c3b7b3cfb81aa5ed424528b87d7683f51c81dad626fd417f6c97de7\",\"749a9dc712431143e94d2bbe14bc76716dd6a2300486d1502c92196d9267cbe4\",\"8e0198c261993818763c847d04976fc7142ab551088ea0ebb508400eb8e2e525\",\"1314d7b0b0cf29c4b7293613d1e8cb1d87908cbb0c6d44096ceb89f6e0e4bb0e\",\"9927c3ef0a115621b362a92de6ad25e47ff80df11dc3f3e98a4102cca62155ce\",\"67b42eb6f66e3b899a1f173018f62634e96cc9309d10ed7c5067e8b74d458463\",\"c82710bf47f3325b02f4e7dfaae0e9f5745f1abb9b627f759a9946c7a2e7df21\",\"1393b55be7df0cbceeb0803a931f1d746c12604bbd503eb5cd4a448baa023c2f\",\"1c70d734bf7bcefe2fc717df7c7e4e109a2657aac184c7e669176b2a2f696298\",\"5d9600c872fa5fa97cc095fca3613d09ae817b870e51205e80eefc684d462566\",\"d3d33c1ba4b985d501e509c265c784d0f44db932d4eec74ba8d30c5953f03aa3\",\"7b8a0e1d9a84aebde848395a9b11ee491e2e5c030e3bda34e6a0d25549d0d7c0\",\"75b40042d4f4aa9e54820a66c6bec70c6bcb6388c16e741a33db292ee5d5a93d\",\"1375ebac9c8abbf658e3afabd9699aa0866d24dde4ac95ba069116b3bddf8cc5\",\"c51c969b23a94e956d439fa528c25170727f9643613f0407a5e7dcefb52f0886\",\"9a6a91184342b16716cd235756bd81880af00e8118d73fc80f9421f5a1efab22\",\"b359dc467143a8e2820243f593697b57ad3a79ae4952c40ce0992b253bf7090b\",\"445ebab65c8305e7b244e844a49dbc18cb6a20ca43ec490a488cd2248bd18c47\",\"129b2685358fc557459f0004f8f3dc4fd8d822520c8f6d6fa1fd0c5b9e2d8fc1\",\"479574d48bf40fdef9590f3af59d141a572f13f12e77c401711fe71138dd3778\",\"2b7bc311aa21ce67fe1a1ea46e3f2b7f12d5d6bae7889af50b75ad850b3b8aa9\",\"3c8f84c7f831543ecf09b7370c9302736378ba12bc4313b7fd398187856fd313\",\"f6380c0366dfb3d8b0bdb5e619d5cc2af05089dcf365c0118cbbe03bc1f8224f\",\"89a86d187a3894d5c0f46712518a2d6952df24837b5e7b523d3a186bcaee7db3\",\"575a00369ea16cf84aa7b548fa65f04c56bdcc03a4896c283e5a6177633e4eef\",\"319fb431b646c6d8835674d636dd57ac53d006efbcbe9b3fc80a60ead118981a\",\"06fa85942e30cc2c3dbca8731a156f14034dc0573c1496900d0b0c5627f067cd\",\"ca2d604e8c7b3ed3bfe23a9a205bbdc7f47547d4a5b4dc2943fcc1cd96e6fe9e\",\"13c71f6945f4ee040d21a57605102a510a4bae3cd3e1870fcdd544a773cc4e69\",\"bf11727e42d6f675ce6bbca947f5b55f167f584e1167f3b4f38e1ed1dad7273b\",\"02fe87f4ff8f5b48232195cbb4621d5aa371ee8a41a1a067b338f14013ff4c84\",\"e92a9ff5513ccda597c9d54326af13e60e686b972f0d74369a390ba88d5f2284\",\"f0fe057dc264b762dc2fcd5cdfd0bf8b767005730421c3d1bcc1a1b283834927\",\"2648b59e600e7ee516ba40dc4288aeb62a723408cf79291386116fe4ec56f29e\",\"f2c294741e3e49a2f72b71ac58a250dffe2cdf38174bb229682d47e1129bbdb1\",\"472b80717d021681e5b8b71d3518f6ef48f2eb24c3839e86c4a2c4ecb93005fc\",\"5ac5047e8acb23e37e58eb8dce41808a8bb400f59eee57d67523d0d7e7c6baa0\",\"6fee4fe86db1f726b3f86340248d4068649439f2d32272e8d0f36ff1dd7c0c53\",\"13a5b7a7d2cdb2c10a2cae0acc0339ad345cde5571296d95e275d2938018dbf1\",\"fff5a8ee14c08e4825636ccc3e3ee94ffd4227fda084608c40b835be17ed900b\",\"59cc8bfe3ab2a3bb2a91b914edc3353a6a1b0affaca57f41c4b0a5b999836c06\",\"75d5240b1870ca6047371b2e45bf519cf80c808d1e31a82ec83b27659da019c7\",\"c01a230a9614bedc44b1774e3e62b551feca2606d6deb10338b8ec5dd4ba57c4\",\"ad20adf443404d15be80c557bc30847c94124a467cd1a0c85241c02f5b68ffbb\",\"85869a09b3dd6677c45afcd1d604abcce937b48a3fc0e3964ca9c0be179a78a0\",\"8e3ec2a01ef2ca3bb750cd561a3e00534f61cbbf77394fedd571324de9dc6ca1\",\"1f143f45f2c2af67615eb2fec96237b45907a3bfd29aec43965bead1d5fd9b9f\",\"bc72a8932a28222de80f027674d7f35efc994bcafbd07fe0404988cd5094c9d6\",\"621751200a8e3a3517a6307c6443fb2cb1e1bdc6a5f0b8dab990404b231d8d33\",\"c3ef86b06d043d117f4194f95c4fdec5d8fc9927f77b889161f81db37b912342\",\"f424066fc5af625893139b71015df004260b76e0e177051a6f4402ea7ccea048\",\"752ae9a1a23cd3198ea2cd945508a8612bb32351d7ccb58c65c499d2ed55f6b1\",\"bf579dc961c2b68a87a5ec3e420cfcd1ec94ef750a61a4dc7fc0b534cf86ae03\",\"f4fc93671378d61f295dc3aa9072845ebfaa1ef89b78ca5973006bd043d69be7\",\"9808ffb2d828ea4e42784e8ad664cbf4384456fb317788073fef8b40e19e6cd5\",\"7b4dcecfcfacec53d2e9926e161f1f435f64f896d0066b88d5b05e07228185ba\",\"1538d9e11fd26ff153c5af4f55516d7b8fef3296049c28a54104b1fd3a909576\",\"55501dc5d9d479beccc478bb02a071216b1141eaf710d1b03de7fe9d0fed7385\",\"cc9dbb851001082e9237bee46dd771ca94e51d76996da5f8e5262b9bb79fd8d1\",\"437c79d3ce4c59f8aa3bcf6c628ada11b30cdbdc1dd333240c66bd3186afec2b\",\"c0d00bfc9d3c59a63f95d672f78857fea03b318545134f11af40bab4b2b9c7fc\",\"ea5a8e180885905846d5f74e891ffa1b0c136605a7290fcffa0fcf899672483a\",\"77ef54f95f73f3e2b839d31befc9d490e1e584e4ead35c71e210075d572ea808\",\"edb7a232822c667b7cf3971fb0ce73d25d96bfd980cc874447286fea5b129f96\",\"f83a92f3c81a0a8eacc8dbc8957cb48e101692cadd9d223fe835577a1f327da6\",\"9c274e174c658fde1a8ffa005df2401a052949236ed5390320ecc56066bdf3af\",\"f5907af939812376bfc8047a6458ddeb37577c0c6c29da23ea34b699e88b2b82\",\"f1759a0cb6997f55724cbafd900790c34975230dea3aef215bab95f7753e98a8\",\"d9c8eb8e64c54f8be9014a6a78b31777b997a33fc7a38fd82b940d7d25369eb7\",\"24c1485158e774049d9cbbef39716d2ac5c86d00662290894d26395391201ed6\",\"430cc1b27e451e8aa61bf77f3f2585b5afde91d9121e8bf1e4948bac03fcf2fe\",\"fbe9f44fed4fe7d4cb2f037c56897d8fde48646cce974e5c5b40501b3b898980\",\"c7b1fb5cdfeeb3a5ec1c91053d1b013a39dd5946671a8d52dcb88f9d7b51e762\",\"f7b7bcda875baf91db40cfb6e0d27299ad25030567d7bd4cd64c90feab22a251\",\"60ad15bfe67bdfb657d0632f69be56cba3cd5eaf6751e4e6c0f5de05850810f9\",\"84496d26d475e8b8ec78846b2a646df85c2fb888cc534bac03be284992b0f01f\",\"9d5ee6f7f769870a84809374c1d15722cede0cedef4326299317cea9da5552df\",\"b2aff27f0464e427f6f3b964dd3b32a20154ef1eab77466684f95110d2c77cae\",\"219a3ec767cd08674055310eed4d33cc80a3cd46f65c62f1df12d429779fa967\",\"307aa62dd26d83d0826fdb94314762fe8d7f583b5d4aebdb666739d217028184\",\"c9ed4c39cf9c879008471f8c947008ac6059fd002845bcea50000e5af1201a1e\",\"c4438290ad5b79a74120b7d73c3aeaa36920a125bd6504be9d4040417f95ab18\",\"43d2ab2a0e33fd303b88afc8c43962fea122aefe42cdfe84f075cec5619ab03b\",\"5fd4fcbf83739752cdd5f59f9084ebb401f1f9147dff751e3bba054484629701\",\"d0c8ba3bb2942dfb65e1ca09940324ed3a1081ecd1748e9215b1fe394c57a29a\",\"ee3117d64b54cc0fecbafa4ddce14a2e74a48166053bb7e17c1c75e8a915df2d\",\"0684929ae389bf9068b31a96608db4b16da9c91178d0a6ad5ba2a0b125af0f3c\",\"97037b2652ae698686acfd6408ec7571b6e07ff7cc1e2dff8d874619bd856366\",\"29ccfe3f97944c22bf6f962ff7ec491fc121f58ef16bab31d944d15062986b5e\",\"a03f1a38b20fda0bb80b8e16d59839e0c433ec0a8870f4e15d1c522779702ef8\",\"132417379d35f344cd875d7db6b8f57b1893353d46e5632e72c117fa051d6984\",\"8353cd32c02a97b14075e78d4f9c0b9d769e97370dd58caf61627aab0b638a0e\",\"9bdfb47bac80bb95ab70e28ba000cf266299741030fa112c0ea094cfb21feb43\",\"1314d4944e318b0fdb9ff76c9d1fdd402c1d577dbff019fadd64b0d546ee5e6f\",\"94f996078a868875b400e9f0ac22f3269730bbf30620a0043fad055a854f3a92\",\"ad9ec33fb8658ae1d3c7ce1b395f2305cf7fa4378d6bb403f8deffe35e2e0244\",\"051964fdbc6b5c04d942d590dcdaf8e75ec161a33afd4c99326fb942f6ce3c05\",\"ab32af279718dd751e8a04202272db0aa665a16005f31e5b009514a53d594e11\",\"9812712f7fc60e63c87308be9fa975171be6058211d828f7d702d334d7c5143f\",\"48141233daa3fc204f46b9f32424c6ee40899caadcaba6c8e083d8b911ef84c4\",\"eda6d79e0672d0e31dcf582ad4446ae77214a644d27fddc3ab4b004c0b952567\",\"78bb2476740c64e563d33c130f1f453463ff08924f2130e243e1281545ba6633\",\"ac39cccb9d5cac5ee1658f655199f48c17f8c1f39585a42041511a93f11fc426\",\"b4c8295103847bd8f1b83da6ff22d05dc525fd89c01f3116137ad244eba42c5c\",\"733c0f9d8fd1ec826403399cbb7171990a3db5b4f94ad9bfb0449869b63c35eb\",\"c0b1e416e818a8241372c9662dd307190a17ea55d20ba4b295150a2e74eef9b2\",\"29f6e289cd0ec4845a1e61a3405a0245bd7b742fb98c1fc62071e7f3daa24480\",\"05747b0ec221122ecb445a9c491aa328490319fc8c5494456cae8e4e6f480779\",\"60bf2cdb361df4fc584399e219872d9c0bf3186c351d32a8aff480861bd8f5c2\",\"fbeaf6392dcea8d8c47d8db1f3e9a4a8918988116dae8dd31e8f550d454ab71b\",\"035f79437065e303e82be6902d98332754b9eb9d858bf164a016763fd3c9a63f\",\"ae66f758fdcf9867fe84f66d29e5498d42e0d2b00e960c5c9a0799d8683bc7e9\",\"34149eb3ad26f4382a170499362aaf3f9c8ed0ae32719ddeb7a3965a428e925e\",\"ff1a65fdb864ebf47ca55369a610ff1edd0c751e00f499a153756b8e4f09387e\",\"525b959efab63e776867f96e9183cf4a154814cd83aff5ae669a234537fd5bac\",\"ff2834f15d69402f485b9fbeb6748975b8e5c635322902716abc73f2ac8d970a\",\"5d7be1cbe55e82930752ed26e2bd6ab031503a641a3af45ca77c0307d9a933d2\",\"936837f9bd9a2ea5f4a62639e0995ba8a6aa6217620139aaddffd546ce60328a\",\"cce7d4d4ba707693342604a2a435903a2f30188c79b0e9b91d8586991333e3cc\",\"7fe1ee00c256bb5cd80c84127e3911556c76ac34883c5152f930827cb7b905e0\",\"4faaa943df3f342968b53923133e1fe03cbc0a2742f24226331848cd8e58094f\",\"58c0df553dd06dc935270f3d0b6ca50618d969a4db4e0ce3a43f166035366055\",\"743d44f14d443110471c7fe7c7e1ddafbe6748c5cf6d5e1a99e18fb88ba08bcd\",\"b337707cc486cbce3d6e0c157967c14403a533ef266dc8e6bf81508a2dcf54c0\",\"01daac46adb7cb23bfad666aa48f55d189fbf21e2c2b6113528881373a33083d\",\"5b5f241fa8300962cd74ce991ac32566945169337761ab09e563a5ccf8b1e8a5\",\"31fc464c33cfceb5ae01738d3613f59a4ebe97aecfa535f93fce86750738477e\",\"ed9dedbcf28015abcd056c9b9f9751f30d3b2ecc78e396bdb0cc3f19ad75a339\",\"53ff107f3b862fe9ec142c7d3cb91082dec9c99aca999f1813060735ffc5b051\",\"b20874131d862b210278f28519365c851e495276b5bf84f9dfa756bf80679d03\",\"e69142f90902ebb311d21b7466dcff9eaa2048630f59ac2f76d8f6e5bdbe4ae1\",\"19d86fdd3a7e9fba4f63405371e86ddf90cdc67f7490759a70823841000f659b\",\"d2e9d1f74eea4797aea18d3478345b607fcb55b490764eda046d4127f4ebe3ce\",\"6974d058436a61a0f12ecfc4248fd78c897d524922f89e286f3fe19e4d7d1787\",\"30fad864788be0f8d2596c9cf2d257ceedfac926720098e2a8a926a4bf7dbccf\",\"5723e051ee460d228d9731c43f60a55e5e1e304f9ccd16f6d57a347927c19abd\",\"1295003f7004bcb74b87c49d00b25daa07702c23f210a9b1b58ed4ce01488754\",\"f83504aaafe26e3a9aa5e5f5c77a574b1a25375ecd398b1d7a43f602d2ec1b4c\",\"a464c1a528a9d84085a084ff417ced54d82597602f10528ee9c3e3961b5fab8c\",\"2b182cf22ebe3d2a18e6f90a6c0751779600b5f3a77450a052a80c2066a560e2\",\"3117b42ea128ee79648b284223f70ff8744a67a3758992371b1916f93f4f420b\",\"297407469b40582d04a5fbc6881942626cf7fa6722891e218e552b585ce29ead\",\"ad8713d24a8460a9fdb137342582d20d7430c938542f4eef61326c796963c332\",\"20a762ab01bf9d916d95f7e7b5315891619d5096376565789bcf982f4ef91376\",\"d8e5f65e5aa3b42005e8fa979e193ae3aea0141c42bb56763281b84e52919148\",\"6d37cfeffb43b3b9f167fdbc4b2b7ead5217d08974314f97ff52edb47909d7d0\",\"b77400e0129538b67a9984e3cd5eda08fbddbf39e477f7f8ffbd298a444f1822\",\"b64fcad2832823b8fdcdfc23c8923667fbf58eb63933139d5a611c97a276691c\",\"71890770c5be46d65cc1eb2842bc7f75fb933c14883fe96e791752329fe932de\",\"b46eee5542c54095ae7258d54bc06ae8aa42421fd032f87e0c22ab2fb4342e28\",\"3e5b5b2c7497384f643fef149cb72a66fbecf4b8fe9d52fe2ec935aed36aaa02\",\"6e5786f02d204128e4cb554ae06a77d96eb89c20d0d6559e6762d4d874c79ef8\",\"7f8563a32a08b869f85c777e116354e1f7b7fc078a4280eeec67798eed3541eb\",\"0c71086c754bed18b6bb8b06378f531c5348efe2859dfc570249e5dcb003da82\",\"fe613c23362070249f7163b795b398e0c8fc47f1526cd42d0067a45cf29b2cdb\",\"892045603f58b9613ff0cc7e5409509437463a0313cb06e4e21954de304f5454\",\"823b558afad6e980192c6cbfe2de9bf1e0ae964bf112554ebe8d7e1bfcb4ae2d\",\"0d599a08ea4c96be1229af9869c6d69087d23f636da709981917947373ae1409\",\"67f31d69000f9384fe9ef94399bcbb05154f675b9f1a3e0a590a02f9d4bed314\",\"64932313f0b346ec8408ef0212a39f307ae633720e8dc46051fdd07bcb8c2145\",\"84ba22211491cdb1eee007df65010a541d9fd1b372ac95fe364cb701c0dc8708\",\"abc408a8ec623b73d4f4239a8d60b8adcc7e7a26cd6684cff57009b7d4ac9604\",\"16bacf4bd5252f11547dbd5a7f2e8c56145eaf331231ec1be2a01046a484bfd7\",\"e10191f68793fd91c35348c295442853589b754f1d04bf625c51791330d16478\",\"96ad313e6d51434a3e5e17d6b04d9de77d5e129c3543edbf9bd1c36681b42089\",\"619accece914a945178be99b6b7e7c52b1031706f6ee1e87c120f1889c7fb804\",\"c9b1d693d2b02c0c16f2fcf88774be8c34881536d00364066b30551535d36695\",\"77a8a3d771c4705778e2106053910f397d574b466ff4565a300a12ef47f8f803\",\"7a12bebb741b4f8a001f7623b3b5ecdc286a7d1179a7875d1ddd6e260be3283d\",\"64c1b57b564f1ee14afacef80297b37b213d37d6540f3f9298ecfd4e4296d888\",\"cf56fd7ff70e9a062bca266655a76bb4495f8771917683bcacc1b35ce5b52528\",\"98bd8b74a26d3abc1dbe083aaceac94459545892ea7d30f31a7c293e8640b4b6\",\"cee83fc4b6a94dfb063d1b80695b82bec1d8ae091838cd86a8951bd758e65bb9\",\"354559b67c6170d503044665388e058a0a5cd26ab20c5204b06192cc9829c006\",\"901b6d14211a13e9d32aa81625e5e0bf76d1e6bb00a63d97277f78fd2ee9b995\",\"fabb11442160663b8799f60326c65820ef40011ee57ecf43bf8a384732fdd83c\",\"f893b983fe91f78035531cbae60a3ee1d6be4e25ab62b74ad0ba014229178120\",\"9951e60f5cfe5b08173d585728f194a1d9bb182dea3941985d02f343b42349dc\",\"4ccc54994b8b7ede2f55548380d76e73c0e6c9806223225360e45eb339ec8412\",\"8a7b4184b751e4b2ac39b1b238751ec8f012fd8000be99d2bdde33e9ea6a7dba\",\"d16f8d1bb00351c2b662e5976608dd204227be9255db739bfe78fa80e864de83\",\"fc2380ca1d36a0049987b7d943e7becd746f221ff6cbd04f6149595e50d9cfae\",\"dbb333f3ad399f817228011a37f133d86896a568b68adb9aa026b0041a1d2c82\",\"2f78f123885b85c5e37648b5c0f75a8546761386d4b81c0971d6c0cb063fab9f\",\"9674163d27a4e5d1b039880a95cbd84a7a8b3741ac8534e142f90e1dcc9f59ae\",\"ae778ff3bd1d65ab5292fecb57baed984848b6bc38e54df6996be269d64aced4\",\"ae5e09d81b2e355219eba132abcad0cbcd20fbcc8beb6be8a39b7d88b150bf31\",\"8877c349c43a5420dc9f04c167de4daa95926c103e4e49202b72611786820b23\",\"f15a911f62cc648f9baeef3dfe310e7e4930efde1235e04b077dfa05488d8b02\",\"bc3806576684ac378ec5c607f1a5a8bf2c2c3639e7a88292b79d60c65a169019\",\"b633f9e7a460e8d9b2cee915433b50750b0371b6e8f540bc7465515301ccb966\",\"389c80dde1a101d70d3b4fc54b312da0cd80f39635be7ca216e75990341e1c01\",\"57acfcce92009c46859e7eff926c5bfcf0622dbceedef172a3930b409175dcf4\",\"781da72477b5bb22fc28fe331f412c7a7364281600d650f7fd1bd82c6e53083a\",\"a20dd859a6150c669870d6a10b4dc9071ae018b5ad294c8a0d2289cc899731c7\",\"6fe8886e9e6be1667ad31b8e720078638f5ba5351044a91d7f37fff3160fdf0a\",\"efbb885a4e7dabd8b673afe1592131030feae9f5ba68aef2ad0d86c99d6fed2d\",\"60777a3ba6b4131afd37b78162cf3ee6214a8006d765828b32ef52ac5cde0cea\",\"110dd45288dc9984cec59c33689545b2f7bb8aa50d62b59ebb0f0a75457525f7\",\"172c37121a58947e535fd57e22cb43648847573bdd2553c4eec232549fae6d23\",\"65d24a51a776fd30e2aea95836765e8f1e8746b5052a6a45af1e0aee5b4d786c\",\"6d7c06dec6f6bf4658ef0e3c65e9779862ed26fba66785309e2bfdfcf94ef019\",\"3314dceec6b29ede99cf3df18ea316476deb7754226e19591c7af009bc663c65\",\"f37384530ae5e1fe91dfdaae343b64d10cc1fead8c13dd2314f05bf5ee2d0437\",\"ffa7fbb3f73937ecdfd62256f700a46f5b2e400852b1f3e099f23e4f4581cf52\",\"298b6e773efcd0e6ff2998b68b7974d0dac9270abfdcf0b1cb7c4bfc1ce7983b\",\"64e503b8d722352a11a3fdb9d149033643fab98f8f7dada688ed598a60d7d09e\",\"c9fdbf385a7e5765af914b8dc33e59a75a50d2b768dc32ef725811d9fa4720b1\",\"8052b41cf73a8d250557cefdbfd6225f976d0f8246c226644eb6a230fe7c9a23\",\"a08442c10ab27e1e9760818ecf005eefb3a5a03350ababec876fa5be860c0ae8\",\"e18e5b47f2dca846eb9d3e9fac2ba8c97cc5bed807f68debfcb533140284b53e\",\"b7adc5acfb0c90043bfb321207b28462d85703bfe476cf5e571c0a5e5b90be2d\",\"683ab4e63faf83af65035ab4e4a3d6bae08f2838a14f456d42bf265686b113aa\",\"a4fc741adc85915c1a286f0f2d4d9ca828b3ab77ccbbab42deffdd98f544d6b5\",\"a7bf35b40c4ab405c9a1fedb435a3e0618248e9e1b8fe1c48843a1a8a7020e77\",\"bcb47638dfd11393b3f388d9fbe79817304050a1807ad215788c47cb462474bd\",\"6512c6de21d9ddb3ae894cf3f1b257ef061a86a2c4460767ef121a84f62126ef\",\"d4443b423350e364ab5284f97a83e3e5dae26f3c559fe8a5706e98946ca54442\",\"dd7bc6414a50c55b2a2518b2224bc7f3d0f61a947292450e49ccdf88fee9da51\",\"353b5a6dc393ed38dde2b8f566b3a84b674507b7a0cdc354a69237a796197d04\",\"2d4196b6eecc69475fadf4e449422f83d3e79ebdc1ac3e2478e29cac0d29aee7\",\"9d5ec8ac04e6bada48d327054e53a71e0a9f9b841c3e96911f90ac98ecfc9412\",\"e2d2acf7f7f5d612afb20f13e63d3d18d8c6dd280fa74d186717d34141e8ac77\",\"791927108c6223c24c0796072f74d399a9b977703fc265aadf14287a66d2dc08\",\"aadc85763b42a5c0f91e47c489953a5194941685123cc14132b05abbd4c38a1f\",\"5e9061fb80b08c1794094d8ababa457db54c1666157013a4170be63d82fae02c\",\"cbbe4b2ca325f712dffef310578e8697ca693a5446ff09c17ccca53ba5001c41\",\"72d910ebd3cb50cf9c0c233f8411b4c271e3bef24d25f97d9b8f7eb48968ced9\",\"b12fda041a3abbad3ca658dae7fc485c518396fb69fa9a32f463fb99cc807249\",\"0d3f260b4c6e64ba84ec475f4730c7e2abe60cca30e4bd32bc01e5602a7072fc\",\"aabefb8fc925c28c3059f4dd08672cc074ef41a1cd82a680ba5b7f0614612302\",\"603ffbaf4a6278572b9fd5d0890667684fb932c1e4c1c3e8f1b32c58d86d0fa2\",\"1414e1aabebb5df9b17767776bfb7549538388d433c404331323cb3dafbfdaa9\",\"8037d1c322a5960650b02cdd288804d20424cc62c660aea8bbbab357d388c45d\",\"fec75a7b8670706863f468a9bb4410a79e22208b729fc97e391ec9870043d6a9\",\"ffb06590d50bd48d878a412b181f467dc90b1c6db58fc2d51769e7c0655df524\",\"c534e35e95fbcaedb25e62d463e22026abc362303148d3bb3a563da8620eb404\",\"9cd12090c53aa7d659af77c5084f57c10b748f434ef6438d0c243a0ac8698cff\",\"a60f21b24cbb2cbb26ac9c365874f325c73f0d280d72ae3f6977771156be6622\",\"bf489a9132005e14880885e7796a1448418247d77fcebee7fa0e8ea74a26ef18\",\"eba153c2a342dd459c223e939e64e5df33fc8c502981f86083d2b927788e3ea1\",\"c70305af233543ca90663e9aba7a7e0405f908119cf5aeffcd145df9823d31e8\",\"e2680021866732ddbfb09c9a2b7e19dd20078cd303456c1b98623922c51dc735\",\"b3badcce821c6b28c2709420f21b4a4a08f5f095946c951aeaaa1c78cf43f791\",\"ddb3ec60df1cd28d1670e0e3f39e2c6f3578dcd5d722ad80485a9c4dcc9b7271\",\"4aa397e92c903f977e50155b9ef0315af327c801a438f57fb420683585eb9c22\",\"51f0743b44ecdaca67f8906491e4056fb7699046a395dbef36139699fd5b9183\",\"522ab1429cf696e2bb28964350354b68613f0c94962e60e354e206759c4a454f\",\"67e531e4afa1963bfa25b21bb03f0b5f3b5fdf4fc643b3c0d1c16b9edec7beab\",\"ec2fb139c1e5603e16fee95642d3eb362649b0d8a78bd99a38fb79a6bc0032d3\",\"5665033a803750d94d4c96a63dd48db07866a8678200217a9e2db15752760fe2\",\"767ce900a91db6d6cbb0ec5fd809bf8a0d52395312c66ac761a290689772593d\",\"69626ffa9c112b24a6614b354c8520cf5034e8ca5c757fb4b0906b7226151ae1\",\"b091e6fdb6488c49f5f44c42de0a76ac83d5f74c1365fb7eb84b221b3c258373\",\"10b6a2b24cc5189736632bcb26e82cd16d22a344980d6005d3ce1b6cd891ebe0\",\"4b5b0b68265f65649c1db9745938e0e1e873a3cf58e42f564b064c9dd3f5ccfb\",\"2f61d3e10a2341be9a862484e81a9312852798c3b855e07789ce10c77cc5e41b\",\"531ae3fca8bd085d236cc61958885b0bb15eb9431c718a9b3f86dc545599cc93\",\"be7fa51a1a7230ff530d484e1579408c4dd9d5ecc8271e06a29efb6585be5362\",\"50b95e4ef8a24fb97fe5b8e17a36dc1df0b89d7799952868630c8c59fd2336e6\",\"c9a7c892f3509f3f21f98e81e623d4c1d9eea5140e902babdca7ffcfa42e7b7a\",\"ba06a5845f1c680df576407339e30540afbd7b3979f4ca05201ab101d83bc33d\",\"8dcfc0bf9b7fd4fee3cd4c7f07acb232da7dcd146ca15871a0bc16bfe52da447\",\"0ddeb88e17e30cdd89a67fb632862b74d06e7c138ffb44408de02f5268c6780b\",\"d746dea8dc4769b52ccd06bd6cce38f5456628de2ef21288c8f022a389a3f1c9\",\"ee260b5a602616ecd7a44bb96335119261c6bf9ece70094ff806185cbf363a7f\",\"718b9bd5e6da4919cfa8e1fe54a9cc713a407cd14e8d5089bcde2e8f110c6f47\",\"890ce009058a4ee153b879318d0011fdebc20f3e7fccf0c71e2b411613c75931\",\"ed1777f7e324b0cae40764c529f65511ea09ef51e701ef36629a7e92845493f0\",\"3ffc5df8bdef0d78586e33e2cd3592b4f397ef1330f0497265b5eac8c07114b2\",\"bd3692dde5c1d0618d069bf9b2accf8f9938e8c9ac1abdf61315165e70360271\",\"6fd4c1d0fcbf40028e10ec6979348d198864e970ee4cea4c3aa1285eaa9f917d\",\"3391c025bee8b38d567c5c934a5a71b04d56c6c3952f09dcdcd023ea7198c876\",\"07fad6ec4d85e551879b442d5d3b90ff7a57a1bfad796866f50949908cae4397\",\"3acf321ddf7a2b6a7609ea124915cc3039eb02e6993d550fb7bb4959d0efef0c\",\"2aa91eed439034ed021d01919cb59bc82e8102c18f9e47939ccab8923dc4915b\",\"12bfd97e3031fd1475ced8939620a210da1693abf37501f6bdfe8bf67556de98\",\"1bd9222846ca36e4cf112733d0df46e42d75a4fdd4fb8856659bc068fa7e2815\",\"8b400d5fad71d7c706a11dbacd7c8ec5dc2d955e7df5aa48531b1b0e64b1c553\",\"6dcb6c5dd9c6363ccbd1ff5813aa80b552f3ef667bcdd82b1d55b591ed483fbd\",\"9cb8d43e07b75b33127745914817f97ec2e13e08c279e47a0ef81fda8498b863\",\"2da564f730c13f765bb15b662c66a12eefb8229f58adc1618d29ef1109dabaca\",\"d695f3659ac42af902d6db323da07a05333ca39123a2e424273b1884dbafeb52\",\"d6596dcb9fee66760d5b581665c7ac81e83459bd1bce12820a587a8b1f807b73\",\"4a9f785a42ef2f1e4528b61b54c5f802ecbcda7cc3b2ce22ed12f7c04c2a1c49\",\"647ea191e4cfbeceec8789cb5e719e2a176a487f72683aeb77b0ade010f41080\",\"c9eae25e9240663fb784a2933124859ef6ee81dd18ba134aa752ee1cc0e7163b\",\"5b5bb6c6a214934c3240aec23f8033fb77886f98e09abe0ca0b0ec39db0d9658\",\"25eb1fe2fc4c8395cce60e6852bb379c7593799381dfe84ad5062bc0ccd5ba55\",\"43c4c7660817eca89842dc52a8a105a2372b9114f0b312675f220a7b160e567e\",\"1be2dd3866b7e8a80eee382e17de05c90c779d1697d661de58a9e76f5c8ff675\",\"5a03628342a19e04c98ef4720863ade42846068ccae8bf42bcb615b1620bb127\",\"df57222a7a4ad5f47818a451e188c5582bb66e11cde2399e24bf3e5a2250a806\",\"9bf460911c160971fc673ae371ffce3fc8d85d0c56257e6bc6d8ea200afe4eaf\",\"0cf8883878c03f87db938a504199099bdc1e72c85394dfc379481bf55833f255\",\"3bda7b2975814cf482af8f52a0394557f8bbfcaa6a0c554b58316472ffc4c171\",\"27daa29ba53e057d34ff3117e50c74ff30a24cab529291e8a8c313ac918007f0\",\"9c9264e64b5b6c1c13cc26c4304fe5b4d4b8cb75ee98d5f5b9a3167aaea56771\",\"66b2c4b0cff4267b940919f1957d2b2760e3341c637fd4bcc363f022f5ff4d0d\",\"9a2925bbe9a59c861531f6eb49c2ee473a510cbe6053879e1c7592e481dc60ce\",\"4ddbeaf5fa4310c265e0d83cef4ca3e6f9372b5e8dee06d3cda76013eb5f0459\",\"151f85533b1d2821607b571b625a909a6b739bd1f780657cc2d9059f19420036\",\"e5dae3832a2da0d1f32d4a45d0e0ba29c073f7922980675d120f2ec7e62fd3c5\",\"08e815ad0779225ab58ac189a49b0d1da13b8490f8a6ae99a7aec427a3d4658a\",\"ea8b55491619e0ab6aa55eae552664183df932c3528df7d6e95a4c7a1938049e\",\"4c300b4f988ead4e66839658142c5047760cfdf35fa7cc432995064858c0270a\",\"0c86eae50632ebb1e04702fda96e848185ebdbdf99cae5f57bd8159d45d00d31\",\"8049a9b0c67450e45473d5e96e98abe55bd0659198449eadfb266df798433c7e\",\"73c2482a45c1ef9a0a3ef4724d4207d137a89e60f4d778183acb4adb4bac6a1b\",\"b562f95791571f3114a85a50c96241b46f3cd8e57b34c1e7292f0330937ebd26\",\"eb219a5a2a71ba68eae65944766efce2656fdd28437c06de6768740719198983\",\"41eb75b6fe4e1b2b48d7a8e1caf1a7adc96547d028ffa6ac076678a757ab85e7\",\"ed71352d9dd36e7ab035b50e8fdc9410d8782bfd6a3397bd94cd1a5d61a1eeb2\",\"86c118fda0b7b810e2df418e2c81dd2b7df2b4895d8fb029f6aac79a78197971\",\"c56d9696d4931957a972bcd80784144f70c1071ff810df38c3598fe286ff6ae7\",\"2f5f7359934126588274a2f163c0cdf59787c23460a719fa437d65dc7f7052c4\",\"89f83d551633ddc0e88e32946ba673b08d9e46047e06d250d40bacbf440b17ef\",\"6dab572d3627d1450ad27d16981bba05a6f6a8370bdafaf473a5f78eba785b6f\",\"7aa3a57e1e3e7f4a336b00c8d8413225daab7e92bf0e101934db1a219ab15249\",\"315dc6c135ffe2cf401a2c4612c4df393063609bef9290e2cf13cdf2b90b5ba2\",\"31c67e34b93a5a27b3cdfe1da377b2f61322148db2b0ae46719b5785c0a6535a\",\"993133b5f0e3fe4169bb10276aa4381ce2e9b2682991417804330bc5b9d0f57d\",\"a9adc96c6f79b0c642dfe79fc4a7c2b7d3c16d577baf0dbca9ac410fa3a5b86d\",\"0d5e87529b02c10d7d5e695dfdf032b3f39bf8461dcfae5b3f04dbd6b0e3f22d\",\"c9fd6ee6e27ae8401dca2af12a30fa0cd12dc71f7feda0af7444bd10eaef3167\",\"9577f22e414d2a6fd2904a20c202bdb2aac2f88fccd4f1465d98b46b67792750\",\"b42432ae2613f9c103b14a16a05af05739cefb9306d35e607befab34dca7422d\",\"cff37917f9a68a236abcf85e21640d64be139850171b16727d3de976c3bf6a67\",\"86b5e11241d46bfff6c107164a4170cc658bfaa360c6be001943b200e0592096\",\"bd6bbcb922cac3381a7ec98bd976878a838bbb0a7fa1815261860866d06de288\",\"0dbc7c5b5cc950424b8ac01d2f12dbcab69c714a36c65f261d9f764f401bb2fd\",\"a4bb51e62fdb5264c469634cf56b7553e5f9239d0c5e0ad2441250ae0b763474\",\"28eacdd8d39aecdd7bea78651ac6fae44d1e6556fdc06ad71669da2f8035a0ec\",\"3fe2f303817a8aec338f3d01770fb032c6651257c2fffc15080643a87f40e102\",\"e7cab896944c9f79a7d7925571a8f3fe0a954cea36fc644d5a66d9b7b84fdcb7\",\"90be0343fb46e31449bef4dcf8ea8b381d5a43d6efc6fca7706082b4c6f23117\",\"34c5100a56a1fb122b3d3be09c3dadab17b22db630eb8c0061bcd9e0ea214f04\",\"85444bf0c139c116332b6034a494bf627f6d034fc699ff3c44df84b23512a53c\",\"1706347dc452f808ff566faff7935209419b350080c715329144895d31d0845f\",\"ce94c9c93dfd10c7dec017a41e9bc112a13526337f19f22969cb2a2ad7001ba1\",\"61ae8ddc8c8e7ca0a048cceb3884364ef989e5125ad95eaa29bd053a5a44fbd7\",\"c28ca9c0341e7696793f754d7060f4dff40de82c22e708af31f9f375389afa65\",\"a2620c33acffa797d6aa83efaba82f65dab2043ff85e7332c9a52869ba008906\",\"45416b71665d0c43b6df4aec111cd3da5c273cfc06cdb9a82ae54ed5411cf1c0\",\"d13800f63236cfc607fc5d95d4183554bc9c791701bbd265558d5b1a74c580c5\",\"961a00f9bfa4e9148eb4912078e589bc69b58d38688473d54a3934f03c33c60d\",\"4fc0595685f1403843d1766ec90a4bd2be137ca2bd578c437eafcf684c5e4bbe\",\"8b0db60b0491bca38ea7b1e2b4ec3f076a074fe7038df696312498ab2ae083a9\",\"6a29f5e91c3c307a9635b8ef66580dacb59507668bf0d13e9fa0867feddf9249\",\"4bd3cf80bdf3ab9c0dd5cdd3f4ad5ff98cfff8a9fa2065f10a5e714e2bc895ae\",\"3306090cfed075706398de017789ab44f1ff3e8f0235b251918de73a504d4a67\",\"57c71084bbffd2c1a568705ad9e599ede8a7492dc12b0897093646dc3b6db784\",\"60a858d7a737a01f0e5a41a5c76a99cd948ca646331b11813928b0a2c9fdd254\",\"cd29a9c86e15c8efdf389cd11f660b324a76ce0fe88cb3e9c39242ddc5aa89c1\",\"be24dafd7efafb3fff960e2da0013d0403f076ca9a0a210f223f8627da769be3\",\"78ea35fcfad3ecaff7ca854e2f32ac43b5bb4f0a1779af13d5208b5eba878c0d\",\"376547d43a3f7471d2aa06226e8634cacd7930a425b49067389ecc45aab00d13\",\"fa5c116384a7d5f621a36e78872b193158c5bcaf2bb4bd4cd38332e3d5d13615\",\"fdbc6c4980a490e98e15d912c0fa2e9dfab9345cc3506e40268e4b8cf40ef531\",\"adfc885befb2ed2058276bb3264e8ee838e2c5fba6d84ef4ce3963a6d41b3b50\",\"128324fb22974a4ebebed0d4687714941801f02744c752627f3d82434cec4af1\",\"f13a878b50e32a49326e85d1bb6058490a169dedc23ef0b624105118df58cb27\",\"7f4bdc37062276f80dc8b4a365c27efede4abd73d93ea8a6360e41521bcf3987\",\"ccbe8c92d007b3f2e1338f0bf5d731c63e26fbb8160479cc0c2df15ea066b2db\",\"d8b638d26c7f72454484944955e24b8bfe285dd9eba141700c1ab23415f28880\",\"5b332b0277bf88ca8fd8d26861b3b60db810c8ade9fd450fb8680c5d5cba10c9\",\"511815d582988e1115fc766730e49f717cd8a6c93869ef33896df05b00c2bd58\",\"147cf3fbfa5bb25d51570127e3147b0719fe10409924b24821eb40aae175622e\",\"0f06b1469b32181ffad3597b600b95ca0a817fd84548c13b8d08c336eb8ef429\",\"75adc90f38a74fe8b89be147b3cae74263f20d309e93081ce280ce53a77abf23\",\"09076c249b7709ec5b65fe52c20b94c0afe7dcdfcfd99416dca7bdedee6444fa\",\"4d7fe4bf1e40e24317941c8127e9c46e9f2d180e4dc2359631119eb32b05eee4\",\"1808b809196e2888cccd5cf371e887640e758a6b603a39324374df7dbe86069a\",\"b8ef2580a3ed0a48851850416af66dd6ae890deeb714cffa7e20b51f45baaaba\",\"fe4c6d479bf188e1466c932b31a652b7e8caa1a5df44bfbc2fa07dd3c1a1921e\",\"6144a68a7ddf1fd3c1b870f9e30efa12bf7775f1a905f99bef871a909fbd8538\",\"b6c114c314578bc297ed79c3a3b33f00c49d4fe0c6c04f9d78d5905428e1d149\",\"95f40adb6d9714871f8a9619a7e733a2a782885ecd2fc01544d39c8f0413de60\",\"e9134ea59f9f52ad84c0628515c57f37a0dd144d656258e24d78e9dfbbf36ca8\",\"cefb1f81821e583c15653e2d118a15fcbc2f102ea6562071207ac03094aa480a\",\"f130b2da03c236f59795f9f55b8c46fa1d9492f455ae93dc694c5a0300516a41\",\"f761101934cddebbe7dc81ca9e6dae220c0d6972665e0e0c2e7e57b09f77728c\",\"96b1f04d9586a59717a5faa8ad6789fbaff863c79dcde5cf1fe2a2b08dae3315\",\"b30003113218573a195749d535d5c425b1edcb0e6f1ee4c98797af151df40a28\",\"56c921fa309ba47287f6b7dc9eed53b7839a387f09198046eb8637e3706f8060\",\"d5546d2c49ff69e9ada1e4fe237392ebd702f998d06036870b22da2a5a4834e2\",\"9bb9ddb141cb9a56ec4ac7a5d541e0bab5e91d45897250fd24f03ff201d54b98\",\"1440ffeca8f15269b40c5a6683884866a776424c30d62fe2301f6dff26334d1c\",\"ea680b712cfbce859f793044a7d58eef1a69759e5a0bde193231eae9317ee6e1\",\"7538ed81cf19fcf946b388a5cc104e97de09c8492922c50569efd6e0f096f1a0\",\"0c67ea5dd87ae1c940dcb845bbc4b5c372670f5bc95be47d928c9cb66c00e7c6\",\"e954787ee11d059ae1e392002b28b5f81b3010f5573bd80239e4d5a86ffb92d8\",\"8ba487adcb6742eeac94efe247a1974e35684e29e26530a27077332199b9e39b\",\"d2e85cdc5e06baac49328c439b462cb3011669b008853b1015be3bab5f29817c\",\"04bab7641a8502765e83d25443029a87aa518f89039a42dec1b4325e62ad99e8\",\"7eda8e677306302a3b5663a2563f35c857304e44659793cb9594899100e22c42\",\"2bdfd650e6c2d50e18268f1a2a0d2de48dc260543c10f3dcd919df7337b61317\",\"d5c932e685609d34d0527e79fb48d103d966914870cbff35a4ac0de941c98f27\",\"f9df4420adb32f1823cfebb35f62b019ef3f19a5099cb6124ed8ef8e58108ac8\",\"f2c301224fcf12e9c770975f8aca4ea8031a67a6149f77cbfaa278988630adb9\",\"357efb617b8abaffdc94a769315da2a2e2f2cbbd68e6e2622adda4e9c42f7e11\",\"7fa06750eb10ebd0f37b355995b3695e4d01e5f61c5adb4ffd4ec182295b0e55\",\"f829fc42dbcff3f79d172e9f1d12a7e3c7d5347e2fba9fa0d98415e58c75f31e\",\"07f2580aeff7c3b5fe73c38b1cce8497d9cac862ecc6fd0cecbf09b2e28efb6b\",\"6d7f5d8d662ec0ef162e94fcadc927c8d9d92596d2f0147b7489a340f9a2ed64\",\"1a50d5277880f700d12c3be47d8ad1582ecbbc65a9f74f50f55a03f2c6eac9a3\",\"d5f8bd6c3c755922ac10d6c74eb0ed019b566a5c649205edbb7d832305ed8463\",\"480ffb80b29562ac4792005b920b68ed36a648aed1f2847c523ed64fdbb6e9e0\",\"0d33dfb4149fdcf48036b5ef332fbfc99598094d7c8339416cc55507bdfc8285\",\"2ca504d02e088e7c168c01850966c3aba07c5461ee63ea167ca630decdcf33fd\",\"761fccb45b187612e27230aaaadd917f7aa515227489986d27dfcc55c54d1bbc\",\"c8244e0d74f9f8f27c7c4df9a772c371c927f75fe69dfa1ac38d5f448d64a563\",\"818370e412d7d739c78200409ecbfabdf2f42bbf82837ed0cbfcc7f5cd5ae032\",\"b66ef594b227509a84ca9152866165e9d5f1e5ad0c51f6e56fffacf4e9b2deb5\",\"0e0318e83baf9e86dba126c27c0fbac2b15672c6bd0bc23d972e0991685e3c05\",\"1226acf9e966ad23ac601f351f3379e6572fb09d537833602e0204dc2b15243e\",\"ff598df2aeac8f2cd9ce7480fd91bcf27cc4cc85a6313e616e0e62f42e58fc86\",\"a9af535fb3ebc8b4e7b081e81f457a315917813b090c9ffdd110d11f1bd4806d\",\"86040a2fba0ded816cc094d43e52ed7d3b15f63883eb83fe1fe09cab15bb8a03\",\"3baa06e2f9cf0fe8122f3748638920192a1a7492180ced072bdfcadb6879943e\",\"1c7ff37ddd9661a9a3491518e86716d62fb334de79e86e6973b1d9e69a7a992e\",\"a1c6e0ff15c99f6fe5208e3e754a882810e9ee4140fbc5b895ded0ec018663bf\",\"82369c87d2245ef1663e5ef335e7400554e070d78b5736283501c2524defc59f\",\"1f0eb2892e48cf8accc7f3fb01f170141cf88840990e36e94e3c66214aa54dcd\",\"eb528f4f813d2060466a33695cf45cd6985da1d6a7242d3c6eb7ae6cef0255de\",\"fd057da68f5996d6709650536c53dc44784136984f7f192920aacd082be744c6\",\"f91a94940aab46672e2b10542dbc357499c4aec6ba86fd92dced3e6d3e67d2b0\",\"9c666d7fbc5da785fb30656bc9fa9fd1582922c41b51176391b5018e57559c46\",\"87b48a0d5282d489b76b6bc6afd0b2350f4f0a7d89702da51c10b7857344cf4a\",\"5bf079689677e9eb299435ff55f243e41baaec4cc3d191c5afd57c6d8dcd2e47\",\"4e035f24180f63d6166dd9db5cabdf92dd60311603136dde8e42ad5af9652335\",\"2b395118a62ec63f0679051600986a289b418de210fa15fcdda22b6e219770a9\",\"16afaecc3221bdbc501e6de1f1e8066fa1e5055871e8f5a2f3a3e1fdc6d11df8\",\"a9ae467f93881e63c991fbec1ab18794418780efeb93245332fd5f09632018c6\",\"ee490fe3e4e25d5b6c71bb64ec083fc50d3a59c8df6753bd09a53da7592456d5\",\"1cbdc1244c95c13deb847056264a112924a61e571c374ece0925f18e3a6b3326\",\"5ed2892d0138fdff23c92850409fb9ff55066aa16ed9d4dafb4b895266197679\",\"d01f6f2314a50d22aa2ee13fd60db21c6d1e555d52c7caec38236d3612dd2985\",\"344c8a993a94284e435f070a16d5630620476b85ee71360507225903363e2c6f\",\"2cc61df23a96c8aa267f32f5f9dcf912e33133fa06d5ce6fb82dea4391214382\",\"9e6e00cc465f401ce4fcbf31e711ff6b9a525e871010277b008097d10cc21d56\",\"8823dbcda11da7ac1d51754858213a5e143ede80c471f2880741906d76312b66\",\"96db8ac3ebdef2511f98e90aaa0c92f6f360a525fdcbb2813ddb5febbe126d82\",\"4eaf74ce171b7f8542c448a515472f88323306f8080aa77f85b81be699758d92\",\"3d288008d1cabe311114678eabbd2120d597ab3164af7d41d85362bd99ff70f6\",\"b2da75ac2b7e7bf39e0fa1e3874dfb53f2ae2128f9a6007ff73d89ecf7fda7d6\",\"21d060c58dc26056a2c09867049f6fee8f15605d1e56c1bc6369c1db77fc7511\",\"f86e8db32c2dce98ce14a2009291c3cc82c87bd672b8aa13ccbeece1c9f75625\",\"e43f4f757dc1d60a2faeccabc9504b0d2b344d0399fec0361eec46b0f7cb98e4\",\"8e3c5cd46ddbfb4dcb18114ab5621f04cb65a0f23cf40016ee4a6f7f242ccaf7\",\"258b643ce8cd128d776886ba5822049fe00fa89ad2545cf45dcc07d9e4871ca4\",\"da6f92a337a1dd60eef8f2f4919ccaa13d4419c1ce981861e7235fa1a1aabe1e\",\"cd0ff0b47506638c5cf62f4f8afefdba6e177dced664f8c3f180af5bf9ccf77e\",\"e6bfea330b6e0e5b431d5336938fb29882b963909a694041455f9d035cf67ac6\",\"aeb33991b54ffcc9d85d39b13f540c658a9ff41202ef0ce1988345cc9eb3b59c\",\"4e7388ede1581845851bd6e00d4781fef04c4d7896d53e2abc1f34bccda8b790\",\"fdfaf8f75cbfc180eda93e63f158ee28cb8951ec3bfe77d82eed3facfb8ce1b8\",\"f129f69eb09bd469c654aba86c5956fef2de6a87151d5854dcdc3afbd9ffc656\",\"7415668b73e292ea043fdf448915b33cac93389523145e6e549ddaa0ac2c7f12\",\"5d979f641a43b0b3e5004051e95f079cb20ac0b296817cf9244c5e924dcb406a\",\"70da3fa5d775e62c4f46e6a6cf8e66f4a2d9036e92dcc44c6aab61047a4fbf1e\",\"83323ac87f0e0a877867a4591d0c6b1a93334da9e31ff0d69d2f47590b042a95\",\"5bd7ff7be0e0eee113f6c9ba5423195a37c1bd8a99dcdca77a01d35e7a2b2259\",\"4f41045d14d1be6ad214fa9fd9eab44aaf38057f259e39a4eaa8448139bf6ec9\",\"3fc1e8222de07915be7527371b29a4e61d9539791c64b68b2f059bb0767d1e32\",\"369c9a04dc1f1e1ae1dcd3a7a81ef98f5db307f550c9bd0740f7df17090feb29\",\"a8c37d0e7351c417b52609c2b42f1ba0230f71b6f43ab9d84d18dd10617b7f00\",\"c39b34814ee8ed2c13173994e568054be6bb93e0040af7b0c5891c99b18dbb01\",\"50da84f6a7ae6674e33d362dc8e53833c8fa8ab5e13e08e0e4d5c5ae40b589e1\",\"159f1e1bbd0b20a86a3d55de9849dee89fe535497f73aea9ea8f94e84c5e3d7e\",\"a8899d1db400de2e4bb535f6e8e601375aee5c543f99b343d28c01986e37322e\",\"815123835799e845922978065fc3a8a26f9ec60f9a299229a4caf2768522d785\",\"3d0865889e55ca4ad1bb95469037208bfe2c6ffeb26cc9ae6c7ef453f43f857a\",\"f6d52434c105845a2f806f7323020882e0aaa78644db6b5eb3465dad119c01fa\",\"3f7fb1acc256d03630ce72e28df5ec415710a6697335c01a0c1a06502b9d12ff\",\"af219a52454b708c0beab8612cbe85064c9c92acb07a2cda0fb12c406b8a2568\",\"d15f663400de527bc33f1d834b1ff81a9ebc906a4c96989d169655c483bdb163\",\"d68135131f17e7d7ee0549de33bc9aee20110d79249477d67202d0348a8c9a1e\",\"d26cd48a67eee85bcc35596c4102509c0cc2b661a789b23318a28c329fbdef65\",\"0940151e3536bc7169a0fda36206a38d57f4f298742843c9bde9092da426d00c\",\"f04773fe22bf5517f1aadf6d59e3c7e50a68c31754e0ec36b36e220da81cdc3c\",\"fd737c6b644a47520fa7b84183833224bcf91a457a0fb8e87617652e8386c814\",\"ecfaa0b2cc5c6ccd244998a7b42017cb18ba5df91581dbc2e031114cdbd1cfcc\",\"6943291e8cce233b66ad65e9fd80877a96efc8a1a0ad7a3fe320fb42658db811\",\"96f68fd2433df93a3db467af58ae2f011db73969009584f70ddfb3e79e547a2b\",\"ec59176b7419e5c9043faf2c39f87b1e4df899b4b60b2091f2849482eb36d4f8\",\"fa0f5bec7d0ab8b128dce0062f8b80d6cd1a1f980f824245f349d99e85686148\",\"b65f5ed67c8f5fe787a9c75af930a4477c1edfa4dffc302be71a7702842b2baa\",\"20321aceb629b58073f579603f20500b7736dd614d73177efaef347bac1300e1\",\"6d86451710b81a7b62babc31461b94a743b3dc8d257662b1ca23148cae86e4ff\",\"8a118b25adb167895158f09a9a83ceee1655a43c56624a9729d624307ffa0112\",\"a93e39772e2a005beabc6621aeb39c95942e0bcca6a59d9567705f11de1601f3\",\"04f4094a1698b1edc3c52adb134308224414440cefbbc5bef5e476e70fdf9a40\",\"0ed652b56fbf79d4adb2dd56297f6c1945454e19ed6f85e3cb88d66d58d5a6d1\",\"93ff3964480f68aa8dae6ae0293e5963f56313e6af7d6fe8733fcd9a73e03f34\",\"aa7d6b0510fea5e35d8b00ad9739b72a56423b93e52f325ae05207005b6f6b24\",\"4ad259e1f9fe3740f2d5b2570a42e5940c7de64afdf02281cded9e37e8966a5c\",\"9c8ac2ab4e84e1e209a24291ca8f842c6991ecb585979a8a432683c24d639789\",\"3836b3dc5f2119e9be1012999c8ea5ee2d2c16955f349e52d76efd098ebc497c\",\"dfd84359933ae8f2c3374f367d95a85dc7a15818195443932e677817403bb42a\",\"52ea06dae1d41133df471a558516adab2c091ab9ebc7b8d8e931a86510488428\",\"ae0f8cc7cd536ad1659719ecd85c87cd07986d775087871f459738e26167a251\",\"ef93540cc1c95aac5e990140126d37100228d50d5dcd6a72868b855865d085b3\",\"d5700e250686dfd1bf021984cbfe59c6e4627b4fe34f11be2c8c0e6bd7d4836b\",\"ba576b7f6059a98b3d39112274572a0732aa483801cb4cacebc453f2cd0f51f6\",\"806b1b46795dfd83946ceb6d394569baff0d51978a1c1560452054899b9830af\",\"b2427c3879988554c733977015f4b4596a025a3c3b90a382d5535f760abd2d8e\",\"8f8b2db92a5525bbbe81519655ab544928106b5b077725518d6405d4fd03924d\",\"c14ca3f5702524e4426d3054b2deec3ac241781b202e6a05ca464761099ded7b\",\"017447715936c57ac0b5a30c0a5ac5f21ab1a5f2d17d4ea15952a8a1fe23371e\",\"f4ad685fcb625147fa0cf873f70d9dddd4a0b5ae3eba61cb24f7a6e3cf891e3c\",\"622f8bd3c26dabe78d5f80272baf99f160155fb8977a116ea477a59013a3c272\",\"315be5bd3b85728c7722362225e98036f8e1929ace6779a2acc6b35955e404f8\",\"282e281c1f6a36449fa153075012fe5dce52a5871e4b81076caae0444b6b9fc2\",\"c2b1f1c9e5006c2810609b530c7686a7769f60488ba411f2cff239b4ac77b60a\",\"8b57285cd7e1078d39dc0f3bf6f91b12b8ebc62b01f2f89656b91b60b19707ce\",\"d28f13d2e6406e66ed6f8f7a313843c778f5f4a196d00c198301032f7e493be8\",\"5ad68d73b7b24ce79a369c0b0a8e2f39c15e71a6dfea292749cd493d80b71908\",\"3da86ad2d9e770816ed7997efe8e845ecf4293806ca85252bd560b382887d787\",\"b2deddf5aaf0e481b7b0303ed8d5031189aa3de7c114d927ec1da61cb3c41b73\",\"24e2ca785e2f49b2feb170231c29fdfaf9f7b86c384e9868f926e2d5c7204c5f\",\"b42f198c5db833c11f7a49e316b2470eeb6907f510039dce3a6324f518a2e7cd\",\"6bd20d7e9f0b4dfb20b10696d24b8c82117c9fe86f15f2fb7a50ce1f26b092ae\",\"e4334f5c17ef7b2583e8e4216b6dbcb9732a159d6062da8a1d1752dbf923476a\",\"9671d36e9589de5fdedbe47927c01dfa6c9f52e2e3725dac3397f84cc14ccbba\",\"c1f3a2d92ee36269f31d6a9ef8c6c8436e28af7a99f11e385adc201c7ab46b52\",\"f4e0c61ddc40e71932d6b7d4cf2bc493f90c5e87fc55189abbc9251f9756c9c3\",\"66416c89fcffbb926173d00a946b563ed9163442c8ec27d7a8a68a7206bf4bd0\",\"9eba3ffbcb79c951fb8f5d8b9b5808b8fec3ec79708b35390185acf2766ecf8d\",\"e24ed4be2ae274242717e38be37786ad218d4cad89017e67572dae2bea693140\",\"861adf51673d1bcd71a42c023974f2d4e21035948b070a84a648d682d0b5e121\",\"974ca7feb0b35f5378962e4bfc2f4729b3658e917824f6649c50eaad1074ff13\",\"0dad071e3ae311a43d623a9330156792bfc2670a3d457d783dcc423652b5f59d\",\"4a2d86611e2cfc0cdbd0c7accd1904252ab833b3232c3da4339581d220b757f5\",\"82d336525457e194a8275c12916ad828b3c51d6d6947d67ce5b343c0ee922408\",\"85c19df7423e1ac2575f38bc6d7ed7bcdbb0ecba2d47eb6b57297302ce670c6c\",\"3430e30c69cd6b78977f69e305163ff0929a9e2de3f8d7a09b2c3b16d66f0e52\",\"8d919edcff219ef4e76bfb094afb809a00f0f7150413a4e95bf7414cf521692f\",\"4a43333867cc6547c9532efd974686eabccafc5f0674b268a7120c43c80754ee\",\"0641c547d2c17b0f7c139fec132b4f93c5e4ff0361cbe8c9e857ec4b23b98921\",\"b90dbbb9a8f2e0be6e455a0cc78796ea5cc39ff98696e3332648473bb8b73d67\",\"c74d1d40966ec212051f874803ddc95d77dc280b43f4d903c5b7fced02faa7ab\",\"37823c034688ecc61d46c6d956e3491d1cb5fbabd0210ee446b478a79f6d243b\",\"a67c1399c37e08851039848fc8bf2b7cfdb3c7dda4cd67251ef43df7742cc792\",\"46c7892335187a166b8685c5aa41e1f18eaf784357d5d65b0744746168bd6e58\",\"dd9c223da4e0e2230a5e80ab36074b6101664178945387e7366a42b49f833853\",\"422010299ca661b825602a3dfc95b620c8da6dfc9aacfb1835c6dfa7cb080bd2\",\"2057965aff20f7b228b61b108b08f12d864173a4b0c44b63de16604f309b18a1\",\"51dee81b5f20f418c103e7ebd4181e0e27262fd7aced0967a6665f773cbe2809\",\"f491ff52ade176b78394821b19ed29386cbb79d12fafc8400ec6697aeb7e7473\",\"94e3ae0ce1b68cb29ae7788782448e4f43fd13b0f0025b41bddb15016098cea5\",\"bc2c2b142b500a4bfcab5622c3d0ac75335d6b9186a195049f56ade235b8cd60\",\"1c0046d0cdf842d91a028dafb58ec914812c0834d4ae4b1a1c0e058bb75ff877\",\"326d082855bedd13a528b89b88db7713c325d79d4ff360947d0c63b04fb982d0\",\"0e3614b15c13e06395ae3a2709adf86d6cab95cf577a1220e61a3e865a690f6b\",\"b841f69f44ff6a7867569c92ee7fcc77a271b40ecdf0360ca137885e2fd7e1c5\",\"92bed7d2e1541b6c027747898ab0f6f7996ae018e42fceaddf2c29cbd5c45da2\",\"3eb900410b736a7bc7390458053b695d841ea66589369c55821a851bf6e3cf8c\",\"c97c8c97a34843c207bc7bd16683a5b3fa7fdb721b5bb5c10b8ed1b28d9bffbd\",\"ef069026de8fb3f0094dbfe82b2c46a16a9b6fbac08e1952ed791eb51c0b16a9\",\"89bac31ad658cb7d5440a294d9e3174baf54536c8fce3f482593ad9ade7d697a\",\"8962a3fce3901f124299fba1c22912d9f13378e67beb220195404253a10cab07\",\"23fec4c8880d1f36d04f07aa6963857a83cdb120627b66dc5f38a52795311885\",\"1d0e3f280db394aa41598f0fda725c556fb9c6a5f14bcc9c85903163410f7553\",\"36c2a2e313ad34bae122336c79cef43c85dea5de5104d01e3aec92531771acf1\",\"89aacf47a8384c2acdea1a0392c3dd03255992eb75b2aff14fa5d2432d081b02\",\"1e6293a38dee2a2967582a101631ada7e6564a54b80621a8cd1133766f53d999\",\"759c0854fd83fa5dd4cff3d6ae1a744ad1b21676e555ab135eda38c877f17668\",\"12a7f33810b0953be731b349ec0ca9593a4619323556459161a8bf4c6486342f\",\"28810c8e453fb24095ed6f13fcd3ca00f038cba63b2e5f88243109358f03ba91\",\"3679b61e9e88007a8f6accc02a55f92b87b71f32e0ec61db7533b9544b1daaa8\",\"4de48ef13f4a8fd5e32dbb0abfe4013cab4d8e6cd222626732e5b22d63234bd9\",\"8775f8593a4a4708d0e98a61351d3c26dd8dbabb4e289c28a11437ae391589b2\",\"33ebed65bfc0f2f7e21e229853ed9ecb954f71263035da7776743e4322a07ac7\",\"3dfbccfd023817f01f2f7b0494a5f8dcf117c56b530bc9264b3560a6d4652323\",\"0cbf4260f65b96ff5448937da95e84cfa512d9616f5ffd158f2a789f34644443\",\"541cd8407beb9eab8b14e569758daed90a5c4f7285586ffa04b9cb911f71592f\",\"f822431d68a332935e0823ef2fb244cf32077664c7fe01ee6a0cf8408fa4cc93\",\"871341d6f40a5c574f8bd2255c36f8325f3a27ea3e501c3b44ca4ee6cc9a7db9\",\"b6aebd407fb527e5ccee104192d2c1767d2e67e49177e08e271adfcd101c3d55\",\"d6a6b7566a833fd9d751ed31fad31352ccf3fc593064b647a55e55ddd2e776c0\",\"1892936d49c2a70de9c9542e74f541a0d34e1fbb279e8bd84de5759c63ade817\",\"425f25b8cfa3b3847dfa6eba07f5e088ab83a1779cb6125825524e4a71d5df9a\",\"bcd09de09f0a96f9e517e732001bc5d5a708be5982b538d345ca105cfeeb7867\",\"e522401d1183d80c666d2ff1c988d8e9b96da5f8302df9e5ffbd6474967db66b\",\"9cb1406051893307bc792e3a64dbc0d224cc09d3af1fdd5a025a873f9b04996c\",\"29f911687edb2082179eb0bfb6e76be117677ca8feed2913eee3fdd2bc1e0464\",\"ea172dd5d8855a2bd30824d61e0abee9316857d67098c0c0391f1252906f7159\",\"64d1e7f3a5c8141ae3ad40b6919bdc6ace6c2aeb46dc908d8838f746e303f969\",\"7df60b04ed9d9f9ffcf37888a1d27d836fceff174da49c19034dd160187fb154\",\"45abfbd0fbd4efdd2165ef2c5b60b73979e08a662b3ba63c01653aceec89014b\",\"710ca47b866ec1814ebb6a5312f3ad567ff2db90d837ff4b3fbb43786f21d3b1\",\"c62db737f9f8f5b3b413b80f9f3a774d6991d0e2b6de908a6606c76bd98ab42e\",\"1eec28bb97a2eb6bd0fde4de844c9f367a464aedfca07e3c5b3574e52040cf91\",\"0800ea612b9502fc80dbabc18efc857f9a9300bf960212838e08c8aa62db8c58\",\"f3d4e67b3e03b9b52f720814e30ebba40b1212c9e5ba94ab023542d143b5c378\",\"2f8a8a2e7543e9c8c152b1422564440a8af9944926517cc69e44f458a7847313\",\"8f999fe6ba99d34a5db434cfdd5da2226bf0cdabeaa00e959f81bb8f3550a80f\",\"ca3dccdd55b3053c39cd3a7b4be92ddc1fa35a42c46c18a1bf03d3f20ed74579\",\"6aad72cc889656ef3ff11367d575f27423bd9f5824f9d3b7ffecabc183b693e1\",\"38139eb15b8de688c0f7215d96131408e404873e8d2962a1886753a443c3282d\",\"b60f6ba5b309c210ac5f4008d985e7eaf9c46b55b99b9bf56704210ee8bdeef1\",\"9c6a080ad17cac6794af9b9c4aaf2051423842d043387acd58a11cf660d8126e\",\"48279900feda5a92bcdf6552ea3c418bd415939c53d04f43869e4c70ef9cefd3\",\"387c64c3dd41dfdd4292c459e15da032e41d810dfabd07d48a2773d7951a5f2a\",\"1883feab30d607a7ad5422aa7c86a14e229acb7b0e626ab735cc6847a5f318af\",\"8ebba9e00ec8cb1efadcad52ff44de0b062f9792aef704cbdfb34926c30b34ad\",\"3963fe5bbdcca4bf7adf0fed13483170db28a8c1a751a6c03b8efb42b71fd813\",\"7399e4f2909bfc94e92cc2c68243437dd99bc9c34408f131e81ee11a5bbbacd5\",\"1796f69155934e62aefbad734b3720b7817133577de1c1d3fef0f2b65f23349b\",\"dda83dca2b96a3ee84f2a9317413d67fabd4cc7be0b1139f996c271dcb8d6e75\",\"d4e691b0877ce610c58cb44f57c4b820d9ccb9a4ff431ea5248704f31f4b2427\",\"73c6f65407b11bac919d25ea2091ddd3da3ce12b621400941e72cd22dee1afe9\",\"5743638e8524d408051f27ba5db7f33810b4a2777c85853c1b2db348ee39399a\",\"19a12ec0cc01ad3a704b585b5435c8b7ada85ab3fefa3cea6327efa6668e9df5\",\"a1b9c7c5ea17e76513298032109294d0172eaf9926be4966f980787e3fee33d3\",\"1659c3d40549e7c80d2612c58b0ce12d244840f252aa312ffba757dd56cf375c\",\"e298f56bc41367809f06047a6a572cba0cd979889808d48d1b0ed2d5630d4a68\",\"d372a090f38209339f503009703df25712384d540a047acdff8ebd258c42cf9e\",\"28930ecc2873233ab91a10c46ca41d3521af666ac2a496774db2559dc33c6265\",\"607c993a86172f1e563ad29bc643ea7b4de4da290c1bdba4f2f0a1fecad86f24\",\"9ad63af3127243728c8e3e9d6a2a052cc5b183038f955f9f612617d4fb01b90a\",\"22e78dfcf41cb8cb33311ec90164ffe2da95ab45b61745396c0708944f042a13\",\"8b7116d720fa32da820f00265a349f619d497619ba4e62853d17300a791193a6\",\"9c0059a3417bec4b7289bd1d2859831d1029bb18d6dd884a534d89901192ba44\",\"c9bf10fefb00264f8ea2f915c31c089ea49f2f77b0efeed25ac535235cde43c1\",\"f80ee19b7d6d1d3917eb1923671767655b66f12eff21836751c963d48be5daef\",\"79f297c6d2b38984fa760f4879b02aa933690352d84ae153dac2767ba0dc2974\",\"8afe83a9d8de3db946653ad6ff8973c1b46f5f0ab11fb9239580a3e2ae141bd9\",\"fdd8bcb2eebfe445dbcde8624638cdc34163a7c19fd5020c1e09f6ef663ad28c\",\"e5aff2297fbe18dec2bc91e48107afec89f33f573b927720d8f5ca608b98b681\",\"3efc7b1e00d86bef19d29a62164b84008a8b2a20c596ec6dcc179b80b1c130d4\",\"0d9fdff75b694039265af46157da07782dcddf27ee705299d8df1f11c9364cb7\",\"e4f173bfa4f9438a45c3dc631166ef6b3ae71e4133570f09064f6fece0f0b01f\",\"9f604e1e334c784c7c528a635e3df4b6b307c42d85504bfe19562fe95cca6a9e\",\"760cbadcce2a2362ff116f491783ebfe7298882a4970d92e0ba9ef4d892c1132\",\"821e0a06f15557bfeb062a3029fdabb7a7698d5578a229c76b3e45753b0ce9ae\",\"6cf841eb82e07dce7493a53d018d1ed4c13af6b8a1d24f41c4f433c3872ac288\",\"d302585a3ff68c8349c3daf0fec992da356fdf08bad7e5082a6980c9c5b503b6\",\"dbf6f1ae0ff8b01c245f45c80c42b24663a8011dac54638d67122e09a871e926\",\"cc757c25b5475c40cb800126a03d2f835c81bb8e0afee41a5add58f099a7638b\",\"18f84781f84ca61929075db46e56de8749e3359a191c6c29ae0e217d8ec1df73\",\"dae84c3e15e3d11544808afcc103032c13e1ccfbc348a9b9579a1a6130dbe2fc\",\"ad363a5b42f528e5e20fa5aa1597374360560b8b70aafda25fbc70f5050cadba\",\"38c2d6a693fc9a982f4e1154514200e289f57dd17db10b24f8a3fa234075f30f\",\"1b7957274b566c83b8da1b28e66b529289a147b3ae5cc36b700cc08266bfcc75\",\"f89a4c6755d5462a8e614d35482fb33748e6574d8e8ce6f92b1ef114a1f9fafa\",\"e6b1b3b3dc255bf8f3ba9d72bdecdcbb33d70c9095c7199044a2c1b7a9dbdf42\",\"2edc42ab3802a22c56e1b269810c6327e9beabffb30b08181db31e54b82602d0\",\"0e104ec82e3ea5f9ba4d744f2a74bd9662e3010a75cbbeb5568e4567ee9ef210\",\"b0693812978662866bc201592f1c316da2cc1d627780503a732b4735c1bc1ba3\",\"4e55b278a7bb7902c4f321a15ae58b798f7cc8a96485e9d660e9caa6a62e8140\",\"f8da89d25c2e322ccb4c48aeb514eb381cb0fc3ae986ebc34993b27ff0fe9ebb\",\"29fc5644530a042637324645eefafe80ff04619d5caeba82e45609f9495cd481\",\"24a9a20cc6c3225375191fbc87ef3eba9bd6490f0accd8bd2855dab1fa747c9d\",\"771e5184515274dc4dca6bd7e5774c8a4e8d49329bce00562044cb2c2900591e\",\"771059c3b5fdb4294f3c388a0984fed01d62d2718d7c1f0916a1687f6e934e67\",\"6c4783752453b1113ba0ee88acb8a7cb6541cdb1a5513844b9fc0e4c1113dd61\",\"869954920d9285c7c2671864aa53c60b5c272074017327cf27b11308765448b7\",\"90b0c22c4110e07cde9767de38b996a5c6435f350737f2b57959fab7baa36f5f\",\"1b87554ca0e8e51d59faedf479ddec5b65b51ff68be4dfb7806800ba92eeb9c8\",\"a0be04b1c40074eaf212ab915194a579e16d9542dec8a12e93f549b1b825439f\",\"bc6d113af899aa5431393e9725d7f5f9af8047bc317a73e35b1d6aa1b184f7dc\",\"d73bfd8b0ec9c71e2067342c3fd40a1686314dd3ebe0a1689c93f035d049f273\",\"7744dd8afa20379150ff1c0c5b4bf506e665f1f81b9391999aca01065a723326\",\"d56d53bc5fe9be39fa6f5bd07bda4d996f88cf8736d9bdeea0700d2c403270e1\",\"ec7cb7bcd9723597dad592b9db7423946266467a2c9d61291491abaf5251c493\",\"176b9771110c20729138881532bab7716075bf0c593934ab735e2b8e5c455ef1\",\"c2994015df0128eceb69ea10e8a49302ec3eaf415617982a01151e28a5a56213\",\"c657f4b489a123113cf8a707c2dde321be4834c20c6e99bf856895e5f1d9f2c3\",\"c9b6eb2b5975c6be70828909976ba086a4bd41a28e7ff06161ed556aa44cab0e\",\"364f37ad720b85b3de8fe8b8fed1f2cada0dae63f9fe73fd75116e542eb2ef8e\",\"4d01dd148b1512e41e0921bee9a9cd49255fc4ae174de0572303f1922aae642a\",\"736b5fd0975dd47e6b9b2186af73abacf03a04f57c3153605af86e974de895a7\",\"3610d2ba0e545d1e7d6f9fe92ff28135d0b30f318d3bc42d1f3afd3a5acf5d65\",\"409c42467105c88d8390616fd12c99b595e19d4413ad60b31c5d50fa430153ef\",\"ca5935a3dfede26b804419d6634dfad8782f613fab3509265da0afe5d1f0fbec\",\"1af359382a6ec568fa3bd0d4181cec99969217a9a1da33bf916909febda10b23\",\"257d273ccb6cc87af84fcd31f7df78ec7b8457ba6eac1e2c99cfac0a941f1d6f\",\"4b22e5dbd601a5e34eb4c07fb50525dbff0d5113067dacfeb18bb070c9a5b96f\",\"4f636aeacc440e3be83aa59dca91ca592e064da61b9f7253e98ce61e688fd427\",\"de8e2191821bc3bcaac6e26b9ea6c83eb2d0578bbc51117d719833890595e91c\",\"86250a1fee68958e3c5a97185b5ee45249e8ebc8334ec1392068c1ceeff3e471\",\"2ef5879e8b7b0f46ee449343811e67c978e3a81f1f253f36d164941a805ca48e\",\"55facae6ed64cddb6916a3145e452000a3a05b2d05109b019744d37f054bb992\",\"0e7b1e55720af6c5d4a356b3c406f1f09c822ab0fa95b88dee4ad353b8a8967b\",\"33b3662c8a1ba5daaf98fd23a1ef86ca7f73de27321fb4205bf3b15b50850623\",\"8ed6604d794d23d5660a5016941f49fed3e88fff5169c6697eed8f9b028249e4\",\"b8aeeb809042b7a64d7be212c977e2294b3bd2e778b71f040c97f3f1c66fcb6f\",\"aef40d3808cbb8c393acbae81935794cb41988fbf8f399f094a479b298fa2efd\",\"398cf5280f2d2f70301e1c77bf68e49c4c7ee8e664f35c97c36dfce600c1bf01\",\"53bf39f9e933b5504a402c52f4fc21c8f5fa67f1b6bbfc98ba563d8e7364ffa1\",\"ad1350a956c5a5d95af9346e452d81aeb2b623c3c214043bacfa5daf6881c890\",\"aaba01fa1d9d4370111b25a4905d01dbc5938a572536ff1b1a517437a2c41514\",\"c5cef45f3266328a6162e3038052bcd5483c53fc7f1e4ad957c93ebd63c5f746\",\"8d30735f70cfa9f0b2d9a4932d09a3edc3b6aa99455d24bfdbb0d3c29b7e31f4\",\"bc881edca932e65d02d5c21b8dcdd5468671c7d4948af2d701b5142b65cd6aba\",\"e8e4c76753e2cb57b8e3a9b1c99492eae11909ae1d78cb23ff38f4f7271669f6\",\"b18ea13e3f67dcd615407cfd775e8d6bf7147c7126d333c17abca3f37e37d029\",\"8fee5b4615f164ee563c9757f98785563ebbddcd9eb96435e198581e990dacb1\",\"32b0fad7cf3a6faad9b758608aa000023d31c2acdf888bcdd7c85b76e36da388\",\"e1b7627b6e1151014a4d3315ac617feabe0240b19e9981069a9e91654e273f38\",\"7f5fc37cb1407db5e25b5143347bf2da9463ea5c3d848601c7dcc37e386dc994\",\"958deeb4540cbd9bf5df36dac96c90cb3eeef9a1acb945716343163772de37e4\",\"0f4003a636b430702cea24339cf4acfe630fdb7e230c76b4e45463d6c636a38f\",\"5fe9fdcade2d887504f49499d7e35b9e8c311f09b683961f2404ce3d6eee163e\",\"5169327c584f8e3137379fe3a079b04d1ac844817491535c20ba06fc4c82ddfe\",\"85e380915d2459fa79f5f784a33582ae95be28472cf40502bf5d2750346a021d\",\"80ee65513a8c9febe625aad5af0b2bb8c230f0210703b3fd2d2cf842e62ebcb2\",\"08a694b3ad2e5d257087c1480f3f9e01974aa5d83818421c298b2ffddfa3ea8e\",\"14f1c77b9c075eb1772fa628ad2375a88f12e2f70835fa9c5e55dea84b247d03\",\"645d6dffe2ff307d27e506ad27f00f537a8ff88d8c262003e599fb56cf5f36a2\",\"d4d54aaecfe9473a060b724304b09db56ea3f062af045e37d2f1e5bf928ce3e3\",\"bbeba82d7e2646a8e6e4ed5a58e9a4452962e8d5e9acd1a59999644d8b342d67\",\"cfc0f3ebef98e96f3516bc7f477f6f47b10da5c10f845d79384e5d93733bdebc\",\"e466bcf0f8c93ccf214327dfa8dae072256922afaf6fe2ad8dbc40bb99c599b6\",\"cc3590fd0a599cdc2066a9d3ea822d245d6fb3aea322a43a0fe812cb5c11f27c\",\"56c42b3aa5eadd9dba34b80361d33871ead9185a017c0048e2fa651ca6efb5d1\",\"8b3317f670777cbf90798cfa4bc7a141bde95f7b06733fc2a1e93d7b19fc7c2a\",\"e152ced5423df2bc6430819ceebf67f7a4a56c34e53c0500de9ef19f94eca5e2\",\"a42b71fee5e32c9f3860e7dfd0a66ea07a95dd654819fe53a7347f9351cd15ca\",\"ac57da9be87fc75fa7a7701a137097fd7d3345747a56f23d7344d205a7bd14ee\",\"890590034da63372039ca00c5fe5a3f7da0c1a1481c417bed39dc132a1102d02\",\"e2365d2b95e1c42e3045ad3341592d1c629e19ebce5e17be4e8f17492551e349\",\"411d9e5d4587d7d0aca624f53e6393d0bc44ccf8e61e2ca8fdf983568cc64305\",\"27ad5495cded8e30802f3ac3d1a5839136a5b39c4a086d4ad46ac6e6054e71aa\",\"9c3024c95252bbd35ed8b5fea27a2ea1dc355742fb6e4817b36aefef791b8a20\",\"aa78d2b5f4d79aeb11a5af933a6148c8be935b6248ad0d43c452d9c5ee80bb49\",\"4a1241056d0fc5b00d133eb86a2e18a3c471b61762a7be59109d37ce7197ff0c\",\"7e6427a54d150b6b6cd76949569c42dbd1c0f0fc466bdeae0525f6f372f20751\",\"55e5019db6c047b0f33f36415b6b1402e8818c907621acedac89e25e81faa3cf\",\"d614ecdf1be5239bec9ece7331a41734a40fd7a2810a4870c7528e0e53b11f5e\",\"33a243580c01041770ac639dcd45c10d61092df2bb094165727bef8b3b3ca795\",\"b03c53b4a812fdbafb47178788e02e7e7a272fd40fdf86594afeb47125b04767\",\"a5d53fd03021a850a7d6267ef407edb1b875835d418e2b395e36b83fabaa9028\",\"443eecbf36884fd3de330485fe278b4e8edad7a3880574dc5f8c86dee7c8fa5f\",\"333606f12298a4dee161e73e7f856410cd02ca5d2709e072f9c7521be8bb39f6\",\"7d2a2d504a84d1b24b55406601faa5f3a307d9d576e3b0a591cd65c7e69788a1\",\"ca3b4d39ec6da5305ae025daae8ed21057316e2973e3b53034744191d4c7c55c\",\"39e130676e92af3492f4036e88df35974d299e7297d222fb13b651645e0e1386\",\"8bce099102653af16039d769c790aa2eb8eca2b450780efdcbecd0b2b088abc2\",\"b2c96a6af1efb07d34a2792fbdaa95eac3a03f078c8a7df81a21ee7fc655b057\",\"af05cdb95653b21b6f26c8edb130b0a7dae2cb401b55cf165234fd831aae972e\",\"806bd7e0c88f5f4091fa186b2b2b8317925d300e0e1e2f3069cc59260609fe8d\",\"77c76a2d1a7a45f1f4ba461cda9c1401120dce4185a6f3c2795ea43695a00bfc\",\"e9928726ebd5a81ea7999aabaf00ba010af208df4146438be0fee64fe6e9e855\",\"48061c71365bdccf00413fc71dfaf397df698097827d28a341ac4f837770eb33\",\"a90e6c34885332e460adb82ef7dbc51e753fb79a1e726890395b9fde463b3ea0\",\"30746c55c269a0025ded9b4eedc36cf2c346ed073387c07a40d7863e156a1a5e\",\"5862f50dca6bfce6179dea9f4dd9c64b974a50d50ef2575a862abba44579738a\",\"6573e72b97b3b9d3e0901d452548b3c74837c75c9c56804a2ed83f557f40e20f\",\"2c2f80ce81fe6be8be746cdca9490f461e024111c9cf09cbc46b58e4f00afe34\",\"be37203c2bd4ca5301a7839537f7ef24987dfe94780de9f4779312d38485b47a\",\"cd38c29dcdf64cdde9e2b1460adc8d81248ee4952a33d09ba06d041c84545b73\",\"639789a9761d550558e2413e8b68ac2a478e2027cf684c68968d14cb46faa2d1\",\"48a32d83a2fbe0205f444ab128aa5c0da99c5b1e774a269d43bb660f6d97861b\",\"6b28b80ca26c1bbf9a1e528a65041246fceda1374cc7df8b0b17471f6d06ad87\",\"3f4914564664718c11fc1a01dd0563d61ed96af5e94dde3638d24901f2e0c3e6\",\"6a84740cfc393f51bdd906a48fa6be344a1afb484eceb25ed462ebfb7e3bd1eb\",\"4b12fe89d2e6b5ec9733a563ce4a84025935a37710d5bb05d88b6f0891c8beec\",\"78ff2c11451f4e7cd8d7e0a4863199815e98f9df2a498168e97e984f62a1fcd9\",\"34a467d3ac305ef2ba39266580e3c7915213d929fc9121bff12847f8819825c3\",\"ac85495c7137e73692df0ad6bdcede893354d9829f9ebb92243d90bd3ba495b2\",\"e9482e436858b2e385ac050f5778006adab0ce3e07fccbc98f61550ed6c5f6ca\",\"8209cf2272734fae7238cccc2e9af740ded1130e0a8329f52303a85169da714b\",\"61d1a48bac3d0e8cea52c86504859b4255aef7165b4cf3bf73c3a36037b729cc\",\"fdc87c07f911ad14ee17abdb75c8224ca9424e5352a774281bd9141f719c399a\",\"6dd1c7b35231fac7c45fe886b8d8a92bbb04ed9b355fec1ee86bc007b6d7bad3\",\"52357ac55a4f7c1178714f57c0ff1b8084783c0981b43f5f2efb485cfef318c5\",\"4abcca0b8f11c63256ca95b8b127f03fa5f15848d4d64034a4f8c655bc546bb0\",\"943fa59b9b3e3b02dc9566c16b84f064b893ea75839e5dd17bb14e669665e519\",\"39c553baca8ca451c3f6201f581c2194b15ccc3365b6ae1a0ec91d0a06c7c0de\",\"e874b767e35ce6bf9284bf4db4d667c19f8f81c5c19afcec57955431b4f65cf1\",\"6df773e6c2d2bab04510883cc11f435993bf07a3b7db1e0f731e606bcb5a28df\",\"f4dec2dc92765470931957095aa27e7a890aeaa3525c303d9812eac860211bab\",\"21112f769c33d297051cdb3c407381a55f72f68e5cd9bb6fe113b944301dc7de\",\"0ce3f889ffeb78549f543283b819be815bb85eac9eda896f42197de174fb1d15\",\"fe39494ac5161c134d5b8e3cd29ae07a29f64175b3cda96f76a8dc55ea1b39f3\",\"26e2255ea66f8cf2cec72907ded0aead8f228067dd2ed509e4fd60f64b5dfc83\",\"83936d4dcafbe37c1c6dd6b601b8c83b30ce0c65a01de565f6ad2b1c39d07b51\",\"a646f0805998d071c30c83748cc102f6f72cba19db969f1925b482becd844ae3\",\"413473deae5175d4dd7a11dbf98ce3e67415c0d34492400e824eff019adaec3f\",\"0ed1325976f73519ecd08af78db7ec92f52231540c9b36df59f773ab3e53c02a\",\"b71d644f1c10839096707e9b38bb7d3546487b66d7cc6023148b48f430198676\",\"fb4d958e2a2c6e97d5b301d57e851a1af502ea7a2e0a535e953b822f2bf837eb\",\"a2a43c448fa92e33b841c023ccd6159872e82bba801ec9eb7d8b89470785569c\",\"f16927a29d2a10eac7e9caed20331ef701b6e26edeb7416b6047f873dc39e149\",\"e20d0ad9a10e24fe3644fb246809f801b7f850ad27baee5b5ff6dacf51d888e6\",\"6c5b314d323ebdba3be449106614c1e781e2f93fd33194a93f3918bf67c6eee3\",\"a039808bf261af6c53181102a0bea58bd8cd64b1c6ffffdb2f1e13d025ff1e91\",\"16915934bd99bca15847f9c7d46c5da92901a9667b60be966daebb2587f043cd\",\"d64ba0f2ffc3590415a8f62cd6b109eb707e302d1660b409030fefe4bb3af4df\",\"3f777d326e9e1a91dc39d8021f9b7091fb8cfb23a7c7501e267dd1aea980bc26\",\"f837dfb3d8fa0bdfb9e7c2d848f2fd2416db5656231ad3d4fe1d842d5d0063b3\",\"122a494a1a1dd40e28e499d297c2d3b2c5ae3b7c847ba8581bce4fd1bf39d905\",\"5a4bcf3921bd45e302509198bb7ec481ece4dfaec5080540da104468b956fce4\",\"f380f68cca4bc7390eb7a552129dd65044ca99a4e5301c5c5ec3cefa104d2af9\",\"e520cd294c7ff41656db85fd01c324f1825b5c6c67eaafb5300a7d7f2d95804d\",\"5e3daa7e07b5fae92060944c326409a4e39bbf6a258c97b129b1529424974ef3\",\"eeb0a299d490c5d845a820f8853e6a73230a0668f97b05942e0c3289076ddcbf\",\"74cfd3cca20f39936928f8063c38ef6cfc2adfe8c87ccb90bbdb16c2a7377a8f\",\"6e15030b15896c2ab1e7050845039a7ada5a236d0f854558028eb37f2dbb551a\",\"5844a543fff8c7354258f39db8399d15dfc909ac863165c8c27db5f6a2182586\",\"27fd0c94e36dfd9f73f3adaae82db4b8eb5bf109201fa5794dc77ec372c2d936\",\"2e56fce492d8e1459905565d04e89ebb6e1f517c3f18a0395f913ce9f425195c\",\"dc6d9fcb3830aa47fb326896f24102005eabf410a568d234153e41a9b636a4a9\",\"92d4320bab2261acf92030e9d5ab33febf68acf64cdecc34dd19108920a1a56b\",\"965892aeca02eeb61a68635c2309cb79182fc2a6633c724efa015409a2583092\",\"2ba658b2555fff23493de1244bdf8df17e5c9bcede1fd1a21395bb23adb8d197\",\"158079d2ebaaf5f82e903c6b4ed1c5c9396a0d9b16870c6f21838e746ebb8207\",\"473b2528c62240c949fbd92986cf99511388c9f52935c29f10649b5cdde4c422\",\"4e55b456f6c2bd4294863ebb9ddc96ad49793ae6c19cc58cac67dbb4cf0aaa69\",\"318946455ae7dde6a7d9b41de02ddfebfeb237874ce447160a8239dc4384d279\",\"882e0de6440ab711afff8861229ac25e8fd1d791ac108db536a2ebc6d75a199d\",\"19e940438685bddbe93646ba477ee7eb00d0e8123429c459b6db9e8a13af15e5\",\"7f296572157a1ddc580e7718e9a45c2fa8c743b948a25956ec32c32e3f249c4d\",\"592ffc8a3eaf4abb67c388cd00aec27b9d03a21fdcd2d691a0a8a5ad7c88b3cc\",\"fcae8b119beb7c54006014e14a3ba2060fc06e0f9d9b9cd542a9194ce1d85a6c\",\"5de1e8851b3c0a2f85c5d288e7a75cd67639be38718f31ece8d427a156dbda8f\",\"6374be2b6ee50f27a445bede0be1be6571f847a725975abdf3281f02cf501e4f\",\"0998ffad897915987e417004dc404f7464e3272b6d20d5c314b4102504a0302c\",\"9520ab32c61892a77a24eb77332bae4ef4f6fcb87fe893d1fdbd6dcd1745f4a9\",\"fdc2cb296f81df73d46ea85652bf868fd5a2f75fdc132d4b6aee3cf17f2eaf2e\",\"910707e52c268458bd2a027feca49760123828e7420af4194922773562ba0f04\",\"2c7a85874966acaf09faeb5de75dc025c50c71c81bed11a83f434a9963faf3a6\",\"4de515ccaf1db74da335898761a77e6a5d7c2b225dce617355d1dbfd2819fac3\",\"d48b8ee1acb67527093fd63d33ae9033ccf472cf17ca2eb52165e042c6738b42\",\"982d837b2c366dcbcfffb02661afd1549b6598bfea60b86e8f17ea90778a84cc\",\"ea48f17a6e76e584464299c132e162b7828536efac744dcee40dcafb01115530\",\"9bbbaebf041f564081c302333021402fad173543284d6f887c072a1fc472a4a9\",\"4142e5db9276e57dd846632f96b5ee08a964dbc8233cb75a851e44d5de38fac1\",\"fa1c8c7cf4c6e3eabb7ee3e4e4795a660dd87c453b2a2439f50e2c70b7100d05\",\"e346653b4cbe00aea1a3d6940b1d04416ea38e9f0a3bc58c464f67e1531957dd\",\"c7a2fa784c6ddc3b158b6ce049d0e5483cf34abc445441fc8820420f16e9e036\",\"0a82040124053669a552958d23cd826abdc29cae23e7ad46b3be24f450c6ff99\",\"3047ef9dedfcb530b449c2a596ecc4969887b192c4dfedf1bd1c1e0a0785cbd9\",\"0b53c3c22e40e20074b11b5a57769fc6c3467e689af860527805ccebe4ec22b0\",\"d4f1f8b1ec0427de426828343677de9a1e4dbe1a5512a750419790633e9d89a7\",\"f176ad3662e854348bf420075cd48a94d55ef36a9c5f3382f94aa3e40219ab7a\",\"fce75a017802390f8d71f1719710d61870379256ba5036734a91a1f90f8b51e0\",\"6628e082b64f31bb0cddccab6b3d278b3bbe534382c58315d7895ed76bdd01d3\",\"b0dec07c6f922fea74a65271b980ae00223ae392f1a2f34a87ace2cbe4f99ce9\",\"36dd9cd98e6221b559cee4f016c65f1cd704c936099cec2b15003be1eb34a00d\",\"c1b6b8e3aa52e1f4219c3f61969b45ae4a92a592039a0fd9a8ed81ae8ea32f19\",\"50d567cfa959725579118b535497424ca2a85ccea42c2d158bae70a57c2cd46a\",\"9c2a38284315b4e00ae08547c1b2bfa095376198f61c2fb7054ed1270873f9c7\",\"e9ae8f0a17b9f4432bb44c7239149973581e1e8f5f70d2603a3c8f0f7079651f\",\"3656537c324069aafa76a4d633356aac2979b45d1f472290595fcee2fbce3d62\",\"b8c88928fd671132737d0fbf0d420d47c590abe5c3850f0224802af138309b5d\",\"1016fbf959288dcb2314b6b354c1d883d2befa795998164d15d4b93a5054efb6\",\"789d3f8af16d23f421ed63b3d62aba9f80d68bc7d5138fc52859c8a184fd146d\",\"e46aa13933130cc391f2a65ebfe6791bb33497b44dbdcdd9939d915a69c79c56\",\"cd04a3a25cad363d694da26e10abea783fca8d6808d5136e97e8e39858035665\",\"faee149e5aef73b899840518c014b9be73cfbaccb5c6721070bb8a435e39b83a\",\"ec0621b3f0aab8c834975774beac39a68d08c988363e568e5b6a535537732950\",\"89c938a656c6f1e4e939312f75ce3a0ffc0b384561cc774c88384793ac956091\",\"559edfe180893747029c4ed0e35ca31e0ab7b7035f55c9c7644552a319416c8b\",\"4e4b70247e9684884273b63826b632ada5e5f8f5609aaa09a663c39959c5acb0\",\"40c73cde674ed591fa43e9d548ec628806f1e25ce72527d9e7fedc429aecdddc\",\"3dce24f154062fc2b79073ca4839cd4562dbf6004ce6a0b822e07607044d5562\",\"5e53af24dce1b46923129a8412b3dd816115b082292bb8aa394d79a54e087363\",\"d089f402512b0a87c72435f46b420e44cb9887bd885fc15752c8e8e18242f899\",\"a8cdfff1c7fbae2a9664dfb3f9c4a40bf857c96373360a8b431407932740d716\",\"b5dde0ce523d0157f5678780c3526fadcc5474d0fbeb020c074d7d17eff75620\",\"0f72850867308a56bb9a29df285184d2a3dec0508b7ae6d913063dfaafe0fd11\",\"07ad827d22c890f140be743c4ae236c6fba3b4f318111e4ece9045543ff4813d\",\"f9e0d16d6a69cf231924e990c6a1c9e9185f0c9aa4042ee7e3a2e8c7586bea9d\",\"87ab6bd7ad551ffd91cad0c26f53c9925266431657fda99db259bba097b3a502\",\"73125fea6c9f1ab80379be8d8cdb7d6568a356c7513b49528800f0aa5b712b15\",\"3ab16324765bc37dda9e93adf0fccc03f86395ed8fb89b4201316f9a7f3699eb\",\"28bc0769461848e0792d2e06d51663e36759643689b90af8cfc714ed80c7c924\",\"e20472c28c14f424ee9168a9db83bc081cf46a91c0d7ce19a1bbb18c622776fe\",\"9e0ebdede508c73faadb7ced5d347bb1ab29605b63a2ab4f4fdd1a73bf16b979\",\"c5615cf8092f27e3c1fa1898e7a63996d7b0e35c2cb66c04b5356d1f3b5edb5e\",\"2feabd0f869166267f9f31f328b4fb7a0efe4d84f9006c55903848933a5f15f3\",\"81d2ee5aec8c2ee414e5b978446bbeff2d1862ea1cf7cb2638a686f911f457ba\",\"8533aa6ed4606415c5f1a3949fee7d31c9e6ee13d8a378075262ee62adbfc9a0\",\"89379e504192bf3d4406454ddb88647792607072cc7fab4edc560dfb8e10dc4e\",\"17c90403c2ba57400902bf783a2c67fc09b3a54880929f91377b9483f271f8f3\",\"5a1b16fa8decd519a872a9b2eea95d161787e75c56f5d94c13847f96c8474f9a\",\"13f7bd5b4c657fe5b1f3034e32c0ed728ac822fa9c3de75653f204fdfaef7a98\",\"372dcf1c66d36e01f6b6af98ea583f011be3632e89534d78919363012e16c478\",\"3624db24e89eb7909d79a98e1fd4da299cf6fc23a2ea0c806e62a745a3a1b37c\",\"2e7f7c70737ecf3930c323c16f064c8cbce32b32e22831b9b7dc005342b0419e\",\"5fd69a9a01e34277718287712de5c3ea41a953d9c91e0f3c5c04b6c1154b39e4\",\"a70cead0b94ce793f4696844c7860f45a41e6b9601a622485e2a1022161cd57b\",\"43f99fe8923968ce72e5a7b7611a733fedeb34950accece0b122a7375a5e4e84\",\"b9738f7bd440b8e8b3b548ee1544cca9ef2e30cf0bb533f914efdcd3dd35ded8\",\"265e5f8af4e478b445d4cefd778d08a3590ec4a7f07aed44cbbe148d02e9710e\",\"ce4380af6831f069cc4417f723462720fe5be2b0cf4ea579bfa327296d9726dd\",\"41845c94578f8f64bdf221ea5119b57b435cc42708d26639a45e5a4ff22d13f9\",\"9f95bf242fdd3364b8b95216df6cbaa8a68a3cb878ec387e637caaa34de90a84\",\"682962ba80d19de216a63ca6e638c0fd04f4ed735b90c280ecc9e460db7beee3\",\"5732de436297065bcabd01e1f6ff5c81eb9ecaa6a5874c8595dc648f245207a8\",\"aba74620026ae719f00bbc521b03e1c8a22189b557d8b13c8b508b114e8af1d3\",\"8530f3142651e21b2d2a5afc9fb32dbb687b1f3e14f6d01d287af5840d27bc68\",\"69e8236c044b4faeaaba2f47e94238dcec21459e4b65c77b4ca2bd40961ebbd9\",\"901a3f1b7d9e697f5fd841b0ee8c879725a76ef973dc9aeae4e026001e92eebb\",\"60e1a41204f09bc8f971f6e460f6b175a6216ba72c3b4df043402edac62224ac\",\"1bc1d53fc03098bf177558b053a84172cda98cc74114a54fa818e682049d0d8e\",\"e6c9c48726985f7faf569383d637fa7213bf2d161e9d49df445367e187885364\",\"32d2a1822868890e6ce5b48c048d0a83919e00e6d4c44337ade6caca75a8f806\",\"b4e59ea01ad51a37eb6d05bf6d53d28debf8468d43a0ba48482f92287cf0594b\",\"c47920b32144fe4325195f5e9e28bc40cd2b8f4d2d3e304878c8dd6be124d911\",\"52646e4c74d197b7ea8e9b1da69a5a293fbf3ecb55edf9a50dfb2b1a61379221\",\"5c37315b7b7444c067c26fc5fce6fc5d9f880744c9fd9ff82854eccdb8feede1\",\"6a1c9b84285c0e6f1adb343b8e14fbf84a49a7ef3cdcc329503f8f0c2e0e80b4\",\"1508b6e346f716bf68478c9bb065eb738f80a89511371ec09db9878e14cc955d\",\"ad334c1a067f22ac5c3aa9ea9b9064f1e21ec682792165be5ea78d29556f7563\",\"0d21cbc0265fbf0748c8efd3fb3aa9dc2a23a8a2b87815745332237684a78d0c\",\"16de838d42d6bf3f89d95ee0b9c226ebfeb2086afbf3736c04299b26f12deb89\",\"3634a6b648b660e0af405fe7c081627208598942e57b9f7d2d3fb7a4a77b780d\",\"a6bc108e21e00603482d0a2b84bf3493f55db655109690ff27bf815e57a4da44\",\"6c518efe30b91c1c71321cbd7d5477cbfa59d2ed93b3446e9098db523a86d261\",\"3a74fed326c391d664476f0864264a0ff14b2cc05c82c3cbeb0b0a29fe699284\",\"0336d853eae28b3e10c814014bb95528f133656cc3122ee08eba2adbaa75d519\",\"93946beaaeefa4af1bd1f9ba31ba1879c797a7751d094e02886373e8cf9c4fd0\",\"31d35664ca8e99e54f233796af288e6cb41357e1100effe067044dd3aa950c8e\",\"ae322286b7e81e37a42043f8e80e477ca84cf5681fc54039527e4413fa7ada25\",\"1f9e859b21797c7284659478b2c6cac40bf884e6f24f75f7d55ef4b424dcdde6\",\"26a43e972bb289725f872c183b38ed008a97d4a9bee827203ce396214cea2cf5\",\"d1a6bf82725c93bf312bb97bf49728e8baef70780c4efdaa7d431825a8bdecb7\",\"b0bbb374f5ff9ceacbeae4e24b0f6e651f2d1ac9a856b8843d15b2e6fdb66688\",\"7286a0d6bfbc08558afec112022a633590250badf6e7c65aac346c7762b9f0bd\",\"a439d379d1cbad1e01ecaf51ff73ea1047af984a29d6bfc5a70b381773414998\",\"fd418796ae64aceaba37065bbd20c0a34aeefac70ad8a00ee63f510da39066d4\",\"337b7557a306af414337acb88411b1566adaa9319898fef5ca2fb651643f71b5\",\"efd464801c19ffbecae77c6f51ad300d3c19630b5dcba260b7a51ad9efe75b33\",\"d5d8649426498f7ae3c089495e24720b3a2346dbf87b34ed27b8ff523e90546c\",\"33dc225c22a0c51920c2dc0bf7b3d498be8767fca53db566d7aae7327ee70348\",\"163aeb0e981beb4cca6aa7d3de09e4561b1fec5d0d94cb0a414429af8d6d4390\",\"2840708991fdf67c45ad6bea2e95dea327005f12111442701fd08485e01a6da1\",\"deb1b32c04c3e3a20d9f6b08a227dde695c855423e80ed7cf0c34de44837a2ea\",\"925942bf61d90f5c399f4d4a39d0b33155fb20251d5ec4fbb4ba05b3194bbb37\",\"e3d51bd51cd27ddc74f1884cd0d36538ef28e1fd9f36bb1e32cca98e491a4ad4\",\"c7d8dc2e06ca368df3e5f3b4fe6c6ca6865dac2c8b08c9aae66a0a5b27fef73f\",\"27924122c9081744a6da96a3569b6a238bc2f12e98354eb0f73ebc9a1f9cac51\",\"424b41fbe39343f356cd58e1bd1bd9dc0d1c7fe6b59a834fdfdf6f0ea5dca1ec\",\"e95cbdfea563efc2bce61968f5b9eccd79459f0ee5c52882865593f5058b4071\",\"b652ea3dd9a2e321a644db9fb277a12e7782f16cd08847761f96f3de4913c238\",\"bcd2a054000740abdcc48f5f78154817321925ece784869caa07162bcab07b6d\",\"0257ef79df688822b56c9310045faba3f5a992e07b1a6bde19923a936d418f5d\",\"e9e5b77ec6d1c24e709ab8ad201f229fdd368a1ecaf083f7cd0a266229987c1f\",\"927adf67c0bee10e64de5db792bd5008a2f358964932cd99660dc86fb5f95d54\",\"c6600ac642cc4f77d9c4036c1830a4dc373f840e34a1c7d74cc4bc781dfc19ac\",\"f2b53e871968121a73ef85a368dd7b6b2dca77bf448b177a516095d42e7c07ab\",\"c018f076a48526166a0d9aa56e22bf5fc259bb287270616f9a222cc3d69598d7\",\"935c1e1255113f173d7f72daa0bbf2efb91de5e561aefb7625200cb04294a27c\",\"c4e1fa291ddc0057fd12c80dfb88a36d2d1b06eda93394578f2c127db9617049\",\"bf75a86139cecc4664dce24952f1a5c536b50f26079ea325ee351c3091e83b57\",\"aebf0528818c96af3da6e1f21ba2f484a451be134deff9ff75048936816cdbf1\",\"256c01f476d4234b4799d5a1d132ea5874fc80993c7798ee940cccf34968a19d\",\"d308053e5bf7e8c91fbb1c911b5e04d0601e6e95f8d27e1a22165defc7ab2d24\",\"c68632db24075d5f46dcce3eac01d60bc0b79c7f00ff366deb3eb4c9cc6d3de1\",\"c75ef2735268e205de6d1a61a331db45a8504b1233575204aec7ae8e641f1ced\",\"8edd407291ff14392c9318f5578e878a5a46407c3043fc395b98f3f263e0b2d6\",\"1323bc5b7396fcf6ed0ab7a42a1655cbb62d0cf68e2cd407049006516f539e06\",\"421c72df564c727ee750eed973e638b3df461173dda6b2de0db3415bc34559da\",\"54e9aa983defa9dbafba88354c7ae93a60d474ee5033930e208ef0c844b94093\",\"78ac088dfe1c516438d1666f93e33c1fba198db5fec6cc51efa5ddb85f5061f0\",\"94e81dafc915454cbc5f9c8f7fb51be1f13f10c265a5d820980e8ca00493aa11\",\"7c7a554a9b4323238cb15f06b05e0d9d63baee11a9c216972627676a1ecaed89\",\"3cf6a6e769ca5f531f53b5427510af4d7afb0e778bbae7c76a0eb4f5b4b57326\",\"1633693484aaf39b444d1c81d7b0846d080406ba76111a908361c7edbefabc18\",\"c84d8cf4ff18a43bff9078eaa14867a8cebca6859049d44e991c31ac7dd97260\",\"4b208e21266ab8da5f97af7a8bf1e07e0dba22cda739b81fb0705db814ab1c30\",\"607a697d59eba866f575b8bfcdf23a5819c974e91db2b78ede6b06922448bf10\",\"274781904e056fa8578323910451cfdf6297bcd9d5a0767c59db84f104794691\",\"8beb5cec550ce6bf72604da3c376b73ccdbbe75c6f00b9aba1fdc7b2543e45f7\",\"a26ecf3460fcb1c8e86db54abfba172cd5393c9667fc4c02f4b90f473ee4969a\",\"b4be93c69818659fa23dcc647423fa18c57dc416143edce4a2562a1592c4bdbf\",\"48f36468b5ba38b35bcbb754e67c39960c8bd1d0aef38f2b780edce553bf49cc\",\"b9b4804e415f685ed172a744ea66536e9cd60f1397e053a7ee2684afdab23b65\",\"16b193343ac2f9b09c661156129594fc7a9a4dbd4c4f539385841d1b6517c087\",\"83894df77b5fa8bd4f9652697b47535644da5692bda9ff59b6086baf15e8ed51\",\"bc4fc17ffd975ab626c3c952035ad55fe0414f1e00809af413be2589ff00fdeb\",\"3283e22a35229282a01ca07fffdd50989a44e8bb617e608fa8383e402e9b0ed8\",\"e79cf30b933033947016e80ec61776aa13bc61a7c4fc3c55f3aa20952fa5fd46\",\"ae297ecbe5460d802784bc50415a4bc5e126124efd20a4950542a8ead9cd81c5\",\"468e503e2d140e687b82e4142a3d0a365773c5f242c842aaac322432a824d1a6\",\"de32f2ed03bc74c45ced85d738475ebbed4430f4669c457e5b0baa417566deb3\",\"bc6c301d9c80d8b0b9bc19b1cd55a40787eb9760533a11e591311cf7f0097706\",\"e11ca747cf1bf6251d8ef469957a9c7171af622e070cd14280b53cea57748cc7\",\"0a515e2bb9acdcda7b22aca70a1d6d699d898e05424d9b9f21b5f217315122d0\",\"ccbcaf80748397a76d537e2259fbacc5f5f13f54fd106e764b36b4256e60d2b0\",\"b0a676e257a9d2ad003aa332d177bb9ad1befc5597164ce68bb0891459c6beb1\",\"e9f31f9796caaac95fdd3d2f3898659647e133c041d8defc9bfbf95232729833\",\"bd41c54353bf343e3bcee6d113bb8a264717343e50c21406e7fec87729ba8034\",\"206e6c6b586f9cc70bf3e9b4de496467619a75048de052cc3162da5bef83ab16\",\"0ab619157dcd539afb3e435769c5f564a9e12444de37f67fe8275d9a2311c635\",\"b5f66254de5db6ef621c6af0e950cf89d4c9d35fed6c5ae716e447a41dd0005c\",\"40ace3c2f88be2ae8c94b22a973f00a9e1605156885c198279d8f28eedad3d24\",\"46c33851f151625ddebe0f60cc96b5611a7f9105e7f84d6e0396c5d89aac1a8b\",\"dcbf9d1834c29a33042a4b9bbef8fa87cec993562c764d35b7130a7f1b2d939b\",\"b17a6fea4515e8a0769bfbce72d401f72f5abcd3932bff8f7627264ab5f9b4cc\",\"515786c7d92e966801c686cc0e642d58224c4a4e2128eaf55f04fbacdc3f562b\",\"2145e8e6492dee2edd00e507bfbf2b42a91e05bdb2341d90fecc575a4880586c\",\"5e0b49bc3eeca717abb615892061004f9d06e2f17004895ae49ad97e884e9b0e\",\"a785ecbfb69fcfdba26f012c890a958573cd07438e7994d0ad33e4cc9dbba678\",\"6cd777e52f6409e2232aac09905590fa7cb7162fb0e6054c9ff3b18cabcc9ada\",\"ee7c04413697de302bee43b9530024ed51e872d8b46ff63602ec70ce502e1e61\",\"cb2fa41a61d80cf8c3d3b861f36361877824274f653b01ab9704ab4a84adafa0\",\"3326fcd9bdf396cfbefafd5934f7feb78cc21c1d6582be391ad8ef98f6b0f4e5\",\"b10bc8c9789af3df9bc6a2fd54d9c8e0e4208a555bcd66e60a7b4b355518ab9e\",\"1295edd58a2b6ad9d313bc202cfd191a7184c91418cf10f33a11a0dab035d71f\",\"efb5a2596df6d8e3e5e5f67d4289b9bce26dc8bd4ce32e60aa07a6611a75151b\",\"95c848655e202e0d437705d63e1a265e6729ad2c50124237fb3458423a70db1a\",\"b926f080b4e0afb43884980b2afece4b9d83c22836d0d1b3c0f9285932872d1d\",\"f6070a2e519a7a46a7024485a5cbb5c6a845e168764b6149fb6d2fb8657e9458\",\"801e0545915f2664ee768ed2ee6d98c873f880304c1f1da79a01e783e59bf6d3\",\"e9a1a726be04d848371912f6384c7ccd8071f0486ddd80497483f2e98aa1eb54\",\"41b886356ad510f5baab34043e0d913888aea47f2c7de0117e5a3d9a49dcc7c0\",\"d3116d148a17c44b622513f9b0f52263ec2eb32f39cb9bf6a2efe562989161c4\",\"88f544bf85fde669a9719a16959cfa5f9050c52504758f838e0510f2cf30782f\",\"8b2e15e7ea90122a880dbbc5ec837c8c170858796fbc8e03d4ea8dd82f57c262\",\"5c2de21749fa1da22a2c323ca270a6bfdf6c41161fd8566a3701a7ba6509884c\",\"27a7d389101f3ae2f56496f513d455c9e2f4abee972ec729f602ed0f903fb706\",\"5aaf49d21623629ca636515644b4ba16220947d95184cde83019544ee27dacff\",\"2d9b0017ccae0f3ad67cf92edede78052a414ac0f786d142bb0b3a7ae5652769\",\"10f780f84ffbc35e2dce3bc8c803023122db7e87f6f0929c2eb212036c80da0a\",\"d77e644865bf598806ef6ed1790f25a00a8894e9d1c5d4b098c0bae78a63e598\",\"debaeade9f3e04901b8a92426e87d38551150178311853e7fae3e56ec07f9aed\",\"b511c552413c8f0df0134342ed18028daab5eb3cfaa2179b42a44cecaaab5b2c\",\"a6520c47dad25327da895bd94b561e414a7821835c289cdd9ba2e6291b2f4164\",\"851a72ffa505c69f2136968440d09e53654bf541168066b16471128c3f043fb2\",\"8274811f25f05c8ec02ba8efaf1c934e441a2d14a7b0ab5a9a1453defe3ed7f5\",\"316f7a1ad86d2c4ca262d1ccd322fd2f674ec12c32b9e91b3ce71d49320acbf4\",\"c5f915a496b695ae72ac26d80d0e37d1ab82a7f7f5c2723c119b5d5978845746\",\"8f1302543ac0a414855f1d538c2e887c51d1303519c37fd0bc487faa90d4df8d\",\"6c28bbf7e5c861413ae2f5f6b7eab92298ff371d7ddfc1dcb6af60c5663ef27a\",\"a6e315130d7bc561cd1b5bc23fc16aa3c53798106baa8cd09ac625245b1f4872\",\"ced9699f55bdbf3ec88dccd2dab4007739081ff7f9f288f9bbd4d3a1ee278b6f\",\"e60f8f9a5283452398dbeed7e5919a4565769cad7a471c22c7f7eb010218b8f2\",\"45836d1cd056ce9e889e91286c36d24aad739905aac7b736c34e9da7e6b431c0\",\"4a5efa7591cbdf562145cfd4371cc1acdcf6a4a897014e19902c7da4f3e1404f\",\"19157a0993db3a8fc87c6d40bb55fba8225c067db7b948975786372132ca2742\",\"230dec92ea10063b23bccbd4fd6aff8b246b9256de675f707cc2b61ef716240a\",\"2c2809c9a1446630f43fac5b9b84920d59f223a408f48f1f291965a76b729669\",\"71ccb9e0d511cc81f54a60260290f7fc4775fc856c8f2d1d1a37b66bafce5e61\",\"ddee1200399582e9bffa3a3a6f9019d6bd1ad2fa8f7528d67197db75beb24433\",\"1cfb4de7c4282761c260310cdafb91c5e8be5942c56c88817aa8041967c7f0cb\",\"bdf9b2e0595b66ae95cbc6f2cc33cba90f365fbf9b32e56b0a103f8f9f07ab4c\",\"0487f3fe86dfc290a31f5f36a23bf86aff7cd264fe8dd1e9e557fbaa9f73bf4d\",\"6146985d8eb72a6a061d10d7db2f2bfd1365edf5391428ae36b608f8bb47560c\",\"02384dcd33aa30fa81af3bc9993960e872ac4ccd433d4bc41008ca2a66d769ac\",\"79d25ceb69897d0e04cccd78a5bce2dd04df411ecc1e1df2bdc605ba3aaa9a73\",\"a4fc90c42a7486b55c852abe5a7418ecae2ca98c05ce5ef0d5afbfd7f7b871e9\",\"bc1e2059e6889fea9c59044b11c87b20ababe7614ad404b9b99edbbf95637e6d\",\"74e88b3af7d4af7eba78e44587d7aa6270d2b575cfbb9938c825822a14d0262b\",\"9bf01a0e67fb4c5878f279c1e74798570404e9ad7d3e48345cf5b71448e4de74\",\"e0fe8231ef1ccee40dc9b213ca19048eadb115d2157714cee1a188aa03ba3c7e\",\"ef2b88f71f8decc661b6a491964b54c7fd350c13f424e25287f817ad3a388a63\",\"2e0bef74a28ecbcd2b52ef530520e8c2d18e27390e6586741c7380c151a867cf\",\"1d595d952ef70bb23d0db92f7dde09e8f12aac83c3fe88fdb492fdc3330f4179\",\"e5b4dc1ec3cc4359dee3e301f6052d13aeda397d275688223efa81da88b61eba\",\"bd0363dcf3e9989677d1f05ddc8680e76ea169e340cdcd3652dbf26609571782\",\"7e73d617b3eb0ec372a8e33fac32081a434e3b35edfeb727774c1f1094c74a91\",\"cf1d72b2ee6d2a4b49f1a7a0bc735d001523bda7a5490a67b21d22343de3599a\",\"4c9924b4fd4cc505df5db053a492af8f9ea5730355bdb2f7422467a8049a1ca4\",\"8228674bdc3270fb62f32c76c4a43b9834a9563afb3a73028ec288416d661db4\",\"70749ff7a99df48e5934c3522ce06451867d05720b05ea440418dfc4e25ca707\",\"14528282dd09a51f36660a09b6138bc25ea82dde402cdd374ed4cf4b95410742\",\"de1b1d3a391962b485994780fddb7781585998813ff59d6cefa36c08b984f9c4\",\"b16140256c4cc65bcbeec3d7f07276d315456dd37bf86341f42d620ce03601a8\",\"8d65a4abd9cd0cefcb3301990ac34e398d0a1944fd55165a137386c2ddac12a6\",\"830dd8fbdbd97fd461855b9a995926e3efa12399984bfee22bf77004b2118741\",\"f19e46381f4d66857db82f923a2dd966356f77ae2846244411850a45e110a493\",\"48eb7f10f57551ef7f81ccd0734e1eb5f0a6d9c39a77370535014e2864281b9f\",\"369a8cfba205228c264d261903d210dbd99745848679ab319b55f91189813697\",\"b922981315e3f86d0f449528bd9e0d2c7d53c795ada8d0c249eea93b0787aed6\",\"b5bfc57f4ad6eeb508c19d6da6bbfa58fd2bfb85808c90ca34cb1885198e66bb\",\"c0123c057218452f11462aa5bf7574a2ba86a6924af5ed7f5a09aead665dfdce\",\"8b1b98473936dd7308664c8b9bb40b57a74dfeb21c1f705d3d637bc07a1dc745\",\"f10465a025e7f86ed52faa3fbfc3aab4b098e18a3a364b6358ff60657d0a2cdd\",\"a2f98a95e7c9ec29c99e08d90c34262629283e43574ee4c46e52630ac4666ca7\",\"5a8889ceff24a4e5b566c285aba3be498a3cd1eda941bcf5063e0a50ee62abe2\",\"f627dcaba90e66ffd93f8dd4323e2c6fa0f0739400e893e1aa50c1c8a2d52fe8\",\"b9182158ad40ccd5bb283b76c87ee123ce632b579fcd8dc5589d89c99a046b9f\",\"f692ea8d9746a18bd0ab4cce46c8fd3287b3c1cc29ad8bee64050902bd50f96f\",\"8eac10beaad8c2ee1e3ef26e25fae6f6bc3ec8d307207db6d578b8b1f4bbee9c\",\"c524ef8f4ee940b3c46f3af07bc585321a77d66cdcc3ce3e72273b18ef3762bc\",\"e72caf4df50e589c353676f2caf3f6bdfe6879a15d4c2bc36136cc3e968cf75c\",\"157f83f57f9ac918cd3d994326d7e6e1d57bd11ec078014f84efccb9dd103477\",\"ccd1dc12a2a4d97e3e08761ceb5bf2b46e7b28b0f35d97898882a77977b87e65\",\"1933a48cfb1bbd84dc340276e549f7a074fa315b4cd3677577be8bc514bde208\",\"aee40755257009a6e425aca1fe1cf66a269b70e5faa6d16f52f4689a60d8d525\",\"d865a12c96537e07c99a78fe15aff55f9ff5a170a2cd84333e0dda0b04544e30\",\"94c9dc7335bdbdcc71f75170dcd5cceadcb2194e467182710afe059dd26b3864\",\"3f11525ae30c939b9a7b3afc6377756effcc09ccdcf521d77735e42c792f99e8\",\"36b2c6d9d957b65caa539ca7fc229976aa1423b5c53d38c75dbae3cbc2d74822\",\"52d62bc0f66c86d820e873c145c360cb39a09e959f2439c9c422b542dcbaaca8\",\"a7d7ad595d82c9faa6fbd63b0bc47496e2e06f2a54a5bcc285242d2c84b51b14\",\"a9295e14b33db127dae88f71e49f0e40db0f8cc733aa2b0033c441502554b2c2\",\"f68cf6060c3ca55192bc3e9834dd3cc82d7ec7845396d8ed2b0c247a1f2c7ec0\",\"9df38b17f73fc102baa8f1b8fc02afefb022ad8de5441c5b9a90da3810dd6189\",\"83e08fbaadda2a13b33a05cf661c3152d7f16c2a59060f7839d8ec1bcecd1e79\",\"805e503cc205de5d335bb3b18d6ae05f11407b36d7845f80ac9d295bf2e526a7\",\"69918286c6a36cb146b612617cf3f28832f852c3b50346156f215bf799eb4811\",\"c5e6144e75c8d9761038ec586d54ee206d423f47a2732d5bbb27981328a67b44\",\"05b26934be8c724bc999148ec5c7513314dab78e434ca2b9adc1c1ed5242c040\",\"cb1288bfa22c724e885c7c395cdbeb01fb84d77268eafeff5c43831a7beab799\",\"a83dfc5b344e0b9db44f911363f9d0266b8bf6bb290a5d49accfce9494a8ab53\",\"7c62069944badd41035cc73511207ba9292080631e6ee0388438d9e084b086aa\",\"e52c144c9698e83b0b5b7bba2cf365b76cf66cc8fca719aee3d05e8f5b0b1316\",\"1b1556f7027cd85c7f9387b9490c0b41a51651de9dedd34593c933dd590018dc\",\"f2f033c643cdb0a9cd32db88f6099d5d3307afdac2a4992ae725c822d7d29b57\",\"8099dbff0653e200f774c012bcf269bf5a90a442df316968c17b4f2d9900808c\",\"ce36566dad9d3d5e42f0d2ce8a347a4178b2d1b14d773366570ba90c38fb0b6f\",\"3d57cb856d686d2ef65536dd43cd47186df7e97b1803efe7a37b65164ab067bc\",\"f713029bdafe684dbf119b54440df4811f8a1a413f8d8d39b82f0a6cab6a9a83\",\"56e593ff80a4dd6f8ede0cfb8fb5bb50bff754acf9919744c6de7a6a211b9342\",\"277c109dcf93c21e070378ff2e5e3a8d61dddd927b1253f8a9e78d9872219ff6\",\"65ee6ffb809b6ad94371cb93dad38c0bc47d946302b43083db8484f0af55af0e\",\"3bf56d7267ea1f7d07cee01b31d2719c7c6e313f5aa4363c39384fee20fbe0c9\",\"f5bf298e335570100c2f87d34f4669df3f9fdc3915510a3b2653bdcc33a3d13a\",\"34f00b40768643a677d57f0a32a88d7cb267ea72679ffb7a07daac371883133a\",\"71c346e54eab7ebf9431bd75624169aa65d58fd4df42b5d15db5ae2761a5da83\",\"1944eca94b7f260bff17af844e976cc85b4f2f6ac644604ce7fb9b2a72adc9ab\",\"d89630feb0ad4530ec5dbb8c6633e6876db72261dfbf04a3c9ebc203b429c7ed\",\"2b435f531f79b55bef10ee950debaf313c1bb22750b706568a730434cad4506e\",\"e30e25c05e26eb4a1ebdde7bbb342c78553e92b5d58f8205b6bacdea169fcc2f\",\"1512b17ca7f9f401037b85e416cf0e8d3d6f328ab0351b61ebb3b595fc7eeddb\",\"9aedd908bc6611a49113da98af49ff6532f5a11e8f779aea29cdfd56c787bb38\",\"b012887f62d8d33e77d4ed91fc074d85458b291dec6da6cc0f2dccb8388f682b\",\"9b8a4e0e0b1a8103d4ef7e9f051698239a6d22d6eaf6f354755d453dc6cbe07f\",\"0084e6f19e99ca17b6498c392745c9f5af8ad57f5ae9b32add97d46a2203949b\",\"eab3d178d7823676ea289652f84d1b1a64de10d2f40cb9723272769132729f10\",\"cf8721bb845e5eec27c3f578310e4d75f62c6a70c031212220d3c9cb0455c136\",\"7062e2f79cb9df28ffee5759215f64485d72dc62f879f4fafd7bddde548bcc97\",\"41e09d13dc0d919ad78e76edb68bb44678f51eaf144ac418e9d2cb15d1b40ac6\",\"0530cfeb14c69b87cd3a697e54a35de10d32b639a5012ab8920bca20a5674640\",\"c1025a04a9b5cedbeea11dd0282ce90633b3c4803f208f771d8b223f67818101\",\"3c663e9fdd1fab187209e8a3dd048f48f51c896bcac9545bac0d44b79fdc3192\",\"795116f199b4e1e05570be9847640f9528f2688ab2183cc3d067ae166bcccc21\",\"5c97bc60304ebd7f4de420592ca6e71e46760f80dab0eaa37f499dde1ca67df8\",\"9bfa412f1a2f9f778aa781bde249f2b0d4eb0767c16f554258e86c9b91eec6da\",\"68466319b3d4daf9489a773b32da62531893f97a4401868f48445d4efa79d898\",\"e9edd1cb7bee183155a7380f5604098df24759ac681f800173af38271c21a6f5\",\"57eaba1c1807adad1977b62e5e9d733ce4dd931bbe43a73c7180d8023d23189b\",\"724fbb4c8ec51254057ad130a878f7ec751dcfcd20bb29accae60c133e0508df\",\"983bf3dbee6ebf036d2e285e1ec6c52dd4c85172e7ed24703387d2a712bd7b9e\",\"dbe3d46701b85a003bbeacd57fa8dce8eae0c3ec034b5733bf4720f27a06129e\",\"89d04836a67da0d3657b6a104554aae45beb669c904eb328cbf2647b5da827eb\",\"bfa07e494f344b5855936b7d1039746413fa118a1af7130b2845449d072a2216\",\"4edc9963c18a9071f7d2a55706e9ea9865b1f693ed352faef62913b18cd7cd93\",\"efd809a89867a23379ce2b16e294f371a7619459a5ccd8249700fd34445ede9f\",\"aef961c25e489b7746cf786e069ef01f62705dce569e3ca33bc5593cf1a503a2\",\"3f76aedca0d77e4c4c0488b3e86b44804469b13d65531294a075dc810b174c28\",\"2531846ac791fe9aba40217066c5876f2f992d5f235e18afc18e0c1ab658f24d\",\"d0326066c244415be2062cbc7f9696437e19fad0fe77539618b99ef31459d0a0\",\"c9d16af05ded9d2aa5774ea247eaccca100ac8a84e0a39c5ad0b3782aa929182\",\"eb050dabc85e2de90c73a8dd2c56b17a2bf0633feb6e8cf444d0a8936bcc5b6d\",\"89522b43b6c36effca883eec6e4f6352afe10d76fb6284e40c7225ca04b72e9d\",\"5820f3c6d552708780451eb13862867dbaec1e7ed97aa3f4e2dfddf54ecaa6a7\",\"f2615dc67317898d9781162aabda945d8ef2ba81ccff9505f91fd29e99e0a1fa\",\"c71179ec0deb2049f5ca70e05fa1803ae9d529d93a70cbb8627e47402a777a02\",\"d26e0f67e5bbd93f677e811a83edac93350cdcb5fb120e9b25e74439cfa01afb\",\"5a66b1fe9e929bee69d9f4bb7670f21d81ee4fb7524d3a9030b4a939129f4bd3\",\"536612ce83c90d9a827ca8a7cb9200273e8510e21c5e11f6188e78b66e0cd8f9\",\"009c4b2dd89737964412b5871065c8c83c949c129ec4b3aa63247156cd444d2f\",\"97a2ec70030828c9b8909765fa163b4ce521ae6bb32ee7b60ade515f653e02c6\",\"dbc0144f27673204db4f10a7ab11cbb828f627a09a3e69bd5a671a1dc51de91a\",\"57236f27968e78feb1959a4012cb3beea5613f95a7cf0a52119db0735fcb4abf\",\"09efc563bf7533c1943fb87bb4ac93d0675dc4f13b8ea173374f7ace9ed1f9e7\",\"16ab942dfe5a1aedf9820dfc237679871c32fb9aa1b2977c6ba95e7de5c2b103\",\"a8557978e815438da98f726711c2916ca3de8877392f49eaa14c32dbfaedc067\",\"cad7e8979ab7b06b58ed6ba74bf88a01fea1232c953e4a41493b7bba16e8e044\",\"2b6b25296530742a21ffcef89e0ad9b370f631e0c05505e02c4366f610e6bfab\",\"8d820798cc6452b073b5a2a30d2b888e3c3f8eb7a8c2736885c9e1a18899fa55\",\"c10e27746158ed769fba637c125cdb15a6902e162ed7a96abd875c7d192bf1a0\",\"c5e2475238e3a20738e3598158eaf5e5c0d19c6f065ae9edb64eb275429be4c8\",\"f94ea5b34f3086da7a12fd4761a65ee7f5d6a8e63d8aa6d4a9e9e0ebb17050b7\",\"fb3b74dbc6e2e85bb0cfbfa44d528315f57967cf8db7320a11e530633496289a\",\"d7d07f8de2efcb71ab8133c30b67208eb4620e7c55998ea35e279bfa7b1b12fc\",\"277a7fc3f6b38906b6ae5b1d1b9b288df1836692ecac5cac9103e1979e213d05\",\"18c1fd1ee8d769b9a5e6048fba87c5752d02cd45eca9781f6bd377187036296f\",\"4c7dced2a2652a93137cce5a29b9732f12fca31b5a761d40e90bb7b249abc65f\",\"fde26245d8f6f7c3eca96c4055a14f7cb77eb693b955e7106b027d9296f02eb1\",\"63d90f7b99ef2796161711d287220bf97f922e5e82728c3121811b22fa329ee4\",\"bffd298ac71ff85f70ac7b24301a94b4aa5823b9444b0728e768d2c935046961\",\"ab72f48732b45156184e88632edb6debdcc8e0271a15752cce207d2cc260c259\",\"929f5764d67eb95514e0dee62896426add7bdb3a25f23c0b0c19fa884763af61\",\"d328bd73a6fe25b8c8ab763b3f8a2a66a733b390fd6c23ed2f5ca8e9f8d409ad\",\"7c4dfacae1bb8539feda0f3bacd68a213d0a2c13d20c1a321b838fc466b0d3d8\",\"83a1bd08e50d0e4d23196bb1158e6b3695c96a32aa34932f5949b7d66de264a3\",\"dae56f903faebecb9005b4ff72564080172e4a84bad4e3ee46c25b3062cde204\",\"5944578782d6c76f9981998c1a0f82d61b858945decd40defb8848ca0931566a\",\"e9700ffae7a8ce4d6b41e502b56175b19644fe66e49ed0ea7153fde05b492c05\",\"f39f27ac2f9dac81a39e09b58f1c8f7062977b3ab541a3f9bbb977aef79e170f\",\"6896734a0a3ded9c73a201062ba45c1b3ccb5ce6a7c6bdd1bb82ef964feb3a7c\",\"993f0c67460621884df5e6daf2aea76bc50ba0cac0b7c21652ef2a3e6d8fbd42\",\"28a98df9638f429a01e82a5a388cb0d424101063f30e8799d9419032bef89ab4\",\"f223aecfbf529d2c9cc833ab3871cab2231ef2b71e9e5844d805a78c4511b139\",\"5802dd9738c84484fb51f30439df30ebaef93eb21403b85d969e3874774f0ab7\",\"bde990081ae9a5d3aef5dbf985790b6c9045c6474973941c150d7ecd2ec90d82\",\"74c7fc212efd800cb0936c3f1f1d805f0354c0942954c9f5b96fe4620ffb7fd0\",\"cdde637d918b6e5dff36eb0d785613462474617a37c362fbd629fee7726198ef\",\"d10252371f5aaea28b31d3eac8abbd71acd8bf540101300a2ca2166923451f20\",\"2bfb44b12380a045ae9fd393e25b059802ae7b7fd2f90cbca581a1c8902720c6\",\"5ee410fa7f91d42de3defdbe0e347f4ee20d6a447e44d57b2031077ef7abb239\",\"8961d963ffd63e2224b3d2abbd35418db7f72011abb19215540bcff01ce80afc\",\"515a1a397a458da42cca6792d520f2a37799769e0451d7a2d793798b901f8bd7\",\"9d6f29c4bbd6587acf56df7daa067f1dc78125a33adb379273defae955be935b\",\"abcdec16c90a682e5232eb5996a36161e1bba118f664dcb6d1c5e70ef2c5b02d\",\"5ca9ac1ba27ecf95a9b9326caee3b5ffa60ac78e2fd43a11aee87a9cf57cb3dc\",\"a9bc89788a39711cdf7602672d7fc4aa981c984a5a1a0a417d8fcc8af251324a\",\"5b521a619c12118011ca9429ce722e2d8ad69b7ee86369d88e408418090e4a0d\",\"ac2374ddd259e5218e36f539c7689ef8b7435c662147138bc1a1775bf3584292\",\"062054382d68b7f7b4f00a2035c03f0a205dcfc6474ff52106f3824be32d4ac0\",\"51597a4a2e1c9c805e27e7f8f11919d07d18b0f78bd55010538cc654ccfea0a7\",\"14d7fedd754a0ee11315b0037fd0a928d33ffc4586bc5f83221662c66c2ed6f5\",\"6e2becc30df1d2a7d9490df4dd1379fe52b604c6960cb09be2eeb67ff7de0e9b\",\"f78fb14c9715c2dc57691d29c92c332ac7b9181d75c8142ed6f731c7c53cfba7\",\"4fff3a44121dccc388ec8c102c948e64eefc2a0300dad3a81cc1edc4a3f9951e\",\"6717f0a1bfd8333868b7251021c82b173d0a8e4c52db63d719b3e8d3ddd0abf1\",\"efa0a06694db431eb84024bdde0b4327acc081937e9af9884a2fbfe007d423e8\",\"bb84a4fdb3f65bc7c149146a52ad608c939723441cddd5870e1b5c36e5611881\",\"d55296f482645a23615081581af6558f67174e59a0381c59d7b3fa18b1a10a4d\",\"b9d1d82302a43d063ad10d38e769d701b642e4dced5700ee7a4bfcd63c86036b\",\"9737c0a2dfe362f2247d8a6a6c272dd7fe1150b7be6f97c60fd251d91d05331f\",\"a15010c63fcf83b569bf54b5b0bb69ec5494575ddae5415b15c165615e353c00\",\"6736b05c60bd5a4549cfa82c328ae3d50af7790593625137e79bb0b1e45a1ab8\",\"47c4275aade456d9d765a2742f848e764eaaadb7c59d1d8382555f9d137f6ba8\",\"060ef0f356d13479139f49738d2cafddf5b966a43ddac00e4fb227bd452615b4\",\"fa4f1bef841655000d9d5ce249415f8f38702d10d80b27b041c603eaed1ccde4\",\"ba9228cd790694ffe67afcb5d4b68b4ddeeefb7346b34ffbcce0feb281cba85d\",\"72cc07ac68e31f9df3d9d3da76259a04b0ffe328e03a5083bb814a3101e2ad42\",\"5c4e925eced5003d4ab3e531f4e1d2ec9cf13f491eed2c535104cf3371d3bd88\",\"930325dc23147b91133b37aefde0b4bf9cf22dcd3fa173b6c78c70443c5dd005\",\"96b3c8d9c2086ea22c9cade6bdcab028c712c45b254b6ad7242a85badc6327e6\",\"cd47eb32a5d226770604c739b68018ea9f2ac0148c1bc108188dcf284dad8eb6\",\"0ab7d4d3d36568c8cf49d1416c893c4ed2d03c89dcd89b025bc9efcd4c0ea67a\",\"71b7bb4573425ae9e1bf243c0a64af82e5d75709aa85a3e8bdbfb0a6f0f7dc59\",\"a4ebc1cc2991cfe4622c3d98c11e6555d2e1213f2799f53fe6b756970e6d25ad\",\"a23bb439989a4b4b1c75ab4a6e1483b55ed59a562b7b7433d93da7263d3dfb61\",\"f31a79e956d9452c341ad8d149e8204fe3ab9be02048754498f4f5c2f12edff7\",\"c807a039647a67d840f4d29f7378c549b18116b6ea54118a6540827ae4a822cc\",\"b866b416013bfcc82ee5838758d5c2461aebf4adc4cf2e5adf00fbc46867cd79\",\"ee03b712eefad09b8f70c5b1114139866bdb76007ce3020319e4d6d525e17a68\",\"84d432d592d0c7d1e9c2b4a99412ca894ea2a0a3c05ae97a4e13a371b41e7043\",\"9f1f8f9de34e356ef722e4c78b0d0be67d21e39099db3cfe162ec7ec6c72cb8a\",\"da64015d22b30290b95018693dcf96494022cd8b74d42b7f07e0d6b923faad8a\",\"462eda2560eadf7643e89d79b38da55e06a7dab871654740ba32fd6c70df233c\",\"b2c834b7373ff0b400c4aa9dc2c3d0214648d712491faa75046f37a684fc6001\",\"345a34ea73f7b5a587ef875412a160e45ea5f6ae87908c9a4a92a5e17aadb4e3\",\"ee44fd531022fcd771a443c065ae1d809c614aa64a7380cbd615c1a1286c0980\",\"f6578a8a2cb108daf95b813dd95fc501a52fb72b1dfef35bba575c9b322f08b7\",\"3c77ce276c5b7f1ee90ca67e4f21536ab5067d8693b95ac07b627901dd40c983\",\"8f0accd724e9d4f5f7ad592a954806a00981a1276a11f883279456f84f2cec75\",\"50cf6e41a42f0d68004d7d393a2af9f1ef09e4c0bc454c80fa8118973427ae15\",\"9cfb339e4bacd994f1fe49cc295586389f00f65a74961df0c85ea5ff6fbcbe16\",\"878c8492864ec7580af971605304aa57f6cbe78af202481b7dd015e214665663\",\"48ff2196bb3dc1cf19680727cc2d1a12be159080f8cbf7d100835226b9116786\",\"80fbb190502f1e3d4ee5987026630f8f616ec635012ac6039659ba1dd945cdd7\",\"fa04cc75f1179b102748b013a688de986c64c015195a867455b56918c02fbb64\",\"8658b35de551c579067ded90614a50e0eccf061a1f454eca13d8aafb926def11\",\"6bb4656ddf08d0ca277459bd0e8c0d280060b6f0f9fbf19d9c88dd4de908bae0\",\"6ab9c7dac6fe6a5b1e01ef9980acebbc4d6863745d9537bb87d6eb634ede5dc3\",\"2fbb301f64d0e418d6506c20bb54352a4444359ee83069177bb7a6410da4a681\",\"07ff583ff6c99918e00eeb99635bc35da5d9e0302ace034565aff6bc0684afae\",\"6f71c7af5ac834e69090630183b66be83f7d3b26d74b57d1b8a8589a41ae887e\",\"ba24bba3f68e2b5537c4d1674e87fc8e99b57a9059ce4b3f7917dbd664b289ab\",\"70db637a9ecacc6c82dcb34d555ff6323b8d39d75d43d0f09b014dc595b09611\",\"c741010a9847a695c0ff12a510d7b621d25bd40bfe388a35b859382ac8dedfcd\",\"6673731ae1002261414ce81b7be80dc1895a82e6b0a284eb9ef2547409279282\",\"8a1258046ffa8c4b2bc63da7efda46017374547562517b145d36c385c38b73be\",\"e099d1e678d65bf4f03f00e5e6d0bf01615aecce88b6b26b908fdd986e7da91c\",\"92056ee272b7a8c8dc7d14e2762800eb8920749d12a79026b28b352566346343\",\"c43b6fbbcd589dc9f0bce4e14c38e1c48d13fb29040e700c6fe560d79af620e2\",\"524071b69ced65a1e9c268a7baede2c5c0fec9c79b05e79eeaa09a36877cd71a\",\"80fc100b7da4b0c6efb4e39aab6d9e8963f40704b08d501ce5ed6a85dcaf6c66\",\"8e9b52d72c45e303c7d64ed302836c7d8f578b1d5af084847ec695d0232e4a7c\",\"142a609bc7ba2474159ba1fd39f4e5e93a0b8af81ba0e73d06f9f8492276d60e\",\"ed4bba294ddf4b7b4ba57c039c25fde98cb8f51b29a434da8d25600b60af14bb\",\"f26344c9263f87bcc903ca5757bc95fe63fb9664f527bce9788d5e15bedc7d73\",\"b96bc71f8e6fad258c971cf26327ad04fcf69af800728b689deaa9e35ec02fb1\",\"56c4561768921cb8f14e5f887e4b6ff91dbb1a4c57277d823ab38406c1490433\",\"25e74548fbad0d2b04fcef9434ce470e8adcca7756e70666212bd16fd3b87374\",\"9210cbac68785a5f6f9de80db7e52f6199888c6210845b8b3ba29f2f185a8a8a\",\"77be60d51ac660eaeb9fb080b136c67831c2ee261cd7198647ed4166b338291d\",\"b4faf5ca78364c6abbe45d5b573cfbfd4c9e3749be7c8fcd3d83915c75fae308\",\"fd2c99788becad6e51bcb7068bdeb856e10360e541ff374e4fda5c2d54d3fe84\",\"48c2ba0f3152944ea52b4a5503be1436bf98b10fa0a7f6a092e351199715bc3b\",\"b6bf25e34f4c2c4a28487856b55dd33802a358a5f50126a18725335035630428\",\"8a8d9fb423f1de2ebc44481e82bc0aff6d80c790ce8f6c6a75f9c3bc577d63fc\",\"8e1708054bac2299fd14d4558c26cebcfeb4a47f3e15ff9258503dbc8bf9fbf2\",\"005875e7bb20651968161d72f7e1b10af1219e0a7f1319eaaeccee6f08e26738\",\"f300d1bd06a251756ac546ca2ebdaef71583951640ee4c1ad87e60bd4ce97dd8\",\"9a2754f72f1440fdeafc1ba7b520d70de2a612037a30a1168a40f42275da74e1\",\"c9958bc034f276202209c44f71d654d91b06b8b51a37bcb8949dfd9dea4adfac\",\"3016534c3d670ed61763da62da1a811be540c9c0404df8b80f3393870673a22a\",\"dd58f7a003b5546bf910e64cdb50e2243f83a08d7b8f213efa9b6c80b837f687\",\"a4b8ba6229ab4471bd4b94f44094815b7205cc90f5c9b52bf9c201cabe3515e7\",\"3d4794104813a9c75fff98086589f7dd915b9cceb855021dc6f38e87352ba3c1\",\"fcc074a4706a93761bce9adf550c96477ee8a8b976f906bd6bbde57ce7c8900c\",\"dbfb20f9d2417caa3b8b7ac42207bb4ce9d612cb0d869dcaa7f6b660287ddbfd\",\"9babfa65029d9a5a9be57556c0567f11db3f19bfe74c9728341eadbb27bd69ae\",\"de9b95c282e854e9bf3e8c6f8200f2eddcd89b20c5bd285ba9f43e2df3101580\",\"da4dd0b1d36c5837232881bc51e4b19f1ec40cd8512cb111363877ce9e9c1eb3\",\"984d43aeb8599e7309f37b10ad4e0f178e70feb97459058dfaaf00d3f687e7f0\",\"618a575df00ed9e203b4ea39286429f19f47525ed1210587ba756ae5b6eb7382\",\"c329f910e9f84a76b08ec4d9289c932283f17b946e38893e3627a6e0bf88b3a8\",\"68a378f12f6e4e97ea9d84156902983690821b7f90733b8bde0fed57b30304aa\",\"50e380aede73f1c97ce8767425e34bcc9ca7b51ad0ef8d12bc2fadc69da5bdaa\",\"eb3295cc0e709f504fe00fb8dcae1af8d16a55f766dad8ff7df1031ba3adb35a\",\"2bc041a69ed79cc943aaeecc4fc11b3b610cc517775893a09cb964420b62c86e\",\"f46a31d994b164c91270ef148b95d108ab4e3a894cce20a5016734044aaa9195\",\"a72ac3e77d497970eb8b7ab5a78318270bacddb6368d50cc3df0b2ebcf768b70\",\"c6c01183ff5cab2a4d65f1be2b86de8d1f93504e824ddb8e5e16e72b89305dc4\",\"eeb4468f6f5bbbb399bc826f3cc4c050878f48940283caeaf0d91b99233288ce\",\"98cc944747e301cd471bf59f02af3fa4efd7df487866416192c6da5f99d283fc\",\"837f7e7954abea7558cf68809c8b463c20e794ea370eec2c1aaf3f5618cf8bde\",\"aa52bcdb926a2220c30f7c9e08b9f01d348c8ce0c5954aaf298377b390636609\",\"9089b10403070fc82ac1db799082214572bf77b654773b75fba537172f037f1a\",\"08dcf9a72ea12ad1991e0c44f22d0ebc4d927c7f988c7423c948ae567f3b1e8c\",\"17bfb788b2cbf436632924de72e57a2f55b186021f6cdc74376834b5385ce52e\",\"413a8be86b806e3e7fbd47d6101395911383d711bd2829cc9d45919f14ee027a\",\"769b727cac3ccb8f03b8edb419e33212b3365408ba481365026c2f95a579c8d0\",\"232f250ce7cd090c0dba3fe6867ca8c3eca6956c862a0ac78f2b5300c3391ed4\",\"26975018b14d89cfd1360561a927bbedf334b2245aa30001c5fb1f877cb999f2\",\"709d709bb1dde5a048db5713a31905a7525330e49707fc0b6b4dde7f9f43b895\",\"45a376a037a08828f6d67c797adbd70201991dd7bd7dce27657b53a8fca353ac\",\"464c40994b138b56a01be8cbf6cce44182fb86975d57340222522e3aa7566187\",\"0e5a6578eeafade7ffbf52dc7f60b08d3258f42b7969191cfb0426710b41e945\",\"35d296aab0016be552126b2610c98e021540103dec50726941b0442ada561bbb\",\"ea5eb43041c7a65904e1f7022a5162cf652a33636027f36de543b505f54a2c08\",\"f1fa0487da77c9fa0862cbfbc2eb2d8d4e5323b2fe8e11e02133c20ed9867a52\",\"40a9ffd66f233691d1f51b11d9f367ac5c79b500f0407d2442b6a2d9987345f9\",\"398387309e2a76ce1a7770edba145da5f499ebced39a8fcad712dbacc6dd472c\",\"de86eb451281291796e8b5ebb39a6ec66e10b988a2ca4f2015127de1b236049a\",\"0c18169df02dedf26b30d3cd44915aa0b01e7a9d43ea2f8b9f0b3f9a28ede0d4\",\"1d24829a0dbf8fd51ee978430017bc7da81c143fdf6f3b56258ad17aa6cb0e06\",\"1a47bbf804c0ed0b17b53f2e6891693379d869fa03d0b43f4f5a884ecd65b39c\",\"c588a03adcf20cfe5f285c242e341007cd01ad6985c322c70360a4cb792aab86\",\"0e8474150406bbf3f7e3893dbcaae4f655ee01efc75f1f010bd9d96aca9443c3\",\"e1332070fb99d4a9b1497c25f28f096a2f93d4b109be33e4ae048d24abe71fdc\",\"ced9407cf1698f80412d3216fdd3a22c6b0c3349244e694827c0bf7a4ca01c58\",\"b6614ad36ce8ae9ed8a44fcfae3768945a56a6736c7e0c8dcfd76ba0134921d3\",\"de06c90fa1ff22794f1b5e7dc5325134097eead695eb7ae94a5f92295f1b858d\",\"4319ffb8c4c921fb2d5f361a2e67dd965e8e4fe97bce890eb835b1e669f7d596\",\"1891944f17927197ad5f4978e967a66786b7d60469835630baed74b338a31705\",\"e457edc4f0d897044e88b161dbb5e472966694e1334385f6c27ffbf962e2db13\",\"d952765ade7c591bc28fdf2e0aca07638b8cd89d317b1575547cf4ef51179207\",\"a907f5c7a27b2a12f9177f4eb85aa6fb024d13a6b277fa152d743789ddf5ed30\",\"27226d2bbd4a6a28f6af2959b6cf1892909c36f9b9232a1d90f588ea7ff172a8\",\"04d428dc83e197df0c152c5173e5151fcad978c6ac4403130f3c4b86f67d612a\",\"c5c050fb36262a129397538c2070df2540efabed3291c84e9b76ff537ed905b0\",\"9c08139bad008f1306ceb9a4291462bdc2aa510bd7ddc396789168643f507137\",\"94307ec56fb977cca454cff879463f6c165d078f7a5a442a70f0118647cc90a4\",\"e01f3161637f6f17281a018cc2e43fab068017013af5bb01c25d849c54be0b38\",\"3809ab39c140a01d5f83240f2177c142e44d077efa9b603fd67501bb782e21f1\",\"750a6520a6068baaf3facbff5d9c8a4921621f1d324fcb2f07601357f76439a8\",\"62a1932871596ca196d04cfb3e299dad620ece7b12a1b43b3ffac15bbb584735\",\"7299dcb9508b92c9532fcec9597beb1a4cde3c879bdb32a58aa395093e8eaf8d\",\"839f316b5719e99aaac4cd847de4ba06d15fe4842646c9ffa1252813b59d5315\",\"181a73c191b42178cf73f4f9ebfbc4e57595f1e105c9895b2c7e16eab6262234\",\"8f6576d106ddcedbbaea6ce7f47168c805c06860c41788601585df0c796b22bf\",\"4bc125b25d23eeb45dbbb3d0650dec2f0aeda702799b14711c41be3ebcd62251\",\"55d755c209ae5e20feb9df3dbc00fe52d3e1fa9214828f4e849aa112af7bf675\",\"bb1e61f8e26fd08c60756f4b1c8ef1753755c627a3d148c5ca44e3d9e5d506e3\",\"1193fceea1acbb0c9100de686db1b5a2ddf458f68685f91ec1bb5eff9618727a\",\"9f837f3f1aa2784780ce910fc5bc1b879abeba823086207628c5a59f2f860993\",\"168e6166079c2301dab3ac2a778bb10bd3c9ccd6c3bfec1c056a205685297bdb\",\"661400d4072c7b9670d1907f8aa9665b4c5c1c06c2ce3f09af6a8e86c6b9955b\",\"ad532740a766a8b9ae7e9c4d2cd07ba329e5ea133dc6329b826bb3c2b2e78850\",\"f3dc5f3e722cff4164c0b8b15352fbbc78d88ebc59f0324e02ff907d8f880afe\",\"a46d2ac79b851c7933f30b3161efe094ad0719bbcc5381fc928b52ecf1625e84\",\"f353e970a47772fd5103f6d82254064c3df31dcd8dbc5cc02c795757a6d11216\",\"56725369f5c4fc7ecb54323ecad0cc3f8447a4a0848f331e5ad45cc78df56205\",\"4a3c16adeb22db242c9e535f838184726e8efb3e573fa0bfb25b48768da141c1\",\"7a18b1ffbd1d96e0017c8c651f0a0f50f288dc3fbf2e985f4d14475231f07ab3\",\"07a8a1cfa7768c0b6286264f5e383f6986b28bf1240eecfe4c68de386b3c922c\",\"2d0c4a42ee7c746b39c7808d5abe46424e1aa0c38a591b265ee78c5c6aba4591\",\"c2eabb0f9664b628bccc32d434a5d3acb4fb09b61d55cb80062399919c17c31c\",\"b4c39954704c632bab40914c0290f57b04ebe6aa21e7bc8bd64135e2e0a905e9\",\"711be6e3b478596ff7bbea22bad598cafa47720b2941828b6218c8d33c271da6\",\"62ff1206a660f6112be0876cd67cb8469f7df86b19f8b4853d9c9fc8dbabc280\",\"e5b64f9f4a0015046afc728385f3077e20fff895ebd65e94bf95b2b9cd5c4335\",\"d663f1493fa98a7232337698a937654a2151c037ff8a3c7978cb597a41b0ef89\",\"c1721c68f652f8a94135de3baf7fbf2b1ed7c9cae3e1fa734b74dd3de5282a16\",\"d3c5b3a56dee35da1058760a92f57829d5ad39f4e208559cc8577f4d84f5d1e8\",\"5dfe60bc400eee64ef338009b3f5c5dca3b546f219d2a661ed840b6809c34b5e\",\"7b41286e46b3917aa456f233fdacab2bcb627e2fd8cccbd4c86bc3596d285e8e\",\"3f8dc6996ac373dd53c8619d91360ef571940ba2c1edc13814907f9c5752a9bf\",\"54cc2e4f63796bc267392f4658d46e34f2478ff70ada2b9a922bb07a7eac5dae\",\"183751c9c4701597e630ff4fb45da4486630d5f1e0f4bcfd9b01b10c0ce65679\",\"c5328c088bbcf760d55ad1c45be2a06b9f8fa3c749ca1d30abe6d7d3a2753271\",\"7ace27891dad215bcf010315060e72b56e0e647d38d908a604f21467d145e73e\",\"ecfd1c39c53e8a6495cffd0b7cac432e794acd8cdd8423563f768f4ed9287749\",\"efc61784fe8c99aeab27d1fb5c955c6228f1e1c385aafbee3d0e4298bfb263da\",\"b8c190db4054e0ca2e2a57673b60ac76996a7aa4cd850e467152db9708dfef51\",\"60cdfb93909730afbcb84fd05658bcc4508f626b02779220d1e86caa8e1522f0\",\"0f5f5bb48de03968b05961dbf541434622ce1e4595836c71a9d88b894d20a4e6\",\"2b2f6ea6f17a93362f4d87ed6957495838d76b45f706784ed75ee18ec859b4d8\",\"3aa9eeb2b52b88b23c1f4172455b29816d9904b067460a89c6034d38ec159e0d\",\"d063b1e807169181288ebed5cbd5a1354692faa9a7e1f42c4776d0545e29b309\",\"a63119e3ff0e943e8314ddfd68eb379262f4ee28bc149024aa995d0bbb8f47ea\",\"2a1b54c76297ec9c3ac8d217b98dcc70aa265c6eeb85c0940163a534f9a5cd3e\",\"c533f8b70a0b51e3890a5456c0cc01370dda41253aa5b417ded02b4d99e2800b\",\"4eb6c89d81a713c41691404b591328879edc3f8fcb501dcaff6675065a60e92d\",\"ded576b41d7648baa3d5368127137f26985cf8053d7547c8b8f8bd79100c9a3a\",\"dbaed082e389096d9949cf9673f363bd6714b94bf08ce20cc407090e01d882ee\",\"fd78a9726409bf31503e85e2710d5b87c73963b09380c8f5c39493fb54b34c59\",\"578318e86a565b52be2e24ac5f422c50e5298a6ae29fe3a7d8bdc0fe285d908c\",\"9b89c56a688b4ae2d11d61ba151a0868b30e10f907a323f1a222b738d2bae8c4\",\"d8ea1ed71926cdaa7b6b113bb68186ef96153a0a8f0c03d059e0e70719baf0c4\",\"0014118684541a80c9277fd0d88c125624da09e56c93450c30170c87fcd1155b\",\"c3fc3472d61e50a2d3c5be2a77999fdcf99591e53b184e7ce4511f48f35e7e94\",\"8262ef46203c90cce2312fba9f4a1ef713af27749ba4f77e6b87e192b7ade590\",\"aca192aaa95d7d3be7c83eae56d672cbfc41fb73389ca95d071022cfeb1bd974\",\"28d933dd11bea8105ad969db7d69263ca51b47fdead0139f43ebb9fb84300023\",\"2adce2b7a5d11df6706eec6ddac9851b993058089f6b1bf932aec03c58177c44\",\"218166e658d76aee473504231791b4372f0db22918a49fc3650de6633c929719\",\"2170b616df5a20371cbb5322acfd47e569dfa0efd24d6a61d62ed500aa83cb90\",\"499477077df998261b764e54fd03b957d99ba9615ab4bcebc012560df4dbe052\",\"223159c31bc61694b46ed6a02c0e1cfb282fe934d4ce3b53ffbbcba631507afa\",\"9c1c9c263466a961e19d1e9e4ec63058250e5b1590a8982b586bd4fd7ec1d1c0\",\"c7b8ca78e967936fd7b9ef30f0947b4d9b99113efc26558c5a0a25d8bad29315\",\"e69d6fd7b46a0d12779b341e22e31998fb93f6b8b1f2f30985e860b8816bf3d6\",\"cc6b17809c468ae2d9a1493c6042fbc71e3100b73293834ee7e441b6843cf673\",\"2b6f6d1df0c39a71fb26d97bff4ac6fbd5ace7e1b172dc657f5332f521ea3bbd\",\"d749a168c3bed1dd633bb337f82710f54c049bd0596c31a9ff34fe7648260211\",\"4b6dc1f4d999d4451a13cfaeb0e7b0782e063c77f1773d027140e8ec3186929d\",\"a34808ce5f9cfb676169e0083deb02afcbcc535ce3cb1636aa410200629ab00d\",\"c09ebd812f80ce141cd738c2d3f81cd33c5d45cbf5cccb7271b4842de827d720\",\"596c50c04f78d9f58a854deb54a75c9ffdd3e378c8014de2df06877ec8b6e4cc\",\"1513dbbe00b118ed40411749c40174c7993ef4f550e65d7a5e25be5809b1a77c\",\"f0a7c229215763db1ba6ec5d6ea578d68467a2966784f580a63c5fab50abb2ee\",\"a91ec2bbfd24702504ff68d04d11b3c7af8ac6d912d3b6c5ea96c9cb5bf6dffd\",\"e6f977b9034043fb38571eca9fa4af24bc199a14cb4eec1c1c2799564948107c\",\"f2600b9bb1048924cb7796699ac38e7d419822dc4b60711079b10b20e14f73f2\",\"dd949e52bf1fbfb9841b0773d434e9041173dd3e03fab5ac4761271d043f2a43\",\"64383008e7ecdeb747551bbedaf6808a94257e3f8a7ea4fdf3c460e908f06efc\",\"f4061bfbc96ea26893eb40f059531401efb9c81c09a9803bb436ba36f12ea26f\",\"2f58006222247649ca6ce32a868afe8f2e4bdf9d60c37d4cf4d828f4a8db940e\",\"99b6e228077343cae70b0e86c65a9638c3c5413b21355b2d470816409fb58cc5\",\"af8b1a7deb4af2329a574ccb3916f61931f7b48b3044c78314a72667106ca53d\",\"94e4a8fe0885e4882d5d41664f0a33b5eb1cdc71592c8a18de60ab17505a0a62\",\"2e7d4b28bda2840f2518a0c1acf2f37706c651bb4fb5cc2ed276e90e40a7208e\",\"5fe7817366ddd9010f85733462cede62d2864edb029420185bb40a32e792204b\",\"520ade5a35f903f9d1d06c00deafd05893c4d3135ed77f2b9744e30146a40883\",\"e801990f00f3d4b3f630842caa92008c0e8d31b37cd7a4c5bdb51aa54ae91d15\",\"8c9626ec7148811a15e7da16f20ba6001b1b060329d1fdd3cf31989ce62262a2\",\"4742cd0ca850abffbf2ceeb63da46f581c2e556f883f0c405b75dca49f392519\",\"7091e8704d731d22ea2e6ccc683199ccf7183d74227dc33bfd6a9329f3f49b0a\",\"3162c3868da92feb6eb2207e2d2fd7a519f20a5c25ab9ae49df356d5dd9fece5\",\"4149e44022791616c9edc17615a83d239b24489d7b1181ad8e4c84890af47981\",\"c413ed8ba0fededb1f02de40aeed56861de67177eb8ef8d514d5e4cd1d4b3e8a\",\"b698ee565bafe027ac88f62be9b435fb4209c359a80318b47c4ba822b94acdfe\",\"a1a39cde7faf96d9c9a5b90c4311a3724e3dda0bb12384c4f390bb366b3ecc31\",\"55e7b2cfb28210f1befa802dc68b52062727679fcaac1ecdd0393f14988779b5\",\"83a243605e35278504a0839353f3d2aeda6d83cf9564652b096efabe669ce416\",\"046324836cf80ee12f9a5960b259a8f23170e9e2cb2c5a42f64342b20451a671\",\"b1c3bb72c15212fd3d48718fbf1b06b7a561dd50a39b9309bab58dacc711d349\",\"57ffc1c50c63c1fdbb5a3554459aaad2e2cc87c2ca5ebb7186d0e4e30cab9fcb\",\"1d085c177370256d17670d26558573172f6c5681b37c3218cc0571dfd80f9c5b\",\"cfa73f6d994dca091b1d4181920211073ba01a4f92d0541941d42e50d8b56420\",\"e9a24b1c8a279f0dbdac070cbf2c47e58ae592051d0701698606d1764b3f021f\",\"0bc9d477f2d074cac95272125c174fd73050755ac2b28e3e267afcaf86867988\",\"961ee7976b6590df0130762d9fe7c4dc32446f9ec4841737c02474a4afa51343\",\"919d2279baf4449a0132351cd26e5199864e0947f51d13f28f3aa2337d13ad71\",\"d1b47faf92e4ddc675d3ca425bb87f22a86962a179c37df70a339d8e18684281\",\"7bc3e2cbce2627935f95d0703ce558e44063f4488d2d3c40f66faf16f32261fd\",\"c5276a36b3ff763b7d2ef47442a4829f16024d1bbfb8de6e0b8c356994d050cf\",\"670741ba1c9e11173c03ab749b66ab3faf014c1c884d8e2383b4f6126f5a26ed\",\"4ce55ac1db46cf1da4a98a24359a745e87e141307a0502e25e1e6b21c46ea23d\",\"11152187db6fcd643d7b24934b01e9a8668a2b1d61302ea393a578e96a98be6b\",\"42b09a884992cb333a32c493b5bbd2b918569c61effb9dfb1f8ab070f5639443\",\"34415880488a0b6371f22ec7e9e38f39257bb3e18dbf07d3775adfde69c18e1f\",\"ee7cfb55e47a3f58d07d4f7f167db8c2527a9f7b40f6d008ab1b918567d31dd4\",\"210e99655a518868b05ca6a26ae746e94b255b44ab2903a7b17f839d7248fd89\",\"90fb1888e18b6a88601f240b1760e842edb566b1a18f95e3860a099e52c4f69a\",\"90997d75e59dbe0883e88c2e3f18c30e7235c51e441c074a364c901e233d0fac\",\"d562e26826d170fe81d5cd41f84936596cf289ace48422399a93202c402a412c\",\"ac1e0e72fe3130f069c1c596c3a6dd65bc29ba9023304a11bebfba94419e5508\",\"95dae17da9d7239280e5b7e27657cd0e045cf2c0f761786c86e66dd43d45d439\",\"1e2891d7323e18431a04c662a57be8ecf1b4a99c11380bfe5711297e9150b70e\",\"2455a1141554a48dc3f4e2e4a72dfb5d8d8a799e60fa683bea3206873b6caa43\",\"5122d9b747f0ba23b1eb34836a654b4591028c5cea8ddcc315d8a6394f9d57fc\",\"955596737de97754eec908301b1746b1e20cb841975b58fd30009629144382db\",\"0becc14ba670a8bb0c0d1fe41ef268f887121e15884ecda76f88e6609fb7bc11\",\"d9132e6a1482d5c6162dcc3f77d38b9c72fcae55ff5b610a284c55137eb65a1c\",\"310b74b1877c549c7cd28391efa743c38aca38687d7a11b3df80a1fd048c13c3\",\"81e2c4ec55504d99e5c0804038a405a707daf9e475b15cda283d2a889e6f93e9\",\"f2d77de01a148b8603f2fbb94d38a7edc2a2e86e24d1303bb31171e4ceda60f1\",\"6f3942f45ac0232faa9d745cd45118ff605da1007993ee8cb0f60e395954b37f\",\"84e05ffac692301b74a7c784abd39e9efe5b1aa9b4ab4f143d64419f66b666e0\",\"d3080afd20004a3c1dab7ab29d61f919a81d6a0a57a97ed1e892b5b2501537c1\",\"850ec5d93e70dec76264ead594c21604a7cd73ce09c179f94965c716fb1abf7b\",\"b91f6ed43c1832c708438140ba66f7a44b6710cac7e5013aadb346effe0ef173\",\"81a17b7457edcdc83579ae102884afa141cae202c28c6950b086c4529ab2b2e2\",\"8303ecafa0edd55fdb788002c39e3cfee7dc7f8c2118d1c88b9dc2de90ec1967\",\"21917adcd2f4aafb1a086a66262566c377aafdf0825ea8022db3043ba7da0fcb\",\"f85a0ead8a8c9b48825791717af77c14624aae62e3b82589f5eefeb77890cc84\",\"86554a8a86372b95dda67c7a1e5059e4d04545da5b6e83e9ca21be4361625f91\",\"9b1a1b301d67a364ca95bf61057d838cf857fc7e8d76aefebb6095562c43da90\",\"93cbcfc2233eab29ef7f73205f157fae559eb9632f999df5c78718eca2bfe6eb\",\"bc380a15fc245e385a852aaa67a216c7cf7883d8b7d694f202fba48cbb8f9b1a\",\"6e46db515ee49cf30cd09f32a75a65aeba755626a5735eb204f1b7e2f298fb6a\",\"1ffa43803c9bf1384b899e5e0c241d813f486b379b245c70703a6435d87b8f92\",\"cd6b3998163b5c2cd9eb9c989db61d655e11d7ec681eafd835d5b0595bff7c6d\",\"1eb0d709db81400ad30b2601fe5d9e8197ab189258c8ff4e2b2f8e63f5942408\",\"32b783c06ab5d47d3d0123a83d34386fd33ee1b0d9ec0e2c61a69395fd6db720\",\"e1a84e3048bfda4611ea607ba761047229e5c4c766ad29e2c1dafb23bc48032a\",\"c132fc9fe416916b409f60b0813d198d34b28c752164fee4684e0dd9639cb01c\",\"9398f4d9f8331d64884f7179596dba3547899349caf5913709c5431cee59446d\",\"833607db0f3adef7b9d458ddba8f607d6e95dbd6dc862c1462ae2b0e3b07a9b0\",\"eb02e1cccbf9b5952879839cdd5add8adb17a43db16b2894039b3165ee4e9592\",\"5eeb6727993da65cf8c0e6ffe22acabf5a0fbfc488561ae3ff506fbe4093d5ca\",\"7e10f5341c110511328e9035768f1f9906aebe6141211073636bb0724340474e\",\"8c8a107cf4bc58e8ecf206c23a0c242237bdb0442ea52197a9063c668814473c\",\"762f3a82d6e7615feb4b5a30024bea2f1eb0df7763b48f45860f2667f4644d38\",\"b8e51ddb2f0b925a4ec89aed6530866c2877f23e0d5123aeec7c1d7c04930b06\",\"4a1b6737391cc9cea554bae14cd94303ff462acf4f76e096b96b3c24a28a2d23\",\"6c363b625cd417c9ee7c36a47a6051a8e8618dd298018b972ce9286805bc3007\",\"88ff3401f60c9f433ee31a32869336af8d35f4dc750dbb3cfa54e6d91be0f2c3\",\"333c7f63adb7e96e7a5eec2c1f876afbe950b3ccb7e6a69b9c2ac891cefed21e\",\"7df4810b3c5d5c1780d137091e8a280c98f779bb434d0c82622b03b0a04edf9c\",\"8dea77daadba19d59af6fb01fed12e0ad590ad28aa81bb43caf3bf50fbdd9572\",\"93cb93c596dd7850a375a068e6b6cc6278cff1c0fe31201df1b54d8411a5e021\",\"8e4ff5d4cefa3bfbf7e7d5996cdaca288fa8377565272341d0f5885480492730\",\"aeafcb8b6f90357e9171daaa8ca80fccbb546f13843abb41a9517dfcf60d8677\",\"34fff90cb805e63ca99a64a9fa475ca82020237ea2d88dc34173e0363a2b0662\",\"bf380ea5943997f64b9ece5b5447125f40f0ab223b043e96bc5975372b362448\",\"a95bc06081a06ca457c9d8b84815e23bdb5e511ee5a5635affd5d7ab22417407\",\"de238c4777f9c547805b34adf74db5bffdd8eaa522f23b389e638aa3c49bfa1c\",\"dc03fce11e21fa59e1c1f9ac36474a5006e1e60987fb82333dc376f373dc7fdc\",\"cb028e70d5038782b7deef1be3c520b552ebab55879a2c09a6a99d30d8ada634\",\"f9f67d596053a93ab515369c7f7fa3745cf58fcc62120f0e7c58020381ad0bb5\",\"688111a6fd040b59916cc95ec9cb84f4108b9991b7885f10854caa5759f31efb\",\"e638c49f9bcc640ff71a746ecbd7e8741ea059dc5d7e3d931ab07a6b98ded150\",\"2de34aab25019c7148cae772ba9799225b8087693878da7e6654259f01dd574f\",\"36ceb19cd55e4ff98e68442043da379cc25a4ad9dfb7a664fc93dc91a6cdd43c\",\"ab87d011eb5725a439a8cb7fa30cfcde89408d5dd16946185694963c954e4e62\",\"6b89f624c08dd8e93cdff2198d6f82e5d8dc33edcd27a45a04f3567764c9fc30\",\"a4125f4d8d841d323d35c46b20d183af5a6d70a0b04c805993ea5d2c71f1503d\",\"6ca37b8488e28ad2f9a4deacd7a49987f2d4d3b1b22dc1466cb06cb05c3eee70\",\"8f8182218e4862ef219413b1f3e2295f4b8d837ffa713587fc0425dd18dcdcdd\",\"5e65c45062edc1bf738f026a5ba6bac4dcd263bc980bd1fe4af0ff82a59aa07e\",\"7d394c2ad28a3f3fc16abf70aecec6f7c48b688a576b9911daa5603535f29539\",\"66a06b6e088b9b2b2df465552d97b00cac627d2bfd4a7811b4656d8bc95243c1\",\"28b1d0d0136c744fd86c411da549cecb881250ed39c29aa910e5e1f3e218bc8e\",\"3ebae25898d49ca9ec87334769977ad48772acaa4bd5c23e1a96f15f7d63eb2f\",\"2ee67296e7ab65649d64af774c0946350f8254e1fd980ee55cd46a86f61708e2\",\"b14e2a5b316850df55f5eda98e146c13e27c5e7b1cee242d0ce8668994c13f1c\",\"1b8d24003667488824ba48b179528af9ff75dfb3401cbb93679c4541b4aabc81\",\"56c6276b3b4565678a78824cb33a38591cf6035035c72d372fe68788bf5b66f1\",\"3d26a4c13cb0fcf406f4e2402d0d47bb0004e7fa6cb7d1afb7e457eec3441385\",\"f3aa0c0cdf36983d4f2089849e6deab1e89506d11f3b8643ee5dbf5571d77e4a\",\"8fefd420da72f6515b07213fe1c8f153e4b6ffbfff9a2ba9022a877e975a222e\",\"505ea4014f7c13617c24c69cee09fd70ea9bdb6a46e41c6ce06d1e42a98459c6\",\"9075920dc51505cf6107e173826f394ecdf54784bc41114e6ae3fcbce10457d8\",\"e168f1e8ee718299cc84305e60e627ec7979e8e14d72a54919a998f62a2b57eb\",\"ee2c276e0db8b8d57f8b400af9ec966b6611681cab39f34d7dbf3c9a4d493415\",\"d3240f87ef46eb2474e5e517d822ad212b6965754be8e8dd960d49e6fad74a9b\",\"9af523938c84d530883f1f95a3d74403d3e094cdef8f6428ce68136acaa78e8c\",\"3122d70b87dc752d78a45bdd4c1731bc105990f3308eafb1c920d98a8fa8cc8f\",\"bd194cc053315945128be57fec5a9be9fad29c2bfc36eabd89ea64520c8741b8\",\"35451a72f9884bcee0b0243bc7be892089481c16939f514f076cba13a01578b6\",\"e67576f2875a0840dad40a1790f9a0bea0b3abc337193039c61cbdfbbbfd831e\",\"a2a935c32d6f9df63334b18b857ccaf11f4162a9b90cc9cf9d6a5f8e508c431a\",\"cc2945d6952339bd59c759fa5c37ca7c38b75290a57f1dc790cc5adf89ca3cb5\",\"25ff908e1fa9bd6b4959b781ea396a055e7e4a2503c84ded977d1dd330354f39\",\"918fb49fc908f8edc346864466b52d9a00c8208e0e2bf700418bab1a242adab8\",\"5dd3c33a30148cadfc5923c0e7aaf3899dadc78a3416d628573fe111788eafb2\",\"04a88aabb0a1a391f31f830f62f78b8e6a742b06eb56145cf566e2e31e5d37fe\",\"b9c04e3b1b7865a4de9056a1ec7078a82bb21a43480b19b81cbbc30570e7ff35\",\"3890a02919f54135c3c389600549746882e2410a05c1677587336f806b23ba48\",\"d50cf662f2d6230428aaf00ca267676484b8dc42f2e3c24bda7a16218d84949c\",\"641d3f79b78862f46e7e567bb7559886f2879c97a2d723775b2e4b265f80a7bc\",\"4e24901d9109f1c6ee5a568e0484d43786a89b02b3c44fe6bad166e756f59b89\",\"32b47ecf47474aa5f1a42dbc47d4aa315b4b9c7e42ec95c1d8f621720b850ea6\",\"e58cdf8637ce228784f7867b46e97bdeb989ef2a8463e954508bd003b6683bab\",\"f46614747cf8dfc58c4783a5efc903bda9976a8fd8ad070af257801d5f870a65\",\"8a818e8193e587fbd49cdc8bfef7a84cc78915fe61c97c56e0937e25cb6f1549\",\"db46c92e7bf93ef265aa2812b2e9456d58c65ea0001763fc233e8ced22b7344b\",\"ddd3eabad9a45ecbdcbbed44c4861f298b7c41490afe1cbaec03c0c07ee033fb\",\"c8691bcb457c99ac5bd90865065caa858454473959e5d270d3a34425fe663d2b\",\"89ff6b7475da1f311216ace544166115a8c4360710108a2ad005e6548591fad6\",\"6fd70a79f9aebf1cc28ffd572f7778844bde3b3031158004e7a2a71a241ec6f8\",\"1ea21cbbdad282460e23f69646123e5f6cd2562e83ce3d8592020096216aac1f\",\"73b76ec2eabeded5de6fcec33d755aed6a5c8f51045c0ce7157749ffade14a7a\",\"e0fa697c6b7f9f8903a623d813897191c3c1e894a4208d66639bb6c102aa469a\",\"f7163e5488c0c4b59230c911840991d4ac9503981e7b5c4f78d0e854e017aaba\",\"7aad70c287b77de7f8197aea736f5827d5dd8ece52c1364be563e9070d47d913\",\"d5208a3cda121ff67de08fc6dcb2fe1c6ac99804c4d0dcd5e9354e0163dcf433\",\"c678cabd9cfd3610ffa71d8bce8bb66ede44eb1351fb9c05a4b31145b3272d51\",\"2a0bb957918dcc09f4752d9bbd65383f2b302a00891e2fb056212270da368937\",\"8f9a0f4213d5d2275b457372cc266ea00ad605d6671b0330e3cb1479dbf7a8d6\",\"8bc54f9a5926276539ab7eb8072a74cff6391a48da9646edea1b34e5c95749be\",\"5a02ff65937f4530cde7371699e7a0e688ef20106026e8f576c530a016881b0e\",\"a783861dfc1d7c8684312bf6a57d8819b14686cba0232a98e05fbefeefbb85fb\",\"345042b024141a1ca99c5883e579bfa53ff05904f316d8d5abf3c5c9a9b544b2\",\"ac28b8a231d1528bce10df1d14cecfedde2282a741aca05e66201d3ca2d2f470\",\"b2eeae86639218338605ab9cad43144dfdb16daa9d21b650b9ac29973b90a5b4\",\"2634de557a0fd9476b3b495d5ee2146f67a1d74b7b5b1d2f3336863056a1956f\",\"06b645578d9a8eaa3c30198ae6374bf3d552f9b26b83c5b72b6e2853041b5834\",\"57e7d0a11200c76fcd4ea15ea4e1c664504a8f4c435af11f142d5c406af5eb67\",\"43a2b56d5b3d9ede6ca27c56f853b34e6e8eaf6680fb8c35c5f15a9b5623c34b\",\"9729a7339311bb95050738df754e9d74388160deab3151952db8726d5a6467ba\",\"e1b4ef847f9e2fb45d84724102474d1178a4767dc163aa9d2e40769fea7cb6e9\",\"94493a55c55e7921154821977abc5a8c44115e152a67c7652c94d8be51c471ff\",\"65a0d1ee6e6949627e06cb506f87903164c88ae0ef0b3129c37f7ee4a4ccf462\",\"9f061d4e0c21ae6e9d72a5355e385d663761c3e5be36a3d9f69f2950abedd0b7\",\"821695bc55c16e5e55e63f77fbec2a47411f1ea4c498a87382b5cb8a9a7a5e10\",\"279474a26ed311e01a21d73cf1f5f4bda0c60c0c340290883e71bbcc70a65cd8\",\"1855367d99ef8e6e5e928990b664e40db4368be195e52bb4fada7d169523be79\",\"28df7d66134a5fd17ab0937ce74c190026fc1445347a6aef9a31371eea57e60d\",\"a26e84eb0a1dd8661c14fca518c839f71e3ad82e8101207d66508688dd556e22\",\"090b121f4fbc6fe00b1092eff3b2ba1958c6a8439a9f5fca89f9b3e030206209\",\"b68aaa7f09577f5f6196d026b12d0628859b115c1b2edb91e8ba0c3dd19ff649\",\"a87e706026cbcd0d4fb0acabc36ecfea99018f484c38238f19c5da4b54523d9a\",\"b2a84466065bd0992987be67b3ba5dd29c0520837c20385064c77278e9521af1\",\"5e760c63ae742e1d87e7e798dde3bd27f1e52d888f7c543ac283ea31feba4b24\",\"e0ddb86d038b0a45d8953774efe3b57b5b4701a8363bc7f25f55086ac52ad78a\",\"85c09c36626a2e58e3560861692e214d2264255b572bd31bfc9172ce2c074371\",\"f25deb4ac70a82a168d8d8aff408026580f752544fa610c163828b09d8d44d88\",\"aef041e25b899e99f4a96614665efa0027346a2768e5a341d50b5c388a593a76\",\"6a54283580e53d10f32bd2a1d855bd65ac562ae925c826448afd8fc238929e5a\",\"5b511204b4776799b8366be6447707b2ea025bedab859c22a908fea342736154\",\"e88e28dc1c1683a945c9c44568f28222db4b302c069a1f23eacaa4527f7c7cc4\",\"4962a2ac7e4efde1d489fb97535cb2c3ff8ab63919509b7a896e44330ac916d1\",\"1e3d07d4b539433833a0b716b145e5087054c4bac4398287a8eb37b3e424de98\",\"c23cf57a86383410b1d87f1044191cdecc1f592aa04f9c12ac81c44cf83eb3d2\",\"87ccdbaae47f3978b4799a75e1808bf3cf4f972479f92ee56d78569229bfef92\",\"cfaf3da278ddb6fd2b6aa0322ed559e151b251924c3db0b7a603eab4ecae064e\",\"5338e5c91476bd93d9d98000a9ce0e567af07781a31e390442212a67bc2c1e78\",\"840d9e90d99fcc2658132af48fa14c23799f4769c34a2b0908c7da1d7ff0df6b\",\"02fddb6e98a198b4bc380e23525f7703a8d3a47ba7090b208d532022ae608245\",\"c445cded6cbf669d5c654b8480d34a652ae6965a348d94ad46bb7caf31254ee0\",\"4bdbf445f2080b6bff382cf4a61b1a3ca0ea72af147e3eca553b090386f4b708\",\"2c4d88ce1963d74394bcc208a29b60b20184487c16fbc9d4c71fa3f86bf4d17b\",\"2cf5ca09232ea8574d23ada74c70fc1e74c56526f2ef6e42c5507c1c78ac79e7\",\"2006cb35f21322f17016946c894fc5d06cc4b8d9cecad36f20d8705ad095ea0b\",\"d36296fd65730ed0f1f9fc75e3b10583566ac08e63873def8729eb5928f17950\",\"79d2d9cd9a972c3a2f8819b9a1afd2134224e567588516b4cb46ece2611a5b4d\",\"26807eebb74d8fe013eafd71f91b4a1584ce6751abb08e58330870f1f1817f31\",\"d0b4971f59d3ff7280b60e4c97e7bafcaf9a70951d2d2e3fa734c84fcc150e6d\",\"c27bc1f93983854e1ed8dc14240897370ace0cbdad59746188fcaeaff81447b2\",\"0ff9b5c0d88aee28a535d44409499685a867b52f7024c885eee31cb2e972d4e9\",\"0b7961b01bc543289bab2fbc0d3e5538759a308a8ad8a3dc7611d75d60c0f244\",\"9bc4f4ba58e170afdee126f4e3d8143f3417c5dc31f50f46e4acb9689e6db794\",\"a9c2c969efb05c1d753fa9906859614588c9c1100b880c97707b99c16ddd023f\",\"4107247fd50f5ffbf23cb438c479d8a24e3872c5fac473b4ada97cc375b9e729\",\"4fb432efaed3d3e3f5bdde66be554ae57f7d90ac6680b31aabf5c3952df8c7cb\",\"7a5099a35b4b0da23c31d1e8de7ac926cabe5d8b997de4ff2658145645889e8a\",\"0e677e9ce33197662df38bbe51cf5e595b94cc203ca07d2ef8364f21884df0c2\",\"f0938427272e67adfa05ee32dce5aed54d1cde128eea57515b593dca327b8972\",\"53f5390a77f83ca2c7a5044023e4769e0e55cf1fd7fa8620040e11f86fc94675\",\"6bdc4fdc196659edcef174fe07e2fe71416e970851f11b5e7d06aa060580b90c\",\"032cf29394a304c1f68fe6206a93f6aef45f4dfbdefa3c544cd3cc441f9e289c\",\"85a07b138ed422f8636005e2ae9645a232ed6bb01588653ee65b1a9f99c98ab8\",\"da2ea04670024f5d52418bc07bbef0d41cb48f9d34608111ca867d04eea5cdae\",\"d65bd33f83e59a1bb172515ad0076b6f81f633af9f394c54c5250c47e7865ad4\",\"d9ae9cf0697a607e4fa9c65ca527018d383c1e3f1c4316a5dda14057fe1d54b4\",\"29f0e4d43dd699f82bd9415f7c8ceb9fa27cec22ebdefb339b848d91a8445e42\",\"443651bd7dfe0d20cae67b8be2071f9c5659a8a477bec77d341000aeeddb87a1\",\"011f379a719cb36df5afa0842873eba3adffb37f0c3edeb1a40dd99a896094c1\",\"b3768af32e142b4a08b3d74f81e34485a779660cea8d1f640e5e9ddd4bbc0885\",\"32cde742c8f5e4f596bca8632de5f295d0b4dfe4d9e660d1aff8a39359c63eb2\",\"896b58bb54013591487829eae5b3dc3e7b93be0072f10da99c07a1ee744da14b\",\"5436b998a955d85123e413a180e3872d114a4ae7dccc5b09bacdb7c9b1cfd596\",\"865f981bb039172a5725ef149312948e01257ef40431f99db0abc2022bc7359f\",\"fba6e1f56b57ca4e0e2f569d98c4914279d58e826ab9cf4dfbf50825924507c5\",\"76f2abdf4b2355b4933e5c6bd45f0b5129f990e46cfdf372f2680c123c728ac5\",\"52c2f1905996b01ebb46aacb0e7c0fe35895a662768eb1d9bea7d775f6fe0cd8\",\"fbb0bd01c081d2a6d7a3314c7f7e1bedbd8bc16796944a74fcdb47885ed681f4\",\"aae037d3f1344b708279c6ea2305596ec1253bebc8f6c77dd178f1d8a1b4f360\",\"0d8c3eaa12c5a1c2348c7f5bd845805b5b348ec85d5f1efbaa064fdfa6efbc2f\",\"f6eb2b9e4a523e3c944cf224ddb48fa1a80a6c3775b139591adb5cfb1fac2790\",\"50b6d066203ce59c59641dc516a4aaad8d6304ea5b723a7a94cc596826da7e48\",\"4f6c58a61e22681aaf0948d4529e9bd3f1dac4db3b546d1217d91c34ae4f25a7\",\"fbbf1d9e8db5dcf58d46a62bb79d6270cd15e89423a598922058e9750d398e60\",\"b2cb9ee54278ae8b77db01d2fb58e404320a069cfd1469352cad9a3851fd05df\",\"f676d968b938c9fdd9f5a37cb34dc5271354aef65042dcaf6b5c1db7b7104592\",\"f3a9cbd2bc8420dc5d7d51161d556dc070db268c495eba61dd1ecfb658b1825d\",\"b24fd0bbdf92c872d225a83afeb8532616d2ce77b3ed8fb3ee1545a4a20103b9\",\"f0f4c0cfc33135b6917cfe7b1c9f2754dd6e7a57cec42be764fee79e02b32436\",\"b9d50d79fe47020695c27b22cf772ebe5d3c2d910733f09c387022fad7255143\",\"bc70a1ad049a1ba1d562330cda4e1a46e55711b8cd04fb0732e3c14397c68539\",\"d51a38bb70415e9b246a1b9b0d31a0a219abbc4fea3d62566fba73027b151d02\",\"c6d1c539513b637c5559e0ce9986b8c6cf6df5b2c414b2d0c7b99fce10260148\",\"90063ffa2beddf384d35f60c49e9a048be18f98cf1c5c7ef4a7b104df1495022\",\"24fe855c8586e278801e171e07f7330961a43697119822ce876730ba43d64bf7\",\"0a1f145c52ede1cb7cf700cebfd4fea0f2239ef4e88958e4933fbf955622eb99\",\"5b2dd451303bbe62cee8f91fc049590f3d2ac94aaba951f66f2dfaaa8f30ddbd\",\"5fbb15102e35e8b913923f3c37b26eabbf7b2076c0e737ce2ceb0fffa5f663d5\",\"a41d35c7639325620bbf17f432ce1af6acca5a58c41047dcc33d8f727a901b28\",\"27cd2b222e0c8151034b84f5d3491dce18a4982b6c4eeee0f80771b2129ee9e4\",\"93c485d4a58922c5e827e2f96f32e616aaeb2325caeaf1378a1c0c576952e04f\",\"4209796385f644a896cfa2c093d02625d2a86e035c5acd24cf10c8167beb86ca\",\"af9ae59b864cc347e361a0643d8d00bac09762ae73c5c99ea0de81df97600799\",\"97a9a2ad0b32f2bb87d7585274cf24d35b854ddf506d694151a130761d8fbe24\",\"eb177ec63b0ada57c1013cb64136bbf58758d0e80bbe85ce442cf55f5aaf76e9\",\"4e1e9e5264d2d240ba921bad485f67a48582ecc69394c273b76d8229d9439cad\",\"dc29b5c301849f57e42c9629fcba42e4a95040b09eecabb6021225e4d1e7c6bd\",\"505df5112412380e8a92d7878beaf40396915120933776765f4aad791257bbb7\",\"cd160bb9b937d39f91be656c446fefb39dcacc35218a161768e95fd82d6c8a8e\",\"b2ff80fcb3ca70849110d0745ed4086365583bb98ac6b03e3df8eb5e92b60792\",\"8ba84b87625a58751c7659392a1cee8794cfbabc0104172e5ea7a8aee047b0cd\",\"2fadba327f23e999df1ba90df60a4f59b36407f639be259a07fea85e666d3f54\",\"7815d1863d4bd2559730ed30a2c2777bdec9d4c4c5fa7e073d323578246316ab\",\"92e6c199048530e7e0f83a7e215ba08f7acab19f253fe7ae2aecc086c3f53211\",\"d1dc15769818823e19bcbb1fdf4b6ad11f15918959bdf20d8f1a1caeddaf7a97\",\"a407e22f19d2bf80861da0e1ad1ea1927b902f086e2a9062ff26d530c6437a4e\",\"78a1b20abcbbcdcf07c0977285bea562df1f3b5790d6ebb968bcffae597746fc\",\"091d9f3e281fb0916d91457490bf5134dfd19edcca88ce7a13b46ec3247c6ce1\",\"9ab488ab41c0ad482a92eaeee72aa1fa516535ae764447963781593c083761ac\",\"98a81089036ff8a893f5646f34d2e9686a9325a59e34b05d507df4d18c89f0e0\",\"ed4cb5c7c52bb57ea6a74bb9ae1fae82f0bfc918ce93d92a4b7965c391717bb0\",\"db054e262b2f869e51d2410515c74987871a140bdec85997dc588bb7bf962215\",\"12079c75bb4ab498a238c3a0b7f75f52dfea2eee9c476300fcacaae71b8c297f\",\"f3cf95a9fb84ca64ee12135c5b3a8f05f43855623eebb6b85d1cb49c48c095c3\",\"b9d359cfdf697d5dd79436e63ed7f97e0a5373c0386d14cef122e23c1f19027c\",\"c273325d9ac1b8a42699ebdd1271fde0f7df82eff2e3c8b5f62063dea5d06ee5\",\"cd0eeb860c47f415d57ba3cdb237dc6c4ec2a461187b90e49d198f7f5f83c64b\",\"72ef933cda43f97766d6253b45eb3a635c44fb27020dbca05320e24450453317\",\"b3632f6addc8079388df64401d814b4e7c39b08683a167c790c100a9d804fffa\",\"594faa858e40600a802457e0a0b3036e0ca1b5beebbea43e541bc94f9fdbe2e5\",\"61929800334ee6319d3cb6f2c5df8e9a44d25499393a5d45ab129313877e53b8\",\"980163c8d47c0da46e3362cfc20a304817f1d1a390d43b237f9f7e421576e618\",\"27547406307202657a79ec2990806132834136e3b843fb348e7d3be8b8f4434a\",\"a4831cfe88a2b698c05834d4430971a5b5af1e3d9cb10ae789f24b8c715e8b60\",\"dfc1d1a7eec8d372f112749d90063a504e91af5591ae3613a45adb53626309ac\",\"4831bb9fda9c15dd0768a4a143b059112ee4dfb98e44f780f0de8393dde1c410\",\"c2ec91c50476bb520f40c4d373204cd8ddb9824772a503cc88966a853584588d\",\"74d923e5f2eb4ca94b7c444c1b22180f0c309afc6f3303ebfee431020d4f9544\",\"5b02605c19c8bc7dfdf9f749070312a8a60ea37ddf12d0602b17c25dc4276f09\",\"27b5ed6e1378ccb5f067650541d3bfe8b34e45da8152d91f16873babd86b5883\",\"8a9b2b3b3a57f3deb41759f6549e1276f014ed3ee4329af2ab6dc5d5fc987bf5\",\"5f7f6d66d2c59cabffafd74b9b435f10750b659eb3dab1e17320ef69414a07c0\",\"ee9801a07050d85f41084c7cc146c9b8a2626ad371edbacd351ac5db10ce4640\",\"63ee10e1fa14f70f139a2ca8efe737b7c3c48c21f3e46321483767fadeb56869\",\"4b71030a7237ccaa2a0281db6c15d87d2ded294cffdd7ca849ee9445bf787eb8\",\"87083e21df0af8db1cb4a537f3d0bb835e9ddab8fb343ef513aba21ceab51246\",\"8b1eb69f4c2b2fe8594b139b8dad9e1960816561a511bd23299872ea3807dac0\",\"da056d200491db00b320041c89aaf359388c3005c991e74b0d9bc739a3eb4563\",\"9a06d2c85475615fdd46620635d9ac8eff89f35cec889285c354a754abacb183\",\"5e2df8bd3d16949fc51b923da05668caa719f5270024a6b9aeac36038578c545\",\"e8489839141aa3d034e61806ffea6c9e7c9b3cd947808a0bb8768c34f2f05baf\",\"c5ef049043bda0fe91ada710504374a480df56a7f8349920fbcced958113f389\",\"691879bf44b28e799131e698df8b1f02ec77b37d3dbd34a413409e4d74a17e61\",\"871b8985c80c61e6ec4009f3b6e35d4e8990bf5c01113c007391360c5449327f\",\"274f916209b0fb84c2740886d11887cec4d88bac33455f3770b254823858caa5\",\"569fd7a1e85dcd1c43e6079e3c761e0a3c0978dc5c478b6272c1d6ede1bf0817\",\"0784fb8ba0f1bdd4011304d98c67996c75f76e6a9d2e39c1959a186b7696e689\",\"e17e7d7bfef959e917ebf871b14359d3d58b84c3ce1cca89fe3dee28c44e3482\",\"4638e7275064f95e46872e4fac08725ac5967b4c675e7dff67f3419b5a6953b4\",\"4965e04e3b30a2fe722fa3cb8b1ce2a277261b16e137ffdd05ce0248471a2eb3\",\"3a5645b93dc7c8d026ea718a5e569fa1071527050f24b2af8ddece1b04ee309e\",\"8b9d6b33d3b1b84067ce044a915c32e43027d91ac52c8d27fca51260c4064d09\",\"1cf556ab886eddcc1ee1894a3f8eb84ac9ce3be1b57464e70e546628ff33b32a\",\"1ef07433b9eef4211a1d12f184cd3bfe1f905e8763c146a2f9c21cc60888137a\",\"8bf2ca85f4e961d6075ccb17fb0ef169044006640f521e63566d67faa00a0aae\",\"399cdc03cf2f7cdb0c6e2ac9cea4e35fdeefb943ae063c74f34b49d8ecede094\",\"73d0a2d5a6208f9caa3dd1519fff63e4ae5487334c06ca854d7750757cd73a9a\",\"7aa73f2a770867eb13ff3c711695235cb6b4bc45f7669c023d37740c29bbb450\",\"8893eac79173ad56c34b8fe562507a83815aa11e874e8eceba019beabf20e66e\",\"5e3ed308fbfe17eb38bd5a88545ca08e2df7d175bb01b3a46b9d6aeafe0c4f6e\",\"1fbd13ede3cdd9ef0830fa2335e88b6952154a55ca16d5587087d1064e911800\",\"35527f9508cfb956434c0a1923fd29522c1f4f7eafcb381a1cbffa1bffa1ebbd\",\"3a3d8fe2df4805632ada89ccd297d13e95274f5cbfdfbc4d4a47afcc541a1a62\",\"4788d4f6bb6087df8d6aa4bb737075018f124c171b7cdbdb6a1e1448274d8111\",\"1d2858c08da3043636c68e88f052778dda4e09a26c28948a923722de36f90f0e\",\"5382ac4ab59e5d3f121de032ada6fef5a66dd8316900bf5426e3bc9a56dfa35e\",\"9b652005025c7234c07b8805c0e132d9dc0ea387a51b5242dfe98dc7c6a8f737\",\"178bf2aad1bc69e5c6010ff4e9f5ad178878ac9fd903c5f734905035610132aa\",\"e7084382e94542f4f9d4df83b6c675ee37b3f1160da8d061988e0fd1edd1b514\",\"489087b55b0c5965bb9c3a913b89af5b5ee72d3fb92e04732820fa475eba2ec3\",\"d867485f3d58de93dca80e52d7a27007833cf22307c5aa07bfb45e12d8b30b3a\",\"b0beaef0a15fe3dfda97d825249a044338fb36ac7cbbd647e8373a7047248d79\",\"26b45a055cd324b579ab4c72ede2279578ab924f9728e929eeedc66f734ca3bd\",\"5da58639f47577b3887cfa7847f001335714f805a2eb0abbaeaeebb37a88454b\",\"4de1fb331f3eff357198166ae068203145f5e3c1c9cae701aabfbb37d375945d\",\"e4d6fa8c53c3095d633edb6f639709fc1a2af3f66b60c4a7d90435d8f1bed5f1\",\"793511c28b8a77a56cf387b6d5f1547b34a1a7b4ff1d4d27b087e9f940bb4c47\",\"8d496edccb25805ceafff36515f672502eaeeec1d199c51e576ce77a37f4b147\",\"b6c66810902beafabd86613fdeecce20ab629c776a437f28443d3a28e9493e9e\",\"340ecd2a6ebc644f3bb36707302a98ad312abb6bbed3f9d0fb890b0f0bbfba3f\",\"9174c2972a2f872325817806a72db5358b170ce007d19df11f83bf87bd7fbaec\",\"feda8e1905006984a83ed225e1d83ac25dfeb129f11d1f89fa66261d3b140d0f\",\"8e3607236b3c595dd27630c1e113a7d631a6efd0794e908572f61874ed6aa728\",\"45b9aed21a68e1e6f7200d28ad18e5e2d12ddf44f132e7a466b1326efcdeea4a\",\"67e4066d92f7d3ac1f3ce746ca5463fff698f02c05735325e62a69893011405d\",\"8b3a07724f31445e69deca51c3fab726222bc62b9a66308ac9c88c4ab315e7af\",\"ec5e536ace60d2eda0c38d25528db3d4e3742a889316020e292f2e56249b5faa\",\"19a8f0b31abfe477dd9927d54c8f5d8aa8c7d615f619153c979b3a026037b1a6\",\"e37732139de0d73c879f236a13048a26d4fae4325950e9d60b1d47f75fc455e0\",\"61be97b113a0e893c5e15b88c50847fece467a906a3a6cbadc1e93d4d909c377\",\"3388e6c798f6486789a3ed1228ede56fd16517026dcff1a4e96b6d328caf671b\",\"f099a407e1e8c4538e23b1ec6de6bcd6284fb62f819a4fd1586881fb020287cd\",\"4fe3f9d59fdd2fb7af097b93040dc65338b3a9ffc754794c55fa023ebf2ad968\",\"c7f3e346f98a9d50b6696c0897b7eb294a749b69ea5ae6659619e0ede02d0561\",\"8c2ee5e2de0a9ecbb893c9b798bcf9cbe1e59dacec90b854a63f6f0bb3bb7e96\",\"69d238236e8b6f6125e7ae7e5df60e26aa723b3743b87e2d0dffc3d26269880c\",\"50a5849068861f470b9cbd1402a25e41d6926664a8e8beb062d63f3b5543c541\",\"985a77c13dfbb67a9da295d3bf2ed1eab3a88e9675a0001ac448a27766215f86\",\"8dbe70e60594273a06059940ad99481f2fe9b2c1558c774fc308f7b0e51bebaa\",\"8f95132632e521e7ce7c31b85cc0a793757cd8b5c180e41bef44eb2244ae18d5\",\"c81e93354e3131878fb9438f5b43ae205c3e81d2e535e3e12b430b4625ec14ee\",\"24f32212ad4dee48f867c706913b288a3150958c3726fe6a5d20c7fc0a6f8444\",\"1e6a2a315660a32bcdb0e5f90e5a18cd0bb0ef68e8a13805e1600c5b55906b2e\",\"846fbbb702c3c3bde678aa4bbef6d94e230d73c07e1d118b06fef5b13a31f02f\",\"b0aee29a25f0e684410e551d88c7289101feef62e0f338162c0232ed5e864e00\",\"32481c3641e091dea467a23e5c0b16f52a7e3c263af1dcb9003d037ba2f9080f\",\"9092f442bdb853705aa67f23374664a05d9aa2e41fc6cf8082000902f6de4c18\",\"a3e354ed343073c591988d7d8f064c9aa2b63201cb214f17df28ea86cdd8a7bc\",\"cca072ede352dbbe7e764e9db4acb0b4f1a9a0206318bfd9ca85d8035eab8787\",\"336082740d0cb9fa8713608d65dcf311062412d93af12999881468632bf6767f\",\"1eb0c7e8ff1dd915ce13e799708282692f671923dfde539854a11e0ef98bd474\",\"7bdb63419bb247df432d62d5a79d9731bbcdd6b9b642231d53439dfd9e33d32c\",\"e5e812e3d6ef924db1fa0e6ad7ca35659f66d92a4c945bcb445ca3da73a556ad\",\"d2a71ab90d047c85d4580501f96c01cdf54b5dbace027d98d51058d217f622c9\",\"f88b58cf0ca624e93239932e79d0a4f7941de143e97361a1c0e95ba2420c32d7\",\"567acf466ca4f68d51238b88f3538b35b83605172800001c9b3110789985a3e5\",\"06ddd62601b9ae97bd3ebb1f3290d71bc95a67ea62b1f6777a71b0972b3f411f\",\"1e930bd0a18e1219b067c551889275631b2042f01008783912aece806cc69ec2\",\"06be6fc9f71d4c4af58508cedec72b2408f99191af40d18ffea6303b2894db2c\",\"02f1e7021cf96f223b1b4e79c64dde48091fc5175bada3e1493d377d6d7f86d9\",\"1c5da1ed6d02c6d1e478b12e410b5ed0d11893dec747da92f3d1409c8fbd3865\",\"73e7f2e646712a40e2965bf3d40a25eafec9e8d6695d39df1798c87a796cfb8d\",\"49d7de0dc0d3b05b784380044b8c3535d4269cf9a7573f026d20ee428efde1dd\",\"bd98bf14d292cd797de400af8f23b1639e13016bacde5f57fd9aab6dc4b3f1aa\",\"3f1655ecc82df4d1478766289ff2e955d8421cdb0c171a795864b52eab62542e\",\"8d3a176f12e245f9fb5409621b3d2b74160fe20a636ebfbccbe5c5a7d8eef83d\",\"41a51390a98e11619828ca8ba2043e2b728f243312785cb9970b841b812cc079\",\"90386306a2c29f53757ffccd89495d14bc7a5ed05449a4c98fe54c81dca7a02d\",\"f2b2e0d91cbbf80a0a7316389f9e8612f0b6f4f11d62d13391a3b63cdee0d9b5\",\"b09acea030e4e13549be58946b47caa8748a9398d968a576479c66e9dde4ad60\",\"9b41c01898c7a0056591de1a287b01b5115cbb0c41feb6c6e1601a31fdcee803\",\"07fed58e5d02be39132e02f69339051cb17071968b69d05743eb604ded2c1883\",\"0143389f9d4a742204baeff9503e657e399a602173bb73443a9dccb7094eefae\",\"4af01181ce3f4152260616f10bea742a9551e50c25357f79e247fb09f21d7595\",\"a1b6c361d78f9ec1bc4f7e834aadf5f09e31e91d5a9e62410df6fd828661a23c\",\"1f1fd104a888821de54119a3b9d970f2ba735661dccf11a392037eebe42a366d\",\"bb644b170f4a54dcd28430210a9abbd924413c220ce9d278f676d52248e2f6bf\",\"ef5b701f6404adfdc449d36179d2b8f6b6b073c24b02e6c59cbdca3911d6964b\",\"174ce4da59f0bee10d55ca0b5fd15bfabf0c96b799a92940b68694875f22bc57\",\"7f7b026ca5b38db0b4e39ed3842af3543a5720b256d01627ef0a2eeca4cd6766\",\"566d5f9452fc38a6ab45dd7933ed9be1e98bc92972124a5cdf590ee6388dd0d0\",\"ac5ce254d4ae0290cd5076028a1cfa0508c836a155f7fec8778e46250f69379b\",\"d1cd472c315f813de8f571518ebcc6bc13e2f73a294b007568f7ef677e1dd97b\",\"6559ae4e2d992eefdb84ba712f65e7330a1816d6dbee5551910e9c6180feb1d8\",\"5581f8cab291f0fde7edee70f5dcb3f45bbc381e104ea143583814291c878bca\",\"88ee46afa181997583b5086ef65bf3f98cd0be20f66e4d6cb7f0908d1f1cb882\",\"3fb5b75423618cc6d39b5a3db0c306d55f000f488aa75cc38d111863dc263fae\",\"4e213a433455fa6bb35f3d511c904f2042c1650154a1f95027630010441d5ceb\",\"747690eb3d954b60139853912d110323359140c645337aa311b233942255ac66\",\"c7c4f24e21f1d15cb68d349ff9d22e7d14afcd3d54be0dfc645ee212d39123db\",\"f187c5def9dc2c0b429b07aa075574fbf9d323989e75d540e7d242ca07c9bbda\",\"ef8115c1a8c962bcc50d0597e005e87bf3efc673d75b58e7ebcb1f8fee721011\",\"08255a78fd75436551c4c5fd3194e010b8d53c684bc4e9633ce06820e5464f0b\",\"ea839dc835ea4b43b36827c60c092092c04501259d2dd64c85ba8d2e05720a88\",\"424391a0aff54ab5667d397516077a6be03e01ba979cc8f89f53ce1bba41b1fd\",\"77d9c0a60bb06929e6330e7b0221ea66c5cabdbe33f33919f4f0787cbea7afed\",\"8a6d8d2e6d073bab671afc324f432c95ef0e7a00dc0581fcfb04948399137f74\",\"00301666a8415964653c42bb148052a046dd1e07101334398a67463237a18347\",\"c35189f8ac2b6a6f3e290464228bb174be082f3e4aa501cf87084364c5f758b5\",\"a7a39e257b3f9857cadaa25de5368698e379bbba704699b88b5f4f963b04c2be\",\"1d9854e977f836c5a9a5e9cef5bb2fa64aa30c8881f64e3249c3076584bb371e\",\"8a961dbc9da115a44da91fed09fa908a2136585e7abf0573e5e918c0af06acf2\",\"bb6add53af2584ee4dd72b2fbddf4627bdc3d9493a840e80b6c93f6bb0c274e4\",\"45c248426ace15f23deb2e95488776b2fa95fb57a4066645cbf9e4552b10da3c\",\"5fea9093ba37e7cafbc88760f19057bd3a0de7696acf45c16a646812655c24c3\",\"dd8a63041288d26f2ba7f2c65effad144e2a2dc38d284ae9a3f6957d1eb9e36a\",\"dc3212357c2c5554ecf2b14f7e2f84cbe6801d4f596456965be15cd1a08ba715\",\"580e905442cd8daf8dc78d7fdf065cdc2f0cd3832eb688006ec8bb9bbae8c1fd\",\"f2cda8931d24aaaec7e5406cdaa2e5d9387bc820ab4347f33a4e075630880517\",\"31d6ddffb7df0545c6634e486594b6df4183240fc7ed2d52e550731ed99ca60e\",\"7aaef98233b8028d7b68a790b3e1b1e67d68ee888e24c52646acc1c8da5978f6\",\"bc5630024462e89657046495ac1571d8fdc7657a04cb53a6465f5ff06954979d\",\"217d71e9c7d0ee4a379e676932316c93a0ec53e79e25a44785ea58bd4ef094c8\",\"07880f9549061d907f0c146088b9b760d3e35a80ed311c6b5cb872188ce24f9b\",\"cc5d561d71d73d6deda4af8c68ec87bda6a8bbec531e7a4dc5a292deebc37350\",\"dc790e8b794b56bff2b183b545d6ed1751299fd37b21e0860c27c5e406f6313d\",\"ac0067a41a5e4babcb16f54301a0f55cef1145771c91c32c9a769538d15b692c\",\"42bcad6d6883e55b3f609cafef7dc2a31521df830c67e59cc9fa903a21cfef4d\",\"8945138e23d88bef67c17be941df749bb00f3787a532f7086ce0ef9c8a9211a6\",\"8c99b426e7a4833424990d6df562f3dca780b6eff83d142879884fa001270c30\",\"dfcc56ae5ef68f93550dbd1a8d0580c23a063a860f2c7d3a6ce1e8b7ada62e01\",\"78728d63484628056dea37559cc6658dc7f7307680fc97971b05e911cb30555a\",\"216ff4157380749114a06f96ae4fdfca83ae05fe0b6a1338cc732a33b89e6622\",\"f216d0f0a0154454ccfb238d5d78cffa294fcbee4c1a7a4ca52032ee03819159\",\"b0e738a70746fb18ab2899fa71868d0faa9fbcfe7d59d8f71b574fcba0e49035\",\"e98d05def8e565d8a29210b91a2e78c6b70538cb4210ecbbedba45158866d91b\",\"51aae126dc47393d59a34831d26d0967e0832af80215e66de7ceb68b41744160\",\"cf9d7c066f3ccf5bbd770085a25b7a0b6474c46fcd05e7ff04a5ed7766f22e75\",\"66dbfe1719c8b6bdb01116e58ce471ba9fe8d21ed21b4fb098ea3dfefe8a3355\",\"b16a26d1a16990814187362e4a862926b2ffbc5a041d3dc5197d44fe07b31fc7\",\"ca7f7632e399bf4334698f53328eb906b8a4dd85d4c50c9c5da5a93c48606aec\",\"09b48ea51acd845f5942a55793c1a970d46c72b77f74e852ec45bc9f127fa022\",\"7926378797c066a9f63999220d8a51a4eb3bcc67a2246b251d22579234a274d7\",\"c268ddc58c4cce03379124c52b9908bc78cf9f36af8bb315c208e82803eae7e3\",\"39eac45e48fb8f646fc59ee3556ab24a36de3e70a9a2a02dbd8dac45dd1c9f62\",\"3376f250d09e2ee3c75f65424012b2ab72970bc79023c32fd0f0874e0036716c\",\"e9421cc7c9b0d7bd3dccf81952c5d1bf4e627dff595de6030be07a2c1d398076\",\"815645cd02b7612da12f9d4d772f1edda5278166903867a58e5f8a26569bf5db\",\"ae939d589414450871f399946bc83d47777db9e053db7ac123b83afd799bf12a\",\"6d551dc32b2d44612a45ef8ede2ab224d3fac87c1e37e6dfe209160be6f68554\",\"d3ae9dd378b864a68fa366b955cee85af91fbc978fff6b1fdd9edc0e379dd268\",\"5a421b423e6d8821c1ca1bdc69255815173abab30c2d49e84e67b57574a66815\",\"3ab205a0070c7deecd33f3d6c582af91ea13dad966b907b3dcc6b24d5ac1e381\",\"2bb3f875a7811d1921b354f98afbb20cd4f822a9cdb219f95270189d4c42c573\",\"8321117cccbbf8da18abe2aa2714293df8a7a40cc4a4a587d50facd62b745185\",\"82df3e203e40a5c00d89f589abc0ec11aa5ec3e4129354ad67279390a0228bf5\",\"3d7910548bb470604b1910625799385e73b58d763f8dd6cb8e2c90b89b63acfb\",\"4843797e3f599541444597897d2b6263f2f962d84a7f7a4af038fe4064826983\",\"f7f6793a99d8f785b99eb9af61a6c003098cda96988912aabc70d532731fb3b7\",\"3f2bc00d3d6ca90b7fcd411a7a0c5f333ceb4b78bf27978afbeac1a8bf01d438\",\"af9b8e1a4d86c71cb8f0d1169ce4e6586ad85b7239de8513aaa1437437f401f4\",\"6d9d9660f53d80bc6e4530585438c44e0382338d2e67f38ba3a738e871b0b8b0\",\"834647dc327a821237fabf57634a344a868b21e326d7ad487ec01d12fcdb49df\",\"a2ce1aa8bbccd91131f15133e24965a2064988ecab144c1da80f32e0595eb563\",\"dc31b58eb028711c7ccccfc91d4ffed7b766546bf3baf9fa4a55752f456e8eba\",\"7deb983e4c7cb4d4591adb7261fb680a20471c230d7eee23ba73e18aee5230f4\",\"8c73da2466f471f9c8d727f48797d99c340650cf40a59091f4c103fe550ef375\",\"8a6268cb75425e2b8b401c739f4a5dda12e200ed412b1dea624fdf23ad870f22\",\"4ded1659e8239e3f4d53c7cfa2dedacbf28f42bd25ade16ac072971a04b83973\",\"d6c057bc7e8acd76a931d33f0e157517a7ff2bbcab60ae8f308a8d2f174b309c\",\"9667d26b63de939bc102cacb5aa6958871595f6c41a3f9d69da47c679d520e40\",\"5c32365075589ad14d260a0c55b8918f970775f604357e5bc43320193d5d4ceb\",\"fb3b097ac3fbf839382ae7298b3779ca330e2ab317d3d5863fc2d762ef82caba\",\"12d29509b7b50721c88b76a97c9d218915eabd9a607dd6310a0c5ffdf566a32c\",\"1bf6a7347c28420d4ce3d774e1a9229da408eaf80c30de888bdb8422d14f45ae\",\"a69318333cae744d79ffa1be968ce1a5e12f6a714143f47dc30f84a03ac82466\",\"e6e4fd9ddfc49efba0d7273d848c3fe84a4cccaa0ce393ba1ac3a1c5088d9e5e\",\"4dc74125316e756254eae0bb1987efc0c41ab72ad09308547d7308275d7b8d71\",\"c434dc324e222209db83b76c19aca8cafbd7b90cf97b91c768fef8957f34981c\",\"4eab1993bfb54e1b7dada3fbc78591aacff3a32e77dfb80d934bba69c3a66ebe\",\"eb0a886d7ec1079600ab39c93ab9bd3ae3abc4aa9135518e3c67db94e7477bc0\",\"993c20eb772bfb81f012ae1f189ff881896c875092221524358d4df99463f15a\",\"44dcd07bae6d3a721a7acf7be4845d6bd969db69d6ed298669a54d7f4e4522be\",\"31fa33eec183012ae60b0b00f72f3bd765665614ced0c310eed4632245672255\",\"e97cd325cf2bc11d504e39416dc7dc347492addc384b303d0407f70839e8ea58\",\"4d5eafe52715483eb1cbf1ee812ee6dcf4d358792a7080043e4d1e70706de1d5\",\"b8034a2ee4b965df23cd40e8bc12fd3ae45c943068d05f5457cabc5a6c765c23\",\"9214a7675015cfb7b38ff56269ab2e6f60fd3795a8b776f5bfaedd884fc868b8\",\"68b21c507d6a2bf4cf3567dda2b93d5971fd1816a19b996e74eda06cf9ca7b4b\",\"03471ae0b684d8c442888d5540f1022348e3623728bdbd6763cefaf1691ec4b3\",\"aad219f0edc50bad55a84ecbe53cbd6b720f7638bdcc08859751da3054644378\",\"ff1390c9ce88e0b6f830118f95001af5f1f125a0e65ffbbcf88572d971428e66\",\"e88674acf8039cb0b4e703ff8293abc1aa0f0acdc837544694cd28840760f72e\",\"e32570ecf3f06f2183fe1dc806c0870b4b3348cc6cdaaf3e43c41a7951a426f0\",\"880d1b7c37965ddc9b3c5a92840a6d8abdf1905a4c73a877aa607321c0371d12\",\"1a2cc5053348f753e5233e3e56efd894cd21ae7bfca0a84f9fd58b3df7ca147b\",\"6763ec1a37ec69f94f11a575cef8b61640a8d8e589e4be4c93ef486bcd4f1c2c\",\"de11e9e2645d0c5dd2a66e1b7551cb0b7e4c3a1965be1426f70b25a98843fd2e\",\"6c6b66e2b5852e8d5ac81f919d251934a9532bc6d4377ead4c5e41900db304c6\",\"551d7530f483a37de299be401ec16af8df54965a6f168763fdf4d67a6fcded14\",\"be233088e9fed1663dfc748983b7dbb938e35062681d14c815a6d3510c53dce7\",\"32f6c850694c090832eef10baa49d861d8a0fad03405d9d6a867bbacdadde3a0\",\"b139661536850c6e064ec2a19511e533dd6f5ed6789d69d8136bbd440b50bb93\",\"19833ff032e46af9e61384e179a00fcb7948f6f02758b570bada061eae9a1424\",\"0ee11b074a0fc9ef6e414dc0e54443e1d112e74c1cfcf105edc4117f37f07d54\",\"9fd7f82508bac6fe36857a88e5850155e1f7ebd73d1212d0e885f2b042835062\",\"7f64fb8ff48b52326588cec088e643f804c8a9acbd1466de634da2bada8a3655\",\"c08dfe49fa6607c556c042af17fb44e93f5a3128b0953aa19c379a4d49853747\",\"f7f1c81b6a3a722857802c09dad6835132eb0e7b27e0763e1e55ef2593064338\",\"6b468001fc31305ed279f59b9f4718b34d0a117787873a52ceb1a99ce32f9aa4\",\"c0c7cf8a3d279ae7932e6c2b14ae617da7707ba41dae15fde1845f28dd6e9e91\",\"5c071bded9cd455f94b9b79f7c52dd95f8102ccbd2187669ee2dec56937587ce\",\"f9968a12303a59382eadb6552dd79ff340734102dd07c3ecf26dcc7d98dd8c28\",\"c61fb74cdca1ad0d77547146b094334ed8d7c3369f9486a4450dd758b973c521\",\"ec447e48b21b305f745c4f89ec9f7609c335dcf7b7db46e508550770a7012d4a\",\"44ec9499e8ec2f48895b8bd87189275d58ad795443d4f522421856a64bd6dd6e\",\"8c1ac3eef3bd09e2656b436f28374de59a67424fd59d676adffa22e731f5be51\",\"52f1bc84a64584bdad956f444ab02a008fb3d29724d080e5522a1d881d989d20\",\"d635e132c865bac2e81cc82b436c68986d27026e3a9c6880b8ff1d313dfaab9e\",\"7ace2c99f08176930435426dd3fa9d57f8de6346811fdfe540fefdbfcd6c2704\",\"858a28d6bd9e0afb49b6cef0e6a296dc8d8a0976a594365905f7041ad12ca2c6\",\"cc13057dc978343dec5832883fbe497c34c6a3f13af4537e4575e4b4004ddfc8\",\"0d4fab2b2df7278b223e4f0509bba503b3f1e8ca386bb15c2d3971b43722f9aa\",\"53db13d58d36525455e475cea15bb670c7f16b51a0a09c5e3eb53871fdba7968\",\"07cd8cbf0e57f097c95e81a0cdca398b85f1784f44e6ac7d982cec9a9470c0ee\",\"c87abc87b08f374cf76c29cdc4ff662f34a9d2ac7fd78336d54958b136c12567\",\"e16de947e719de2a4a38e6b35efc9c7241ba7bbb49383286f7b0543f2096c7e2\",\"d7f6db935c2e6727476c885bf37cb86fc8e2a824f43189d8781e7598d14be9f4\",\"6d0d4fc68967f4a7d89a742401ce758557b557fdafb0b52a69e5b2c419a20de7\",\"474f38b1763893e636b84dd37bbd4817d7335f871527104734094c819b841d34\",\"e906dc1910d918b32e93e8bf8f8ae63435f3e5aa0b4112c1a416096458281195\",\"0bee92a66e63081e59312d27bb299cc251d0400527e620f47a409be33be0a2eb\",\"0bdd624749d8adf73487bec346d996a14ddc957c0ad3b75b73cfe9f5de67943e\",\"1a9bba5f51a818f8510de93492aef1ffa1f5896a2b9a300d5bb00052ec7cd884\",\"13fda2f62b801d08f37682759f7bb70a774920848268766e9e1bf3963c114fbf\",\"cfba2e18263a255386cfb01d48499a4518d9e21e9d5e9e46b4b2524bedd9cdae\",\"a09378ee62910bc02eabe59310ccf73ee9cb0b69d418b5d97762313f859a9288\",\"6b6e333c03c15d72a52ed7d4cb937a1c28089dcb39e68f431600fafe6e764c05\",\"a38ef2185e7e7ad52076ec81f2e7713f4c4377e9dc36dc2380892c9e7fce6d48\",\"44423af715305f5847f8e0ee07a782b1a7e07bf5af0ab1051c17f3f15eef2c36\",\"f176344333869ae58b1649b95bb3e97a7d49e17054e38270ee54b06c4a6e6025\",\"7bbd677f24b833f5eac0eb12b0be4e27898b52dc9843f14175f07613d7e78637\",\"bf7e7a36e747c0efc8a72c9323f3aaf889bec272b8c9352f4ea51a2c881fa7da\",\"18b51a5b3345a119e812f5a38862417a91c35a5af87e11a48f2e001754cb15eb\",\"685632752cb2dc4fe9201169593de7ea27a9423d5cf90947f9ca4270ba6981cf\",\"a522d5c764e5d4d0e757b19aa8a18da1d456551a2be8c9a82d950ed693515653\",\"95e7e1d4cace14b49e4a4040468461aeaf381784b538d7f171ccda7d2bf202c0\",\"bbfed9209ceea11c4394a007b808bee1f885736141e6073ca9e3e05feb3d67d6\",\"653dcfe875b0d23ec795370aba3de1baefa513737f8520155ebac95978e485ee\",\"7bc2c57ed86eb73d2df4ca9ec70fd719115b145f3136106527a273bfd9bf7e22\",\"96d8cac5b671246c51a76bb4aca19577dfb5fd60ba7fabf36f3ddc77bddb1d58\",\"2f3e0473718612fab67f22d49c28f66a38c2ff5b021aa38d488bbcce4ce43c44\",\"1fdcf33e9a09a71fedbc07dafd35695da74a13d9778f1d48a9103538bfb11939\",\"d557460b61563d3fe029eb7bf1946de08a99bec253b90668fa20c55e9c265aa9\",\"3773b102c155e6bf03b5318e7fa2959bfd6b463002bbd43b2d6aa9ebe23f3e82\",\"5192309cc8682ff3a07d1729c3fc9339ae13d0244f94214dc769f30f32d22057\",\"0b2748f0a13da122badc68ca6b9df335aac6ad92945c6acdb4cfbe393ff4cff8\",\"211eb27be563bc8dfe62947440b32991ee343a61fa351c5cc2943d1c27c66622\",\"efb57576960a1649385135101fb19c3214a73efec88fe9483505522b81f4c5d6\",\"b6d450a0061dd1f5a876d2f7e0060fe1e56cf520148f30af2b6059a43dcf6957\",\"3c63b1018ac56337b9651f44813b575039538cee43f66faf57e8c0f1b817ec8f\",\"85212a02b45a4551fe5e6372659ee92a71e05d93dcd0d28e786c546b7c3999ae\",\"b8964d0948e40862c2fc2fc4946584f6336a86746f73a8d695982c5dd12c9b9f\",\"59521a0077802053d6ed7dbc9d935ed786b3e6a73a1a018ad01205569ec2092e\",\"5978830f796e9c7e3706eb84d7161b81b955a7d4fe3a3b49fcf93fa5029d2e54\",\"51d1598192b06523d5b2b6aae99352c50eb181a6bd0b601e896dc48c25131f13\",\"163aec13f2c895f6e85ac7b1d67a1a7ac00ef5898287fbf2c489025c92fecee2\",\"88a892fee73f795ef544606ab8983386fdab426cb507f61d5839e488e30d7568\",\"8520d36693e32435d7052013ab8fd8f83af2485b5c6a8c5c13981de08a60c3dc\",\"64c4a92283768f8066d08e8a5b60afa62dd054ae5dd104e0ac471f22058f36eb\",\"795e8fff096918263b1306d7a6950f8248ac24e7f299ebadec47e1d0a2a2d1e5\",\"95609875315073c63d85a6855245e856e9ffa9e47b480343dfab4a086f93ee66\",\"a401812babce47e17ab475a23d2fec64926b16fc78ef9b53577ffc018e56ea77\",\"4d772bd148df9ab0d8c06aa4b7f8d08df2d84513296a6d6b872e3cd76cf6146b\",\"849eabf6f59139db3e57aaf9a010dc4fc7061e6194e05d1e9df9fd817137a1ff\",\"ec9fa5ce2a2698e8867ef9692d11947fbe848e2fdf16e327ff4d668e3ff57e87\",\"3332fad893fcc98a5ceaf0aad168cdb13123677b746ede250e6d1a5f7e67e382\",\"d0824ed473af4c3207aee10a6383f49857ebacd852fba8e719a6059b741e87fe\",\"a6b30409c36df6ab379a4bd7549a31db88f0cc2e2c0f6c387438425e0e37686f\",\"8e9551d376ae5004ffd12182c110b96d9f42a86979917ac314a0eda5a1e1986d\",\"75ae978b03885c8349d887ceb2f47898ce2fa05c56e1e80682aa339f0372d843\",\"6cd200dc7efdbec355fe0a3c78c6835d3828c0bdb89fcf7e3aa6a12a29c95ea1\",\"b1a53b1089f4c7465ddb3aee2f919921ed82f3ae1835f75697c9c066592c9601\",\"37b2a10aaf58e4601cfd9fd9dd79384ebb35d86905d33fd0b6c0f433671ab119\",\"5321ec0592d2255f4348db1bdece6d7c9964eb900b4153ca950e0674abbe4fd7\",\"9f4038c513036b43310ecbd857d9ed8d52ba1f169ed22987d5d25b1f379e1349\",\"46aefe18ca9156ba488259f591e72eece2a73cb1f2f2dd2cb64fb10a85a988f2\",\"3cd732ba35fd4d86fb7f083a68c783deadf04dd39c220db571371a5936090c1b\",\"3c4f327a97f073dd0d5cac6ea17c42339cdf25ae66d6b72de640a5a75b27776d\",\"e126f0dcd849b1e8f9d555a705ecb8ba72b4e50923ccc83b7eecb3105424833f\",\"683201ca47c1f130eef10ff983fa37a75f61275e8031a30cba2fceea6e31ee23\",\"36ea2bd95907bb25c9a46fbe52a46621cd0c9e7d9822f87bc45827fa111aa474\",\"e23793535fb8cdefef8a818e30ede3d8ede406fa0a03d63c1c92c409fab7b83c\",\"873d39595eab30ee1845f81d6b7db576639421c72dd4b566e5ef11b22e67b9e1\",\"cd0daa7add36ae3eeb926a7f197a8b6f1871e83034ac5506faad07d38245e2d8\",\"2a86a705ddf6bd13af791bcd87e7d993b1506d8d8eb5a455d006783400717b62\",\"bd14fa8ea0148331fbd4e82505e6f26b77823f6025fe055eb90b1afaea527098\",\"3035a0d7021a1c0af08960adf3b37efca26d85c7f60c6a235eea95f021109d35\",\"07b6dbab6df4aba67380e12bb74df24156eea7939211ff556837b80553a2b44e\",\"8f603188b7613f4620f785824ff06257454f05176ea7a26d065da4d9c2ce84ec\",\"272c728c5d546bfcc3e49acfa893bca9fc162cefe6b0ef039e380889c2cc5a06\",\"7e9d9dd2454b6a0fa6d97829bd8e82fd5a6e2aacc7c6de9d7b02b8a2a077d8af\",\"005eaf2c409acd807b6ccbaa179eff50a565b5490d2fd49fe045666ee5ec6185\",\"9937c8d2973458c753ac9502e8dece8f1642509937657cbee0de12e40379dbe7\",\"424b86d137b3c4a790e865948da3095c801fd9276b23dd5d9d5dfb9dc25de546\",\"d0ea0eeb0d1a84d1080bc536ad4bf76d4790f7506ecf27b65c55a255c9ef1e64\",\"970c83be1244d1d00f5b10b18c33219b0c40e2553e6b8a95b46e536cdfd27030\",\"b0557b2ebad14a642b973426e79ec56784deab587f9eca766b82f581345bc63a\",\"638ceb7525deb90ad266b2e44282df53609344179b5d7efe7504294f5ee74a19\",\"34bffbc2d8c86681cff9e13fc4865f50c0331abaf67abca6bf8be84001bf252f\",\"c40d5262ca0f3f932a66f9531174979318fb9718ac5044709444860606cc5209\",\"835e68484b18941914fd5f04e613ce4528a3336dd625d2fe4d7bee5ecf2e0e6b\",\"23c1a75852e3e4b5b4d60506638120bed9e196c52c81633b491f68a444b413d5\",\"ba5ee2a00529034e0cb8f7f046d67e12b302c542bd5509a0b8e0362d8a2419cf\",\"419963cdc904ddca846c49a031e516e4d84327e9f78e03cd8a8da770b2b8ab36\",\"a1df2b2dd6edf9203126ad53df0ea02effcd49e10a20b4770c75ad6e8e4f9668\",\"0c91a26b90c29146cafc1bc7092c823918dcdbb7ad973a99c4a861cf65a40d68\",\"ed6ff0a8fc000c8abb49a2babec1b9e1840927b14d1a975cd19bb91a5d9ec04e\",\"59ea920678ae9b037fb6b14fdfb9821d933a2a6d7565da0ed6a9652ea7a11c2e\",\"46b14a1fb9b12384ae3e9a44dc870ae19ab9300ef7f61656245e043639453a90\",\"26958b709879befe26161192848d6b5734d9d54f54ef94ff31d2e226b13888c3\",\"c62d1e436cee13798e9a51b7d95389b772915d65d0eb60cad20eaf8b1ec6dcc4\",\"944f17deacad4d34efec31fd34bd4f037cacec3232e57cdb89f7f4540cd73ec6\",\"0308f571e6432617931670685421e9ba0c4caa8a0ac98fd597f8cc2c5335352d\",\"5035a35351c3ad7c08b9cf4d942e4a5818547ae6e63d13ab490145c4b0927735\",\"24501a1d207a5abc0c47fc9b6977d99c9e6ed411fe60512e2a2598c2f2ecc93e\",\"3b47c6e094b03745b9a8ab99c2466e5186558bb89ceebf8e8237ac005309c317\",\"03ebdd284badd6bd3535457e6d83d1ca138110353d839028ee97f25a277b2bd2\",\"bdc212cc4abd4c7083d72f348192c9e70737909a3e6a44b71072a22d8d6bb680\",\"06dd4a91159c34f9f224d1dbabae1153ae2b8d2224e46bd1b3535c72329d885b\",\"af3c915f88ef9e39752d6eeac618bcef16af836952706049903c451245d1a313\",\"436db2d9126fb7a1dec32ff1d13988d23fded91018e151486c18cf4e5f49403a\",\"e9260a1307035be5ddc7f5fe7fb5c5aeb39e2a6936a7c2e663a2744ef0594db4\",\"21aa4e1d08d858d1a639b50b24829a6a8923edafc2fa16684959deea8df1b5cb\",\"79029785a48124b2a83aef12b8cff8e9a9240b8b85fab2733beb7e06fea34892\",\"364f8ad1b07b3ebf3ddfa5495591df38c82f228b701be26a9661ecc6789c5232\",\"4e99ae6c059429da232b44d39456180a09772c177490a6688cadcb8100612eea\",\"6d49d99c32ccaa9797d35d336428ee77d61cee9c8f6bdafb6df85d0717e4d1d8\",\"a47b622ea12fbb9463b74386b9f02144660ceb04e54b7013ec51b155184b57c6\",\"20755c533a3a2a65561c82131e8eb1e8061264680230ffb55ab43262f0a4b1a6\",\"cb9e275049439ab4050e1b12a856f4a2eb103404af15b669751a48f7ab7a11fa\",\"2bb6615784b68b4cad6109828669033ed8e85bac0ca06e91b0aca161c24e42eb\",\"30afa5ac666e48d8d33d81959a32ca87a962e058c136d97a76079971359a8e36\",\"1715d661ae2227f53f792b65204b141c8f3d7a7adaa09e6df359b5ae7b480de0\",\"e345c4d3b86ea502a0b1b29c5974ed5144c625e74457481d04742ee2bd9d3772\",\"d92e5a274a6aba74254dbd000ceacad1bd76eb9d0d667e3f5ee8f38db39b065b\",\"7dd58dafe2384c3f71f39a1f76d79e37d7559b1e3a8786e92698f21592aeb30c\",\"2f4f04a41db00ee0ca68c5247ba5c125cecb65c0483c3344546cae03e5d5e819\",\"acd7e6746a91c50fe817a4d68fd1a4e38588ed6ca77184becf82f184439db449\",\"3deec490e2fce564c58bd702a7e29aaf0ea6a18bdcf88f3355f03d0da1f4441e\",\"1581c2e642bf04232e5f6b27922db3360116bad50a528ac25190c7c957896bff\",\"10c9d3216b7922329f9d9127a8c25f74a2d00d2855cb9f7dca6dadc9f75b87e6\",\"7147b876818cd272ef7b92e1ad4a656c65662a55f879e13ca4e382245561078a\",\"b3d8065bc0617a6a7bd1eede9ec88b277cada7b8f0455fe2773d53182bd9d64a\",\"a911463689c856bd7c6243c4ac8bb716e9e5e8a823d32ebde528feae054c02b8\",\"97c4fe20064e6b2dc0f80bb6729fb19c2d90b9d7e1baedb38ba7d9b44ec150f1\",\"e10aae74350f33efccfc5d9b9e2890b480af0bc45a2563aa6f43dde01bde16bc\",\"bd760d0880f841dbb27aef8236f50df71d21dc4d9e0f4d5dba7d42d69126cca7\",\"83d62be1dd111ba74d03d444c7aa4f7109ef7fa1d3f352b097fa80d2b4c48ed5\",\"79cb3b7cdb5125200957a78b49d46557a2a19725f035d73d41cab3c5fde44ecf\",\"9ca14ed73dbef0fa2b962b3b8cc57f0efa2e7940f55798b4809676be30f317b0\",\"040f0e49fed935ccca8e9c8a5dfee179348445f73d5fc97ceae4ddf687ffcec1\",\"ad8819506c8f14816e97e8d1ffd6cf43641b3b9658e9b6dccd047af8a9aa2d4f\",\"1930a9e23a6a6bf52f86ec62efb1486ae79df9f72073c6490e9e8042ff32cf25\",\"2b697fe96ea26f2b4c375fcb9f8bc2a99d8a89391a33e1299db973a84512406a\",\"743bc618fa4cc20b20e562be830f8827523077ba552e068e681742afd18c636a\",\"f9ee054314810ca7eaa18ee28e21da53603b3cb8254b3bedc262f730bd54274d\",\"ee2fec490e0bacecb55bc4d6f81763be4f6e00bd3efeeb7d190be3fa1c64d6fc\",\"a559c4fdd47f31298d14ca3cafb922f176363b6f94212ebbd2f2341ce90e9176\",\"1b738cacb68cd8bd79b6c425760b0f49c75e65632822cf97a918ee989aca5f4a\",\"74ec2a47b1eb9106712da06b3e2835ca9d4e0506069c24b519508cd06e41eb48\",\"6fc350811d7fdd384328c48c0cc86f7552169f356181a276c0377d773014e787\",\"b3a03e4f90c51be529093bc81136b9f4acb65e5df1374c16fbf67ea83fd5fe21\",\"587769057d501eec885dbb5a97389cddcbeb962497c62232a5f5fbf58daadc80\",\"4df6f41ef33d99121e9d1bcfe4b761db3878fff0336fe79b62fa5ed2657bcb4d\",\"066d53adf708a3af9fe29df7ba4d9e7b230aa99ee9aba095ab6be9062e80a62b\",\"19ec6f24056032a1a2d3b5e01a94a2538ca612ad93a972e59d53b6f7261f5178\",\"07d73a987d1c15dbcbd210f13e097727bf693d65ac30db6b8a7177f0dff7adf9\",\"75e47a685ef9e4519398efb71ceeffeff66f577afc13f58c39d224465a224eb8\",\"b8863516546f8802f260093d036d9406a22942050e089f2fe36e1b5bd1ef77c9\",\"7a14fefe446eb7158a4d8e87163bbea7b6f1df1f22035479b9e8f53b1d8c8320\",\"6de872157447980097c17d8713a00163c1588441578ad8b96f780438c80f52a2\",\"4978094a60ca10806343f0ff6d3053a194ed4689263ff5406ddd84f94088af4a\",\"897155aaf26128a0c119a70194dfa16df8a969c71e69d186e4e4846ce2a26ab8\",\"572385deb45cacf306de0b1a24223bafc06e63cbce6ebd41a0e977c4646d81b5\",\"494adc24abd0efb7ed3eef0e1b24c22c0c8bfd7bc24064e143d03b84e096347a\",\"8414e65b901a6c8ad6483fa51c393214f5ecb31116b81c39e831043fdb5c94f4\",\"8b7b7b3191d080d42b08cbf385414e36972df525c0831f520f757ab2853ecc2f\",\"8bfed5ebadf53ec0b161ab2a007d7e65885050ef0ed27d5c5faaba16fc19c868\",\"53921594076d335b7136cf9a2005925c7c52c7f718cd2405eaf7520ca37d0789\",\"6e93ef5867a84c040d1774e7ff89ece28fe55f11d8e048eb29f241b18ce1fc71\",\"fc34ab0972cf7242d67b53781ac87e59a0707cf22a41c6b2a920bce960516dc8\",\"24fb3ed8dcfa88678d33a6570458294c76a3e9ba470eea9000ae50b09f126c80\",\"1204c5eec378f9149ed45f6bbe9dc92113d3f759e56dd200219c0d204af2f123\",\"ebc077eb4529afa37226d8f7ca6d679c175cc9cf2682236ffa1b30531d703261\",\"f07a7f42c424b9c8e01c2178889aca9b9f274d374a92a4418c9535b6dc7bf61d\",\"6de36725dad737f2cd5a283d26977061249b112b6539e0f833e019f1157698a7\",\"0f540a62c2ee4f43636810d8a704650d60d570f56ea9a7b9e6bb1c4fd7e6893c\",\"8d7fe0962b0a8e53ceeb22b14329de06282e0e50569b14e1aa17a4f1ae4ef994\",\"daee6ded081a40ab7ac9f7f724af2c33a97136e1a427e621e9d5e00f033f18d7\",\"e1949430b65efc4bae40ee9c0e969a06690e5ae86d87770b5941476bd791d2b5\",\"dc68f2262baa3df1fde20e0cb237a83c9455533b5f588946a24b26f5c7f905ab\",\"ff344db919bc60bf3cb8df472f68a320c2cbd5dc7971d4917f4e35eb1c109ffd\",\"3c1be38a898afe5f3a6c07059ca6028c7fc0ce9a2fb272c2e04451278e0de4de\",\"e7e7337a261a83e5eb16ede85485bff42bd06bd928b847dcc6db13655a86146d\",\"15580e4157b734b33ab38d9bda8579cfac1d9276e833195db1e7f08700793ad0\",\"a96e788370075ef9e64ecfd286979009ffb96815a04cb8bd9910a86b20c742f0\",\"f43c1aa2a9a05e22372a650e6fe495f27bd789ec02f901d3760c07cca7bf197e\",\"f0206cb368216fd7e265722b4f97a8cd35e1d65dc22d578443b10d3a6cd429a5\",\"31c4b218a799c2ea651a0274870de4f7d5ff1409dd9ad5b711115d7be7d39aba\",\"e510940338ebc66198c5c15185b9df2ebefcce9c84ff4c5a4e30af165a1a58a8\",\"188ef45f67d815a40e5c0b0ddf9a836b1b52a46ebc7cd451228e85616d8e8822\",\"260f6224c2461307d98b7c85d04185c70257e6974b9aab5232040e09d2daca4e\",\"8b6b59200ccc8fb5eedcfed7e016abb75d3f9974d52a19d226a79e83b9fd0ab2\",\"8681c108d57adeaf66b59ce55791e04ec6786011455868b06878336047b66bb2\",\"36981e25d67d6e8d6e5ff91862b052d516e731ca815a77160b3f961713852f2c\",\"a4dd81a8bb0c014d433fd3a693f700b02fc2ec2b4b610ad57c31eb03c6a0f4c1\",\"426b2eb76edee84d89489bfbbad91807abfea232cd7a4b1b70c7a35cc7bc57db\",\"03aa96c8f435f7b2c1a916ccced6ba128e227ab5cc266d450612106672389e5d\",\"2ac9d7fec60cd687e43130eeb7cf12d2363b3eec32dd862278bc6b112ef04351\",\"1f91ac2aa338d7d0afd64248670fbc3f97deed7eb7f9918d9d3aaf7750aa7d05\",\"bb11a56ac0d59e81bf1d771c9fd82242a6f65fd5537e60e51326debc30df9d4c\",\"9c184587741302b8d201e614cd3236856c78719b70bb557fda8d18e66a283523\",\"095efb25486615ce1861bd7f64ec9e3d067b8e0e4475bd56d58c1c7d263de310\",\"f62a7e11c22fd12eabe1ba6f466b8ada39dc4979f3615c0f0bcd1b3389d676d6\",\"a839496b70ed1610879ab61f15a72d32aaaabd5ba71a17856b5051ee52977293\",\"f6627903228973a36bdbe288048c5dd7b627897394e086f3f73da3b45257b367\",\"7d21c8aa9f27b03c840e82d893d93e2b1525ba64a4b3e28e06eb52c1c0a2e1cc\",\"a422be16954359733e81b9c8b5151379272126ffb5e81c48e1a92d467829e6e6\",\"a22662ce5840b92caf03217d75f103095e6f24e81c0a14d5f118130bcad98292\",\"8f295b04f3f5631fb01b6af42d7feab5b9fb45eb1e3684f5f0adb4158f9e3afa\",\"64c1ba201326d0ea78ad51c0e014b41ac1046583f5675967d8aa174e521d61fa\",\"afee20c387cfff2248a0c322a87e2e55f2d322ea59f654ede57f50ec5b5dd491\",\"742228c24081fc55e4e616de84fe81f16b2215195e4bf4bbab809b6e6ab662a0\",\"caabf71702edbf7d458c2efda4266560be721da4b4a1784d70d695083d0f60a4\",\"65d8238ecc0030180c4716d4d187eeb1e5e37fd2cb437ead3e7c85ddfb131ee6\",\"d70e12cd6595b8066efb4f6151fb6972618a177f58314a6c28488b13ab5dee5a\",\"d36b983ef316522ab00cebda980f21ae9726451d920f6be32e90f61a0832a1a5\",\"2bbf1b77be07691eda2a25f2e997e08ab55c2f1e6589206d4c0e66f09928ca9c\",\"143a287bb9b6a8a11c2bae6fce60eb57860d5be77370b7d07b3adbbbd59f6cc4\",\"52d6a195bfaf9307d41aa8ec1594fca058b895390de564260779a333aff5431c\",\"e88460a0653355726e4fa56c61029a205cc47e4d05f15efa6ecbe995bde7fc94\",\"e7b5789eb697d8aa08e2739638d2b47f10b170127d2af02f376619a842232f52\",\"f86fb2b358b317383f03800aaf0897d28b49d196171675938555377facec9a40\",\"581d3583cfdc10c1ec5b2dbe8c21e4495e7edd60509e9d7829d4391d2efb258d\",\"2b230b88970e43a5ca155717e748e9e02bd1d77f83e54559387c21ccfc3bf325\",\"51163039b0b7f28302a163271a77c3a28b34f4691b1b2620f4b78688964984d2\",\"be44266438bef7b8a303c574ad3c50914e1a5b124645a3cd09e45a8bf7b1caee\",\"1ba2b6735f38b784b4cf939b2ff274a27925990116e20c36020ca115e3eea494\",\"e98121afb7ba0025056e63d243f8aa81016f6b80bce8bfb7f93f7b64ad5a4c39\",\"fa4833484e29d39d3b52506cb8acbd29f64e7bf526a50fc2bfe444e2710688a7\",\"9a345a14144097484de61b0917fd20250c8ff3b6dc49dd71d90ec8956d948e7a\",\"a43b61b5d0838718570cb706501e0b018f7e86d0abc6fbec4cb9bd0d78c895d0\",\"0a208ea8874786f9baa4e625753af14d9646dc158f60872debfb3afa1ad2805f\",\"e2e53810ad744af05560cd487ebdf2a6fba5ea79986e91118ccc0b77fcbba27d\",\"a1728b79ae5c2f4a354f56036e219fdef257db1787b9446839096011d72769bf\",\"9fbe4646264416a08a40662add257dec65d3ba1e0a891d59ca093deb7ad238ad\",\"963af909ade916ade8f33ba5f262719050cd2fde29f721ec432861739ebbea5f\",\"6a745e994d5075cedbb7e52c4f7c224bf4791d3acc22443fd74bb33511714708\",\"d8090ac0b9dca0a270ba1311003119fb4aa645637cc707eeab7ab9dec2a1908d\",\"05ace6e04cbe83108ed0150da2c89849f4290a6e73b10e5c908cda73ea5e0d53\",\"990a5411a219f21bfce2e308a9d8d86ed0f224aefc24b59946c5c4286c3280ee\",\"f69041a17aaabae43f1df336105cbfa3c842f27298e5343156025f367391fe31\",\"b7cc52abcd96cc3321c3e2e2e32362037e8b18a29646f58eed67735c13ef9229\",\"a552ea98ffc2629397472b7bb57f6ac954605b2bf8a524e44e59d9361ed37c06\",\"220997fecd8051505ff17ca5f3f17aa8ef2b18dde3fec1d733f7f0c6243b257f\",\"0e664979229423c085fcc5dfb794130c9abc8d25b269287e84202c90b5e21117\",\"2cf1ac3c8a9fcc5cae06db014d535b26f4dca94ae1117c8210124628ebc72923\",\"e4662a2986a2c903f9b3696cc06cf411a9af3c153a483e3f3233e9c5277f81cb\",\"e14b6236fed3d35d7ef0e01667a1b42cce61338bcb3ab25c1f30cd1ec9cb882e\",\"c3ffda1cdc69357619771d9bc90a630448f27e0074bf675a7c2644cedc528d15\",\"30ffd69df887b6027c7366a1f5973689b7f983e3602ea6e070b779168faf3cca\",\"c8cf12470c504f06602df797f2266cc07c5589f28233ac7827f97e2472dea763\",\"f2f0cc5e44ad43713424dba71852dc85cb82dc0799a03ed308a95ba30c7cf22b\",\"6957b762f080b7c22db675c861989b7fdc31139067f7f9106dbd58c1328b39ed\",\"9272bc5dedb4aa022df8d32cdd8dcf3f24e30a94ea056d8a5e71a9b394d532b1\",\"db069f7174ba39659e4497d7204d41168259214a973121405a67fdf1fb4a2d7c\",\"70a45ce7dcf2d18a1bc0077f3fe113ce84c100d0c9fcffc16b14a90b745988c8\",\"062066cfcd2523452f764c7c22145d82f1b6bddf51fc1cf02d7bbf1b1059c5e7\",\"b9b1056b8d7ed64ca083f714ab9c3d42792376dbcb21e39a7a221831f537853d\",\"54584883177f0974c67824f24267dacb2a65f81b9070db4c9b578a255e9522a8\",\"07f68267f6334288e4270acad85d9aebaf4d5085d61e7f52fa769966b3336440\",\"a7c1407d202f9cafda17a87a3c905cbacdede60d0b76b7115f225c0ddcb5973d\",\"789ecffe59b303526d2efa44f31c21aefd240e8f5a6c9b2e9063e9cf5cbb1097\",\"a461fa6c607c65a1efe8c1d2675230e38c0a531c7ab6417533c34ab333d9c2bc\",\"4807c01cec5948f6522b1c0b4bbb9c24958beb357e02ccd0391266e70c86a19d\",\"a2d9ed194c4f214b751fbd1d4f0e0fe677ec99c32d6584a1a14dd9dad20f79e9\",\"42f70644a2c38d8bebfcfe853d42b4c531b6e81a4213e82203ab588739afafa4\",\"84bb260342e0f71b6d12346b4996557bbeba5daee91392f89fe8dc619021282e\",\"10939928cf4b582f9e193dc9d8e38d00c2a1c3d75cd9a0f05cf2374370329a16\",\"4a0294701b1cc31dd4c36956caa07dadaaa465e67859a00b997ed34b2e828172\",\"80b03e4b54777affbf26ed90e25aa760a32facacabb5c2545b300f418cae8db5\",\"6f33cdb222732e09da761b3bd19094ca04b3bdaf6f1b478bb75c20d254c39970\",\"65f55c22b16987185bf92892c91585a2d6255caf9ce40c4d32d9b8318c24f5c0\",\"db8f14e77de1ece3c0e537ce0ad6856ac0f3c2c1530e23a19e1cf8e69db0cd90\",\"e349d61b8fa081f87e825e50f306eefcecea3d2495b3310f9bf81c9479ec3063\",\"b62f28a39050889259dc9a3fc80afade2cc8a9d608328da739b1021d9f55e4b3\",\"3d7f5684691d090ad8150468768007385b3f04069afbcfa4e9b57d7b0e5bc85c\",\"20ccda64493ce7794e2409e7eda8b68a08822584e6962ea83c63229da99b8a50\",\"f404202ac6263a54674949055bffb22502eaa85a4582777a05c80eda79470b57\",\"7a05c7547a5cc57910686900d83177841b330f2cb49b6c151559ae5aa490f7de\",\"d14e659e904e6cf24fb298d1f2fb9671a3db8d3187e80da9634cd8d73f96416a\",\"bf99c3ebe909adfef2d8160ea39f54620760ad5b5e0c09a26c2ae45f099f5109\",\"494ab4b7533fa748b4de36cb1743c90d5ccf9d20d9c56e95a3193425a2b1aaf5\",\"16291720047340d04c1f89a0a3c248c1aef8ef12322605a51f33a552845aa06f\",\"efa104abf8735945ea25cbec5a56d9b9c7ed58e299699940d2f295a02c0b262e\",\"fe4aaba3eacfbe1602f0ff26c601b90e55a14df702e272d01bb9c28bc896b138\",\"7f066300ed6752ef0c6657c191e57fdedd8c5781fa2b45be407ab6c74ecf3850\",\"40de5600828e2598335819e503e5739feb4abe77d005ec08377917401813c319\",\"68a748b05707265ca877f1dd933744372a779ec5decdd31d4629cd669601bc25\",\"079a20a89143235f536605049085d51f433b1ab6f4d1fea7e7624628b8bc40a0\",\"e024591ae23a90f89b7efd9ecbea85b6f20b8475037d130c5911998374ccc363\",\"1137a2482d2b8074306ce58bd7ee737900ccf60cc9657c44ee0c71875ee544f8\",\"1cff4dbe1df821ed406e539b76267f8be3600d9850875af3097676cfb52ea18f\",\"bcd836ca5d48a8bcd6fad5ad802c9da368283b4fa5b8d493c4ac28bb8032f8f5\",\"787e462a76abba0fdab0b0c096cc9deebc4e0adc04039d5a581ffb38662f1177\",\"4574f69ad96665a6d81bfd22a684685741a7d8fc9158b89aca8207df203edf7f\",\"284915f4d2af21c2310838527b361a43c547abfa9ba4d0d65c2e5c5a4ebc4056\",\"22375d82fcfdd733c137a890587aa5a11ffbb6a5b2de9d9425f52a9250e0d5f2\",\"f5eed1f29305c224c27fa692aa318c4867f7098666921c751fa70cc0a683cc0e\",\"870e7c5c6e94b94f7d87492e22007f5bb038d081aff6b777ef58d27329a2f8ee\",\"411ff9acac08f0ab8c4f51ee1c2cd9c003a06682b64fc7e7bb3338f9c6efa042\",\"469355382087bd444d88b67f3eec133ad2a73fe971f475fa6b6ec28798fe5770\",\"9227fc605c0e0c7596bf8dd28265120bc960a29cb550d2d64d7d80a2780b2014\",\"dc2d7163df8d716a09560b929fa2aabf91d69adef40360bf5871e63257e3b920\",\"90a97a3513fe15b8043393fdfdcacf42d84aeb29770ed3d3109736fe356ad554\",\"5517004fb0e6d11982eab22c1faf69f0b37333449954387365732f957ba43520\",\"a86df96e340e908236a3aff2e1d325bf74a991d2dae0081b4dcac43dc8ba9555\",\"1383ffe503c6b8cc9f510562e704baf561a2a1cef2fec1029fdd99b64484f768\",\"8822fd3cab9307942f04bfe32d96c475b99d05129bdadf5e62b5cfc5050c7a05\",\"74deefe5125dc23f71c7522cd42e8289ebd956ec82c3172042a24ae41a390fff\",\"5356c23937359dc83f7113192832cb2e36119c1cf3246cff34b7106c8ae7abf5\",\"01198bd0f1864c721445da3a3539c2ffa0c356eadf15b3c89eede9892b86fb36\",\"e3560fbb939280807eb05419578b31a61eaa6e63c1cc4136e3e86c898fbf8eb3\",\"b1cc1bad74d0e0def01b749092b73a4a5a9d6ea7d3b294132e626c32ca7f52c0\",\"526fba4ae2d4ff6db760ce69cc684804086191783c35ee99ab14c4cb1aa7d74b\",\"079751c3689bd9f530b2497fc20f83113130b8efdf4cc6ec4fa9e4273f4dbb1d\",\"947274acc8c429042954862217e7b6323599b1599f335f2fa7e9e95e2e8c7735\",\"a85074dab36d9d1f2e232d06e4ce705ab16a9bfa98fc0c598d3429ca1b99aa44\",\"e01b17aa91b9598b5f960e40b007afaa95e7a91e8fdd74fd0a90680242fff375\",\"cdac0cba0ae956b3031d046191bef3c193330b76fc424b43bc78df50a1d66838\",\"7655c4b5c1b00fb675abb26538c646829bd157de4123d27f663e2ac4abbf5608\",\"7feb55eb686d9bb35b3f931322035df1b8f33b89dce08d83e9af66aa92373efb\",\"a505b06d40a2e2ebdb733ec89349132d661e46e7deaf8ed9431adf718eb21b2d\",\"b87c5850c2d00ab357e38ae86ed43898d4e5b40dacc28c044685299b2911c519\",\"ad7ff1aa8225aab671341b5a8579aaf5cd8e6dcc22bc7928dbca5927d4c56566\",\"5c08fa6f506a75352d47d66fe9d32cd0b7d22218547ab42a4f43a0e058547fb6\",\"1ccbc91e70d3feafe4f3929157c1bec63836e75a477a12e22722268159ff3713\",\"7c5927f01fa6dbade42403f66d35eda576fb171343a48d9ee06e12b1e9a3efd0\",\"9ddb69d4e624f9771adb5cfc57f2e994c1bb401a95c51f964ec45186d61ea22c\",\"cb45df7394ff433224eb86cf5e1988172d3dece1feb61fe4336a3cdd6699da59\",\"44d9008fd0269f1222af4132c030407c92902b23cbf6d6dbfd09a1f2015c5fde\",\"1db684ab19bbb20afee0a073a8a4e35861781ecd960512fa54e4d27c771e9bb3\",\"a88864a59e7b3bd265a401216921974299862a5e553ec0b8abe3d88b2fdacfe6\",\"80b2f72ff05ec87841a6691fd48f3d54425a96007ed45079a69d1f4228a1c151\",\"ca8970d410d9517a33a09a812076cabe1e3bb39d67a560c4050b6a754a1accb5\",\"0054683e9780138a82681a9ecbca2ff89b208faf15a3fdd93997383a863e78ec\",\"2b2d92fd5335048818576ffed9cbfe0591bc42bd50aa6aed9a7384a8237e348f\",\"bbb44ce1563bfb3a8e996343d098f9cb11f13d54286583397978469d8f96a7e5\",\"29ba1661aafedf0d66053f2a6ef073f83463cb0a37069b4ae267523136b7512e\",\"cfeda5c11fa44d9c1475e435afced8306867a76bf8b3f584aa40ba1deedaee2f\",\"51d076b15a9dc82b48aabc21672bfb255f471b3cc7d4133d00c87fab7d3f3211\",\"2169858594ebf058b123110921d2395aedcccfd366cfbc500f895784f5fd99d5\",\"c27b877b20bb622d5c3b7e497b15ff99b8c02e35d18d80a13a1f2926344dec4f\",\"8e92cf07b5ea2158fc4e6737b01c30ad734ca0247af44a320c49f9a209d130c8\",\"efee0a733b3e56bda71354639aca3c58014e71f5339cbca3cdd88ec3bb3a9a57\",\"4cfa87222678846fc66debf489fa910982a43ebe888f69cb85525c0c03623dbb\",\"d7728d52987b7b098342d76ba8259ad406bae1457bbd0649dd7a0cf55ad9f5b0\",\"8a588df1d925aa4f977af1aa0f7686ac917a037be0607012b908f352ed992993\",\"4426122d9fcd2318d5f426fd6cacea97f15f83382ee927f7490cabc36c332053\",\"81302a2cd1329df2c0417a1a2523ef412525675cff4c83f7ce0cfaa7a0c4ec24\",\"99bdf2639a1455c5814c7b92509cb125a1426e4d669a134794b87a4614f34454\",\"e2e8fd9a5792c90df614889ae90d71dd929719404a4db36f157ff0d6c62c255d\",\"2727d9cef61436999c64c5233647ab003927b59eea3346d0c1add568795c6e58\",\"1bb9a4cd250f6a8a8a9be79dff26a6de1080075f490792be6550e8144e7dacf8\",\"c39067a75d44c65d78c6a2fa3c16407c5b98627e59768a16b7dde087c161ea87\",\"c18ba39ad2106a5bcfbd4581c5416cd3c65bc407e9e2036b51840500d30ba41c\",\"d051c0b6f0a18ebb8a173fc7755253232cd1f8ab730698c44978abf998f9365b\",\"9b6a494a0ce5567d278bebdbbda9c3ede4a2baa5c32b4d5b0f9ee8aecfb23b6c\",\"9bc6cdb752c43c2cff1c4e75b52c9b0558a3156fae6cfcb883f193d6552110eb\",\"051cdb6b7d60e1bbb2774d635b8e1011b0815c8a96aca5ab608674c62a129ddd\",\"dfe576ccbf300ff2e96772e7c7cf76ca1b304ac6200e014e63c09c1f5507583f\",\"788fb7c6775182f86ecb098056e57cad8ccb7622da816f3690ee9bdfb5558c45\",\"89b09a3692c667b0f71d9d796487b7b227bc26fc586aa00d3a1ca3e2fc7b880c\",\"e5f494e5f912033bb5a13b9706e08badf4932b05b210e94f50f0908aa5f16005\",\"cd7b8e7d1b9c001dc631747f8dcf456808f60c23c72e47a458a623b1b1932f2c\",\"25628ee1a462bd69f926d59411d08b1131f23e84afbfb178558fd13fcba7df8f\",\"44f72ad84c8932a6a5a62277b9d3d9bc6089836196dfa521b61edc9c8a71d235\",\"20f47b9c792202d4f61892cecf0a3596d3266039edbb00b5b95c477968c4d951\",\"3f803615abbc7de154e684d08fec4fc7beb4846cb8a783eeb0459dc4b5e642d0\",\"22551606c4dd577856f0b30bcd2183fae01c16de361f784d1bad846a3d7080cb\",\"e647930e1ba1c254a08c5c55095cea8f7caee2dcb3c812ef5ec969b6bbfa9a09\",\"6b4943785abef77b652e8fff247a4eacf684a209992ef86c3adc3222b39cb257\",\"91c89799219e2d660e36d2c09603e6ecb27e28b2f9b8ab8e6a5acd95b6e62af1\",\"297c86931f1948d3c7088a428cb3603179c5bfe455cb0001428ec72afb2ea7d3\",\"d22567e452fa7cf3dca592d52ea592566e01d56fb1d00eaa34fe111f715d4d8e\",\"433899f8c2dcd7e5e52bc1de631d515d16bd6f9c50b854946cc708f0bf5d2509\",\"4d35ec4fc736d7db442da4e79916d6b80a417e9a6367a10aada345ce99b28023\",\"17ea7e9245012bc5426cae41b128d2d09b5bedc09d6ff54b0237be4684bcc56e\",\"8c215987542acb5be16fbc1f215535668851b7e8bab05ebc8f2154bcb0a5e846\",\"ec64538160e73706a14f5f0656f8cc4ccdddfb03ec1fc3cf6c26c154d59e474a\",\"f8081131c4ac955d49b516d298fb7666287adc8941870a3e0c633f3e5c2360cf\",\"8d341d233501fb5ca87f32694e245c54038fa2342fb80f6851e08446ee1c6f3b\",\"af96fbde1a934e9b3720d99b58ac33250e9a2aa6a16ab525888538b0335874bc\",\"ea4bfac8fb0ba72a87226d706f631b38a0531620a4515ac343ba9f548c1dbb56\",\"e96f905b14f5463c05041d55a3c165104abff424d05cce2c7b60922802073fd7\",\"b508172bc48b3a8621b5a1d9b1db101c62aa4b6ea918d05e1737a439cc2e76f2\",\"9049c57a2fee92592b11639e54b64d0ae11dd63708de30677432fd6f151d1c79\",\"ef211321797f63390a77ef3ad9e1e42a4cc73a57464f47f72f2a691db879b7c3\",\"87fab8ec9e276a8739a553589d5f21b5fde00ce795ee50ba35d125a90718456d\",\"c15380d123e545b221f6df250f72d8aa25ca49aea068b76453643f6d71683280\",\"dbca4be41ffa1f4f5593df00fb439c0663ecf1539f9ba67cb1b34b70fa7f7f69\",\"4a79668ef18bbb21e90dd1278b60cd7ff047e175000eb1dab4d8346bf0f347ec\",\"41880c9e156e23a8fe9c275e30e0b639f6480ee1b53d3c1d538f54eb29e86b26\",\"724da18d2a39346cd6847293831bff5f29ed54094c8fc06b63bfa89592085e4e\",\"0a81b7d5d647d94af4829e4a241f45db578d2bce2972165a8ea5407266ef9fc3\",\"4e57a762477ef7caf760ec559ea58cb7b1545b4fa141291b489c4ad1fc15fbdb\",\"3d67c00da5c6f0e8ecfb726a0694606d39bd4b0afd74a2586923e56671145567\",\"50254100deea4bcbd44d44cf9d9e63405640aa0142e7d5418c3e312676141a2c\",\"232d1ac574be659c6ddeb984e948b1f69c6fce431104003d14c2c1ba984c0074\",\"01e4435a09847be24ae91756f8a5a4fd538d8a40560ac0880dc70ede725b1921\",\"981a51a19c243101ef559bc3b5207ef9aab3708141bd537024271618e331eaee\",\"7e6f30e40598f2510fd59f343af44fb4e7f731d86cdc2cc15cb115642d3a5fd4\",\"074f77e938acd6796294c55ddf65f04a57e002b6b433d8d776eaf2d58ea3c22f\",\"cc6203b1863c60104e84e6e0a706670852d7ba99e8162e09b7a4ddcf293bb524\",\"462b8e2b8dd8cc614c6b25e73811dbd3d57d1c5639e4c193240e82a18fddc1a9\",\"3415015fb8fcfeb0204e71573f5c921c2d7a6c5215987094b4eb527c50344054\",\"d0c832a034d1b4d6576e11be8475cf7a3b0e5c42a0e1e573a80fa80b1ff3c81a\",\"3407271145c1d8d146ef5b59661b638f45b9dc5684fb02e26cbab278b43370c0\",\"36fac5c0862ddacc443eff431b179e21c33df6a1430eed741011049fe4195734\",\"6e488a3ba07cda89de0fa1fdd92d9ad65bb05a34fa4e88090cb40621e2ab88f2\",\"f84f6fc6cab027d51ed6df473288a69d1e78e26060ced3300e52175c0b7821cd\",\"eb295efe0d1b0ea9c0e556328e5598fd0310755ef14ff4794a24a20669884816\",\"10884e191664a02aed9f2ec8ee776f0c3355ac1c72e6331498ab6a4b0c061e64\",\"e32ac9077f569839e4431d2f1c67acc58a5942d4f4ae1f83b795d872b24b94c1\",\"620047d32c96c010a8441a513ccefaa7d96e560948488b3dd784550368fc09df\",\"a77d436149973a16c96ad8e877001767119fa3de987982116eb9f7ec56d3fe56\",\"7c5c4376588fca04097e02484798a367a741f72fa9268a862d6c7954d448f20c\",\"8846678fa3960c7c48bde12031e453488ae6296be35a36ec2a56f331b841c404\",\"518124436af9a84522b4b75ee51451746dce3d6a637986624717e3ae5f4f164a\",\"597577e54b6a233faa81d33fcd0cfd9c39b558949a8b4d33068c684d2cd6b972\",\"29d93304526b31dca1a319f4c5ff690187f51425dbed4b16a254e92de7e78dd3\",\"f332854f8ea517f5c73d8aaf00b1bda6c4e7241c5f01ce930420cac4eff89e09\",\"22e7368a46e974ba5ba5f8359ce8c990d26dd6d7edb22a9077711be3bb274e9b\",\"d392db98293a2555dee3d84b1ece20d66124c03aff77a2426da6389882816522\",\"a4d013d40a4e2dce1f81ab2a90c4190ff65770c1136cfc08e9ff102bdc651b7c\",\"674c578b76fec88d81f8bd7a8326719671564fe64c9129e3352deb005b2010ab\",\"be48fce55f0da2ecbfd00080bbd93d2f1527c9be19fa3604d3ac6edf06b30b22\",\"ecbd1938025ae2680057f36960db67054745f3da09043f1a9198e8c11e03393c\",\"d3dbf29387a52986d720a4fc4c25db7fd3456d75829e68e804a5a6dbecc00c0a\",\"26b81a53ea8bab34915b4de5ebe1a6966fd7ffdd150a4cc038a958726d70cb05\",\"de0c20be6a2cd2ecf135d97e8f5ffd729aa7081f6349ff9081a8c8b4c9e98020\",\"0cdd70a8bf2cc37a2cc8907b631d1db1ee6e0c46916ee22cd11cbd2d58e346c6\",\"af3a0ba837105ee82cd9f6a578ec3b8e875800f7286f1aed557e4781730b069b\",\"63d31cbfd8c833ec64dcf5ab84d6dc2b211ac88d854addf77d06aa7640cc7090\",\"4bda6d3c9546fe0c00c18d1bca6d22cc0d9756fda4697daab90840fd547ace6b\",\"c12f3277b0f20a987e3209d9634b2351b8b100ac49b4f077c77351e4b82aa9d6\",\"a7c477f9f504426b16bea3036976ecd5c8fad0337c6ce173085529da4da77572\",\"7254ad191d9d5fb40ea3d8d60c1b8380b0c0b908bc4482bdc071348d2aad68a9\",\"acd929eda8405739e09f07592f72e545d357d61edd0638c2ec59a412468bd906\",\"1341a44312baea914a3bb04ab0d376e2d5aec4129c27cc3b956ee89493c9af94\",\"40e54c9d385a6f53fd2035d2d3aa03bbbf866bc50be19c1e7725a502f4b412a7\",\"c0f392a37b95b11e59ff8cb4d776fa5c0543e95d86f3769225f1c50c5a444d8a\",\"c7e29361b4957ff20fe8f385eacf22b0c022bb8b818fac1717fd325fceab2909\",\"8b69cf23b6644ddde0c041b9dad3150f5e47ba999cff650bef3e348a9bf98e42\",\"914992d78c9f3985df58a2aa1dae7ccfb50b4624ec172ce4752c6258a73bfcc9\",\"632548af15182f6e791deea51cf687ed5925238f5fee12a1fe4fbb05b92fcde2\",\"ee6b4ae53b93f8befe2e239ded77b4f01e2b544aa5b36e0ead59049a69575dac\",\"1c077bf18e902c5d235c0836dfd2ad22c9884bb83440728ccc19e15248ce8b64\",\"19f419c532f4758992ed3c4d6b4445a18f682d1d6398293efc8cc4763faa6fd3\",\"c250c27776b675098c7ad51328c1412ed05c45b3759751e137b54209f72d63ba\",\"97d0fe574d5b3228e3b8669c98b52fae534197c78cb0e32874d0a15b69fc1257\",\"f7fac434c67896eee605880130c25af61c8e00b4579f25d68d51ad46d666aa27\",\"e602daca90f00c8ebfd374f4ad1dfc17d0b5482fec403ea53c9fd2285d57392d\",\"20571ee822861c310a32c7d4c884e0c6bab546e3bc33c2d75896fa6dfc903311\",\"9cc82d7c53e9610c2404ed92bb586b823843036cf81b877bacbbef3d8448c5d7\",\"a0c1ebb8f6023ac29867c14cb85470a920b0e0c6d000281dd84ddbf5a49eecb9\",\"a38c8cbd6e64898ba39f9fc5d4f35b7c9932133a484a7d4875b26f74ec0c9bcf\",\"c1723ca98939e3ce101560fbab9238c7f26754b576767757f4cc3fe10e9b7440\",\"c69c7175e7ec8cc82a27c675165336b1dff3def1328ad90e62a49f26feb413e5\",\"1a10add8af8c2691ec3fdd39f4fb9643f8eacd471781802d37975fd3bb7683de\",\"60c50ce24ec25b3a9b7459e6f87f170143ee60a2b4d83ce22b721a9dea42e156\",\"384fe6eae7b0976dee878afe72340bf37f7232c7ee302c93eefafa9f1e9c09a8\",\"e70f7e291c0b0c956f4e107f751538528cf8c5adffbaedd45c0ebd6a509c4fc2\",\"96e3595e5e632ef0e83b3d74d3329854dbf3327807c6dc865e483d3df6867cd0\",\"750b9ef9a9225f488844095677971a897297b2b8d601a6f536e71caaec115a7b\",\"1218b651be0b34e97a52d6bdde46234bc90f54549c2832b13a247b74496da670\",\"3c2b5b805046e171dc8f1df4acbee5ed8a0718d910bc2ed937909038f380f52d\",\"b0e49406b848ac6d164348c82b2c46983d54d016db091b3d4c175e42be34272c\",\"8c69ea1ac21676f5dc63ec91662b045803953693c93d1fc5541b4659f6ddc070\",\"82c4223a30f4ef6c492dc8c1e3d9f902d9cad4fe30ee571ac6621e306d647470\",\"13147a2d1897437cf6efba1e4eef3c2d94784ea0d57589e9c9af9ab1b3b89f4c\",\"60a97d3daaa4cfc00afdfa98848eeaac327643ea627712a34c560f378b74e34b\",\"b0b1140b1d78a5e1a7953d0a4789bad8545da02d4f9ce72ed1ed3865e5491e09\",\"0888bd6be274cecfd037d53ea8c3a1ce181c49947407fd0dd6d15aec0cacc953\",\"0ef222d2863a9bf3968598a6ed84db5f072c10fa8e621df13b44a2104e800ea9\",\"07934d24748be908557d133612f06fb3e5759ed05f083604e2167a38281df360\",\"a2b1a588cc574591c59330cb92fb11c9794a3b8f0fbc81d7660418569887aa58\",\"8af8a3f58373a89f90fd06c9c730eec5bcdcd495dde83be71c145dd8d24b10b3\",\"bd633ebb00d948e40d2ddcae76d7f655759d8eb39007552ef56d884331757957\",\"21306fd7ce721c9e896f26593d938761ebe2dac2d7dae7240eb2f5494da3d7f9\",\"848baee577f2750fc30525e7d504e618946740bb5028f1d36af7e099ef623620\",\"838197e99d0659a4931b92dc142bac14a3a7e8772a7df9d11f5a81e72cc03437\",\"e6ef23c1b02da508b6ca9b7734c371b236198b587eeddac203adc15889f52f47\",\"302298aa0eb6e7d4db82feecf38c88c6ffe9767b4f4c31f2a14d7c752a928a18\",\"5fa89e382ff5c7a741a787962b0ee7304a063fa61dfd3103bcfe13df1ef9c37b\",\"15bd33ba26714dd856bd83a02778a4519bd494cde8b9ef444a62b387246c36bf\",\"5625b4d5ae89128aefea7aafced91998de97c0786e86df85cbd265aadd34a333\",\"2df84fdecc24aac55685a132f616e953572322dba8d0b18edfab9d90e583fc74\",\"6e0f54b6a6dc946061e70b79e88c75022fcffa3eae619b250053908257d4dbd6\",\"288c34d35fcceaf828c767cfa9ee5e47d121c9b84dfcf696b307f6fd0dfed949\",\"0a90c8b86f1a0781106af8f7c002f38fdb9c06525ce9b92c6b0bce55434a05b1\",\"96e76f9c4528231b090b98c50da9ab8c21e42739e60b2446132fcd425f877ed3\",\"3a48e0d8bf147fbcc82e66615efd993c4aab69c8fdda24177c8a4f5f7e3eed6a\",\"44e8810939a004a0e94128c28ca60569b73d24ff567e600cde3c041ad5530b89\",\"24bca05476e6f70181bfc1d8e0a4d47061dfa79a0b28fe10a1d25634f649bfa6\",\"e65ab4ed5c3bfda316c36f82c9c1bd76baa4dfb25bb64f8a5978e3238eb45d12\",\"2757cfaee5c0bd5e0f40ad7509e7b3ac56e2192152bc866a7cd58d7a06abcb78\",\"0740a2ef0696a2866818f6912afdb95c1b9aa80e8846d468d03554d23812a3da\",\"57ca8ac9bde6aab1d513fdd657254a8366540ef759f738f22f13a8c6ebaea6e4\",\"f3047a8ff0cff3c99211dd9d65fc361aa9594365636504dbdfcafef47194d44c\",\"675119b5513f90e3d68f9fe2fe7642cd6f2be9d55b289d1a113dd18ff58bea10\",\"656786797616015be9a6d60d940851c38554e90ddc594f25d37a704061675c4d\",\"b8af81d2979d28be0ad431fd268873a73e4a09e298ef170f61c051c5730c7be3\",\"46524f884421557fe520bd717b643fcc819e9cb5a22f1f7ef51ff2b6e482545a\",\"2b3737ac5838e894cfc47d9d15e987c6284ece56ba968bba869eddec04adfeae\",\"1cf12008a0abc068bf4c16d9070c7eb183c7dee33b29ed66157eddf2eb45b5bf\",\"b1ed77744d450320e823cb1814863c8bc27c9f28293b3e2d398b06cc8de68600\",\"f51323d9f881b3b65d431d7dc4e3c306d6a2c5d368f63c39aa8c4b0a4143e7f1\",\"1b6858f85ebd16055b559b5a33276989ce71b2edefc3abef30a817de65f4e064\",\"5e009defa80e4d1719cd35e6f91055923abfeae21e19b21e0bb349cc80d325c7\",\"462131143d71786f3373ae5ac76defe9d03a11b0875e1475dcb0ef04b8d142df\",\"f77e2461acbbf279fa08ecaf7c2811f4914485b6f6be3b3a63f8ea7ddd240f0f\",\"7fda4f50d2d8b27ffb15d1e99ad472a6c8d2aabcb5fbb2ac91b534ac5cb52ce1\",\"75166103c128a9fd9fee58e8435eb2447069ed5210acd677ccb4ec1ac1fa7e36\",\"6fe7163cddb50b9d1379c5760f4106917ace0d99cc9d0358925ee379bde219fa\",\"c7b7571a18949e6c8dd4be22948414d2e86f1411dbbeb36369a645056bc6da8a\",\"2f0b457be3cd3ea816ca46eb9566865aac63194c5e84768d649530b54e7749b0\",\"b9c0a1be3be670cc5e8c902ee152b6ed24f4ea0304d0ddf5a045f5e7a4b978d8\",\"c3a5ef8400c3e0907df19e309c90fc3e16768aeb7f76ceefe9fd2570c27ddd50\",\"929f2f772f48192deb93c66bcd61cc8674bbeec82de121bd018c68209f7b048b\",\"aeb68967a7b14e4479f815ffd3acc7332105fb12640e8e8f7ea55c67ada0feac\",\"c36d89462639b0b87903b419f63d47cedd1dff5515be0efba324180a0641af2d\",\"9e33fc3a926c4fc97a600fd4aa7c8928a1b687aa1209951281e286c96acac52e\",\"95e09fa991bf416e1ce05bec9fe3ac834d8f04b6020f5ea6607905700f430947\",\"ee6d74fd1ccb4ac395c61e84cc2c3da0d86bf8c7507f7df3f32619c8893b1de1\",\"0c4a37554b77b94863cf31d4bd733286a9b8b9a3ed2a5b4e0ca9e0dd73f116ce\",\"041a63b5cd04c3ae6f1c7654bf953fc8c27da95821e2028e4752642fd849394e\",\"9a731ca595a7ffe6fb22f2565c82a66b3e0674cb32ec1a7312e5ec58ca20a17a\",\"0237e97ffc88d46c2ae2f1848a58f0411ba4ac12dbf6f4d2966671b0c7fd2fb7\",\"3368401c98bd57c437fa9ffa06c93a6a89b9395fde032789c42f3718eaef14cf\",\"816d090b5a01e17b35cec78fd2466d8cacb1e6f5c5b093aaf552e3fd84fb88ca\",\"a25aaf722166d98d2bc0e1612763c4ff4851711fca079413403162dfed5f69df\",\"ceeb8005b7c44c73e75aa64ba9bc680cd4496045e8bbe8cabf1e8fd5b48c32d5\",\"63097749c68d6a4e9e9f7e8040c73f8d660b111d241f0bf4eb2f3e57bb4850c3\",\"46c0d11ae71eb0a9bb397c8cd73468af3f3ed89284303828670894994f8a061d\",\"b644a6f023b0f2dcd82053229dd77329c4131c5f74abedb2b097e414dafa3f24\",\"ed5bc29ea61dc4185accc5633bccf0d9157ad7d8d1a69c1c37ff52f23019cc19\",\"c073075d65d72d754ca02b5bff1489002df735ef1d07dceb9ee8823817ba0967\",\"a0e9bdd40e598d2c2a98703d102f6f8afaca976464733b256191b98d3f3e7055\",\"36c6208412c40fa9f7e874dd85ba8b76df1bb6cc90d1962eed519770062596bb\",\"e810296c3e76c83ef97d242bccbbb88f64b0b42c451adb60ea865ffe8fc67be8\",\"7879bd65e4dcb504dda2cd46957ee722d38d43ea35e2adf81884cd9471ba74d6\",\"d3f02ffad2f8b32775d639b6ecb4634eba3ff69cb4f5587df42b19ef66e153c6\",\"8188432793e4ecfc0bca4c5fbf1e11e807d267f11e57718733eb4f568f0587ae\",\"95c727e39b234b17506ba0012e56fe053748b4695bd97e26e99e91741587854b\",\"c766c7f575c609b05363e88a4547773aeffbefd70b032da1156d0fecb03f4b68\",\"50e86c09e699f37f92e9f11b08fb59130c72e5aa2d339d8d53728ac1c4bb34bc\",\"9d2c94de074d905fdc3958e1fa5226f9d9383cb9d1ead26c26bac3c5fda803b5\",\"61ac33ccc32fa9f94ab993ee7717228d59a3443733e7dbeef0db662cd503bb3f\",\"dfa8536a8aa9b34ef4ac5aab19fac1ff810e3a29f01eb32a75bf82e67b491e67\",\"62388201178a3993fc48b1b1514365eb3af6a345c487a8e2f7d0c8a01e058378\",\"a1b6b6d2d7ed11bd545d36834495480b511f0f3dd574853f3910550b2da259a0\",\"921b0eec35e5749c629ad651efe216a8ef65df8cba221b5f62f3ab48fa0cbe65\",\"4ff7d01ba9ccb5c62ea1e4ec9706981d7a43d41ab1907881fa4c51ca1ca2ffd1\",\"74e9f2969430f63b230c3dd68d4305ebc0b724540b9ee2874916bb4932576db8\",\"a53e516cc70e27a1c63751d63db17e0a50481d44e1b2bb985df5d416bcabaaea\",\"89833385f1d6d35f4d1d4215f199b58d1556cf3026359a9b55473460aa8755d8\",\"b730be1688327c0ea64b59f6ef1f075c42bb02569bb6209c586ee2a28893537e\",\"7cde23832774caa41f444b8d463eed9e5d51a12e25329aa7b04e04b570fb2e50\",\"4b5f61fe797e1427e9f773971775735ff47409a95560fb019ecfda2268195003\",\"a8ff1a7c675070d8c1d9b22f285c9468816bda4edac081021e4dc3a217cfe0bd\",\"35ef7f1a4da37264085369f2754a40821affb9053610029caff2ee7a8662b24b\",\"439fb59a11180f4d96af18c041c64f9ad4e6895d77564ed8b66c57365e12437b\",\"19dd35d69551baf67c81b79941760e27dc65fe974002f185fcb16eb73eeffffb\",\"627a562c9f1c2e61e49f20e78f7412729f2e29be074faf4f1fb3d8094c8eb739\",\"de083f2b4786f3cadce1f2764ac909dedb82c4ec1ec548d1f369551fb1ed5884\",\"fd3fd33423f23717b57e98c3c6d52935c445e601fba00693510ccc6b40b05591\",\"e932f8a2773d1a24268b65e09e58c48e2ffb74c4f05088946bbdeb049c97028c\",\"302cd9aed85d7e00ae720bc86e3c94cfa8f6922c921c287bb539f0e44839d55c\",\"70ff42eea72e923f8d681b860a61086f077df4b6c2d4dde0f61bbb8d015a4abf\",\"138b7d3e33db374500cc9724e82661c22d6258d09634fbc19cc72350c24d87c6\",\"bad3d94cc6d77add18d9476ce1a3ad2883a0124f283988fadcb9e71cbc89b84b\",\"1ad8821bdd81703dec231bdd74d131dff72c077f6f619c112e43792414ff8385\",\"ffa11291dd3194546f9cc37b146efd274ad0eaa82a76a93602c37696613dcc82\",\"ba84d46ee8def9d8095718874879ace72307e42d7d2fb5340c20fab9c80ca989\",\"78bdd553f3608823435ff7ebc1571c3babd0796283f01c4b7a26e0c7803218d6\",\"e7e134a01e74e7788f651a3be178b68d63db209c6023a75fe6958216217189c4\",\"8bf567e8f48d9de6c2276d5f3334a56e442a58fb88766859f070057d6d3f0ef6\",\"f5c5a893cce35d3dd811a58f062884f53d8838cf4773d61b81dd746dc0a23596\",\"06770d818cf9604c8d239aacac5089e79d230f60799bb455ff595fd6b031a8f1\",\"3a51fd20aa982d3c10bf43ec3ba684e80fdb0fe883537063501743d379f461f7\",\"72ddb904119ec4e56e05ca8e01fd2dda709357df6b6b78046c9e0562019d5db8\",\"af1763a1312c573c8c24b69074b6c46665c11bbbe9e4b322a86c3baa656681dc\",\"41ad6c5695d44517aa0879046b21cb371594a43e73b550ca2886cf0e98ec61ef\",\"09650e0a2d53cafdbc191d20e6abb043462040835496197d00edc46fcee37f00\",\"fcdab78461ff27a55f05758e12026f60f5c4487fdb8d354a8672226f2581116e\",\"bba6e928c2d694359430de3605f425a2ada5bc5aff66129c8bf72fea4c9bb0d3\",\"0be1f243678b5df0d327ac963c0c56db3625ebfd3853435305e013322a4dd5ae\",\"8d2c54d61b75d48b06d07d5b1235a4196aea37d0d21bb13fb1e5f283fd53c372\",\"a66af446bf50cada637d4e3721315daba8416797f67ca318f304b5508d1d68a3\",\"5269172fdff4033c7f2ba470d8b171e52a057eabab1634992293833cb478b87d\",\"693bdc27a29220773784a5550a4f5450ce151786a2a129504cf1ec4629dd04f8\",\"7e9ef53d3b52dff606571a3337ee8f5d797718577bbe546b596d510357050a8e\",\"cc2046fb3f8446c2100357bdec9e7475331b4121afe02556a98c4560659dc3d4\",\"662f73675f249a160c3ea3806369588a66a75d1cecbd5426493b36f9afe0d83c\",\"b723bcc07cc315ce89a5681fa33e9b52418845ca822779f35bec8dc4849e6de6\",\"664ea9ea8f7dbb6513fbf015f71793a9aaf24b9dfb99910b7bf890f516f708d3\",\"a27b2a60d178a81ba9e7781aeeca83aefd41d1e6d29e5907693ec5c85f4dfbf9\",\"555281ba4a4914cedf0f992e1a6a2acda96e56747e28921e2b65c751afa25c8f\",\"701e0460dfc6c2cc7e3e0422c81a639d812b7c394ed18c736b231a6288847a77\",\"a4f5c617e79c2cdf1bee60e02a33297b42329487da9e881c6e3ea02cbed8c426\",\"d068cd60b757c363ec131119a3c1f32c64180520145d2794a60e673d9017114f\",\"5e5f395fbe4b6646cf9d092f56d2bab449babfc2d325c5c3d3000da32db5aeeb\",\"e02539b9500382b691feaa8dc9423296432ba6b8405ef52bb0daed61a19ef654\",\"6816cf791aac028a1aa6c5a219513f2f95895c848e7be5dee6a9220a2dba8c35\",\"b8c7f1c0da995b1c355a639a0c671a917515397c283a4a29bf30b52454ee27f4\",\"167bc21397a9f89eb391ca0cb26913783f6f767e5164b0f16f8768a5b1882208\",\"7093d75d282849fc8a9d85b76d85059def3551a9af23b71a3ec370521171c712\",\"50c6ecb4870acd4cd6a66a4e43f14fc9da147fe0fad08efda7b831d0d6f37ab6\",\"311a49db7958ecec9f54712163f8c5de660ce93aa11d9881df27d1f17a01505e\",\"2816e718eda22ed66a8d1054a577899be64e7077ba0e6cd223c26a9d17f98fcf\",\"ab8a73915f27aa6b5e4f921c3c341e4727f5f1e4dac45575b38ceba3a4813b7a\",\"9e806c5c107724b1c1ff9b8a4049053cd991a65864c0dd942970f2c460696596\",\"169ee331981dd6c427393ab42dae575d754c986d322a27c577ad1e557a5e3e3a\",\"06f9425cd0221e74745135c6ef4c133bca33db90aabe27eadc2dbf2372decd90\",\"0fa4f984722ad2dab4ba573cb77946bba6970ee90824aa11270cc2e897a2b691\",\"66c55bc3bd69d61d2ae4ea2d749b97afc4ea571209e757058934232a43934b65\",\"3934faa718afe0092d9257d3e7f3eca9ce265ed217b73db5cdb41914ab31212c\",\"8e77e366713086d5c8e4d4742fdce87b7985550aa9a7f9cf6bd35a37675708a3\",\"506994528cfe6b130ec5f133e6c6d106f0183bbc2f2e8918f9abd022215014d0\",\"9d6abd9ba229f95f6db6ee85c16f02f17e43d05a8cd780fba07207ddf1202374\",\"363c5e4485842f5e488c894d44c661d6c2f86ac77f1f24c956d76250743678df\",\"81a8a078de5ed4bb2a703ae04eb1e65a8c905ba26f76644bf78cca4c0c89ec3c\",\"81a0bac58aa88c617b8dae1ff363f98a7e0df6df7929efe2946f68599fa274b3\",\"092df17d55f0a7415a8c93e6bab5fbb6b7750bbac0baa82a6e323f5718af391d\",\"8bb37b46d022dc4683f487a27e48787ae2fba656bcc92ac33b94f5e5ef1ed846\",\"4e991742e573378fb730ad5ebbfd96ed574ee2565c65cdde437537bf59ad75e5\",\"b059e0cc5b89e5293769e8bc447da98d6982b6a1a5cf92d1df8744cc60b6d9ba\",\"1f4fee8ffdd5676cc65ad4327093ce64e295795191ef9ebc39ece8d95c80cd2f\",\"947f317be4fb84af7960cb525c2fefed5873bbfc3732999066eead1f3f5a1e36\",\"b1ffab2fb845603d5dc5f91f8a45ca9908452a2ee9ad98ad131cd87fcb9118ec\",\"9e00dee48a35dd312a454de33ec8aebd29177bfd51dd9a0b36e772f08ad40001\",\"5f97bda28979b35cdc110a77b821638f047b56372cb34b80395b45983f8d9722\",\"e85b1a9810733d1dff3528708fd5854295813901606232af7760438ef923c5b8\",\"8ad181a97222c3dc668762f0738b19823cb87beaebeb6122c6e9c4be990d363d\",\"6b85154e43807c3edb6acb7f6605b1885b8e8421f44be1df6c6dde835bd51797\",\"ccaaecb34a427579a763642729c9516edf634851eab2331d3a9f1e7e9ad18c42\",\"7eca2dad3ee944fb129bc5f049e2ad7a406d26a63ebdd94e8f6ed654230e7e44\",\"6ce309fe168ff7bd7dc2b642deeb33f1c71f4eac5defd6ded948ad32352dbeb6\",\"1775ddb84673e329c44bab23e6f6ec624c2673cb7043e65c32c6bc0f69fca536\",\"0207819d1b0df40b64dd659da3917c1fd04316904be2109c942e8e6b21febea1\",\"270d5960e739ddfc12d623070939eafa07f6c8bc491f67b33d9cedc0f645e9c9\",\"a79d101950eaafaf0abcb52184862f7476be46465a6448c7135dfa82e6f1b124\",\"7c1c5b54579e1771cbd152ebbc413702c25b69e54760f3d6b6a3365b7d6281e9\",\"6c03f06eb4e7bcaf43f6e995a08f50d70b82f651a2b62e8bba8960bd73f4d9ca\",\"2d540f1348ec065f16d6c79c0d83f291efc195ebb0553710347287ebcf2420f2\",\"c4a3d1fd0a60c9d33ff932dc3f03fb156202b6b1c1de465a5dd86946b0b79272\",\"da597f90fbf4ac3f2b2d24baf74805c1f1ecc28779d72f32077d83be0fd76218\",\"e88a6232c14706be27131f9a12653f69c108964ef9ae9ba646f9100c49dc993c\",\"b34c287abe5fee734fa1b4b8c80063a79e42054d1654637802420d2a69744f56\",\"fc64477930dbd4d162d256774b87914681356266a5555525526e058aae01aeab\",\"99cebfd085c85271ffc2392d5bec155c9a2975f9bde0aff88726f4b3e654d24e\",\"ac74443ffd52bbafcd823d72cd7bcde4bb8748d8b63ea1592689847aa04f58b8\",\"53ad5293b0bde4e5a9a56428bdfd0f05a7f3beeea4a423e311048f2506627882\",\"a7f3488df3d4496e1bf0c40d02f682400a4e6767ec2d22f864a91c1ce7bf853e\",\"a0098c5c2d7cc928ddf0d704c001bc515865268c88d8c3390522735929f3e9b7\",\"adb2a5a6621fce754ac70176e94fbd2b00d4f68adc399cea02ac47fcd8f2b61e\",\"e43724d77456cf920b807552d2cda078c4a571ba65b6d4b9f1885236f76eacb0\",\"1950f647077e34cf64aeab23e9e2b4bc8cf4dca900e95440fcbf00ed25689689\",\"4234faa062bbbff9a0c47b1a8a8f4d32e0001a38949a450b56f018a307c64c9a\",\"d09f300ed397415de9fe59a25899adf9ef0a3b76826d3e7f7b0701e4fbd9d528\",\"c3f838d9e887f619a46f676438d59c3de9585c0ef76ec159ff7d2c0d1729e93b\",\"bddb8bcc0f799cfe8cdbc379b17ee11aca25858e3cb910a128861b29ca069cf8\",\"45b416c5bde8fafc088532f0044bebe7ab013a803adf3a5c2e8db7023494ef63\",\"7ddee65f81f9e92d04a59bded077059e8f057f5033ec7f981265f1d59610d5b6\",\"7c2068e4ada7f34b1d739bc64a34c2b40c0c636227d89ecaa5f92f3aced50eaa\",\"c5af2a0c52372b3f1cc966619ba5c8cc22d086bf549c584b5153a3eb31fad399\",\"d183b54788f81994d803869aebe17726ea1af230d51207cd5f7391217acbe5e4\",\"eff77f028206adb892c2e843aa1048dc07ab2d4b9f65e52cdf58c662c32340de\",\"e214771bd030446e21184e8be96f728394371c9174548fb255ddd96923eaa00a\",\"e002b60f14c27decd9d3df626403bd90a84ecb521ec4e62ccec05211504c273d\",\"121433ea332dd8091f3ae0576d32b4d930f8def82bf396b6d21453ccb37fb08c\",\"4a63c37f29fc23ce924abfa293d0efb5043012b12d691cbd5aa9169681c72a4f\",\"6ddf01da3983c826af5f0c96ec3558bf78a2f29bf8d419cdd51673e00aafd7a5\",\"7cd73bd3e7ad50db582f9c040c153f79b750d8bb38cf220742eac9523527bf04\",\"afaa8064d3f668ac006455ec080c0679cf0a30c00767330a6f5f04fc8ff26e17\",\"1171e6c3910d225a59e34c72effe41c2b029628d17712ef5caa18957654445ad\",\"cfee74edf943824f4ff157e975a1dc31ef16d0c49ef1ba1938298858b7801b43\",\"52ece3c6bf5a6c57622eb22d535a57c032cfd026e3262599166fc974f7540c81\",\"789a398f42fd15b99bfc23cdcb959bb3ccfce6bfb3f1d1b39465c8ed267313c1\",\"464de283cad3409a8c133a2884c95e0bb0ed554f0f88ec4f14af9bff420a92fa\",\"21a66e534ecf2f1547fc50c6b2e7a1c3a5855ba9315af23bb9171a69393122c9\",\"55a149b3320eb2bb6eec3fc773cf5ee46fda145c763ab63f30c32b3e775a0d87\",\"2a324d2f34c859b90b4443d57844b70d04ce4e179967c182054076ce29f44707\",\"7529e311914b6a25b82f27ff45fafa0018807ab6374f028a2563da4ff943e11a\",\"ffa89c5f72bde55400fa90550674f2788dc2cb61eebc22326a34931aec84bba0\",\"88b1c3d7c34f94ee670aa4d3f5bc906d67225f6584659df9a08bb397371726d6\",\"7bb16081cb129777048d2243245b41f4c710b17f46304ae952cf0f5737fcd1ef\",\"dd93138f0348f537846279eb772233a023c4a25220795d28600f3693365cc4a9\",\"55d5fcfeda15543c3146b01c5f99c45cab5c64e97db0d8e76d74678c22231601\",\"f231b04b2cdcf430a951765a7da4559f9f31668f3508f31baf6e43aa1fa76cc6\",\"628cbcee1d49b07287298e13630d3c3febf489aac5fef50cc6f857bcaf39933c\",\"fff84ec4fa726f119bb636eeaf463ea3808e486645446c4bd347a7860d01003a\",\"5ebd09fad4e33bb9f96a67a2c74d690cdb2cb985b552f88c5b5294718f6526ab\",\"83e2b40d95ae3a9d888304949c19e9ba3945aafa82e0c127309333745738c5a3\",\"952686d346cc189b5ae14b59d78b086483d187ccc4806e8b3bb0de9240f49dd8\",\"6d4a6499fdc79955062316e81a7c08e4c1dba7cc17975a290a2cbb66518d855d\",\"4c77f9284e4f36c169e79bb18d293c74a25a2aa42ca9158b346980e09bdc6ee1\",\"6edc58a8408958cbe7ecc77363e85f32402477b49f05c14c27fd17a113d5c33b\",\"e60e3c304f7252410f7c78c63917899d28a8c7fbf90b8bf70b80786983b6ba9c\",\"3818f3838511eb9466c6678a25b132ca4f09f5e2442562f8f1c987ba756b3dd2\",\"4e1c89157d2e1e390b4b735fc4564dc93b83456bf33ec0f1fddc4ac149d0c3cf\",\"839b3d68d731b076dd061015024bef5594e51cfc4af0b890a3b34cce32b4face\",\"47ca7046ac46056e3a1e3e3e1f498c4705bc08f2e5bc090f9a59d6694757b33e\",\"b50439c0f6edce57bf84959e41d5c9db95853c0d861e94837ec03fe1778869a7\",\"ab4b13b9fbcaff60cb9eb2974a3e30b1173c4a1495ff0b238c860180e1397ecc\",\"55826f79158b9680726ee1a8f68a9229467a7f07f82d3933a283b1532609371b\",\"0e931e975cff8030a96facfefea55c57b005f73182d875cac805223fb329625c\",\"bfa8a6fb479f37766074120216b12ebcaa27ecca8623b18488b891d2e759a66c\",\"dd50f58efe282bc9e2f29ca1ab15072539c522beb324ac0932f98d8c7cbe5cf9\",\"360a9f64c6ccadcd3270556baa4f42982e5ae7c007d95dd17a6fe7aea4bf3cc6\",\"07a12cbb53402d6dc6da07bfa9fefd0c07f45b239981627edeae49466a72361f\",\"8d380b8d33b53f46d103746acf4ec7732a41b52bf22480ac0768dcf8d24677fe\",\"253ecc5cda1a66c09ede6a81375bda3c4568e0dbcdf0ce14c39eea4f9e5f5756\",\"c37ca1371145753e22f4f4982d0bd0266829c1803bf4257187d0b6e780d3d0e6\",\"cad19b6e721af2aaad25c89532f6200173a676b865cc164494037b926b160f7d\",\"9a29d85abe6157ce3c6e30f2c80cbc1667c65b143624c8c398682d7ad555ba13\",\"65c7815f1899d6a3e7254c38d1f91804f05dce5cf4b53f477beea128203384bb\",\"a223a6dc5872426b540401bfec8766028074d11ce9705299a05575e8dedeabd7\",\"b41d702c0f171e7a5eb0ace34e446800790a4813e948a96fd668e57514adaea8\",\"8d26d39371f96da869f50baa4b89a0565e20761f25458710d6b4b84c8f00fbbc\",\"0227e87c17ea17dc89487ae09ff887660431dd79cb4cfb72fe9a0f755e8ee6df\",\"a6a74be32face6bf971252d42d79850241c2d91cb14131ed17ea7f494d19c90b\",\"6242d2152be6c0078437580b07c4cd5c98eaac7e3e68fdc4498e369959f2f5a5\",\"094f9144bda35d366d7817be22f26bd55b4e4a8cafc552c70ccee14085a13cfe\",\"db605b65fbc0f1f0606d7749e2c79596b6a8e9edab200e8c3fa8124906cc159c\",\"31b3739de10c473f1c92c75da6bc92a0db7010058634d87e3c6ca6bd5ebcb4ab\",\"28e088b4fd06e9b648f23ab014a1e127823bae034a4d93e637093853e8bf77ea\",\"d97a3ad41a7f544262356fb8ca7c423054e6a6ca06a29cca2072f29943fc445c\",\"78c706153452e06a9d3d5623d797634051a2cec60a1bb4a8677ebd7ab09aabe5\",\"e6351f8fde8c5fae7e7b78b8a0fd9e45affca07fd9c9c126b6bd8a2c2c7656aa\",\"7f6cb8309f0a483e7d40c091dca6efd09f3303181263177b01911d440e288b4f\",\"ef37121c14a04afc8374944a76a5179b9e3f74faec6b6f46229e5d0e4bdeb1fd\",\"52ad01aee2af884a7cba334c01878e311264e6f5798325971add5ab049bca55f\",\"50af8c1989c9e76e696c49ab99f678b189ebf9dd0683a4a18dbb2d082e200be0\",\"c296722a60c4108585e4a1b6b3ad25d86bb3c98bd3184ca355d9094f52b74f8a\",\"ea9891b76c7d52f5ea456d6f1c8e400ad6f05cfb9a229dd36f572932fa91bea9\",\"a75e9c0e9494a5c9702dda353ba14bc09ea307c4c3a93337baa16183e5a75fe0\",\"29232c5f46bb41da8eab4052d6116492b4d7844ef70c9f5e33fff9ae132ea47d\",\"c3209e97940126380f9726b9c117504bb803d7463ae6dc12691f58f479882d55\",\"6aacdb10afa6f2442905d4c90838f4c50f16901d73ef2f5f70b8763ebb23a40f\",\"7559e7919bb0021865d2004d8edd3296723c4c32aeed11c070749c641afc6480\",\"3d624a6dd69ac1467c04eaa943c0adbe32cd06bde23029574263c561649be847\",\"6919ba2211495478d85ccddeef94e4591366d0e4776d7ab44fc9ee0674908b92\",\"8694839739e1e82e6af5017f17e94cebc940e549f9566ed3f5facafe2577a562\",\"267716ef37df94fc6b7fd2318aa531ed3a40fac375afb8d63d63e591039d3c2c\",\"0b975f6cbabcf427d16a852aa5a934b34a377d23232b75f73e8c026394109c22\",\"de952b38530993b45c3a87bf8b4da8427a552cb78127fa1e234c52a3f3e53d75\",\"be6ae687e1f621f056b6a3955bfd1bafa463c25498bb4f49714600d7257c36aa\",\"3e6ecda95a2470481dc044e66e5812da4bb6196a03364b9859313a6a84eaf10a\",\"105f3c94474b882dbd8b61b0d069cb4ca1c41eb453a5882dae71e8331a2e4957\",\"8ad54483adce61f19a22ec8a9be7bcacc5876251c656e0b3eae66cc78ea568fe\",\"224fef2cc28134689d184e87b9993a481c87c9f951358d2d352e891242267bac\",\"ea4247454d26d2550ce196e5be3dce8ce10c942c6112bd37bfa5d5280ddc66eb\",\"257fd14be7d2b294a934051fd1f5ccd927ede235a97d62426e80055898ca9a9c\",\"736b3291735d4f69542f379292575b58060ac016851c4b4e8be43ba2ec86cc1d\",\"4da3bec48802420eb03a0359413acd2187dd4cb3c9e8e01e1e05ab627cffc010\",\"63995ebff9e057b438e9f11607949fcda6ee7f56faa12f507ad1a59a41838a59\",\"5ba10ce33a8310c173da80611374f7ad82635c88a4476955a20aa2ae7c3710ea\",\"04226039444a8ef9f71ac9cac0afc36252f9b1bae7328a9429b1496141c97026\",\"25a34d4e69b0fda104018103405813f1c111db39ac127f1a81e161ea7b492694\",\"69cadc88e9738319b2a1d47bc4f821f0bc645d2e9bfa77fd3051b75bd9308f10\",\"ff25c940cdefe05e1fe6ade367a87a0e782dee8e2e04065fd051ccaeed9a6f30\",\"2b636285cc95b6e883074d9b186ca45977a5f64219aaa676ef37aeefedd9dd63\",\"83b2593eea94d3db15453535c2e2910659d6fc4307290c8e3df3a1140402ec2f\",\"cb042ea28ee9dcdb794fe9a51f5bfa251c77e38e2747ef040fc7cd6bea6fd17e\",\"23b76364608e0a073c9d72fcff7995b9711f693b7b17ecaf8e5a14e809091727\",\"5b92b0802c3deb1fa6aefc395bfa55063819b54fde962ca15fdba5eb7e4e847b\",\"ae0828588ba4eda7dd5e021f4b6a2fba3e61aa8b8b667bc14af27ede1cf98d97\",\"f3917594ed8e936798c9be53312d1d2b35d666949aeb0f91a86f12116c1575ab\",\"67ce376c540f99e3a6eb9520c76835bbdb1b37d491485227b5a514e0c369f223\",\"9bf63d0a8e9527509bcafe1b0097a4c79966a2d40f35d1bafd42254e1a64b68d\",\"1ae3f49739f50789d6cdc81cb9da513569373789293ad054b2496f9360c8086a\",\"09fc8200e4e7273f0063b33cbd929f18368864ce3dcba8d3cf0dd6d01cf86337\",\"1838f6e2521f0e5e15faee97f26c52c6cb23b93f624d902869252d6146e5158c\",\"dbc1752f745d6850224a3456eb20e7816bd980dd25ba52073d927667cd9869e7\",\"6839686263d6716c31832e785d8dae77cc47dc9e68235e5504e618210cbe1b6d\",\"b4399319712ade74bbb6b495a2030e8b2758490e91672c18748aba11a6284749\",\"b49c634be6177142d21659ef7c2deb86cfef889ecab4e32fe2bbed2e61ea819a\",\"046b095afcaa990a987f15e51e85a0d4ce01a71f002a438f3aa6809b6a1985be\",\"ed8d0a32706b6270ca2cf49fa098677a0234576402b7a1991ba41abcd224778d\",\"9c2485209da104520f96ef9111046174cb507b0540d25cae3d67b2cc9871c864\",\"4214b66926be7fbeea901599842b4e6e78557572ac87b9d2c4f09e0b8a574038\",\"af62e5d673c228bcdb6313a2a53a9af4f7459a9093942da7e0227ca724e68604\",\"2eef0e49a6df66a2e220f531dedab2bd799e46707f634d8f0443e992a8f76b5a\",\"4808efeafdafff0e67cea0f137c7ffa2267e52ebf4f0398c8d973b86c74ae5ae\",\"97609184b05371b49f3190c99314913ddff37bd3c93ce3542ea32afef7e5802c\",\"a4cc6134c182fe487cd70bbc78dee548e8db356f728c4af6ae5cd0a03d663039\",\"fd7dd0d9661209c2070a83d96847651a0182fc5b098c192aba6c3855e67b9db1\",\"06f534007c7d117b0aa2b2eeae15d8318cfbfd3f40603180ea5a3b075f61106c\",\"00fb1f750d293e08fe679817f1eb0e56d64214f3a2837bb8a7b0f8b168026bda\",\"a47802134159fa48f791d1b54ae9fa3bfd086ca3bb1f7936ebeab1831f4a4d92\",\"fc84a089f9a33d8ec4add6c8a49c2290c885119de3f6ee3b67cb08ccda0bea7f\",\"f2df2b6ac20d2326901d2501f9534d5e17868ff7406c6aed9483bf0d0e3291bf\",\"96c5989ed28504d9c4114258a0b117a6c0f6361dd7112a7178859a77c321212e\",\"c51a36a0c0f29bd1fa91d6a948525ef5781f45d5323d0b467bf6068297b6fc25\",\"aade088b3a16c5baa5e8fee2318516ae3a021a6cf41f9527595d976565c3e9a8\",\"35cfcfe1bed9e2eb4a3ebcb5e827f93bf484a7aae4592604e94690d3f562aaf2\",\"e39215866eed51b148dd7461fc12bd698ffbb6e2a52a874dcddf421f5dffca6f\",\"38403ac2e4d6b01ab3aef8d93bde7b9c45b45d73d5a6028221c21e01bbb474ec\",\"51a53643c636822a0d0261fe34d2983b51fe25c408d239da267c58215062c8c2\",\"62fa73d04b66215b62efd24b3716eab7d61509683eea7e8219ea2b655766113f\",\"e974d3fb63576717ad92a61c82a5f7c9f94dd28acdf14b345cbb9a079fcfbaa1\",\"504e499c730ade8df429536fbf0ebb7f0a1fa78ef2afad029fa5b6281791db0b\",\"fa90a944b3b7929245dee3dfbb4ddee9b46cde73cf06dad84ee00ce0ad1070f3\",\"f4e857426521c6c7c2b87dd028783f78a14ce3f01deea43babd8984304ff5e72\",\"db1108a01374a170790827dd6e7631b445e6a2af07d94fe4fc4e29da222449e5\",\"eaf8d9b74670078215f73ab5c41db72f02999e0729eecc1e7ce396373fc5c7d6\",\"769db0cfcc8b33efb324e3d578b0f5fc91ebed9aa4cd581e9941d8dfa526f853\",\"08679bd4d4a4f677b17f18d21153d0f97097af2f4103ad612ae53048ab151659\",\"27797d8e6cb11469f30f48a3e873df033df57595f92d708944f8b008c2c15220\",\"c36d5d5bdf3d698e51612af04eb37a59972e3c9fe2ca1df114be54ca528887f6\",\"b4fa82c69514023db95b77931be5a7154e0ca8c9e8709c3ac4369df25c3e4da2\",\"27b7ed7003a4d684deeb7add7e60685d8b4b9cffebac3f758ec36786234b4cdb\",\"1f01a2af9b23f5ca1492c1fb68b6b0edcb6e78352c465534764110dd76a956e5\",\"a3dde512496b79536f912da1e98201f8d5d3f7a5561e4197f405fdcb79dd1fa8\",\"138dfe5a1775a4cd761da3b75644ec9a635c0f093b35126b63ede4b1cd5b45bb\",\"69e1d695feb30033b73e684b2d6172f2a5e166a968f14609ca2f591f9e9b9f7c\",\"436529300348259b51e1b08967138dba2e4a9878edc9e7d4b3934ad846e00244\",\"16b996c71242333cce88b98532b3b263e5a1e5d6e345af6f807bbc7a6f4c44eb\",\"266493ad60192e180049a655441cce903c76bd2c00310719918a43b69fbcdf89\",\"19cef429cff80f162013c17cbea83c0de6599590435dfb3122117d4e180c5d62\",\"54866746ff887ca60c51e07f5b8358f8acc68d7a782d7ce3810f0e8aaf9d0883\",\"a1cd2059e62ec439f470fe7aaafacb104f0fe889f9517d57120a9ea3047db4bf\",\"4d44a3b51eda6d8789fa246f4da1e94e8fdc7fc152a9e11729a7a9967bc82d22\",\"84e8a5842bd0557d37a2ea90d74675b5a2a2faf914a681c5ffe7f9bc59674fb3\",\"610f084db03328671c61d494d148a4f10773b570437083f306af357cbafdc7b0\",\"1b97e251937bde7eac9e5ed4d83ea4431c6662123c99419b65eb4266dcef5c45\",\"16852ddc1940a1d2339a58039a6843e7c34f1ce320d47314ec54c5a0cf0eb79b\",\"266784110bb02b0d84136091b2762935f87cf4b9084645426d0e03cb0857cd61\",\"3ad0afbee72534386001fcd9750aa4aaa4751156bc456ee05c39eeff1afc23f6\",\"dc50fcb60c693d1f2569e1ad6cb5b55ffb3e37abccd0e9a759550c0ab3cd2d01\",\"81ce5822dde6b8178b59a57ef990e9b0d8f496549ec9d4743d3490b1545d6dd6\",\"adaba79b65931e18223aab9edd59e44ae33f2b9640079d9616e91cb1775b7e32\",\"10ec8a258b93c41dcbf66c31f265c216f379b28a682ea41162a1360f96bdffb1\",\"a57e98dc6653b29de09045bc881543926a27df9ae2938e96fe2b15720379c429\",\"3696152e390df273241c11aa49b279eb86c3c1b2cc433d8ce74ee87581c1cf33\",\"e1ed4c566bd731aa6600a7d4cb58fdc96b5e68a417c30cece08764c2078d4039\",\"1fb35994d58ad6cf4b9ce17860666097dde1946cf3911aa0e8ffb29ec249600d\",\"3043250f1c7e596e605d16f200d03110eda34c745d715eb0893f5f23779b6909\",\"0a4b1c66f6fc48e4578c3cd2c6d8c5e043c9662d09882837bbcf0bd0a5013d94\",\"1b178d640002d8a51d1dde07f0ee5e09f506ced01d32aaf9a9af2827f8c4bb6e\",\"78f6a6129e6db38b147ac0d3f7b6a398fe9841d2f663cc184dd2e13fd9897aba\",\"bb24b57f46ace4985dd4cd9e5b514211bf75806ee5bac23a84bff16461fdbb1d\",\"643b57b0c2edb04c85b284ed1f0e57ae69fe1aa2572717620f9b293561d4b6a5\",\"97149be620eeb7428929ffb8980c7760c89277292c6fb19e242aff0d8c4c3112\",\"bec704b9ec609e5626ae89a89bdf9e83ce1bbf39fb2519b40d7c55d7a0158979\",\"44307a2f24ff162a87a1fd133650021be0c6d1ca7658ac8fd38f6735031038d9\",\"f62bc8ac87448d3cd952dcee3e81b71e0751beeedf840948840d38f8019353a1\",\"de13efd45dfc320177a006d71c1fcd46569e13f5c01359e32415f1fd38a7187c\",\"97df37d5e7df7f40049a1eb0e2f5186f506a352e21b50b5606d1b12dc8767579\",\"cccb79c435181048ce112b03eaea368fd90d385c5b8d5afd9b3603d9a9fe95f8\",\"9ea02d3a8ae0fb3affdb597d717e4b97e8e5b10a77c35ed764f8a9f22faafe02\",\"dad67f7c61a5e6d20cdfe77a0bbf5d163a5dbba6e5e1fea242f03ce8ffa96be2\",\"7b573655d923cc12f833e9a401384564a27f65d7ba92304a1917f0e3ce2dce9a\",\"2db66578e563437b7e5044952c00e7b4ebb3297b506f59b73c0df7e3180e1746\",\"6d3b20a6287e971b8f126527a361b5b2dd8b4b247b36a9f43dd44931ad89786e\",\"d35eb23ecec0451c928b6be7fc4353bc519cf796806aa034fb7ee469caab5c4a\",\"6d6f02bcdd4b9a967973f873b158c7b16d4c32f942accab7c1c7963791dab6af\",\"62bc1c00f6bd8cb833bb795f529904e8f3470b2635e9b9a4f5787bd7733bb1c4\",\"b6f055185a8d57126e716db43ebf7925ad025ac90a50c9b3b2450327639332e1\",\"53fda6f33dba706cf595649f7abc01aa8a997feabdbe5a448714542f3ef08877\",\"cef3148988b02718325a02ed1f4bd567d605b874ec93cd8b098082acb7c01b9d\",\"9ffceb3d289ecf4d1c600a2ffacd7113d42d70223aac8f4d921687aa0f531679\",\"299f6326eaaf4ad475fc69efe376f11e820a9dcaa067a8c2222e823e25f11886\",\"6fcc263505f8ae84d5aec2ec2001f4aad5df93a94d63cf2960bdd0a19a11a868\",\"82498bedddd063baa917ffb6c72e6c72f9af0fc8abe4374653120ee9b98604cb\",\"7c7384d512844701cbc8374cd53cdd23527c98dad51b52529bc81af588aba210\",\"0848ae3ae3c5fa35ce31501a6b8e4f0b4395ef81b405d2f6d9f25544329e9129\",\"fcaf129dd46b76d467e57a8d983e7e8e83681376f3a4ed6559982d25f7bee281\",\"48b6e3f71a10ed7a261033fa371de230ede4d7e94475ce5ea0b90651b01abd2f\",\"395cb4743a81af3cb0a63ec52f1c57884043230ad1eb104fdef1107646fe8759\",\"1fae3e36abf0675dfbeb3643277a38648024daa5b483968726700ceb9cc90cca\",\"4132b8e403ca6c46d127003db12d590b578b8a2eb0681c95e9f6dd139d160451\",\"8408ad1bc2454e446b4362bcbea0d5921842f519f1156623c62a2c96ad466f67\",\"707e5956a497daa93b858742c0609f3f681787ae709880ea7cfd51e5ec9fe945\",\"a196f11f17f116f9c5f04c0c8e0d92ba0c137919615737e68f86d4fda62aca9a\",\"e9d6053e323c20e8ecdc5b95ed5d1800f33c62019be3b6b1c6375ae46c161fad\",\"de5f275597c143d6a23df18897c909fb456fdc816b391352a282a2ba188aa0c8\",\"e4bf77b7fdafaf8e8cbfe787aa53247d554d5e1f271ad4eaaf523f7f2a5df5f6\",\"56c53e858db2a83fe210f33f01f69a16d0eb33d25a9214416227dc1b5c6c9ec0\",\"6ac614db64c57b92dc678b9b3495f3d393a3d3d1a6aef7c6c3ed69f040d002b5\",\"cedfc84319a56f518681b7888a112c17b93589b564b72f0c3939b6c9ce873177\",\"299a12bef8a9dde869e863824a56a943ffbf7d6887932d3037cfdead8e8d3dfd\",\"a10f259d7c2f5e67400c44f8105e51169627d9cc59539faff59779816270e0b1\",\"e85c4e7294314015846d40f01911e12a59b34108dd6dac44806e4889dea26e08\",\"9c280f38680b2bb07a82deca83b769f40b9662cf84bc1ff493bf9f4c7c0d7639\",\"9efe33bdd45ff42e932106bf2dd0c737961e0f0c28566d2cd940ca85a3201bea\",\"219f5fecf9ee70c62cb7b899a3605b642829ef251009ae72439c1269341b1cd9\",\"d1feb19bd1f254b9ad77fe0c85e6ea268144435092706dcd1421ac74b651c19b\",\"7aee5ced14b5a65d527582ab6204909113406d1c8d5ac565032ec196e880ee49\",\"d790c950abfb5895fe5e90973ad9aa0d9f094abb369ef7f3a96326dae3dd9c3d\",\"bcb74a732ffbcac4221e96a2cb7afb42ed7e99009b30e31b4297838249bea124\",\"b8cbcfa7dcc3850e4fce5062480bf8b625fc307469ae1267a0e7adce67c781d5\",\"68e122428c6ea786c4215d2528ad181f01c6dfdfd8cfdcfb678788e6f6232101\",\"d7dae1295ee03b79992742eb11c28cac6a2412b8ea592002c2265074581aee3b\",\"d2f09c2142e7cb13be3e6527f13439af7c7990a86e3593ce5cbb9b217e718bee\",\"5e0d80ca7f45c4b6117bd1f98fae10abfa7da19b22292efd9d947db5b20dbb16\",\"158347b9a10069e7482408c0e721b18327b1f2d1c00cdf97a73c0860c3e38e8a\",\"7aee477d75f256d7e54137ea2312d46cf6231b814cc26eff2d4d2b92ed0be1db\",\"d299d4b8e82e3270c60e00a637b00de14f4c27022ee8c8bffb295aa384081f6a\",\"90b6b840aa069cb31883cd25bf1e746624302516e740b024cd45eb586e1dd282\",\"547c16de8d2b6fcfb34fe2092fa7e26a2579fd6600cc271c74e07bd31d9b2e39\",\"3c97ee85fff4a290078fa5665017a3451ff8fc03ddf7b07e6d0b4eac6ebe4c2f\",\"d2bdaa039fb75f200fad34bdadb89ee12c7f09f32b24054095edc098ae3e66a5\",\"e97a5c060e25fadd0eb215e96c7c4c40e5c5b591affa598b8541ddd28945617a\",\"c21ca574d84014056f40c39d9c39a1d832a6daed77ac79bf3645085c6cd3d228\",\"e61ea585ef52832ea65dbe0da0260771763741daa90d60fe24eb5333d5df63b5\",\"e5d6cf059c99d3efd5e5dff774fbff8ca8aa306560fb09ea0846905019e46a11\",\"d2e3bf179a0d443f3286502ffcb40306ce097006453aa19539115ab929051c56\",\"fdb5a6130ec877fb0da54d4e98e78d29c763b465d65b8b71a0269760c922c960\",\"3f95a578394e94dc774dde7e14b6ad5f35abb4c7a96ec16a9d9b247f4a62346f\",\"36be71d2d9bfbed3882fc82623e35e16e50ce5f55300f6dab9d2102722a464da\",\"02fece2332a4562ed0f3ff33f8e81a6575936d7de12382a816dc4fb3d434ebef\",\"e4c2d0d3002b01f9c63778c04487b65ce1cfcbc6737a4c10c5b2739ae787d767\",\"a7179469d70f66be2b6f88bf4e1e980d62899f9f8a8013f65fbb81406bf33c8f\",\"37b56c0136e41c20d988f6e8a87e048c606d10035abe910d0ed37632ffa3a17b\",\"3f1bffd0336fa6fdc46623cf94cce39b522a4ecd8a506cff700c24f3392ff3c8\",\"9b2c25f499912518bd32edf580b0788eef2b7debc430da7bb94112c5d00eb1bb\",\"01ccb3de79fee9a820e11c7b9ddd7bb3a477655a7cf0263da3bb9394dd384ec0\",\"6a5799651b7497676a5962719685d374b9851b7159c00505fe374e8588b901df\",\"cf2e706f1036060383bc68ed107b60caaef7e664deea1f0626aeccaa55eaa00f\",\"09e29e48101b801aacc5a0bf52b13ee3fcffc9742d929ed22b8878e0c0dcb4db\",\"0c840efcf14dba9310363f48940dbe09e0d1fb10729289ea835b4a3e441a3091\",\"cfd54dd1a23a608cafc9bfcb394b816b05e3f2b9167a09c955b4fa5fc89316bb\",\"64a3941d695045047deafd9c5639e7c377cdd1dcb7e750922d0d55fd9f906795\",\"c340089fd248d62da351179cfa746c6825407f376f0641cc482cf12f5e2326a4\",\"331cdf42da44203cae29b23cd69f0402e4a9fb4dd0672a7c8e04525f767f3a34\",\"9f3949e524e94076424b3a67fd01e12829eb56d7c138d72ce88ecc8ed22486d5\",\"2fb961088a2ec4b039c6c1f4812e6381c793f95d4d4bb97194d37cc3973f9d0c\",\"e7b744bdfebb43151960264cdfa227e10bf83baef4390f80598bf2f657d4aa0b\",\"e5c89f777b8b576a739ed142d7df856c978cd5635201d37b035d549387c17d14\",\"f8d429d98a3d973efd7143fe170d4d9475b7ebbd691eb3bb51b57d1d4cf2e9be\",\"781f3c10bf7a1bcbbd2b2352613f0df058719e6ce67205de41527afe1b644670\",\"9729f1f22264bfc51130eae9f848a22115ff10ceac42052915bcb7e0eceefe8d\",\"f0882fb5c828c0c89438c0942235b810f84555a6a485637485d6eff53c8a22c4\",\"7db543ab7be899f0f367cb3e60bb8afb400be58110ae56fc7e25619a6da8e1bc\",\"c42032d0927fc46050ba3dfdc8e259e2bb1b0928e0e193339555af2df40d062d\",\"5fe5e49d625c72615ce218a8e2d0eef5d889387c698e0afcd1b18c1bf7d54182\",\"63560ce1c9eecf7d043b85b3c31822afa3bd05aca77bf526b7f5e8347135acf9\",\"f1f3ee4c10422e922a85a48f5af77874bc14efde4e8ae650ec244e05c5f6df62\",\"224dc20403b1e3a0380b60e13ef4ff6f6cf30c57081bf41962c5916e29b10863\",\"431c48154aba713e6c53153c7ca0c3376a160eea9872eee7312210b244589d5a\",\"e77d684862bc57a79f63cddca77d81b5122ebfcddba8e7410d64343a1e70ad81\",\"f59fb0015090e2daea24038189868697aac05d8758cad9804e590053dce9aadb\",\"8a6bddd6262fd87129ae4e9a5e239e2436a00a3276af8d7815c39ac22f8f3a81\",\"2ecf060b05e95db3f6b5fb929b6615184db0375b8f7723ae457714178e646ff9\",\"5011a975f4c037016fd0ab6ae4407e553eae5f6623a07ba6c6a6f0713b2388e8\",\"6d387a54aae505d4397f81108b24a6be878349a2e70b27a0edf1c7d3e82ac958\",\"2f5d9d8af6d4fab85e6cffed54553fea4d7f7863fb24c7c70d1e4e0ec11e64b1\",\"331faad6894989736386ddef5c7dbe92be80a4660b31c3ff0827937907364186\",\"6faa24364113ec671a5a494b313a3d93b650bf656cb6bddf24f13db37d03e193\",\"c5a75a557f3008144d22a452ebdff4b2239a5abf4c4e96a261f5d9e5dda7fc2b\",\"223831bb37ee259f1bfd270b19397bcdaf740db60f3a7b6258c0921aacd1533a\",\"bed12a6d4afd5fc6c0bac12a0ba120f0d7ace8d36ae9a1871ea3af98cdfdd0c6\",\"5713f420a40fab3662289d07a87245910ebc323a28b8762f5e20a5f9b329ee7e\",\"20449820a950b53f1ce2bee3a90d5c6256f2007f8b2b4f1930aa4e2854ddf11e\",\"13084e8bfdeecfce5063c5833926c0f3b6696a3212a0ce6d0f60df84c3b706ae\",\"1abc7e3c4717a4ef410fb7fa44a4bf53e7a6993dca119b4d8de3faf0710a2fbe\",\"d841625dfecfddb6b757de453b65043877c6da617fe9792cc25fed4ead74f1c9\",\"79a561bf7595411898085ab257b90aace7fccb4fcb1152ccde69a4fb93bef712\",\"8280e15fc8401420b160b0b2ff0775ec4a98b0f5347aa3813f3980afb75d2ec1\",\"02610b2f90e9cf41aaae47d1f0049e950796fb07f648cbf7457267fb87753c9a\",\"608b9f6b9e3014f0005be2ff1616656fe84565a29deccd40a068dfcee7c51f14\",\"1b937154e8fbc5b085b69a7b2396a9f9ef3de1f39f0bb77bd1cee5638b61d993\",\"36d610011f91f66e4b3f4ab2cdfe20163277fa3690aebe3e69e11f914f1cdebf\",\"a607266a546e3e490175a271db94a15fee37a13cbed417e01ff39da69cbf79b0\",\"c8f5b6d652c78fc23fe8b57e4519c513e1a982050637357dde5aae78dae58919\",\"2c92273b3696452565310ba01f3b602abae88f547bdc17b42c0eef666cf73e39\",\"448d0d0c3efd862b4c03f114e630031bc450982df3f6e3f86d115eff16a903e3\",\"995c05f3982df60d1b12409dbf3270af7a865221a4081430c94610657ff29e40\",\"c1c4b76b68a4ba71129b93f07076bc59d5cf15c29ef7f00140ab870504b5e9ac\",\"9e58529cac9b9007a68099987f4c728cfd54b144550b61ae278f10c058b84a44\",\"a6397f70412fb45a38b49e631c9941a10b4003bf333ac96393ae37985e82e2cf\",\"e6ab45a74d6e9b51a0af2564140467408f8f18ede490ed5589861a64881be497\",\"5c00ced687454f11949cb59aeb8d69522811c21572895c559131d022a0497335\",\"5fa404f940f8dfc1e0b6dd40c6d61cd50771d70b3fd74040dfd88d578ca06140\",\"5b0d4ccff39638749b4b2b77f874b5ea319bceb9574bd7f4c63f0eb24c5d7747\",\"3fe2e98696165b6d33d5240d3d1aed4d12f3f008b0c4b8aacc0a7d80ad48edcb\",\"6c6cf44e57aeacf98d89497b633d1bb666cb68db0b6a52adbebe6ccffc31944c\",\"8b72ea187a63f0244ae67d6792880e55407756653badef646afb1ed5c2ed69f4\",\"05d0f1a8b316b7e32c170d9aa047970ef025b7b336d7e67c4af9d51a14359f53\",\"1c012f322dd85795532046984ddd8e2db6f71c992a963056eeb456f4537576ee\",\"c295f96dc17a9aad9c6939d32663a12dc1187285c62e20235e1b29f5b1521bf7\",\"0734aa9f7153559cf237b29932b705de152a82929e1d39d7b3388c409b850334\",\"5bd95db8c2c77cd5a87f38788dcdfe5bec2c82da7a920d7aaf988ead291b8955\",\"09e49cb58be8de123d8c2aafcfc73c4e14b9ec7c8b4e66b0043d5022b77fad15\",\"e4a5455ba0e7791ab8e345d2d5624e529488503a4577320e9fb626f0f22f9726\",\"bd56c4d2f4fafdfb62de75df76ccff7b80007a7190063276b433bcdaa9d90871\",\"3f4c4e557af55e91754cbadae11a056bbdeb8f8eb370ceb267dc0416082c0b36\",\"28930d9f4f1c6a77790676f2d3a3bbbb8c9e6dec4d1a0e7fa6cf952742fb9bd5\",\"b9e1884666a3b4e6bfaea1c15967e04381e48316460089f1565e90b359f41021\",\"bf2ba5d01a0194dccef85038b4901693b2bbd29850b34703f4c76d0aed2f31b0\",\"ab8a04c10997a708ef36d43d42b4ac3829e01109626bbdf8aa8a3a2ec8f74d77\",\"69d0d9f616613a24357ae9bfb39bb9e600db6b631dca1fd3ceedd6b8fa7cd7df\",\"150382ffed00b872bc284ebf610e91b7e6d96c5d663043e37f4343600a50a827\",\"a768312cd2e2cd2e5484ce82c1775256570f04ce0e1d6d435231c4d396883989\",\"d7fbeb7add41e6d872a1861fa06348433f681aa45a225a3104fdb83500897619\",\"b74388e841bcb2c6a8d7032f777b1937e4c7ba432ba0ce5145c7ce6cb83c276a\",\"d3ec6e7d810bb9f39de00ebfa060fbade12f04951959e1a7f14782c4f83ab495\",\"6f4e1e07de7b684ce4336a27935e1d884dc31328295acf237838180ae81a17c8\",\"21c060bb179c8dcca5883a3d43c2636fd1e21b75126a9c45fe1c813a4468fdb3\",\"204f64307e70719f5c53330db5ea9b363f6a7572b3c20a87d0b060e56aa5e912\",\"8833f3d771e6030e10634745ce0f9aa328040ff62d042a1e68699c2fab1bbbf8\",\"27d4d4b7cb843a2e728fb1de6f9d34fe485e0bbae4afefe95c4799c0fbe06f3e\",\"0d9b0fd318e9ffce57839acf2c3966dd29afba1b5bc62d4f40d4bb3b5960c8c8\",\"47c7d9a228f40d2a63c3f8d5b7b18f8a3e55cbd31be8f0fd577cd3b899a771d4\",\"d943b0cac94623947331425e0c9b4e296d21f12c49eaa86237e29f0e8fe491cd\",\"13059e2bc6aca69720f57d3f15cc9a2e7f70cb7567ef41152b85579bacad6494\",\"e3f447c9fef8454b6b392f6d27b5df92a4b17a675cd696ae2e6d31d5896f5ee3\",\"d160d1f36312c70a81e767cd6c2e2bcef0ebd7e17784ba875f77a801286175c7\",\"d5ca43adb5414a932e7ed22871ca590e704de09a57952c62702443ded35e7545\",\"f3dec8e61c22674e212eeda059f41bcde9e879aa868c0e4f46b7112221c9846d\",\"9986c532bf322571fd38870efc637dc489af14856246d09318e64b5f76f94e8e\",\"9e9cf907914f873b0a03a39cb74183ab3b7cd4970f63d536d7cdd677d6e59b42\",\"4c0b87c8b780e959a7784366f5e3aa2db788e37c35f4f6eea0e75f8f463905e9\",\"5680fca5db34667623fd74e1074ebeb3234990cf0ab152134619023a7f456af2\",\"57cd295d1281df1aa49f754c306c6e96edeb602628ffd3c44d7b2370ba72567f\",\"c7e79204b4570baff88734aa84fc2e7c3c039cb65bcf550f5922e9c14fb89e20\",\"5650268a2124244b53210e426e94644ead7e327b569e386ac4c5f39e534a5b36\",\"39f0e565817a3179d8d31320e39400e72f2f10c87927923432bf6c7251f0a356\",\"fe1bf6937da0dcdae36c0b579915c326441a11c7ae7bcbe6b28d95e295b3d311\",\"63e6593bb9c5fbfc8f7bf5365c2f80892ff320fe2d54f8232bfa66f420c6f17f\",\"a15f4d6ce29867db21414ee769d90d3156e91c91b972b689b39715de8ad4a32d\",\"daedade83ea8b59fc51f1c37fa82e3fa20ae2e12e4541b5a8746bfea92f87f18\",\"37f95019d28a33899857b61db2b58a71abf3565425c180b6e74893af7d40f8fb\",\"daf43b9103c5f62d63fa82f264e7ce216aa46c33bd281f41bbd60ff749d34c9b\",\"01dddfeae0b7f2465947ad6ed67ba830b69fcea04d64c9ffc010d4be5c2c5811\",\"0a4b35c6d30767b16c6843ea6f26bca177713dac617acc088544ae63ad88344a\",\"b4290797c1e0b64320b566f18087956cd6709d24e76beeb7319d5dc46945ea31\",\"eaa037d09a617dfac22d019e010195813bf09c0fa67e0e14c61cd12c7a304be3\",\"298013707aedf4ab53e509cc7253170274e18a40faaf19c8c937cda79dfd9fd2\",\"b9a4dc456017824e47fedb1d21a7954692cbf597f28d4d50d96f17de66ea7410\",\"7624b500927aadfe7a7820cb6c855daf846c5e0ccc619260353e8c1fa9c25633\",\"50391770946b56baf3b9142be832c120dd915dffe7a6b9c3a6f93e41f576678d\",\"89cfb3ff1c73b7e61460d69ec9ed41dffda6f1735eacc749704a629efaeeba73\",\"231569101288167d980edbbffc2af2aaf9c5d70945afefa79796e53b115294a6\",\"c5cd9b4c0c7b66db5e8670ab657abc0b5c5ec3c3ee8760344ac367626c4d0f8e\",\"182da9c3927ab57fde158e68c101e9e2dc2a860df9352da606371a0f5b948009\",\"e09c9f7732738bb3b07eb35b6aa86f4c0372e49a739c8215e0c3a88b07b4ee22\",\"412eb0359390dfaa3c797bc9c209af8c7f91980a3d5a341749738e603ad5cdd2\",\"4f3ee711b2df7c14812f4c183a5a16e3acc1d0d73b3fce7199ed5b297e3b3db1\",\"2004aa55920bc0e4b2142a3745aaa5eda91af3be0eb2591acfc17639951126be\",\"65f2657f1339c871f9283539f0f292adca1bf17e6cb85b80472421650fb2f622\",\"e7936a7a88d58718121ab91b8080e813a00fbe9bd8828cfe44e1e202adc533e5\",\"c768fca6f7b3f895ffa8875653ba9201764c3e61f3de899e99fbdbdd1966de1c\",\"7302926cdbc6565ada90bcd5869f5232bb71309d2a8ad8f4d8de6a1d5c522ae1\",\"cc0a6825b4e06d40efc8e7eccc874a585ea5b56d8de3349deaa11462a5c70df6\",\"38a90bb11903b9f039900e7c179e091852bb5ed24ea2e1433525f1dd13de8734\",\"fb702fda34de9bebba3486757fc78b0b8ead47b98dc986917803fbc9114d6ed6\",\"2676168390c87fba03da6c7541f25b6e3932a4b65c6ca6f489860998bea2d264\",\"e9e07201ffdfa7ab6004c6c32e9a4e46a74516936c7b536db809336c3293b7bb\",\"4f260692bd9a6682c5c1896068e26a283610ca44e90bac86c8bbd3279b976a5f\",\"57050ffac3cade60a9b1f223e14caf34c6a139a70de16d81fe6c195b65fd65af\",\"af91774518652ea1b9c91ee0f441268a0381ff08d110fc2e722fe15c97b92dc3\",\"0a7254a9bcc3ae1879fb98ac029e0607dc16622c4fbedb144a44205cc0dfd8d5\",\"6e38dfff8de54ff97965269bbd5723b717043f15f4313735c640b4da6921596d\",\"109e2503de5925b087a9162ea2004598446d9aed240d7a6389fa078543bc0072\",\"cd453408cca03d01f97fe6b47536f0b67fe98b9f27023a9d2358c862efc0002c\",\"157b3926fc2f8c07158f680489c36c5dd2f6896df213a31404c2a5e059ad3471\",\"3277f8a7faa3134e85dc4f81822924c3f5ac2ea6cd75288ad69da6e86082dcac\",\"1876c5c4f2d2a5783839893d9b92306cb227fb4a7833568b426a74083437c7be\",\"588d405d555fadd2e8a6e36e9bb5055a8d3ae20277c8bd7d43971671d1d5acc8\",\"c573a3a9b9906bd0b0c604a844d1263a21741b25c3ae41a2bedbb5eb0fe26afc\",\"0fcb60aa7dfa83ab5402f024e67fe21ef0acc3972523da9837ed4dcdaaf5a53d\",\"70d552e9049e5b1acded3986671f39330bd0405e6b1a6b5a3a76b608ebe23b06\",\"4d7be3642c28916d236348599ad908ee6766c44d5a1e4d858003f60d053b87e3\",\"410311c63a96c7733ff0b32af385426593151b969daa7b2a386b85652ce0e987\",\"395790f45bbd3716cac4abddff1474d1e5cdc48a5ea7ff46cbf98d0d685493f6\",\"d6372d26c73b9a8649516dc21ac1cc476d4829e895fa9b9c363d13670976e461\",\"9710fc96e05c5d7fc4824fbd29d22d0a1bc9c3b381f9a73afa8226e641aa6db6\",\"028553456db7b93a4bf5dc9657c6881d92d97c904e4e1c6fe22d973b691b7006\",\"ce543525b1017825e56307db000173e6c4b08bc8e170d72d3bd12713636d6dbb\",\"03c76815da279cc459bf7a83396605a7c97dea27a3d2191e99778b90750f5efa\",\"a9ce9d6dd61d4a97f2d00bd335fef4ea5741b936a6607c53bb58439694c0c604\",\"8b62fdb81abb1b63f5749e831c47cd9b446de4102bf02eabc652d872d9637a95\",\"c1700e177aef2023b0b086d7083927db588e49625752394decb59f3b062bb3ad\",\"b843df423a2b6fdff56447150dbb8477bd4713281b9bb02482d581473379f9f5\",\"a44868729ff5e27d7b7a2f7a7f8d6ec8c5cf3c9f74de6095f4e82202a3e89f58\",\"def7db451a81594691e3bbf6b05182afba0ed683cd0d2564216230dffbfad961\",\"12b6679bb6297074e08a9d80fc80b7286335f5b98537fb95102dd3f70591a9a5\",\"8b08fd53da915fa1d5d2e5711f94dec09fff6e3abf03a2daa112c46862306f0b\",\"cddad730b4207a0287d5c423b6fbd8b198b523eb0741f7fb5f8d2a27f1ce8b21\",\"61c5dc393b0d047bd10a422135ee0b00a3f5fdc90119715273bf6e753887988f\",\"0a4df86e9b72bb8cf8f4e66e4d5ce0884752906c804e4ea26e6005801c9d20f5\",\"72c83e404b4d4bb35beac92daf96b909319cf10aacc9d4b0c2f41fa85f0e611f\",\"9bc5114129cdf769b2a93ec3ac9fbc9da2017e26c3fb3ef3de5f21516a527f9a\",\"4d3040b2cf6d5dfd9c66c0236c5e3dc5912894b61534b1fa34f5fd0843abc191\",\"34d5a397d54a82ead148055dcdf27488570c269eca8dcac496b46ed5387cd90f\",\"59963ef477467f58c0a26b64ac638cecc8af4cd71bda6b1d446937fe848afaae\",\"f2446bf71a5bf4819db207d677a75188e36d0db349fc36605baaa27dcc24f8dc\",\"9a3a061f7b64f53d29006f53b80c3791a8ad1bbf21146d6a853ef9bb372d3958\",\"d7883edc9f75d218af7908a939c8e3bbe242b757ea245bf6bcade31c8e7d1105\",\"8aadb8bb46a7754c736ebd15130076753f8fb1997bc3a22b9709499e837db5af\",\"ba15e68f3395dcc50d2f578b8b445c9fe541024e75e8c32aa09031712709c230\",\"23172b3514af2bcdd243e2b012c548664f0c7dafb23cf4eb2e068320ac3f4830\",\"1a38a35da2c88ae5304f86715f054bec4df1f2e6b33cbc59c4da6b0b76a9305c\",\"bad408f0471373abbdfd78e65edf0b0f9b383040589f53eeee6766372e9ca775\",\"e891e60da3644b4aefca70789484c1da85e1141a9bd60aad295efaf5a4133816\",\"73ba8d610915ff659ad0f7c82560dd04a667728872899a22ac90847b1cc49970\",\"3d7c9207e7d1c4c06d9f62a589628027f9393f6f91a1eb89c9fcd238a3143d84\",\"900cbcad08e93d4d09cf8a84aac3dbe47c8a87b8410b4dd41e172a390645d9e9\",\"f58dcfb3c6afb8b825d0bd24d8f2366d35c894692eb15a954d5758bca2fd3024\",\"0ead57433f61c25994eb4e14a1894e233e0cf9f27961ad5c05596916c2b7a6f4\",\"b747f022a45a0d915161c0ad854e167e63d6a6145db8579db89f2a81176339a9\",\"89efa04f0516582d31207a3e24202e5af347d0241eb771e7cb3372b88426e77e\",\"d229f000dbcf1d9d2a1e1327e55f6ec13a6101c16daa73a20ba87fb6b2b0b032\",\"71787b6aaf0ef09e94db95eb358a8d43b6f9bd5d52aff5d2033a083e270e6747\",\"8aec39c3f6100caa0c986426ba75e4561e919ed462b10c8160780aa535288048\",\"cd1d3c48e8b04cfdd920724ec380d0369604a2d7eb503fd6c4dba573e3407160\",\"1043d3d52fa549c118bb3647056d9e6e1678e77bcef158aeed526f16fc3d1214\",\"d3b4d10d809f585dadfae6feec57c48bf5d8c051f025196bc3f9c8121fded953\",\"7bc044fc791cdda538adf5ac0c1a88844a9e3490d440866c92d7633a38a24a19\",\"3be72fe7f12daa4e7518fd25f52d62f2c56309ecadf2069808bd80ac90bfcf1f\",\"5478c5d63496ab9d9a8cd8978c5ea28f06581fd24ac047b10b081d95bb56ce83\",\"41f7e8dc56e7ac88ee4b2ced079a4d737c4be96c61f822947106d40414d87342\",\"f78b32be99377a42c08f9d286c01ab88155cd46d3bde39235936e81393303a7e\",\"6e9982f47a453aff4a402693fa642057a0f7fb8cad0a8f2a806e315d9db533d2\",\"b447488ad9a47029a7ef5b4ee70bd14aef5013e84dcc48af8d9f03cee31a74ba\",\"a123d8d86e39c7072d6e00c9c6ce1d525686b63a777500e7cabd5b8befb0fe28\",\"99180010b4116095bf6a925c020e4a0a35b7424942d8fa5b679ef5cbb5979869\",\"9bef88bcc23e1d1a502c338890006be38b14d3a731d5117d11b1845dad341c20\",\"e256e769ad0d958a27f913b5f4bd3c0c0ba8cfd3740a2e4ede14ad686d69c593\",\"d4c5f277b992d5ea011308b94e484506dd9ad11c289a55885442d1d3a5bdbb61\",\"953f333f6ffeb678545698ea8c78ea17df1dfbd3cafc87001fb762e6da187d00\",\"a5c33c02d92b9683f5b8f3ed8a9f9352a007ff53981ff9d248c6ad1a04816af9\",\"d3c5cf5f19c30e4c683ca4b632547c58e29f5316d0ee3f7dd918573f52a5e272\",\"43ca536c81b0c5caca843f22f429c0d67d285b660b67a6025684fc25877aceaa\",\"9131ddaba31b6d49b66602331540b51d060f70c9820e546ee80b3f0603e3ee63\",\"5338315c9cfe32a6a23a92c1e5306a463cbb0e96b1dc7997c8c97323a2897d27\",\"8b687fd54229d43fa6dc462c37dce71a9f6d9bbda3950270da9f474c14b83ffe\",\"76e3fab7b07eb32bcd568e177284a2dda2c36f2953610987d9d5df452b4d6b1c\",\"fb3db21b4c6f2c6084607567d70b11506dfb60e111e45c8d625c7e2e429b7a71\",\"0a3bba0750b9872e708328bf9b3eece1e08ca70e5f7d8fd24eaf68c71dcc43b5\",\"3f0c6e70847596e562ab5b99596e6caa6bf2bd5b95517eedac352f1d33035f9e\",\"339991e8c36ab1534a40b8e502c6a4c44d26fae5360c1c5895a22d8c0e3335bb\",\"08ce83cba5dcebe20915abd79c1c8e0f3972b6bb9aa23acf80e6be0ba9bab10b\",\"e192cf1225a1077bb6d7909fac5bf0fe0362831f2af946ae11f5ced38da010a3\",\"4900530e9263437219788a2f762eea4bfe2d834ab48a6b6bae6286c0dfa912be\",\"625493d2b4d60a514a7f605a57eb6d90a4ad8e3929aca68b966f7d8664ce53b6\",\"ccde359f101b0665c85f65557b75fe73f5043a9dbb05a00c00ff4b62a76c0f69\",\"3f530930699cb5fb16063d682d874963be2781a4eaa7d7d72b856db3110f6dec\",\"cc3d482eb9fedc0ecb290a0ebb107e4bc5b67638a6a47bbdc834c4508d9d6004\",\"60cba512e65efdbaf71862123ace23cc459fd8af02eeb858dd1c11a0ebec2647\",\"0652f7fe17a1366b89d20fb5b223fbb4861aa2c6482deb22c6231d3fcc1bf51b\",\"71316a412160bb3a7c49833753fde126bd6ffc9fbb20294f5cf5740b1e8fc448\",\"57af882a83f7a160cd00fda86e3918d0475853cd8e3fdf79f3e40ba8b1cb8956\",\"d5d10ef41dece8d1050fc19d91c2ea77a4532ddb0176c094fac8b3d8f1248146\",\"0b7d403900470a73e9839f36c208a7c75619eb9dbc6ee3e309f13a141297e2e3\",\"76f76fe4f348147bd106cde7b0a61030c922bcf3df98186056b459f151b9cbf1\",\"94be8bfd8de5f2d909906af612afb312beab2bdeae3ef665022a1cf049331b94\",\"3ac71ff0344744f023a09b148aad5d6280c6fb27d69ad5bbe0ab5a11afefa32f\",\"c5b80181aca4d73bb0fc8675b0297d9d22f603351100dafe6be0093b12475558\",\"16a3d4c10cf620bed2633e876090de378ba41b76510b7315fb55584e3ecae4bd\",\"6af66bbdeb39410fa69d433c08139919a49e1de69d47625a1f784a7112bd344d\",\"48940bbd490e7e7f39300e7c3401084edd7f7973ac228d21380c4f20ed95ac6c\",\"b6faa180d4afeefef9ef69389f618c86cc90104c69111cfbeaf4ac3fdc63c1d5\",\"6e4dcca1df8c70659fedf97a6e2ea2ea4233caab975c67f6e965df581e221c7a\",\"89069c459d1afe9850965c630ce1881db83e4d719b8d89153cdce32d8f4954f7\",\"efe781945b641dcbc0c8bc77f9d062724036f8920e584b108f389ffb3d169903\",\"4218bae55f743729eda8258ec914ad7a9089829d67bffb9da485f6ac7a7ad458\",\"154b4de7babcf9fa7f836970fd33bfe6394d37c90ba6e1b0de7a8197a99d8e52\",\"9f3429dc6fc443562597c566a81e2ae764c3dcd849d650efe9c735ae02f58c1f\",\"0b65615b09fecccd38a8d841f03651d97f0c6bcbb49332a094f8a24361956a96\",\"15d3139679b8bf0a1a4d086b4978ce241efb7fd5df16589bec25794c71d56a3e\",\"ed6e1ede79b115a777333f5d82dea1b45f0b071cb797c120f2ba795a105be9bc\",\"29cc157bd6b490f4aeb92c595cf32d2f6ac29bf50b4084e1a2b04385b932886a\",\"bd1eec427d2cf490e98b050f7714cfe43f35bea71bc5494f0faa6116266c65f9\",\"2071f4b2d7053d79edf93a7c9bb57d3349fd0611c81198ad3f9c749b06781bb5\",\"a03be34bdbf7eae8b87812ff2093cb98357e19f88452bdd6b5fcf89d8680c0c5\",\"0ee14764157196d2f8cc329a70856f4146b542168bd770f727a85f0a3ae2ae13\",\"0f33ade1d62784a7827e75090659a4bb30aab4529a69429b91489bc8b3803600\",\"e96529b95a268921719fd270493b191b749bc1f409a48d0d9eba13787f947a8f\",\"c64f1bca748d4ce948d9688e85f5638457089504361d5cd02772dd1ea634e29f\",\"54502d226c03f2eb6515c0b75fa5c88426426172658a512d87d84a791d0235f2\",\"d292a38460fd80c345ba643286c8f8abaa2d29fe2ff8ed43990d9e3859991767\",\"a59598e6c625427498784ff27d4097e4f4b70762c20c4479635c3bdf4ad0901e\",\"f7e0551cbce24d91e83baac8a4c54c03a8620c1b743646ace0eddcdee14ab6c1\",\"caca5e423d571d9efc59ed92d031a00bbf34fd0bf61f5091d2869b1003ef6e5f\",\"867d98fc5ce2d422bc4afb1bbb9f846e872bda6ce9d3e7010be4452aee789cba\",\"1a91588ab0f0f8e39588c3833e283f94804148b1a7a50c2256be95b02e00343f\",\"c6f15ab287d4ac798927a313d3c320d641bf1944caf9d475cf30d2ed34f3501f\",\"fe92792c5fe574ae416f2681edf39e3d4c3dcd9c4ef3c2aaaed4e55bbb5121fa\",\"c6edebe52a636a3f018fd3f1ad7bd2d0d7c97fd68db077794c6ab5c2c26cceb9\",\"fcf3b20e36b7dc053217844c205db86f475700976bf37c05d322511338d5d9dc\",\"c111d8e9ece54a523063ca0675de827a9ee41c7b595871d12e515f550fc3388c\",\"85b4180c8698961896269a3c814cf5cb13218983860497207931a4cdfc7afcb7\",\"23cb3ca0927978ec9c0c9f0cb00ab852cb4473d0497b07e13907a7bb42b83d58\",\"a0126ee2d52ad46c9d5a6d971fa899c40994150a8f9032e07c9ce57243f1d482\",\"126a379220437ebcaaa544f0d4d078834ca348ac34f33c8ba6510351e3fbadeb\",\"e76b5369c5f3df7cd567214e906127168d349464146a4177c2bdc5bba8d869d7\",\"e04c8af03b73eb32b50744c7850b76a49deed281d34bd4b43448de5f18332920\",\"618f8978a49552fb5500d9a799a13c9e5c187ce046e87d7be74e856f4bee927e\",\"d91ae5efc29682fb9dceb61bdec9ded87ca1ece20feba813a9852bd9e702a2c1\",\"7b0d29d044583bedf973ad93e126a2466097f29bc2d06ef34d8a4d23bfafcc7b\",\"3c7e90713f618064aa41dac192738f19e0f17b3c15a3ed900ddaa4a427b692ef\",\"35c067a429de269cbb620c5c0579dcf8732a052ae1c6e90614c7336c00d7d709\",\"3ad1d6848829d9fbf5285e3bc64b8251d439baa01cc3a3af503d737f718ecca7\",\"59300dba04e8f2567c7a747b769aed7b23a1439a22d0b05615c95705723e2020\",\"39e421ae1137fa58df04e212f1eba5207c4a9a48efd863800e4725d627f93f9d\",\"107e5a990dc9c06e8155b8cdf20ed6b861a96440d93ed4f4bbe041f1f32e0450\",\"529fb5bdbb0163ae424eafdf64105dd32a1aca10496f5138f45d47a0a69f6f95\",\"6e21659ab1cc97e578bd2e5523994e9e1b37c4efe7d6c3d99c2f9e3e1b5caa62\",\"1037601d17f873077c545fae62967a1715e6a9daf9d8e61364640f06fe3152a6\",\"75745a836b118c03ba4f519732a404d0da70ccb5811ed4ecd4848124c04912a3\",\"1db5c8dfcfda087921a449bcae3d0254f702f0977b6d331e258f1d33ceaca837\",\"2b14e960343758ea1b2f120f503e6747eaa10af2b439f83fd9394c696c8f5617\",\"8f480252af499caac50bdc7671965f960d3357806adc261c5e85ef29c5d86a37\",\"9517b5abe43f293f3da47f37c0b3d1325acb7c508b232b5f6ae63e92e56651d0\",\"57add1cb41ca020a474b9f4520f2b566dc3e220e6efc600d8b1a0865ab870c6c\",\"e92aa2cfb550820a72ba844feef096d4c57dc8484b71fa0ec81dde9c0386043c\",\"8d1f3e507cba74790f033e9f3baec71d3a1b85bf3e11003817ea16fb14f7fb66\",\"e3aa3de1459cc5e955a8fc0c569839997f884c9a38a76d31cf092345992623fd\",\"7bd893f5682ba113a2f0eac5546ed18e7bf6c0c361e747ee41eab8f0bc45f6d5\",\"2552f07744bb6ca12c034a51ce0de01bce040e6512b96599b57c5694f696f3c5\",\"cfc51a16df956aa439e3126ae9a43abb7acd5b5296b5edc51ccdff826257906d\",\"bbb834aebd8b54c7ad9490a41dae62cc318d9e4e51e18264a033703467da3957\",\"b1dedf3e6fba842a12ab7ec4dd456340a80cd2feaa868fdcdd87c431c2f70992\",\"a65c86ac42a464fd4e242aeaf71cdcb51451a550b389ef328036743dea5a2371\",\"bc7f3292cb7144e1dfd8311519e54094038abd0bdbd32caefa5de648dd31b7db\",\"c73885cb228b10a586fc2dd3ef6b6b13b86f267646f5821eef0f9954efb697b2\",\"39004db612e10c907678cb007ad4192fedaa8dfbe139b61f083f02d5ff0cfa6b\",\"22bc9e334485149f0fc323043b5703faa342b35584d309ff61dad10ba3da179e\",\"33e01bf06da2b1f70a76b732fcdd60b7a2089ea3cb2cdd6c8b200a76203287da\",\"4b66ff6add9ee7a4ae6333aa39cc5dad51b9f2e7611cf9df8c68b4af203a0f1f\",\"aef7820a3aa570798ff8718b0591b2732512a675f39ece9786fc0c5803a5042b\",\"65ec9692f0cc5f433889789aa22f6629a1e33eef05bebb966ced337f984078f2\",\"2f379c28fd4d9f557e5663a5c3f9a6ebc36ea769971183649b063628c90176ea\",\"bdd4910f8cdf8733ae84efca8df1cc48b7afce7242252ce8538d58f1318be53c\",\"c9d57a07fd63f689a2fea053fdd674ea3c8224edf4cad071dafb2154cee25b57\",\"3c1bed2bd5b9494ae85ee7ebcc3a672a356e314aa679a6effc3c331cd2195c34\",\"0e67aad737a6b0f73441b49cb54a2b77044e89370c2e223f5091dafe4dc2f514\",\"bbd5f44ae6b447a5a0df2be2cc81ca05805d57b5955d20958e7ae171a6032019\",\"7163551f254997b28e83f8c70724406cafe4118e89dd999b9741ac0369ca6c74\",\"77da2a32f92375db4b35f6ca5c91a6d0da78e4f4ce50e70083bd6477517efddb\",\"1a8339a58d6aa5a16b9946518b93109c587195f35a44254bd3ac3f270abc388d\",\"8e8b578fadecf98f74569856d03fb531b2f493d76ebe2729976239362698bbab\",\"178c03306a1926bcdf44cc9bf5000d20304cee2df84b5be474cae714e8016b8b\",\"f8e33c2b6b07672414f467cc983f8ba6fe2947c02014d7f251d204fb46d750d0\",\"1c56fba634c59ddd0eadc8c145964400a90d15a72c3d25b7fc93dcbf5e0f5a91\",\"77cf234a987ce842a140bbe592406b862a65c07f512762a87dfd8d0f43c9b034\",\"5033ac1e3dbbba34db0fb6a68b3ca90a1192207f82ec6c6389335533af3a4632\",\"2ac497a38e816662df88228fe765d18cfe1f81d2d1c2110f6f2d055298ab19a5\",\"9478a1991289f645fe28e9396738fb38cc79c593ef37d4399eece31445bd780d\",\"db4251615252a0bb804107ab21baf15a6ee9febe42e0414e1610a944b1c98c73\",\"772805688fd1be23d2892962a08c088c9c6e1b3e71cbf5e915096cc0afd005aa\",\"78cfbf2e52a2d11178848b789e1b220bf2266e620596021e13d325894ec8c115\",\"b638d2b1ae26ba8ae2fb3a2bf59802c75cb75907477d80af456bbf09b4c85dc5\",\"5a7d6740973c0dfabac55b45bc22f6993389ec32dd21cc537c2e8b58485b72ef\",\"a9bea35b8152cfd8c4ebb613865594ed9a88992077f569e1414b605f39233ec6\",\"669d777d545c5dd18f30fc158b2490664ee90d7b9e477d106fa44ddcfb559950\",\"e3eb9c53d8de8a95b2c983a002febd91b7fa05f255b3a41da54b3aaf8df96d61\",\"9145047ba70fdac811f03592faebc82120fd8ea884c3c7eb607aba87c6d2db39\",\"85b04e3392efab4f91b7652af66cc9e945cbc9f0d99537bd5650792c306c47bc\",\"a532eebc636b7a7ffbb1ebc8cc4b00080afdacfa876a8295c9b25e9482b3c8d5\",\"3f52e25fe0a5cf996e498b53b51bf29b8096b5848db76342729afdc0616c0ade\",\"ed1144d739248b92e280d6d2eb8f40049facd4a2afd0bf8d1f8ed5df017ab89b\",\"6bc73761e49922735d0a7926a339400da7075d4b71066d72e0b385b6c6047e05\",\"64c3aa8fc32e5709cad2b90b521b172dc40bcb85f00c76529f0924b72ad44a1c\",\"458b4fb5f94bf2e265f0637104c40e9c855cb714f8ecabd363dc263163d0ee8d\",\"b44f1aca79dea10be49a9a96ac10c37fee0dd6fd3c80baaf03166cbbef9069b8\",\"8dfab751bb389dd9bb917e4535ef1236069ac0c659e8fed5327dcc0f9974c4ea\",\"5eddeed5508af425622bafc2006a308f4c307400df71c0ea5366211c35d69117\",\"a325de670472b8a43b225d9a97bc8192d8f0f74d9cb61884e83bbddcb640d06e\",\"12b6ebd3df2896554318eb45f921ac199e92b06c558be5583d583ba2a45ecad8\",\"a3510c9744544842d88d509fac1274139af50c478fafd6240e6c7460527bba6e\",\"b8b65375a3933f56748150c64c5391e33b570fc2fd66d9c4890a5eccccc68298\",\"75e1a2ed74f0dea1ab807f1971e7965d7d1e31e42e35c5266d91adef9ed0ff67\",\"9ab5b219c6afcad6765e4925a7a38f4bf00eff86b5aef21af680186468d1eeff\",\"a4581ff9650f8b8866409ab80d18997e48bf4017dfaaeffa27cea375b7f6f28f\",\"3ad9894cd7c68f75203a9d7774bfcd6bbf60db0d12a80551540774cee4290527\",\"50c178fda6fe22e3c9bf17cafd3d72936c22cfe0abb2487f5b3d98a7ca29d2a1\",\"ea26e65b759e4a4a4051a496d0684e65cf7fdfd4dbac92d116907b8e0de5e743\",\"b71fbb83841e5a9a23993d5821116a6f394913c2e9d0aecf3e5f389c30999c81\",\"79bbc54e81707779296dfaef5832096f8b9405b44180e3e1c38dc5b1ea75a905\",\"9b294d6cc7f377f8ec888ee9d98f01d31290d0334e7ef8a38eb7f1089512fe4f\",\"4f2ab1acd178587ad8d50d9c59f98e170ebc6706cc8d948a95fa388332e7a1e7\",\"22606bbd85003d1be0289c7cd674f4d7121c33a14c82317bb74a39e684e66146\",\"540e2c5dbec48332f89aa5b1f27b851f721d3e3c76385946931e20a667340ea7\",\"8ecf25b7d180ad45aa4a7f6dcbcf79b8cf119a7d08c9444a41cdde11ff5a00ad\",\"0f9006d13f25df61eeca7cbfd240b5e38e71c0b0b7449861c433bee86d1d7de2\",\"34ec444ab9716bb5d101b857cd8607ad9c7573cec086fdb6bb957920ba1246b7\",\"f74bcecda9773be14a94b42c14c534718c47ee7bda0ae5317150f56a13156842\",\"42b2612de521d18da4068a16582b8f6b6106fccd654f615b58417acdcea242a4\",\"24bdc195c7cc83e9f914e9c34c5bf66a85eaf10253e4dfc275c3d797e2a9a293\",\"78a22f25f0ba4bd40b2eb0671d75eaabe8b2bff32ec11d533809c884bf34e8e5\",\"8a770bc3bf140f0106f13a348a21012a13312c6b75975226d2e25cfb662bf864\",\"9f527a3a10033b00741854e1048d77f1612a2ef945f491c7e32dae5b93176544\",\"bab8af3a4c00576ce3ea766bc4ddd4943d02a780dac6340a46aa010f0e5c4c06\",\"c2945876707e840c5d96c4d3dc67ecb1f290197dd25546317b4a95596f54c9e5\",\"5496eb7a184fc240249f3cffc550330564ecd2903d751f7587d46e34e3501de7\",\"bacdc34a3e35a6c0b3635bd8545085b1af27686f021b7f4a278a4c244c4a1961\",\"57ec37c773479d0a5f80c01735df760c1b02696124939676a4355cd31547db2a\",\"3701af2a6cb2e66660ae0536e1221af153b4f80d682239c9651267907beb98bf\",\"4c8f8b9471b4620fe7b0d0d6a89a06c9dab926474c427c14762af71750029df5\",\"97f2d6d971bbf2cb841001dc8426a931332149de18fdb8ccc26d22a42af86129\",\"36731f9108c9a2f606fe3e315346eaf701406b4d2ab1a1382028b481a111a6d2\",\"8a563a57f2501b8e7bfffce8b33f2e4310dd9bf9c0bdc7037d1c892a047f274f\",\"0a18a110ec75e2444e60f74864cc7bb3a86dd771d5802dd92da9d2c96e10cd65\",\"ddef1ba31e644aabb09bb92bde529411a9aeb5c987659e00f5e499f019106bc6\",\"a924ad0eacec7edbacfe180a893ffbd30ca3e14140c9e80e164a9506da0c17c1\",\"4192866e9298012423f5b2ed88d329ef8d1bf12828747a21ba113802f41f9481\",\"23e9aa2a9581d9ad5dc6eb12105bbfd2e771af8a3870c50776e5b60b967b1578\",\"53c7205be7387d79fe1b11c07ff13fb1de99c4e2eabfcbb600ed06429b65cab3\",\"f80d84c713d1f9400ff21b66e8a7ed493ef5d862c307bedfb4a9cec378be54a5\",\"c1c291912437da4bc0d724144c0f049c95b2338af826f331fdb78f8a7ecaf36f\",\"551d6ac538a4f167be913cd9c511ec7602b9be476dbde6c14e328234cef649a1\",\"59458a709f143052bc93bd5b39ca4388bc49b8c5224a53545b3e53614a971eaa\",\"c3c392355c86453267cd12476a060467660985c65ed2f2ece3b387f5a5ff54c0\",\"28c1e4f04791f7bba6888a6573b5370f8a97179673a5911c7dd0e9bab5efef7b\",\"640bd10f765a40bbaca381317857d8c4a84152f3797c661cb0ef3601090dd7c0\",\"f398609203a3956818a312c97633053061ab00eb992b8a8b3b4ad897760ddf97\",\"b9aa4b7e0a43daf003a967e080d07aa547651504e0521605c06df34427c80b0d\",\"0ddba2b89661062393ab41462fd517ca285b76526eb3480995ef16afb6554c8c\",\"2f0078d2a5ca9c222e20cb45638e5d70fd7c8a8aafeea1c18008028e52ba9fd6\",\"cbb96ce85551a076cd6bbf8428933c29bb439de886fe1bdc04edd687f13c2040\",\"56fc279c302ccebb774c8f485264d03cbfd4e54b6b8b0f6b7d14811ef1af5b02\",\"b0b8115e88106f20f6d127209c795587872d918741582de95f2f13b5fbacf9d1\",\"4558715fc2bcacb87159bba5505b39b230cd99c8a5af7b4b60e26fe36480474a\",\"d3e853ad58fbb4c374933561e8e4629e9c00f9612ae8755f32ffcbc88a4ee217\",\"f62dc73fe06f7bc29b14a1601a2826165645721f0fd0e2b3dd6cc197bb81ceee\",\"24c45ceb7c8ae6bd53ee1bd5d91fcd8c8f6822ec0db2b6551e5940e5e7514b37\",\"c062ba12849b4739891ec56875d550149a93820a57613c023004cbe7979606cf\",\"6fb54d1b9e780bf8c9909c0a58a1164becc915a4dc375885604d52ed57dd3d5c\",\"e9c9fe774afae36d53416c375f2009e0f8238efb72b46d09aa90d2565a64f6e1\",\"1cc84d1cc4d894279e3b690e312191056d45240bd8951113bc389dec7f58816a\",\"fe66c2fc452f508237e38763724ff684e20bc4b6c49f9d5fd28b72d5a9e8a7b9\",\"df09ff0abc51848b6ca7d2c4205da0b7b77c57c318f3ea8a4704a17c15307bb3\",\"9d418616396c8b7a6002db87dc50bf540571b623250a8f49e3bf31086ac83318\",\"b01ae4a57e3d78052247f3cc03f513635873fcc855ddc5adf715d071517706b3\",\"e0dad32e4f9e156ffc8cac684fb2bf8d6525b638e34b9590aa7e11ed28f6c7b0\",\"930457a780427c2df7cbbaf07732c50059bb8e6a49ab8926821d78108427cc2f\",\"0a9ceb8d0a90dd1d18b9da10f44087e3885050592a0f5f5e2af94af15b7a70b0\",\"e5dae6403544cd6c537df478ee848cc904359c552e55183b9acfb744042cdc54\",\"d7c4b9e645beae78b9bfe88def467a814e26184e8f31857c0221dd232fccacff\",\"f62681f198cde9427aa16a4f6a3b5031f56bae83407461af0dde59079c1197ad\",\"a92ef158a278e9d6555d08c8a1f0ccde1655439e57a95033cf87b53248bd2c61\",\"69bfc40c4b90d666aa0a194b54d7136565ec6c18fff7e7549a55168893ea560d\",\"6175f8f1987eeb621e8d2783a64c3c7e64559da0b1ac37b0d70ebeb05009cb9d\",\"ff63f36b6ed69d1cb03b4c567696baa32876564a63703f4be8e341e7659dcb50\",\"6efa8dab14e929283c6a4f0f9f4954897fb53aa718e6e2fd7b2f2018b28d300b\",\"71c193996350a16926db2cebea65cfcee29b10b8e9dcedbe6ddf0ef8354b7833\",\"dcfa9d666253316654f406a6f452c003b7e1c6580d9f8d466ef1a785f12a6754\",\"622b808bc48d19c5eb630af99a1975fa156cb4755a0b4f79ea6e327a942cd382\",\"8d5013255aef2a5335e962b8f6174d1f87f1b7810de756ede68e19665a4d65c4\",\"cfed57133a8cf25ef4b1784c82d8b09881bc1e53d9a9c9c823a656bebba1beba\",\"406939ef9ce3c28903a325a2c5d3d7d6a9781ef4b84587c8d8452203d558b674\",\"db10d1ee2e96974ba3c2e8ed1502484e34dcfa26d0fe2a4daa9a3253d335773d\",\"6149372b235275707c3ebb7a0011afef6ea9e4d439425cee06d0ab42ecfb3cc0\",\"24588e00a60c10edc4af96ecaa4679540376408bebb2d634cd646bbfa7228e89\",\"614a2f0f328a0d41f4eb2996d6739ca85af7eadc03fa030bd84810aa38111ac9\",\"f801d701c690ea9aa83f38a4702e519d60ca8e7d564ab03e3e12c006afd25840\",\"f6ce91ebb21a50e857b8915c47b7bd3e04377e53f6704f0172d89fdaa9d7013b\",\"e379a13e6bddfdbe555a5d814e82928fb183f50510d180b1846de1a0e95f62f1\",\"32e16c5ae44927603f9efa42a2e963c369b4ba99b17734441159ac0e72aafe51\",\"586daf853d1c1301c0488ecfcb46d100dc041091a51f7a9f2cf2e6b867d541fd\",\"1a89ec39431d22b1f93137dcc3db8e4b88d8cb727f832b85e3a2c1c040fe8199\",\"5dfd06dea8df4540cfbee0ea731e3158c5a1a843bdee36e8e81c41d1ec0ae855\",\"822f007a0d655f7c63ed6cdf723182e34447100b366b40f450714a80e96f8b2e\",\"573b5856c30b9c01427bea700d881436226cfaf771d0f0358b77963c56bbc022\",\"e1a6ecf332fa66ccd556bd95ba2df1d1743295d6d2ab1283aed09b485b7aee27\",\"102c529ea696e54d9185705f4e516ea0a34e7d437cfcbfce895d1de8f3343a99\",\"f2a302bb49ee25293664d62d550acff286157a89ad050b875ff063a808be721b\",\"8deb86d515b630cdc92bca79eb4a988ce67dea3f43656eee360ceaadf6909d88\",\"6a8dbe548d535469934b4045dd974f59ccfc585d913adbd10bba7da04999980e\",\"ff60530fe2b7f77004bd75417e80f03517f69466b54d39b70654f434400402cd\",\"2fa85b5f3718a33292b7e2327dbf6af6552f03d8d7fa8c3ddbef8820a49fc05c\",\"4561073fcd02001b99edfe9bd1c64f8eac00787c05c8a0a6b11d57921585b336\",\"309af1026ec62f15fd8385ef223e3fe113e1ed77cf03d63d5f9bf5e8fbf8da5d\",\"70c8c249ae430d643255d773f8777037d906dbb7355291518b7004dba9c167d3\",\"5c8765ac31f5ee4263c835cec13ccf0762188d3b0557983cf9213c4f30e3668a\",\"107fa05466e42049f100b97989b4084f441e8ab86cfe6a31639b1338d2631e52\",\"d02a067b9083b942cb4cf58e00983ff984cd8a02ce05a701af2ee36c35727277\",\"9247633040831ef81b6670bb0f32392fe617f6289efe66dca8992e295d360831\",\"cd61da75a5ececb9427b18085347d00df702a476a987a203bb80225455169111\",\"42d6d9af05a7829898a6ccc2695cc4b7e4254185fd088378315a43f45e455de0\",\"ee810fbd8d85af5af2ec83ac57d4c45cf44d7a06fa1b54c17246c5d47de4208f\",\"e89ce4852db4998df5652465190d1b44297860f719d4b374961512b5813b7b9e\",\"3bcacabb80bd2d4de577b18de4f227c800392f2c374fd1d0cf8b842550160344\",\"2fbc55fdd8ffb208dc64f370cb17123131bb6f8865de08e75da85d3ca3ce606e\",\"ed453650f14c7f16dd0bf0b02e5909d89568826da568f301669682511dcc1233\",\"082495237adca923b729d6dca35790f7ddb12b9c2092da70c9e848b8f5130c2f\",\"9f325c86efb1b5628d66a8bb34cc4d4b23f7317d6181642ed961ff64f7d85239\",\"27c91813279de89dac9a4a629f92af5f8c6931ee7706718d5382438c6b29f5e1\",\"cb35e95bf5387426cd5372ded890efe407ef9b4da7c0777f4511c5afd2486855\",\"bfe812498dfaa6f8f41795160a19059121f7c1e917055e6d9aeaa0a83b18222a\",\"c6324f68090377fc21b8e2a2c096204340ce1e87fcefcf1e682d883eaf29cbba\",\"33c6e77243a8e8466bbd7a042ce60665e5cf766d82843da366d16b97cffbe670\",\"50f0a6b8d9f0b472eb09afa73385a16eaf1b1218495dbfc5bf53f6cc3f679aab\",\"cdec1ad1a68c942462b9b1fb12306f534ffb500a164705739d582fd04b3283d5\",\"e480d4bcc106c7f8bce65802da2a29786dbd0957c8bd6fa2c9d8a2af31e322bd\",\"2f3901c85d3232da527a64ff5abda345443f87031b390ac47941a1ab4907c2a8\",\"c1a71bf2dbf0f8a0b57a0cb7ea8e0e8d9c2ef2871fc1fce94f5f9374a2bbe65d\",\"2a848f887a6acfd09936b5de5fd8440cc640581c38d24f50fccc741465c56d62\",\"e9011e0848381efb6a86a7707c7c66533fbeab314fde6c35edf113dcd04577f9\",\"6159f546fffa82d23dde97707353a239de3dda925671d423c873d0f6b2961703\",\"88fd91ccd9801689b9297283dd6f9f9fe41e28fe48ec685f55ad4765cd9d9988\",\"49a08e653fbf773197c9bf587f9223257dea592f821d7832f1267f77e3b37d16\",\"d1f94188f3a24f4ea66e865c01203c635b883719cbd9e577fa1578f506e9ddab\",\"85fdff8bce7aadc3babd0509169762e271c4c61c8777f97e075884fa78a2ec75\",\"abdd384772cb333ffffef7c66af00d7306f7b9198b6f029e7a08a9971d116dfe\",\"c069bf7dcc2709f85ccec36e6a664d48f9ba494d070da5b2df3f7804ef31a93b\",\"2a2a73a11d8d9271534a4027ddda45f91d7a71e8c150619f82e0add77ab6d8fb\",\"4569fb6a9cc6b948e6e786c691d32e140ecb34aa2850f8e8abac18c00f0a238a\",\"78f55c2dc102ee9aef1b6bf845223bdfa1c59c2ab552c0074b34834bc8ef8c69\",\"0fb3d7b0e3daa348426ed82797bced99e7440ceb6bceae313716122e098a21f9\",\"2ffb462dbcf2042023cfa1c0fb6dd2066ca593be7f389e8276a11fa676f81c60\",\"6ffffa2dc5b0385c6d95846323d029cd63962a8cde30ce3c98cd4dededc45edc\",\"59a355078f0d4596295c5e5ef396cc1cd8d21d1eb70739f183016139b4e1e2ad\",\"c827b94b0cf9fef04bd27be59319e2a3d2441bc37a59ba2261befe5eb8c7fa5a\",\"67b2ef8951edf6011250f0a8c10a2634ee2c4000fd23f049515f9de59cd5e2ab\",\"3c3398de5f3ae708c335b02c7bf9a511b9fd1619c870fa3c01429069bb1e2d3f\",\"6f7689853774ec5b549dfbdf45f4eb1974eb37db3ee9b0613e50f879325d96ce\",\"0f39272bc31567ed059792f5338fe03624c1ce21380b46bf6ef01f734bc6af5c\",\"fe59fe08ce3b06da2207d00008466f6dd385c8c14fccd1f375c9750ba53fd5d2\",\"4f90f59dab6fc10e45e4d68faf3b97012c1feefe2ee4c4d9011ddbc54c8d3d3a\",\"fc0a6901df422790d0292222f435267d074bc9ba563f4224db79e66131944715\",\"142ca9e32f8c60de5f51f7117dd95995a53cf7b0e9a73263ea0fe1019fabef3e\",\"da7b07e82044035e4907d5fe8c06695be2801b3155e2f2ef75a694c95aa5bc29\",\"206d9da7823fc05f94db085e736221d203bf9e382dc797dce3b881c14f99bd1f\",\"f99b9910f33d922fa2bc2826f221550de3b3c6f18c11306aa85f2909ea4a15dc\",\"a23f47d21f5f03a4687c594e44169fcfaa1595d92e626e2e08eb7afea2c4176d\",\"23639a9e7436745ff6d2f67185e6b5dced2e67ba8f5c012a07ffffd290c940fb\",\"c9962ca0b55eac80b07ae8ec38cf0edf39b473bc527a02e97b85c8f7e5ddb734\",\"c8459242d05c9e77f93e12217632aaea663cd910acf302b49fdb08c6b6b42878\",\"9d9379f118e11247c879c26af529cc065d904612add29ad0b0364bbcdfec5359\",\"64db494cce78fcf026f9b4402a28fc0b32653f6407118af8259c9dac34b6681a\",\"e50e05b28078a502b131b7f9fb5801bd4380ea59da53c10e0fed454e7c22a5e4\",\"9087ef2b64b14f3857f4e6bb964aab1c48b2a610ad189e00dc68608861da44aa\",\"1f33e49f31f22651b1c1b37e20442897cab94cbc44a5b338d3c94ba5de835293\",\"9cd4a09258355a0dfcc03bbe1f5b02cceac7c356c53a4f7c3250324d83324b7c\",\"dce6b4570ba33daa8b78eb17bbb4d111aedb5b7c7d2b395f0d6239958f8b5600\",\"9ff5d5a2ba3e5ffc71136ecea97158481c40278313bf181c8f71d08c04ade49c\",\"b6fdb559a4c715f8789d0f1aace7b9549f8d16739e30e9c0993b4eb1382e6023\",\"d3214e614831e9d613f30f72c787fa1c2de4929a1922809ae8fc745b525b8896\",\"5d3a114a956604a65cd5776967b54642d15f714792184a178cda03e65cde7a56\",\"62b592d72436ce0aee206ec83088e19f770afcafcab0b2309d29e6594bbb86ad\",\"5295b4b5ed26ab13b41c74b1f541af595f3c6f200b634fc269864947cb8b92d9\",\"2969423df4bfc24bee1bde8c8d5c3188b9fd859931a632fd3a8f9a3dd8c5dec3\",\"066d733d9af98c9583f6d5dd14be9bd0d766a17a82fe219e1cde81054baff631\",\"bde143efeb74e0f7f8a1e0517fdb9838ceddcfd7ecbd6a518d410a5e88721f57\",\"f240eec84c038ccf2b0cd060a45d5d1cc7f6f65c60a79eefeedd5564a8d4e509\",\"45bd7fd946f9af55dabbd2b5e65efc406de1f28b8b9e1f03310db0192853f279\",\"e91acf49d89fe4714613edf15991ae22f20733aa0e6b64a8899e109762592a5b\",\"2a2ae0534188967ba92df5eb099d63e751bc75db85f4030cf39371d3d2066d2d\",\"3fd2194e5c1c2eaf893f5c80f31fcf96bd3365222a13240eb297b0638e397a63\",\"427f7fbd155fabf65a2890803f358223c7712ecb925197112fc246a47b2db574\",\"772666184afbee996553778ea54bd05dbe5a5dee11810dbf2a2535a54efb7708\",\"5ac4cbdf12f2857ffde7ed38bd1226b72f357672ea8d31567f263d881aa7e883\",\"96574908cd715f94dcc359af25fd508f22f57ab15848f34e89b8c150114ccbdf\",\"2a91f89c916d4e4157ef71c6fb22d14e9c1d6047a0b74f37ed53be45ef70c054\",\"4e019d6f39bf321aa65ad09045ba7209de47a205c670b4189d6eada8a9018327\",\"58ebaa9511606987f02dde0161518997eac4995dcb112e2dd105b5029eef5810\",\"3bff97ae4ee4d0ca151133f5ee0a9277cd0913b1b3f99d3a49345053e1783295\",\"87beb2a68b46c3eea41409e7f01dfae6d180bf76dd9524da35fb8d01ee7e000b\",\"44d8b524a3d35b970b83fb54c4d8fed57fca0563cb6bfd9aa2837c9979328e84\",\"4c4fb1e19f6bb14aa1f0cc27d684e9461658cca4d25bd60548eb9ca188a3508e\",\"44d58eee50ad35982e8617bd14dce4423c79b9685de8b61a103f3aa326d6f9c3\",\"73db1a8dd491291a0784879d6a50e11830a45a59d84150f7f55a2710a39bd3d1\",\"dd2cb48dca8aca1f32eabec9a6d94f0e5c821912778a95dc26e50c17f7f45ebe\",\"8d112f5d7b79bf34ae171b9cf589ce0afda10f63560eb17235403a0dae2517c2\",\"92d4a7496255fa8929cb3a26f6459016e523751817cff6d9d2c7af5a5ff59fb9\",\"ca0e6e0f55e4185003e3140391223df212a11099f7167cd5839547d1d7a8d7d2\",\"323b7537c22070f8660c3205db7671ce9ae948f19422d112bfa255a83d6aa7fc\",\"1dcf2d657915d42bd12a9ba0ed1f34977a5f30e99fc0a94a7027409263346a4f\",\"7ba19e9772a853d638a7db0be784fadda320b42632bf47f8ef52ed925388b012\",\"47df52aa3930addb131bee536a449410eeac0941e5012604b3c367c099a64b5c\",\"ffa005c4a45fde76fab779b6bdd176e83d7b8f1d865313407a51f661c7fed51e\",\"e4a686c659c1549284d33c479a08dd93599e9477a78a68ff206c65b5f1a03fcf\",\"632d220511bb41cfd26dec6377c4fc121d8ef77f7c5c12a4751a97f9d63deee0\",\"350ca54fcc7e97e80a73f45b8f1e587266ca77151ccdf6643d107a3e0463fa4a\",\"2b8a4c4502e784d88509665fbe4469decc6796b1c86c1c1a67847fad43d7ea26\",\"936db400ef20bd8963b47b433680fcbe7879d87eaa49d07886d21ea430e09567\",\"83d52ad0aa480aa449d5dda3914a590ef50ebb08eaf09c389d652a857cbce1a4\",\"e0f808f8b6de81a9f7ce7be31d63b940150e7414e90ea40ac5919d433f3f10fd\",\"1a765055259a35b2ea47997abac5dd1f1e737b1d619cc5427ae13d3e8179b581\",\"b3888f5a91d872eef4daa8d52e04e651be8c6cd3bf3f61f508fffb1527602007\",\"4ba8a206a11097de0732fef889fe01b6862b4529c94a803566743736550f58ef\",\"3b9aed50023beb4424a58561f0e6094dd9bc2149d3fd20333f1cf27b24ad7b66\",\"43e9035ad29626eb77c8e9b3a23388b683e01a9b61807bad00e8c483bcf30902\",\"a452ed995872780a0b574341424bb01f1d4cb164cca15216750f9e85dd859078\",\"54f2bbe2662f4cc4f32859dcd244bb0e5a8954bcccc855462bfb46f9ee86db18\",\"9d36a8d753fd4fe8ea5bf2c98bf6b79fdc14fc8fb2182f0f5c364d4f4d3201af\",\"93d146e0e49ae78df61708687fb63635090b7392cf0c58d84b869f56737ae0d4\",\"a12c0e88002213f85900e5f643ff535e9c23a4a30af9660a007440f639a7ee5c\",\"bce441f407a11017cc7085d331963db4a7467eec75210e12a33a587577c470ba\",\"58b417cb97aa3b560d33e6af39afbc6c84c565f35097a6f55087e9a19ca063ba\",\"75c4a0e079805f71b547b750a5bb1f87a358d732537880d9ae59647446d93f98\",\"04b111b13427f32a028a51aef78ee046c67a795bf07381a46db6b5091dc11fa9\",\"2ab0c35913469d63da3aef5860c25db4cec8c1c84d97c00b9fc0468a92bd4e42\",\"596389bd1b664798d6bdfd13c4e59553ae1f9816ffce8b370daba7b49ecd87e3\",\"c0a270d2831eca42dc51f638c2303e6cfcca9d184326396e3ff673525e21cdee\",\"df5f7ff88af0fd45becff8f06d3dc61cf0e2b1348dacea670ff1f1e13747a649\",\"15e431f6dee99027ed5ac7d0d7d5782cbc5fc72c58196809a735408bc7d5ecfc\",\"ed773d5d78c536abae12c0fa55a319eaa9bca9f185f870a9b1389d18b6023248\",\"614cabb18693a394e369189a13553e60b276afb138c5fd298c5be0cde81b24fe\",\"d88ad6bb48ab23d32b0780fc713b11a161b8dfe644f634e26e9cd1b343d82942\",\"c557db05c5334181b73f9234bbd2c59184630e83ccc9793296d95b89d33ec1cc\",\"e7f0f58ba538bd37ec5bbfed0b5f470396418b51784586b537dce7c89d7e79c5\",\"b4474eec69c96b117e7fa26b92e462beb1de67a3c85c47a1454dcc36361f8a32\",\"9c0a8dee0e487d84d9e2fd6d8957bd4c885e28295fb0dc1798c32c866f38886b\",\"2bd2a8e1933c70516d4cecb0a9441cfff708ca7a5c6d0af3bfc9035b415327ac\",\"dc3b1bc31e48c45264784dcd72544f33da7559f12fdc361db2ad023f91241701\",\"ce0785ce9148d4b479d67339903f354896c72bc42b89a9782617faa11927e9a2\",\"07630608de2124a8ae745faff6d5109e5738e64ad32cfcf6b0c58d35d23d7c79\",\"70f75a23f5c4bce64064932e767d34bba3114911673ff38da21ab2000659ae74\",\"8c00c331c596d13afcbde7d3d25aff7b308c615ac929d2166048534aff72f826\",\"20832fc9971499f11488bf05b4df837c091d68e2a4755dc861f526d653159e2e\",\"3756fbc34a57ef924abee465a316189a1be41a2a1eead728fcfcb0544838a75f\",\"e78a129e269b1dcbec090aae752762f5bf113780dc6b49e39e649bf5c65cf153\",\"ac60be43a21d0810386687fb2d549bf61bea7f953216da26eb7b5be638e85275\",\"87de7dfd8d4b3ed8b402da00ca463b6ecb21c9d534fb91d0d700bff43f6130a6\",\"2a204465e169a910011a787de28933f30b862b5d7856edc591f07967bcde1a8d\",\"bd92162241a6ebcc3411518ad4363672719a469e827d9a1bc5b539e668016eaf\",\"316fb2def3a534ce42a0815886ca0b900b7aa3523ef5d51dfef9bb929ec656fd\",\"16ad829bc290a72ccf8034ee9e2fae0427cb689f2ad273ab244fe65ab2b95ef2\",\"8792ef2ab09fc5656c7554d28280355d250af0aa46085edc89d7121e1a85c763\",\"b35e0972e3a756e8bc72f9dc8afc08d19113a4754556b8369c7048918f4760c7\",\"144c57d32656c434269263d8c709a0e8c9f3e9432905d15c39abd921799491a4\",\"ecca9c883926541011f74caef5030cd12e34c26265016d76eac48417fc23b19b\",\"3bd8a5ad3bda96c5074a760dce5bf3acb73d76ed46564bc8547af6df5d84cbf2\",\"758c36d4c59c2fe9728b2d8839a9edeba09b7544b3c747ab6523f72f94fd1ed3\",\"78d0d8fe1cce219484cd29d08701c82f7eebf648c90f0efbc73a585d1d6b28f4\",\"b4959461444c998dd1223f9f9c0131c9291b9988acbbe3028f8bb84c2a277c97\",\"67adbe34ce4d41e6fa2d56570325033fcc074bb69f92c8c78a873433ea021314\",\"7b41b2ca5cc8b477e5ebd43d3d254775695b783eb3b33d623bed71bb2699e744\",\"95ffc9613e82eb9c628d68fe329a5e4a7b1d243266621b1c914f83b9a9468f12\",\"985b6698e16e1ade3fc25214c34c842285af151a163a27c70587973bddb6152b\",\"ad1db4de2543a617706e4c0208e0f4c5d762b872abb60fe552b8e7d88e37af6c\",\"f7c0670a50fbf1c7191af8895ea459513c4d2b672f64882dcdd67f1d92cbe884\",\"d9cc598354edcdecc160ce0bd9c29f9c6dcc29830d9f457943c2410581477043\",\"f81e408e35e82a326b5c98c0fa841a9952dafda5185413a0507c0567c8935bde\",\"635b59cb06cd6b4479b1204320f25950dbcaad5a977abe1b11500748d2b4ca3f\",\"31f7129506c47f63b0466bef51106b1a284d346b02c465d068c5e4a25cee3412\",\"cfbedd09397cb5d2d4bb2dc044b186a2872839fb87e49a69314823e504284966\",\"d3e920918d1a4008fdf411a3fc15b973ebd7841fb83c64fb39a476f687d555c8\",\"2e52d64f6704d2178d846d715c30365deee404911fbac32cab4753811987d534\",\"355607ce3c9f746db6857ca4c75ece1af0b817c886408754ca949c17bd1785e1\",\"cf9903f12a202ba6e8bde5e0d5ee88e210ff8d2a67f88d184ea2a376b02492c0\",\"5a06f8687ecbf1f5e4c2e1f55392b13e59530364539685bc399ecec479447cf3\",\"f61670aa108663212b37a8199e768e093296e5db268024f74830750c9b77b069\",\"53bca9d594401255c9cde9e03a90b809c34042dda6b4b99b1c21c1d3368fb640\",\"db046a9e6a4640f143dc50885841b9108e62c0430ba064ede5d15877e9f618d4\",\"42715602e7a0502e153d0edb1ee9ff2645d4d4bcbf200e3bc4267f00ccf5f08c\",\"80d2221fdefc4bfb8f9f5b249d6a685f98ab111b7d997bade02fbb9998731551\",\"c723aefeb5f04f9a0a0c734f5461062b3dc92ee41149f4e1fda86710c2ea5dc6\",\"4425e5337dd16367b9ddfdcfb838934c524b2e52d0f6f563c5a841b9aa43d542\",\"f21e4498a914925f088575e10e46c21bc7aba263029164c92e7699435182085f\",\"e17290a114ad2ec87e80e555663e47ebb10da2827ccf51d68a05c1a45c3cf619\",\"fc74c2760e1fcc49fd308ddca4faa2337cf574268c5251cd1ca2b5b139bd140a\",\"1517ae58cda3d57006aca5c28edf7eb1b16fa5f594462b0d311cffdbe21bf2b4\",\"e95c3226d4b17a6bcbdc54d41a439378ba0ab611b3c8fb3c3332396f236bede8\",\"64e993e74acf4b331450f8a3ca939d80d0176b3f6d1b1a7e3ad04b292ab23000\",\"edb43244ccf75357f1a21bcedb3405c7487f9bd03b515563a3844110668bb546\",\"09641ab5f184b0909f45ba6e375fdf7f397c61941c19021b4c655524bd5ef005\",\"3130d4fb36f8a40f6ed1c49a27e4d8c28b896cfd8095e65ce3526c1df59ad1e5\",\"219aa2a0f129afbf51cdeb4eb301aea95bb93ddb5c7f80a281b2bb5aec5644c6\",\"1f6a757ce0301bba9ae037a3a97c567a2cd3859bbc9eee7051ba2f631d47a440\",\"ef579262f725fee039e46551fe0c1d78629e437050f993bd1c3d547b295d5332\",\"c5db30c68863f547e9edcf6a41b3b89889edc723b0392b77867dbc661257537e\",\"437839682d3c0d600e9561f6104a31e3af58738daa5f6cd96e53ddd45734179e\",\"3a7f5b19c02b9aca3b61633a60133cede2525317936d70bfb6bde24c2ad98b51\",\"7c0a46b515c2ab883ada5888e70e2ddbf471dbd888c12d168cbe5bd329c93fad\",\"5539293eff66a998a27af5b799f1754903cc01a76695a308d0219e8e5fb97a94\",\"a841ea6c6eb686d79b1c29325d70eda6168615d1f065855aa6f8459ee336d9d2\",\"7f57d7b38f963efc30f88d1fcb748f9344187f99e75626727730d0bf9f3bb5e2\",\"eb0d027fc5fc95a24a4f3f94dbbb2f91aa1940f36eab49fd973a54915343d0c1\",\"c4e3aabad38c974e6943fd8fb774ad19e8f8145162c741fe3abdcb6dfcb84a30\",\"9684932f02f3028f6f93fb28422034ae4ede6071c4dc89f85510fac45ceb0a08\",\"88626c8c3c620f459bc393105082b30e512c9d7e6bdadc1fe299e592c1479094\",\"ada7e128b22321466bbd90d1388015b114435cd8579f814cd58ce72e96c12663\",\"ad200fc2ff143f7cccd2505d26a52708eaab64659873adecb43c13b5d89b9070\",\"ad65627c0efb4ae5b93322db6b809904802b57c1094a01aae8dc70e3c3874ee7\",\"7d410abcc726bdd3f10aa7eceb6445710f74a6808145e46159db7964db649579\",\"017df8fd2196f38dcf80de00cf06b222f97cd5d25d2b392eb20fea0ada68f97d\",\"7eb0a22297cebc2cf6c9f69d8bc7a44fdf8c580c409f2110e98362a9e0cbb8da\",\"b3972dd919e29acc4ca4990a88c43066471dc27652479db04de6da32753531db\",\"4425bad534dc869bf8b583f900e207042ca441594327c959533ecf2b13bb5682\",\"31200aac9830ac8792dc297b7a09ce5d4504b94556d07e81dda68cb00f48e61d\",\"83f990461de5d2892d9474eef17328cca57cc162f7a79c32b203aa4c20705586\",\"efe042c3bd2c9ac52a974f10c8cc50a195eae214aaf01a70b2b2a93f7c8217f5\",\"3856153b347582290f612290bf1e832f8c073c80bc90d270dce6bbf545e6e724\",\"a7a690b2d8944a2ed08bc572f9aa8b1ae5fc0d36f600f5421071e3d9cc6f4411\",\"aa8acf1c07f4da293727d91a30873f0e913a09564d21c794faaf49a0a9ce066b\",\"2ddb551488fef6eb4a2e0c6ab877d9abe5c8e0771ccab36e4a1b36962abf7a02\",\"bc0d93d8deb3545fea6b1d06c9e0e1543111ab07d230f6837511d3acd23241f8\",\"a6467b9387fbcf4153d9c82fc4332c7e762fc5b134a4e99674cb53398d42600d\",\"824a0fd6644aa95f7c33af4bc7011b275f51ffa62865147aeac476124d429b6f\",\"690b4b35ac24aec579818420131298e96d46114f235934147e218d25e85fd99c\",\"c538d9420f5a1bfed2225360277ca1dff8a6a717ef9841af8acceb0202b2b51b\",\"9b85ec2c4f0b915ff607c32a15d1ef16d6948d5a2999d83f3d4491cb8962548f\",\"6e7d139e29283cc8e6dbd66e1a0de3503f9fa9c919f04deb4a16f3c86e623c1f\",\"b06162b40b700d831c07ff1c751bae83a594b1d6bbd2d8b29d1c863a924f5e0c\",\"e8857053476251dd26c7a49e8a1ac8927b011ed28e1b5db84dec0279c7cee94c\",\"0e793fe3fc0afb8becf2976d0ac1cf05ebaf27bfc76a77140f0db756f9f9e686\",\"1ea54b07c189688032e3761e303cf18ea5b50dc25b3963b4b40a48e8527c13bb\",\"c3efb59d3a2deaffc1ef9e8851bc969108c6fdd1340a3ebc7c83083ee558e0fa\",\"b1a8fde041447de3f93ad2a599b1c14cd057c7b08e5cf921436ba923c318e6af\",\"b032620fc06b1a5b47fb1fe3c63a9b87d9caf62821a9fa24503a7db5caf400c8\",\"5a2d03ad1f6a170efb767c3e3e37351a89623c2d576f50d7b3bd153d2dc650db\",\"90ef64285817b6212af313cb5bc491b7d793b6534bc6729d07afc67e63194558\",\"0eabc30a8540a365e44244b1dbea9a0fbb63da5c545408cb0f66aa134bbf0dc2\",\"2c2116a4ba7add0ea0b741af5730e757031134ce1e08ca0a251967299e326c12\",\"059da12359dd94f80c7123d04ac2478e975c794febfc9b7943e5ea8fe5f189a3\",\"30ddd3846abbcd01a249e5d901e9ca40e7410ceed611ca4d721e9683b68bdf46\",\"10d3c57cd35ea67c951dccf50e671a7f5c25e67cc32637802f38b66a4a7dc997\",\"8b4732a38857a63637d0c06bfe9c0f2b1151a56427e8f4c316d3aef02aab6ab6\",\"d5291b12407870b91cc21608b00c3643c952548ad09da852b24ae741e8d92875\",\"cd75b2cb1ae7eaed954bcc88cd3b5a7ae116243e66662804b446fac4729dcc6e\",\"098b8a35596d370350660dc76f0e1805fcbdb67127be7dec70b5bf9089ac1b6c\",\"92d20cc16e4cc8820d3ffed5ec1be281f51ddcbcb614c870aee2c1e280a9fab4\",\"91c948403366b1acafd5778053919544bc6dd7333610c493274dbcc135f1e685\",\"c954fead3f1d17b383b68b293dd1adf6255ad8b3b22b133623cfd23f52287ac6\",\"a46b55064347e6e5fc653cf633e25a99ced25fdbec2862fe8d3496cd5097fe5c\",\"2f3e20746120185473dcb5b10f227a6e18359eb3a6da908c1196d38676616d1f\",\"c05a7b13463adde26842f266ad2d4cfd38489a64c469487721c11e27f088431c\",\"0e5ca88697b7bd9272682e3e1c4cc79590aadb576dd7fec2179eeb486f9c0094\",\"217c830f9ddeb4d1185e7b4e98ec83cbbf8e13b68a72a1e4c69d10cc01a81493\",\"08c6216ca536d1c87af83c3dc154031eebeff002a5c13e86991487898ef94b89\",\"794a619078afc2a6ef759ae5a238373f038835024849466056c55662a7bfe3d7\",\"fa3dec879a4d4f7330ea0cc524d8cb37362677db82bb57046472abb2ef4dd48b\",\"697c30d68c8bddd836bade399a3d0c7f6be34451ba9e373fd11787702a0ef8ec\",\"56a280b3bced9316713e0b396eb3acf0ad98981472beaadc9758835cf27c77f0\",\"e3ff37d1e3787a2713093079eb3353e87f9600bbd601470df22fde5911c7456f\",\"4d790a98bd44e3e8be167521164fcc443c3ccff790edfdbdb39009525a9bb58f\",\"2652e19a3f07e9180a880bacb7bf4f33f50784c4c821d5c9c57f1a214ef0f3d1\",\"091c19963e3b1caafe0eae8f55450b55c2da2d399badc7b88695a2b675014744\",\"e0bf28534faf22b17981eb92ef594e1f44f226cfd6943dbe40ebc0b4b4a7e8da\",\"428c4fbf1287c40e2aba4791796f13fb159049fa66a2f9cf9ebacdc6fc014ccc\",\"780f1e720f053f1f007f953a267675966158b7ed5aeccbf2bd8f320bd8201538\",\"8ec4ff28f90ed23e0c9fb4926a812f54a4f03950e8004863cd889276ff5e8937\",\"8e332bb04de5584bcfbb7e89e5ecb49d8560897973ec83b7998dc4b4d59252ac\",\"f7785a554eebc790fbfcc7ab0fb581f2b07260b38e48728dfcde9e4942138940\",\"e8eef59c4aa0cedde6164c01dc30435669ae2f525a137109b3ec38a3bf90f814\",\"ce253e5c8698ec8479aa4225b6458640cf6fa21e6261f75b37d3d796bc85813a\",\"b1ae31261314090cc4b37b3481f1ced8a12a7ebd29401f466d24bc44e1852438\",\"c2a3278f0187f44d034ca8959399e1152c853381ff317166fb3703fe6336ad13\",\"92dbf9d223a5044265fb573d359e9df93620617ea4ceafc895b6e6193ff15afd\",\"5065c1983834961b2c16583054fa4cbabf7d456263859b510d7686640e74a102\",\"f2f465ebd057eb58ddabe5546ab54cc984db079e9fc3425722c830c62aaeb6d9\",\"74c8938c7cc2da922f564aae6fdb6eeec42dd8cdc35fded466a49e42620ea58a\",\"63c7bc50d656f09f31d44240dab3a6f7571c921ba3a5825ef7f079ce5a717253\",\"5a6c08c42ed80742a003bf76c0cfeeeca4d18e7dad3f76238850680c1d3d20fb\",\"dedc9e467c5f99208111afd0f035215ddaedad101c1df5e13dd806e7eae6b1cd\",\"d76c5b842452230c536aefc33ebcebb677b98aba7035cb8b5f8daa9bd9ff8dfd\",\"656295ce83a25fc9020c00dce7749eb77f75fb8ab0cff03d23b19160a59731b8\",\"50bafdba8a9061f7f19f3efec1e1f65f5e9aa5ba584696f23ba1f91412571bfe\",\"0bcaabc299970800e178dcbce070e2dee50d0270523f01d160e5861196cae091\",\"2625819cf8b0b16b4dd78c487f904c3fe54497ae6270c7865e0bf6b86c1d00e9\",\"ff43261db7d0ad7ca811c5005a853603e3adcea242f4d2776097093f6eff1f61\",\"5aa0697b566bac4440616411920e90fd90ae10f7ac1fbb85597068ecc459f93d\",\"62555fa22b993f9eb0dec2ddf0641eed587a32fb53eb020624363619d0bdca12\",\"56d57d0e3cbbd6b32abd99f1ef686a34c545ba827513f9c73016271663f7b773\",\"475f93090133c0a3620b72dec941bbb374ccaf2884cca96d934aca66d70cbbf6\",\"35fd9d8c3747911d6e4d3132fd4a107e1777678d11a7a7271bb2f9033f1c24a0\",\"4ba4407433917f15488a325374f152b5e4f3e767a56b43edbd373d5670416054\",\"b5d1e0d8c09c52849f5e746f61800faf8770cb1ac505b07011c4a5406a9f0a62\",\"68cb0a186499130362f2bdcaf36ec7d9ad5141818cbd3ef6e49a91923d84327f\",\"dfc74e77da44b87ffe5b5d3525893c317978906b68bb2bf386f4e82956a6146a\",\"a261ab7c6bfd950ba0e7818594512685a0941ead115c06e0de4bac32a2f67438\",\"52152803b66d173dae1abbb7fc6a74d50472100796e90bb5e4cfe14b140679de\",\"c070fd7ef1f32c956efaac73cfb259fff3cebaf6ecdbba7d930716a021efff05\",\"5b49b82384ae18cbee5ea04117ba4acbbf4b8058835324ae299cd75d307efaec\",\"631f2d84f6c9d5568da44f0fcc2388b46352b9d689989cb3c98456093e96c901\",\"2fd7191e98042276a93f04b2575afae52ab81963d28cf741d4ae97e373126d2a\",\"506bb344e0b69239995b838241164521dc1f5f8dfc14a5d3225e2c1940a77392\",\"bd6993c041a3f44148a46d32c6cf57ba756c71d9b9bf7ac3ae4c7dac734631e5\",\"939a515dc75d08894a635dc63b44f3426662f21c3d0bccc551ff8c5600ea9aac\",\"67940494f3a9bf4632ab906480275146bcf653c3e40dc5e44dc9585e653ee296\",\"9be18afa291b19b91e501877e01f80760174dbf1a3b00a4254316ecd64c78ce6\",\"044448ea0f3a6813f3d8fc4fdcf53bf50a25ba8755cdcd175f28b05bf44d2f3b\",\"9884ca0d4964982073ae4aa32da51e7a9f4b919e36af070e891070cea4e722f2\",\"3f7fb0b7b0e1f3f13c4b514e3920abfe8f54e093a4bf57457bcb608e663d59ab\",\"d13acbd15d50b071887b29a6dce615271929f8b713d13d0357dc95e247693612\",\"2a5928ef30295162051eed61c43c7e6224a9b33cca45ca75c3962a55942893af\",\"2ff15f116e8b30ea8f7d0a91dca5ff3c86728d968e60a9a0e6bc1ab834cf8a74\",\"6d023219f552bb83c81618625df2601ed52044501a2246b87039f9933dd17996\",\"06d912121657b18d092d0b158fc9f36036742f7b007832600d0b44731d70714b\",\"1305d0d8c640572aebf8dde52a29430d37bd1524796277b189cb6d3ee277a789\",\"9748a414538af2dafde3f4a6e332af575183e20cacc19e70e052d303875662bc\",\"6b4ddf7d8ce51d33026c1cf3947c0375b0d71f5d7a979d417c8c4f7a170f3eaf\",\"ef6bfaecf4132d5ffe51494bb4b676aa3f9bbc92c5af3315659570caf211473e\",\"646e1a4780b4317cdf6d6c4995e634103a9a723082a0fd961fa9e7980bc50280\",\"78ee206a802d4de2a6743f184d405dc5d9a7c5abd770a2840bf7a9f4ef66bb51\",\"7effd1d1375e9b4d43f4118a99f51793141b1d5c3a33aada525d108c01b90775\",\"79b35021543e19a5ef7f8fd1f015b42d8de7fe1a790915cd6f87e135093ce490\",\"8ae632c6b3b52e1d98f9c1f366bf87d6b3e9e3ecedd48fb08d07b496e41f2b98\",\"c0bf066e5a278597cb083aae73a67dd100892e935aac4d02b82c1d65101ea739\",\"f1f92f9e72353f20d854101524155b3f0d33a00df1560f7eeef8117947b0b1f6\",\"c1068aaf6cf0b4eaf868ff3e98ad2defd594d762684d00a2771b3b6a422c7209\",\"ecb3c1135cbf5bce4120179e33968372184ff73e60300bd101cdf56121d6d4f6\",\"1d00825ee85893196e0fa3ecbd86c53cae15684affd7d84544d514122b9614f5\",\"f89411e89a238b3c9369ea384f7ef0decac81ed00ffe171a428a7d03fe9d45cc\",\"01e37315084847dce7b0782d16c2a64d0c33e9f1deebb0c530bd4c17e6799d38\",\"97859a9493eb6c0dbba5d013678edfdea6e577daa833ecbbe993e749a57899cc\",\"4102fc560e877e955eae6f0721dd8266d680b5da90aec98977bdadee43a00de9\",\"b8ac7dd5da77329061f67d382b4c71f94b407e91feed35de00325af46e156878\",\"385cc5ecad19e14c1102c81900354ef2f4a907641c7e425646f13ec673113db9\",\"f64835ceea07aa7eae1f755422e55fd9df864915c92ec8fe73e5e64c7014fb20\",\"4eb2fa71625c09351a74efc12df3b8860b5c5dd74f8e811f3a5f396c25472fbb\",\"d09889a18a55a4ff3d98c96555ab9e16ee4cd83a61dd0cddfbe1500fe1355bf7\",\"6471bc266e5686ec1f0c10a25b51e3413b58f72eaa6b21417e73e1f5f6e4be15\",\"0da9f0ebde90dc5fcfc17b8a134c90b1a3f62a2a9a6219210c0b029a9f79167b\",\"4be2a89df42738e51e1e52c2d7f55c41ef295c0e41c85198c957120b27118063\",\"165142d9364bee5b3dde385af9ee7fbd195b200e4d88e0397d6d49c2bcb13860\",\"038928b0598cded4644f9a48001aaddd436a5815a9fec3e827710098d48d042c\",\"c60909d932ffc7401374f63b9d526dfacd22d0f681a334e19653ac14b6074f97\",\"8c3251f523f0f9cadb48b7f60c6608b92fc3a264f1fb424219f38e2339ee1634\",\"c24ed7251450dc04a0f955556a8b5885a73b8583c46d8b1d1211d96e00452302\",\"60aee9a39437c45e4707c6d3d668e8c0b4847331d003783e4184a13a6fb2e9e2\",\"f009f18ff95350fc63ef30442362c8580698388970e7d56af1d8a982b48963c7\",\"063277d673f6ab0613b0a09cb59da1c5793ebb351e3932fdc34f5f9df5fa5faf\",\"0404efcdb5d5a4da23196bdca613e305ecc60e3079064ba3ddd16a093bbc955f\",\"f0adee303cd80292d5045eb60a430099ebcea573eba21543572283788dc1ec6e\",\"cb514167c728c1a2488f69762a3a83abf5a2db0309aecbaf0bdcdba42129eccc\",\"c8ae419b52810e00cbfd3976a34fd43c6f96dfee7226d794765e507fd3ee0a38\",\"3f7465312ecee8e99f02e0fa95ee55804a3bb8103dfd6874583775d7f3b2ca94\",\"76ab2448066cc6a00233cbff9dc23da0f4769a56d51495f827e777f63b6a7eea\",\"bc2ed9d0f78edaea2e66c4e08d4d5ff7d05c642489ea4d13ea880c2648530a3d\",\"ed1caaadd88fb08a361ab825b0ef0c69dda682959d30951409c33af409b2c6f0\",\"3c0a2a83ed44483ef63e2a98d1ae11f52941fe7187a129c2c67ccd51c682448f\",\"5197fd21630c4f292a7e943e07345165a0c07646b97bbb7c8303d9c6045f81fe\",\"e3ed4787852419d008a540a5f009debb3a902788ec3fe515398f9c5187167bcc\",\"e77a92b29807369db3738aa324cbc7288da95f0f415937fa804b6302c1352134\",\"17b8e1916d17a065a73f4418c3b4753e317a1582bb4fd1b4bb886ab3f2d8239a\",\"b04e79b382f2dac28e245e363913bbd2db968cdc566f9310a46b0f72c3f19d28\",\"00b4825024e852801084831f82731d3ae3b5111f6937647f024245748efb6575\",\"c9fd6f4024ddaae74104df4733368e596bf1dbb948a605ad246ab06293a27c67\",\"7a064de34bb08d622649cdb4657a8339da7296ec05185e55e68dc2f2c20cd71b\",\"4457ad200b11d4d420cbcf27b4ca78e240d9bf2a47d02e73a68b6f11d275226c\",\"9b606e5219a22165ac9af4739d8ff75fb82273c577644f41c408903c19897b32\",\"f2789a3c813c07b9a42e59bc52fcdb2bfa80e160bd2be6d4aba9cb9b8734ee1a\",\"cce817580d13662c945e5adf1b669dcb55d73461d8e99fb19830a569db5ba850\",\"7c01969965eaa63f5296a202fafd6c67ae69e776b654547bbe52c9f280c4e425\",\"b0cfd963e510878090cf51dd8ad0358e5f3e6ac8ad3fae159c1d9f686351fe7d\",\"c3108a9481f6e715e334d45091b5e8c3773ef8d38e861777ef556344763047d7\",\"33c1f186e087e442bc5b53506fe46fbc15f338e916885ea29374e482747a47bb\",\"6622c66e1bce36b971ba658ca5869119ab5c9795450b8935276112e717cf95c3\",\"73a659f2ecfe2d1c2cf503dfd579017e75596c98a7b23b358b23fff4edd7aabf\",\"9406a7cad67d01db3f2bc42adae0341417dd265d1ee94bdc3b47fe5432710c4b\",\"7e8475ac4bc895a3c2ea0a723a235c9da2ad725c6f44b6c56295189df9f5dbd0\",\"2489ffa87099614ec0e6dd2e69ea8e7d7a70fbd87cf59aab1d2f087ab5e6afcf\",\"8f6081ed4e8394c597ac8e800a21a22312162942657efb3cbf158a163ff56e2f\",\"0250ba08264184305c56ef3dea01e376b50480e35b22f1819fbd896c94ad031f\",\"46bb4ffcf1ef535db8e9e5988a7c6174312df253b43372ff114f44dc1a791b07\",\"28b1ec8049b5c8d5d01386c143fa944252093caf78427bbea85c721c124cbe54\",\"502640b1053125ad35ce00c8362f1a92cd0844644adac6fcb18a13c3e5aa55f7\",\"decc06585603128ec181dc1a35065f03711bead4d5fd236dddf304e75240d0b0\",\"a4abe6c4a19de426f293f458cdb72d4edae7e570badcb22c253fd9bc62946587\",\"949a83cf7fad84ef55d2b1908d4819ef09aac9f0df8ba795077d210c2275c6dd\",\"8937b3e7fb97f017d012729b40e5bba1f21cb2bcd344b5bdf4cdf769efb87f48\",\"9e454d178b7e81251cca56bd8d78ba5a7fbc24e9ceba3419c1f263dd70b254fb\",\"d6e5b9603f1243a405a5c17879a62d532b9150f4c669104f3ca0dffb7a1c6535\",\"fbcad6723894fa77286ebd0c6be5ec94afa90a60b7f1898fbcfbdde17bda9a5c\",\"326b11ccc6dd8fc0fdd9c49f38a033b2c0c556d51865c6a50823e99c372996ed\",\"3bea02158e997de643e5f2298db68418852be43f478cf128c2c3f4541e44cd8e\",\"8d65e6994784e50f6b692370bad1792251ea9a3a372d6e165a710606af276e2c\",\"3d57b00d32eb25d7aad2c8d8db6a7f5fe2bbc71c7e3f0269b087fe24daf68e0b\",\"d67a438c14b368393562fb4fd55dc81ed3e5c0f3674d5042489a7e2427630843\",\"5f8a9e634206874366a78f23db1d3114dd2a53f49c6e6a78a56d809512b7504f\",\"63386802d58de7a0c74f04197a85e98c3a45f3e517d1bdfde4cd19dfa94401e3\",\"8962579e93f9009638449aaed1a5f2483ad4e26ab903551cea2e2c6b9a4a29d3\",\"2f9d084a8ba72f997084fc890a4511eb0a1d79668920ea06450a0c1ccd724acd\",\"a93a2a6b842f84d6a8144cef210a937ee57947b83c8e61671258cdf83f802b06\",\"c4d1d46216e030a785c48d617977de6ddce1e7adc331af769d6e911f01bc2853\",\"25d1265e5cc1fd5e4f95c119e99da45e185b6a9c5fce38740629c9655e3999ac\",\"411125fd15baf9046cb05a61bc0b7c4cb3ea7b8fcedf0ef81e2d8b6ad6d5d982\",\"b16fae74f7ab3518f5ab760216788d21095bb8f77b384235bff804b0577fcb01\",\"552536668ec7b1d0a610ad0c242e5c409ea6714826b986ca3799e1a62f6be38b\",\"5df7ee0dfa15fd34303e8e8f9805a0d8f4fc878b3c7b6f9537f3c32dd4479d9d\",\"71096984c62c585501608144c883fa6e1c755a82f865add0cfeae3a1a55765bb\",\"cc2bee471d460cce8e0c750a31d2d32e79b397a2d3cb456f4c08ab1d56c607ac\",\"a37be5068fc6f93e6627d5de2ae47bb5866f9cf040ea88ae1030288404815b16\",\"d6196bd221d0bd3e9553acfb385a00c6550c4894669eb1f9d6010e629a19998c\",\"eab0786b26d8b80d1a817a18b6e44d89c75f1f2151ab9695c20af722dc3fee5a\",\"3a6ee6d116bb890aad606bbebcdd9aee6506af929d556d8280bda3008f22ae00\",\"d5aafa6cebc6ff74308bb7b649f768771c0a06538009b641ca14dba4a0fb5f6c\",\"aa09de518f7967c5660973b177855e072073a751781c60b8e6925cdb3d11cbcd\",\"8d7e71032099a99e8124b86906e24f68832c18bb2a51bc6ff7e50587d2b5dab5\",\"abcb3445633630639c478aa3c0fecc58e04985c5c32fc633a0eb8ff91ca05ec4\",\"d3b0c42fd9aefeb35e6245099df128eecd8e7f91617b95bee39899a2071fe4b2\",\"b8d56e46033b7392d829c523d3e69fd92920411bf4c3ae6d294d50ebdd3adde5\",\"9c76aa2f217ba2d3267de9494b13a535afc0bffa7c1f98f089480c99b0eb1ee7\",\"d19c9fd0886f096eedee85d66662c763250ed6c4631d0ee39fc4c9656f063def\",\"4a7feb806f227bbe12e497bbcfffff956e85809d4a07c637886a7183570e8548\",\"d1bbcc4d89336ff110e8dbb54af30c7d6c513284548c54b3bed6a60f3bd23aa8\",\"8717ee6b3964e97cb0ea71f328b557a0c823be64a9767263ecdc53a770aa4fff\",\"fd50ee6d5fc6e28665f5c3565a6f5e4986819473112f37dba19106392228ef44\",\"81b3e78b0ed606e87f171a63cc689acb709e93d1add866a2f9f7aeaa5f8c0f28\",\"bde9cfd14037dec9cf55f8ba08d8f003fa75bab7fef692f9405cbc1c379b8f37\",\"1d0882575c8451f273d9fa0a218651364e2bca324b7c5bb420762e2ca8e6ff60\",\"b320b4861428b696258b94a960ba123cf574c20dcac2c895e2f65ee2efcb1190\",\"453873f14aa6a1a2d20a181f3d72a920ba0a939645deec615b67c5c32580f5b9\",\"9bf700fd6327aa37eee2810081397fd6b73fb9f290d325cd9ca8f05279839718\",\"359a9f8e215ef18fdcec8034df900baea1a8efd7d4e1dacc905725f69421278a\",\"54002c11344f8248e0ebeca90eea023c88650c95683b2d7678d16cef9f9a49eb\",\"53790257e553d3288619a3251001aed55755fbf6d96ca647c543ebb3e94a75dc\",\"d8d2cd003ad5c668f3a7f393c83408bda40187a53557ca5491f551b47eb6eda8\",\"9566a506f32e398952133ab095dbd2a1841fefecc2b5174b9b14e82b6c3d5e9b\",\"e30400e13c58ab92cfd4f35d59007db25b5df4a393a6064f12174f4f948a8295\",\"935558106cff402a696ae0b079971c333413cbdab4d9d37c5920e67f2a49c9f6\",\"23bebca8c98613641b9b021e6e37974069f8dce75e3fc971ed47967448ee2aec\",\"c3649e280cd6fa4e452b2861d41c86c16d824afbd5902b9c64a426d6fa05a718\",\"6a64a75462960307890c21e5cfa43ab645c474a61286f08e330668b0e7fadfe3\",\"49c31f203a7ec0132b82c9a3cf64065e78ff46c00fd94cd9a751cd86508dfa86\",\"8f8c65aa4a1f237027ab1ce4b670489afbd92741b6ec13394486e6c77f8651b4\",\"f34a5d62dbe14fb73c4396065dfe7a884c4a9f018d43fa1feac36137498bd4af\",\"f2f693d65a9946622bb6bff154de7c5da950f10bbf653264298714ed738de521\",\"175d6bda641d4bb8da8c773d5d1bd51acc7fe969f1df17a280cbd8a288ea1bb7\",\"56f30ad8c3633e72380af7194ba2e1649d538c6d66151325a32fc42a583692d0\",\"ca7cecae79deac8f17fb99bac3db5a70a50ebb2b9cc6db078ce8520e586af5e8\",\"0a5cc5fdc6451e0bd749487ba1c50ef3cd2b7f84ba51fdbd528eb3d3a0fc82c9\",\"1192d1569b94fad5c1de4e3410ae2969139a1abbb2d9e4b11f82d9baeaf3e533\",\"bbdc4a682aec90d8e7af3401d0db435d8d565a7ecf31ae395359e9fbb159c93a\",\"6a36c0c5363c8b7563f34a8bc649ebbf423779edf160e791abe3d7d047202ec0\",\"c4d65c3dcb620b535cc1f538954615db95f61c5793be9e76a03467a92b1451cd\",\"ad68523bf734f06915e910dc47ae0036e303112aa94c2f4050a0eeed652292b0\",\"6465afcfe232d28965d031d19fb657db9ff7da3d9382ce1ffd39b970cdc3342d\",\"3556727c573eaddac0ac1073e2fce4fdb4b3868272d9ec303c1176a2fd8d5419\",\"c99778259dfbdad44e3d89cb94bd53eb90130e8ce0340f5edb807c2ee244d3c4\",\"7ce943221746cc244a5367e3e3f9fa6df0ae5d2f3db072f87d42b4f913bbbce5\",\"416002184a410f9429f44bc0e1e8fd6baf6c5d73571fd22768da79b2fa46ab67\",\"554af9b399b460884ea469d4450223a29d650c967be071fe99aa497269ed3d16\",\"49340406f18c4762862320e0996976f27051e583ad9b02925ff69c4c46b6af0b\",\"080d9c3208c092cf83fb0f084dfed531ea5ecd5a74228229d3b165c7506d4ed8\",\"890261392c8398729abc487b94417a8ed0c7fc862fb21052c26882f73428c16f\",\"8c33415607aa281d540e0751259342c7426d186387dd59ed2e2c783772fe262c\",\"aa0f9cf5e88dee8d11ba8ce14199d7cdb9c76943bbc3d0c0828fc8faae10912b\",\"a27e91ff6ac6684e81f6a4e49df0941e44831ab26f279756d216c5d2fcca177a\",\"b82be2f7f860e586fbf568dd3e3e604919dba2f932d4b64eaddafc3c38d4e6b1\",\"270d6aa42a629f8d33b624504e8a37da76bd100f73d3448519c3c5fc62e79633\",\"0430a69cf3c55c4959af18fc995b0fd93f43e38458c6361a1a1e233887bfba9b\",\"c8decf5b1873179fabd20e757dca5e085feaebbc6d90c792d3f91f5f19194235\",\"68929236955acabdc4db5979b688fd5885b71e7a05dfd40922ae743f00b337e4\",\"8f38c8b38bcb77d280a99d67035919ce892817dfc29c05982f3b3a1eff506d70\",\"b3e1858e7b91c7964ed5d966c22b1c1acb5e0f3f8b1483c96545087650d32384\",\"6c010949f040ec97929e68cc0232fa952d40d172dd33535c70adde44e7983dfe\",\"80a523431b7007bcddc7507823a1231253b0ab179bc011b032e18bfc6a11acbf\",\"972d7780333dab29a3277c507b2353ea12a8cd346e1d45a353e6f70dfee63d96\",\"52c1661ddc610c380a03a2bf81f9af1a23dc054ab3a9e1a1df38a5260dd0f06e\",\"42ae7ba744ed78468d59d7eca79bb15e04225eb1202fa40d008b73d59ee7102a\",\"cda48d5fdce441b5167865e647731b4be36c54230bcf0a2ad4f41771ae3fa548\",\"b532344685d2041a0727f62a1225db4c123eb75c10a12770d671406dcbbfcb44\",\"a91ae94da49807544c8977d947575822d3549a9152582ce7ae28b74aecb05b47\",\"9ae81e8c57382d1b896b925b80c1b01f681ff1f5c2c27f7e7956c996ff5c8fe2\",\"b776f7b185e0ae3da9357b4dc658e12c257ec704cef7ca16d97cd9362c3df56d\",\"a27c9fea49bc1961cadde47da9ea2a0b8ae044a081612a22a2ba684b72e931cc\",\"dae51164a0eee2fee8ae5a5f070d1ea6edfe804a0ac054560824ad668b5fbc3c\",\"40fe4dc6176891a0a7582f4d371b7f03f814304ee414fee33b444c8518cf8cb8\",\"0af56b0c39dce041319e30bdecefba40a186eaf37287d13756885e0592cb4a47\",\"5240c59ecffdd92e16ed91dd452e5e8c05392076e54f66a0622ee29656e9c365\",\"df17438576c3b2e73c392f640547efcb89c946fdbd2467fd221c3d06a8e2cd84\",\"c48f1c5341ea97f292c8740fd757bd13ee50c829474ad253fea87edc2f9f9421\",\"b48aef3c657fd79a831127458cd5c4a415e2fb6b7a98b68ffe331368de3d11ca\",\"a47d49251c3b44f970560f72226755994bffda6fa5c31cc898db8a1fea77139d\",\"9d949d42968761b06f6289bf4e81fd145f2aee8c0d3bf8bf960f3728fe96f7b5\",\"de71bc9293da8bc32e5304b420a007f930ee29d871138fc32322f37ed5217beb\",\"949dc736354eb4048331cf521d63c7356d0ccc699995740ef0ef0e6c0e219f36\",\"10024789e798e94343cb904174972682fd0057ac69931ef129ad3fe3444cff13\",\"7d0dc797fcded1adf0b0b4b55c0a4267b29f012225fe93c9a735f10458aa25af\",\"f4630d66cb25784dd2cad1888b97e675598e23bf3f84ca84ba5c101e69a80e80\",\"1e2da2fc9427f14c2a4234388bd6f98cffc6daee734a603e5c39242b2e67bcab\",\"38c623b31b440447d7da5d0f44f1b17c6470db1ce7d8867600db5182cdc484a0\",\"3b56c34f5ec2c7a510d82b12b5f3f0933872da7e741dd2dc402d2af2b3729c56\",\"966c2d537c73b15bb698fd5c4fabdce3ce52ce5a82eb90e5e1bdc93f67e67c12\",\"142c28efcc705ca45fcf1f4133b0083911eb4e13175c9c0bd4d83f28087cea7a\",\"bf3ea6c187fc758daadd5a97470082a47e666bcdd9478ad94b6e67e20af98021\",\"73bd69f95ab8ab9c3802e41d4e8079d5ddd01b5f0cfb3c06b07a887e4f25e7b0\",\"dcbd05b1f26cae650232af9cfd4cc142f367a5f1f4c16e291d3592b7cc66da6f\",\"984bc58bf9b4cc5b8f912d07b8061ee7b56314df60a08eceae536d1fa17a8f23\",\"f1d35e7c926ac7a53f985482de5d4ad14449a10934268cca5fe4c6f000e67fb6\",\"0d71bfe2689bb67d952c84775cf7edead8603377643bc33b2437361afeb9741e\",\"98601ed0df56a2553ecbd4b9c723fcd2b3d561d13146596be524a43e21dd5687\",\"dfc293b1e8e042497fcc24f2cf318025a56348e33d3ced7c0b949e6e6f1d7648\",\"532d10322aa4936ddaef4e3b50e2879ca8850055b199ce9f9b7be63b4bc1b29b\",\"e2a62270e1ef6c32d07c53f3f8bd5761f11393716162dfd46a5c8cd2c028f59c\",\"0f3c1fb691253f74a586cde783054f843af9f5185ded863226e7afadb32a2e2a\",\"15bed04d8723a82441c77969730864b08a01798f1be80f747cd8d2615a2e3ff2\",\"555925dda2cde1f5be077c2b7414941e0141900c67d0cb40ad32844c42a8e20b\",\"718314ca603c89bac03a4e47a8fd81e28702cb711cdbbc6d73c73686c674ebe2\",\"590290358faa3fe8640b23a371777ec27f21c8c2964933c0f49f3048d0b4f24a\",\"a4e91785649863ffb1a7bc4718adaac3b01be856917eea18fb5d299d1289118f\",\"a3a00795906a48223a94ab1e380a64d42f6b4beb680248361cd147384a86cbc6\",\"d2f92b0c5523f4bc55b4b9e13caf62bdebf0e09d31e431da3313a6e968c158ca\",\"34f2ea51c7e1e2be8bb3ee351ab319dd5a9288fd71bdbfb70adee9925627ae87\",\"b8d54d4e63ee430d463427ef7375340064c7ff7bf7e11fd1261cac8f896dc8c7\",\"e8716f5094c671b5fa45666fd050afa2ca02385bb0787698969a198c2401b28a\",\"16717e220353e2dd36c17d3f7c0d935ceb029cc7ab7c4aa93b75ea095293f2db\",\"8b35719a69b0b0a6c7bc483d9863090f7ce31fa228ac4854759d5e80300c86dc\",\"29a9a4e728451a1e7269c3f4bf89e8818f2a4e2dd5bf40524a0fe834456810a9\",\"0e5078f7f12c706a9a03c22d60d096046ad88918d3cafd1dab8be210f13dfcf8\",\"801a26dd16c2d24ce9dd65da08841ed7b296e12483a9d1ed287bd9c391f112be\",\"c20b7dff9998fb046630cb5b7db0dc4a5f11a512b2f17c7d1ff38ea1c506b8ea\",\"4fbf8c4a110acb9875425c1bcc76f08b2b9770bd2986adc7b756c2ffda26653d\",\"af02a23a9f7b14905fdd402570db3b1655846b8b13bd9d86c434de12e1ace9f3\",\"d6fbce068c07fe66a8f0b2fcb8851769d1306ffb630f85b4286f62bac80ee53c\",\"a2e1c0f3992800ba8cd4edf2a210936156fdc016c28896deec1e0adbafe1d872\",\"9caae1f7b3073a29afd550791de4f5c30a2e0683291e68817392d40cbfef0fcb\",\"3165c605e7d03a23668d055ae22a6e2639606e15b0800340cf8741bd03f358f4\",\"93220f96e68ca38d9c53f0e0c8ccb71ed397cb63ff9ec2ec2d98128af41e80e0\",\"203ae9cc173278de6977383260326e77469a92fde66d9e4acd6eb190ac533c3d\",\"6c5c383dd59db19a665c3716f55325235aed07308b55accf9b74120ae5c9aa2d\",\"a3befa8f3b9b5feb183c65684d30bc926d8c2ba3108e7251dd1330fb45c323fb\",\"c90b76c7c53901d05d307eb4ec2c939a79b2d29a22bd1329d217c143586f0a23\",\"320f6a59fd560f51e23d09ad85e75db99b2d5db1f7d9d261ff48e3ebc2d0f38d\",\"43e6eedf735024d152de0d0a4ef9b3987e71c5541c3ce330464f68b83e061346\",\"5cafd81a4d0d67d6e7e5236e08099b8b9c84d5a127f83cf3d4e26539b3670eb8\",\"1fff8e4cc4957394e66c4123189a8658830a34173016da9abec260157860bc83\",\"06d0702037f70dca0c23e4ccab1f6ed2cf68594f813623c10665657bafb1e61a\",\"c3edb35e5f6041d06413b1334c78e05fa3d1ad6a4a3ae02dbbe646efd5943558\",\"78f914f9e4fd813af707b4484242e1bcfe83bc42b7903adc68235419c1000fe5\",\"2d065e5591a1eb9313cd6cf27be6589b324d3fab2302fabeb47c679534e27dcc\",\"ed0e533e44848f7693bf6f6a692b3097afc9cc6304c547d52553cd3c6dd6c117\",\"b2f9a4cb43de02bdf201dbe8746de350cb24720f88eb9379bc68daab00e097a6\",\"13451284f8f71a4497c25b12e40af0cbb3c62ec46fd25f51c60e0c413c69e3f3\",\"0bb13eaa5bc70edcfff69269c6746a5c9ba588ebf1d842b70a74e339bd922cce\",\"41819485854c1c1e3d0004094eb09aa1dbd399f3fa75eb7ea918c7d311e911a9\",\"d1fd976428ce32c79a8de877a174ea9874db2391acdc94fdf9080868d1fb4490\",\"b5088d6f6655bea36ade53a26ece4323b155a26c713e81c31dc0096d74ed8557\",\"7d063c2ca9360bd76f0b06a0326186e1c68f176d77f2af7140afc6dce10b98d8\",\"649242878649fd1b3508c1a22a7e2f8ae8493d5abd384337b6d3075c3df2a1f8\",\"1f86a37f5f6c512c99449b506c62b998ca5854449463b76cf760f1b34ce84aad\",\"b08977f28f1ae17e4eecdfb367ef772913f1bc9efd81a1a3afa01663922ade68\",\"a38dd042e9f56ec74876bc8544b805926bfe9c0598af2af52064f5ae6d385df9\",\"07b32ac00b406bec77d0296373fdc715683f3708eaf10b8e60153604eca2f695\",\"85cb35b5783b45800ed76bc03e40c9af16e77d27b79600c06feb3bcc10e6b227\",\"638e5d163b36b6b73d79a0b4ff530347f3b25930de01cb7d6a5035b3e98c4eb2\",\"0f072d6a32fd55f001167b16ae37f9bb5ef48869992ec1bb21e9494b707d20e4\",\"e7b3050b19a4c3b1e0b5b4dc925a4264aab1fb6f69e0cc61541f3ec1e6fdaf0d\",\"b92ecdd984d001bf691948a7bb05cffddd0e92ff09c63c99c67c863fc9f90b20\",\"a98d81d9b65d8faf8a5536489d1164256a65f5eb706814c67e5f939dc942de1c\",\"6654bc6675bafc37de2471623ff807a20edb5566f92fd3b5b137500fdf42d99f\",\"5369730d323b8636d5baa00d9be295e2ad215ced478a3ca97dcd00cbb1666252\",\"73b911a53c66356efe33d6de53ae66baa7d9939894aa4542c993d77f2144d988\",\"48af9d0481c60fab3b80a4e2afd63248eb0e8e66d796afb9a3467c77d049b38f\",\"387c0b60b925d7e9b442d0db55b4640356b6a1d5e3804aaf732ed4172ffb277f\",\"43b5de25a8cb831370001ab226fb542002359ea5d845417a8e06c89d28590f41\",\"dcf929ecb8481bc7109602262093b724c96421f3114af4091651fba33ea1ea9e\",\"53695595afb4f5d0e941a789efe47dd07130622caae00ce51d676d7079959cd9\",\"f0c5589a1b9a18678b99a65a4746eaf444875ab9ab5fb8b587325c5bdae3b5dc\",\"03ec07921c79e5c3df9b57fa3e9410369053fa4b26dda494f82ac885d3fa469b\",\"e2cf4528e81ff1489ed59742cece6505401479b2d97e1108533050e2db619dcb\",\"901fc5d048c82f531e51ad918bba7df07c37147ddc59f9721a961d3dd0fa67ce\",\"96b76e1d54788eaf5c4b5b2894f1c0ec48dec59e3511b1fed7fbc75ce483a6d3\",\"eae0d8d5fed05723d27c185782e62a1ebeeeb33869517016aba25be65552fb4a\",\"1f83890185074f9c40be083ca214af3d735c2af1033f4a89ef29f45b84e59a14\",\"286e448685e0f8a3c4813921a2584338ab33d1d6b8e71df9ca67e685a134a37e\",\"d472e385cfcb3f7ce9032091bb3053b054124c5f75d05a1254e29ec9f66e8b26\",\"96b80a6e0969f64e2b0406b47f130251bf9223e75cacfd249524dd2f990955ac\",\"93a42ce8b73bfc4dc560805d8626140d055ae7ddd949ad7271522f7e75f27841\",\"cc444c867246487a1153d5eb42dc1bc3e80fa8a746e16dd4d71062757a915ad6\",\"2dab1558e83c3c54dc2019448041fa7289a9af35d990218ce56c688810a59bd8\",\"cd57eacc64fc5ce74cd4734ff79453af1bed66c0d869fd5fb664cf75adabc815\",\"44566186520c93c4e43bcb22e2b0aa844fba39db84bc04d5b71aa8eef2cb6ee4\",\"842bd26bb519953154010a9771301fbc756b981fd01334a8f5833e6a0c1c6430\",\"7d299d44a3be71ccae34cbc3831ced48c2e7a95569bd2f66f5071da8274f67bd\",\"e9db5c321280b6bf8ed802afcad98bc99f0c14ec331033ef6886d8a834531fc0\",\"2d0d60b9962e28ed978121d99008bb84b2468df37cf118ee0e5a49c27852eff8\",\"3a02b2036abab1e161fa33cccc9114dc0b95fcb1eadccbbb68beeea67d6e2389\",\"2063f9e4d80049c78fa650814a9519a70f4f737bd1e7007d8a091b6321877cff\",\"e928c91c2bb29e33a45472b90ce704865e8ce1a9c3b40c5d802880490d395034\",\"2a0374e68c325c8440e9d7d24975e14c960ed2da8fa3a7cc526079a4ebaaf846\",\"4e54cefe02cf10cbec4ae9f4078d7a7fb0e44be926b6ab86f6770fdbb49ff78f\",\"224bf186ae664583f3bdaedec6156f9b183af107c81ad58b97cacf9cc32f497c\",\"f73ac57b226f37e52d21447326ecd378b8a3795a1378060e24fb6f5af6c3246a\",\"b397019aefa3212535e1b679d09c92b9ac685f720b41cc2a74e79d977c752893\",\"ac1d10b5973d29660badfd366c4dc9037e3520f370673e527fa80704d305bdea\",\"2f6a7d6a25400244ca87e8b76907fbfdd7df9f3e7240e5d97f660adf6785026b\",\"5afc881d6ae90c5370f228b3dce79214a481a09229725e49beca61ca28496865\",\"e87d3389c4d49eddd6a4d5a5f06e57a79de5b37056b51b58a8b1ae823a60a120\",\"11d65842c7ce9dab28795b530b6d87385019d24b44605905e7f529a398d96d4b\",\"f5e5f37018889009ef616c95229a3ec38bd2eab6aec6636e683c15cc30e94066\",\"7c370444a7f67f0d79f441991ec07443f71aedbbacf82eb020994191687915ec\",\"a36fa2725eb61bfd3d98c947618f7a9efd1fe7ca6b0a7b2089897be57094d65c\",\"50b7f1c5ae45d9ee8c6e0a6f04c7f3ffde0b78091d541d5ce4c6be96e60757f5\",\"56b086093258e7d9e4e43b82b2ec1941459fe872d5b0d10726741d1e0a37ae93\",\"a9ad35f6021729616da9222176e7ee567a153e36906ad4d585e48ae6ecefb1c7\",\"4397575525240bd7a176f5a08978b92c404ce3dd56b56b35672aa418be7be4b4\",\"4bc9883198ea4ca797fae758434bd4e471db7c2afad88aed65e1dbc1f705d2e4\",\"64fa8e8d51f3affc9846bd90c0ebd76fcc7f6f368fca1d2f93736f09e00b75cf\",\"0c80de3e8898708d3b54a39ad5207761927eefd81c356a4c2560426d4a085dd4\",\"a562a9bc0139830a66fd61cd92a59e162366f1739eee760e58f8fdf270b84620\",\"37a1a0921bda8e091c95644b8dafd230bbbe6ccb7b45984f0f8c2be47a8ff3e9\",\"5b0fe5325598f0dabdcb72a6048e895fc6d55a1bccf52ae5f9339abe4a2e5817\",\"af5e95cbb47025e4200871b28e7dde031f1c6e9ec0cd71886e6cdc86151ee532\",\"5c9ed02696d8151e43a86d211987642b350653611bcfb388060fcf826b2c75e8\",\"97a10fbaa512a2b11344d31e249e8a1f050fc1a80c338cfc77ea7995bba0fdf9\",\"cd509536ec256a9ce330611968af1bcdc87625fcc1954665cfdb3d16e9b7db5b\",\"e29139095f8a027c4587a9b8b8955e1f5bcafe8ae23c876c16c36477f792a0a8\",\"a3f51bbfddea4df9311869c727add76976490f69055ee7e4aaac848a06c238db\",\"57f376f98a53296665e6b03024867d944d2a064c4a21971cff191a0dc711efa8\",\"4c541febbc2ff3d70f112041818025143b3267b91bf4da5474c8784f982ed802\",\"628802638d8dc5465ed9043aa2efad061f63695663e7d4d2f95d2d14bf3cd258\",\"9c693260a04b13ab9fa4c4667315e6790192208d34cde40c6b213644d533d271\",\"cb4255b9ca600adce55faef5265a32e19191ec60ea3c30f760854d0f100b3bf7\",\"348d069a955814f3f09cf098c2c7922b68b5575b65940a453c4c74de5d4b79f7\",\"120af066cce1fd5f36e48ed2147281a1acd7821f2a4037781fab58c624945e2e\",\"7c83afcb7c13c3cd8057619994ae005ea2867ab6e4ae6c72d07308ce6bd6986b\",\"e1a3efbb8c69421506e5bdbd57b729bd7028ba689e61680121424852d2c4aea7\",\"eec40a32f9865610744bbb90c1744f15ea58e788efd08137a1b7786669182365\",\"49b3f899a4248c31650b7cd6c1118078113734b513a71ac367ce66774c970cbb\",\"43a58e5c1ca30cca623725e8167bcd8e02932171b44c2e65c1e98a0d94a3832f\",\"c76ac24defb094ec25c2aa7926f37a6a12dc8d23c051a8630ff0c3fb51630323\",\"83e20b320f6ae31af693bed31caf6e59cae854e3bad715df701fad3dcb97e981\",\"1412757fd6efaf4f57ea7e95a1843fb076f8be874c70209a6b2b38fc6799966d\",\"98c847a9ab7397bdddbd3fecdd9cefad3fe10d6ca3fcb6ae1e3e6a4126b52991\",\"095af46351c26d66ee22d2fb162a1854b1ed78e3e6eae338b21ea5a824bbf248\",\"8ee0b50a9d57d4dd57127129ff0801d89d3a48701424806891c6526e72c7d184\",\"eb0c23426a81b1cedc48d3140456b74dc0db76a7853da953342cf6463650cfc7\",\"bfa6ad51e4f46c842015e658861c819beadb15bfc7743a0fcf174b01e2d17d22\",\"01d89cc6807c75a845e45164e2276a2e3a93faeb9de547406a6ce23c7c37b8d7\",\"218bb41f7b5d6be53c30d285e5862cf7dc5309f787689027e5ff11842817b12c\",\"95ab960995f8ae83da2c222b291ac32a55d8b23ef2c648fd9d2e96d181b92bec\",\"9eb74e70bcf551a327343e5a79d992daabd5a8c69d2fb3fef226939b88c68658\",\"c672870fda1f72b2540d7ac6181525db73a489010fd679441e1cc11eee36c75a\",\"e2c1894ccec996f88b25198a78945bf05c58458cd6f2240ee5fde71a682871ef\",\"5953a0dbcc216dff4bb60f3285f9a32b1de7417bd5f136121bd63308df8eb849\",\"c5aef0034c95175f2e1627430fbef863a3903ee003b3640e48e8d33d5c1ac498\",\"96f19a9f464a25fe32e80c017228ece64aafdd6786f674307d5577978581e893\",\"db4adfea69fa0ebcba47306da918e6258a3945f9f35240f8088137862085ce61\",\"853cbc0263a6fa00041f02cafb399e4b85aea8856f062f0bbc03df9b26e08523\",\"10f3acaf2200e6030d1b653e0c369e3485f47a9f1da3c0ff87d90d2809e41e40\",\"6752b56c81395c053b572e2229c55b06cdca1eeb1a6a197cd9ea6732ad60ee37\",\"fd809012b050540153490b9b7b7e8e37813a9a18b1bf314f97d996dd17f1afe9\",\"42ec86efaa02cd0cd48bb97f37b39a578608295ac6ea91434eb5ae6fbf10e40a\",\"0125aa8ee62c6ccb7234009286933a42aef60b425d8c23d090fcbfa51d54d38c\",\"6f4a965a7880c13a679b800df873b3002b35fdf6cd57ffb75ed0cd706adbc78e\",\"63f87d75de80a66c6bf960506ad7dbae75f6db0f494cde8e8edf5513a4e9d03d\",\"95f37caaf6c4f4b5f2bdd68aa7132624c55db557d8c0897531514e0348c206e4\",\"0f5afee642208ade51cbe087f1956149cd8023c21da8838e76c01a5db677deae\",\"20ac2f2e050d69e60743487066b58bc29a2bed66700ee7c9396133aa0c443386\",\"92c912c65aa782093472cb504580a61a02f116e0967f54905da8dadfcb3928fe\",\"b72bbaed09752c1c1a2752902ce293a30faa8dcc0bf0ccb98dacec7a90ff5fab\",\"eeb2f391cfe1ddb1e9079e1c60694d565a769a400dbb45bd342ed3ce1538db57\",\"995cd819ca7ca72304e5f687c32e56d0cf78b7eb7287de1d9a6d802bb39e7b5e\",\"531e754c6f243da2e62d292a12492baa69e0a9bc46fcfc6cfd58b874ffe02ff8\",\"0fb146c6c5fe8c62131f4f12f4f10d45c87ee5f758ef985c126ec40a47337b5e\",\"6216f308b351750c95be351f6e73d05d1f45ee639366d996ad745f1e9d629a78\",\"b670f8d3b13a610f4ea3aac1dbf060dddbab8f10076371ad66cca327aad62dfc\",\"8b72d63a00b57f38fb6be677a74b06aa09d5472a6a959fd0c169b6572940276d\",\"3beaaab2fa63fb6d49811babb0d23dfd8e474bf838acac8b60e50cf6170a5ffe\",\"8b9f9c9edae205f83dc3085735066be7d474c530e5910927c9bb32c2978195de\",\"1bc2f4e4f4434fc46d88b64219612d844063cfe954fc7544e2d0b5917a2cd71b\",\"610337017c397d6f5cd057abf939e4e94ed3e1e0327777e624e2b26632897512\",\"b5e3b1f7ce0dfda0c8cd301e5f6460146affa2572ff7903993bc12e6fe7a9172\",\"b84b577a794f4e8e4ce90e0df8e64cd429321b02316078d842ef7be835adc782\",\"2a0db62730e6f922216e17e13c70b6e1b06234035ab0068ebb4c8eadbf18e944\",\"2b5a113ff09aa11683a8faf0ec31ffa0f8916b0053716ce6445a507b4f1901d1\",\"982e528675be88340a9ec08c5c5b03705d92e2c9ae594c7c93e4c5595faad0b2\",\"50cdc42a21a681e05db30cf77e922e67d1f54b50ff908d572d631d9d1af00fa9\",\"68dc003b77d07e568cdfb3b4be61f39bdf8f70c4b138c1c86a8228f444ba86aa\",\"3c3ca2d41d8ba7aea856f68eb77ec9e7d8ade0f12803c8029442fcf5a15059ab\",\"b558aec4ddd211094608e335b21eda4b960652fab86afa784b3b575fa2698419\",\"eba7b251cc1ce5dc93e458a72af1070f93089ede8b3d10c9056948736db0915c\",\"48db387f9f3ea4ef20cfd8aaa967fb49c8b2143a1baa668d9575b0d1b35d0ad7\",\"bce7f383f4c266e531502b784b4ef9c9cf3e4a867e10cbce506459effd70a3ce\",\"07a2893749c6bd1c3e7a7bfea47a945cf19d5d8b45bf7a16a0d18166b5448b9f\",\"beabde7ed3b99637fb8134d6bda6f09bd317d58eb5c6c4d4a75f0749405f0e5a\",\"79d9771e9cce32824f3f06368c591668a47ac92a257298f2536212cb21ffb889\",\"ee825835223a0879b552f47f813e5350d79f6d55518a5458bf1a59b7cd1ec18c\",\"b6f90418e016f640a1e9fc47db63420e2e8b08b50efd8d2cc146d0c4d696872b\",\"5b5a671b350179e330edd342b7dfd62b9c31a98e3bdbb19733b706ac534a2508\",\"418cc1d2964e54c88c1d6a4b0f78a5d926e784fa6b469df4d67be61ca5ea2666\",\"c508e510fe7473e1ff721992970f1b976d42dfbc98f953673bcae36added5cef\",\"65af824092e29bddd1749b6e8fb7ed1dfa10af768b8a993a4e8f5218d6b1ddf4\",\"6836aa1918c04c3d6c0d14c13c31e31a6bcdfdc1e14f794b1f444a6fd45d49da\",\"de549ddeb7b78e3b5377d4ac12d6919a4fb9139e02a923a11f7e3b503320e9f1\",\"5d1f86614243dd9e86389c1777830927bf8bc5d5636da3a450ae642d7aefd781\",\"cb3bba12deaaa64e885ab1b38bdf82830eec1980cfecfc7773409542383e5b70\",\"605cee98628e002befb0453b9119ef052bc623659321e6d66d31976202908118\",\"1dbbb42e698ecf494739855266f8fbf3e7dd243ceac5650b6b3606a1cb0a61b5\",\"13711c39bcefd6b6d7fc3cbee59c247c39fb2bc9efd0dbc998d55d780bf99d15\",\"84b70d9ab132e486ae9a8f2c9220d5744e4cc0822c05b511bdd889f05f5f8d19\",\"6a7e55c4a326a55ee587cec75e384fec0e659a33ef96cb5fe6b23846703129ee\",\"4f3aef2e8b9b3407b8c2d7d01dbee5e18952a28d91db2d9633cb4317c59c238d\",\"4027ae271e75df1827f731b9ac837060d131abadfda0f0c820e6ee48e9f11c81\",\"04baa68271bb0fc32536e4103cc78193b17a627f7375a53a53bc006d69443d1d\",\"c3f2be3332d292116deb1b94a5c1c08ca6f21d42010be708444cd2c16634504f\",\"6f112326784298f8cb120e3587e86220814a3f4943b5634864ebe04b9f21ae74\",\"90710f9654ac200698402f3e94b13f0e5d64e2d6351aa8062671e4258895a541\",\"bf7edd6e2d5a11c86acd4e3367362279e403b396933c9509038d768c049af603\",\"90e75bb2e60d7feb2609c937c83251266e74c2d4d19b6684bbbdf77e7d3934db\",\"c3bbc15e3d9a4a34aacd44573a4de0a3f2acaad681d65b9f74009c9bb79a87e7\",\"6f471a4c5747c7748726d8f617ac0c2c7e92bc0f5d1cf04bb7e5c09713fdd779\",\"55490dc15408ba9194f8dbad8ed15c87dfb100f888edd94f5102f342daf357fc\",\"c02bfbad6d3444314c52641847ca905aec963e6627b639b8c42e7b4240665eb8\",\"14d668a2031b5ae48bbfe90df882052fbe100b94d7cbcf29abf4bdf74dd9c4be\",\"fb5ef6f84d1d265119509292dd2912e3aa349441d97ea02e10b605ba150845c4\",\"36c5e09dd61e51c2737a1f8e39cf0a62a419b2a4fee6d414276dca998abb301f\",\"499e3dd0c1989273706dbf5b5ff4bc4fa15a57db688f39e8888fac121cc3d57a\",\"19dde181e928a8223d4feb610fdbdea0e058c5681f5dd8e4be9fc72175d3cc95\",\"fbd8ac317937133c07a591f335c74b4831b105755e07c4523db04a86b5604a6a\",\"3c896707ac32180103fe73ca79df4030aea2b5ee9c3fdf058984997d0465bb52\",\"abca0738cca65445ea7a9304264fece3baf242508410e7b3ec3090efb97d1b31\",\"532df73cb7117cdaf3d4d30d1e9b516f55cbb665691bfcc6397757df8488fa01\",\"007f03aaa084bb462de51350042896524e9326f3a1f9ea058dbf004c0e27e60d\",\"96484985acc0e2c2370950d79b8de8c7b3b521ed8330a66c3e5eee258899786b\",\"b3c3ee3be2ea42c05649a7ec30c282d8e727c4d856057b96ffb45a5e7d50767b\",\"0a991c8e51e1b99ce43a469a8e3069d063cccc4e80e9412cb09e895f0fad23e7\",\"a4b8910c1e0e4fa04d7cf5d3836f00094fb4487a525060caf0500142c2c357fc\",\"bcc964030904f90a2bf2d4dbcf4e986092b84b016d123437a3d14faa16c83d17\",\"f1ede8f57c2cef9a6a2478cc4602c882efd58c1916017e07c722ee75186fd52c\",\"eef3f22b58f58c076be743cba0e9f76ea3a1477261a91de41bf67bf012864adf\",\"e4a5e24e516678dc368a356244a5e8ad374533d0acf366a84567ad58c45f22db\",\"6b222512b9d0f106382b3fab887a13c62b5c25b54b25cf7152b8bdbff7b18159\",\"add22d36618b394b2abf45816509aa345800da03cd9c80347e7bda5b2a34294e\",\"61dcfc9c1de6af328215dccb36b91b7a90d8c545e68e583a8726ca631c559fca\",\"edd3bf908b76cc3d3517b977ca18b84721ce1af05d600738629c8e47e7f480ef\",\"989d5adc7a4db00f690f7fc4c450df992698ea2a3938ea085be47196da357f42\",\"fe515f4992696dcaaab7f3847c9fd5eb7a1bab38922410b33ae49dfaeabac9f1\",\"6693da197d928c300de0c5249708f901177a4772955012c6adec03fa8f07429f\",\"b2a63d3212a52b40b8b82ddea9c6fa3e596a971ba015d30467a95718743bb7bf\",\"cb17f9ed4443867c0afa3af6b1f355c1e9db1b5cc3a9e325b124e33f62fc5143\",\"098602debfecf09884f9df5be7ba8e7f9d2c0f50eb4eff1cc6601bb80b1dd721\",\"983827ca790e06f2a84205bc539906956a26228d7927ee937782e286d1085c24\",\"508e0459287a840bbe5ce743078d3a3a6a53b07cd2f8d97fa27958a8ecb11c16\",\"d9dab86430f322f4261b12dfb304335fb5590c19dc1d04015e5b373eaa794a03\",\"3f563b8385f6664b228908da227d434d79bb61c61cc4f70243461b69195ef766\",\"8f5d92df510364c3e2e06b2e0c905395aaec145a28d0b9deed6d95e26f46ebac\",\"fc6a12a1c928889c9f3ad051c3f190a3f4e8fbc0f996a697798cee9a3b1c296a\",\"2697bb51fae93c84fa3e21e99471e0bf1428eded7a1eb3b193e73fdb6774191a\",\"237f026a1b306093909c90014469110ada3401a10cad38a337f26373d934c830\",\"dd9d4ebf9088ad8316b717039d628152b004b7d79a3211a1792c7cfb451d1afa\",\"1b83b2aa9c04cd0d02cf99bd81d33b50436178b1c73ee89be0e5aac13bacd593\",\"bc2b2fdaeb93516a7d29c37cf00364cca6e38d5cb58b82cada3cd089f4821f3d\",\"3c2b0be87b34924f6adea850eca4076e751c29f6b8516f2fa395dbf3842190ed\",\"afcfd37c74c5e45c3d753bb4121358bf574460ae7bc695560e08d180e45fcedc\",\"9d081fbfbd351990c8c8bd2b50d1d6a8ad8dfc6f49b879f68064dba22edae309\",\"159b7b876863047c3a529c37f06b87e21047957d6cf93df4cfc709a2f398527a\",\"9f639d10317bf253bb707ac3642f553eb73822573a303c7ee13a559a572d6b57\",\"baf188e1d08c8d8b095afc750f96964e03fedee430a83b57c3f71ec586398889\",\"062e9c86567a3b3ef27906075bda29216c7010fba959e5378bbbe094759e067a\",\"b4784109278acc91f56d266504b86395713a36234addbfa22a7708bf2d9eb83e\",\"ebdd9f18186dd39e2c0de2fc6275b219a48c2cef084672a0377c93a1620e7cc5\",\"e15fd9d26da541e6e691a4799f4d25aa86853c27606373824dc61bb858c66db0\",\"8de1f5664a1f981ff4d5640e5c7299b619a13a299c52403f8ec8ad8bdb2a1811\",\"0b74331cc529a9ff9f7e092d6b8e72d19835eae6d372eb2f93124213925c5d0e\",\"968ffcba6f45b6eac2bcd4fc5308601193396c07f555544123cb496c48ee4e78\",\"a47821aee5b24cfb21c62d838aada60940864194cb7346e2638fab7f3b5240ab\",\"ee4819df2d9e728580cdbdf4da9dc47d3d922c74340c59890d418303dc87a6d8\",\"f752f4e375421bd4318ffe10a02063791b550b4a45560e5f3d5043807b565b36\",\"33102bc76485388f18c83ba5321597a6414bc01dd89ca989dd75a9063fdd3d3a\",\"59c2435f73df1282771d7151a35e589f07584cceaa9d90d69d772960f54dbd6c\",\"b997dfe8c0572cb9a441aef950fd39097b3afc85a1e1a0b1cab1d57bb256f973\",\"5b14e7e1459b9ae46e641f4f64a8ad98639c6fa7b4f97a0eb1483c4fd17f85d5\",\"6cc819aededfd280540f51c3b1065cf2b75d6be1aa77b0efe2810b11a83f9c9c\",\"811ef3079389cd02f43fb076604f1629b29ecd46a3bda99c4c6513142207d4af\",\"0a10c9a0fabdb355f5c65f82b7053c321164c93d17f4cedf289b65e474516af2\",\"4e255283dedec19fce560356bcdb63539d662204f921d5a4a2fa968ee1f9c574\",\"caeefcb8e26cc57ecff6c32aafb18ce2a509064d913b94aa5119b419ac437248\",\"aa6dd49a2e19436c10823bea70be869025bf31eeb9888b2f64af616e964f0550\",\"0d83b49ef9ac9bccabf20757071f383d9b9f3ded5cbeff6e581276cb26f93b0f\",\"9e3373ec62fb6cc22fff500c7d2c49436a72b650b070406c87e9a5ead04174c6\",\"4ca127b3eb9defb705db382907f491f0263c75c1404d8129d4748eeb6538a9e0\",\"40b21ac840f26d0ed59d36c4372fba7b9117fd22953cd9235d1b6000059d1407\",\"443d9852172f4de1a8faeb6ef08f799d1cb1aff05e33a4bd75407c3abd24fc95\",\"0a49e61013fdf994ec9c14a6830a65734f300dc3a25dd6084f15d7a943ad5b5f\",\"41238a4eb425ada06d28cfe905b1e29900b61e316539794e508c437d7bccc884\",\"7a55046e3e970d0af3c8562c144cce81f4fad008e802fa7014bd796586a2d52c\",\"2a229951857d3803d7ecdd4fc5b44c3812c1fc285f5cb98d3131c4f6555ba2b1\",\"7b1cb45000baa6b051eb5bb39ecfa70db412037fc42515abc70f160d0e38f717\",\"2d2fc0b8300e37655bcdde09b6e34e78961d4c6e672ec42e265155c18f39cc2a\",\"0c077d535f7c27075b183991863db22ddfdcc0d81266f1e2e96b0f3706de8c99\",\"501639b8728f6327f34f18768430c5d8e151542aca87da9685528ed16cda5d01\",\"063adae407cbdeb1d642d6a1a7a016ba4cad9313920b873a98c71167d63a78f3\",\"0e8732659b5f5cb80555632a9d18fa016b14427225eef5f1955913ffbd8fbdb8\",\"155a256df8aa496df4431d905261424ecdfc8a6bc3f23fdf5dbcaaa5426177af\",\"5f4efbed453d5055d4ecb86c5a73418ddd4d368a5006fb0acf335cfd65ac7477\",\"c5fdb3829b66e2ae35860d65983b72567dfc8b27d1f65657368d864d6908499b\",\"a5dd5450bf296fbb1f0b3cd2f978e9bc9049f000bd45d5881b866f04903df945\",\"1ddc9107e50df5ef5c7637bfa40a00e3d4e01f64f79f47f74503b9cbfc2b6b9c\",\"f8f5e5a8a619003b5fa54c6757aa27be66a8a9c7dd7567a4092c76883610eaa9\",\"96b0b44b7c16bf413a36442d1a726c1d352e3921da393a49dde0a7f79574b27f\",\"6905feabe94b7f96f91f6fe194446879da02c6fdd1606ab6849f63f96bccf6e2\",\"1f88ce18b4077468aad72a2279aa1909331f52edbb43c79d68f54166f269b5e7\",\"c60545ce6f9f0ea74deb9969c6a1ee18a57a71f50906cdb298ffb126f103f8de\",\"3c30ace953681c4e53a62abdc92e1fddb653668aa1826527419bd77fd4cdd63b\",\"871b4ae15b9e1596cbebb042befdafa3d447afc3a836f69af7c6c5051bcf62f8\",\"6ae73a796ca2116966ce78b6c0688f35523499967651d3670aa8fed660f6b373\",\"33d5e4e08ca62e62ec85aa8f897babe3480fd34fd67e18421caa9668131c2d71\",\"b210cd254c849f792f94e9d0deaafa2156bf953aa5ae04d3b79d3c998dc59708\",\"5d52149913cf7d2e94de4ab850ed7848f6609eb09984d3e2421dc5a917acf389\",\"f6fba283f92a7ef21a6339cf2de9c8151c46ec742947a4f6e6bd4a2bb093eb65\",\"4470ca306ab431fa52817d14d1d114b2a9e7c1a16cdf6497a35e9481f2053235\",\"7d39854874605dfcd40336de9c2be4b06c486b464da8038cbcf861dee6842d5f\",\"7fba7466fdac15650bef20c287e82d03170aeefe7b7e3698f49f1f8d76c4464d\",\"9cc48df8f8e3c623c4df8850b367db29450cfac17cabe4f61343a66cbdf9d202\",\"1bf643676f08a72b4eea6981c6c68acf6fa6f3b47f0c00d427ceb0d71949f718\",\"5f8fef2a650efb914b4634b34485544c57a9b46f30a75be2360109ca7a9c4323\",\"2512e3dcca66ff8cf871a0a40f850943e31547a9d018c741baa2f155392ace38\",\"82d0b85d284b4497701e1bcc664c8ec59719bd4f7c04faafb87eb65a086ed4ec\",\"c39aa3a112504cb11f2ff92bfe76060b16f96afaa8d99fd4ffbaa5899200accb\",\"af2699915cbde343637c170284c7035ae69c53dfa795bef152ee40224f4d3cda\",\"305af985820335f2d6cd7bd86f1d809d6b7b42acc8b16df31f3b0ae7bb60e472\",\"70ff6ff842c8629f1800b5f1544784c236603244faefdb85c74eda42de1985e7\",\"43d401be28ab6e906bdc9822b3cc7687382d5a5cb4ef2acc082a8d91192e5b73\",\"ce5d30f6e3043a8724dbe250356016ce765e99d2573d057483f0ae39004f8ce3\",\"a5ff53929e9d36475ca4c720d2fe38cbf934667895e8fad319de7ed26c5fc046\",\"bef7a90f9120b9fac2a5b8cbb2c54ba9d2dd40921ff6ed2800091f5d6c9f852a\",\"45aa422f69b569fbb4f07b6a6dd016ce7e3d1591929cb991df8f5f86dde1dd42\",\"ec414ea4411e7cea7c5803b4a26dc1003c3b28c33a80745920b43997111b280a\",\"b02727533e4a7c2328341ab67c09b5f282a933880adcf1898dd90174f2722043\",\"cfb76b4a02de1a0c1323236db88dc6874159d88859b23df6fb91b404b846ca36\",\"1c0e972d47ce9b4ec73eca30bd09c84cea44aa82767abf5dd2394964ca34da5c\",\"60877dea1d1aa3d6dc72698194e0883b6b344a3488d8856a68e47bd00b523bd0\",\"a8c44bd9a6e1507f68dcbf45c0e2ce79bcc5dd5d24861b3981f5f3d95ba521b0\",\"f0eb3f69c0b1f4878b0be935acf5d6b620940e721955d0bd138c34efa19a68ae\",\"ae0db538803b090bf64a0fe33e341614d7c44a775b1348922ec31dfdc23b5f70\",\"fb9277568462dce6ce2ab9c1ec08b941e74bd075435f0699a6c3b3f2a6ee01be\",\"20a0f4724e240d3332daae8f89e884b46fa106386e258a25f7d31ad5397e4123\",\"5e6d41d72c84be369a1f049cce990737f7b2a01e3ebe61afbf034d74cd9be983\",\"d94420c72aa30619c6602d5795676c80162408f767307eafe60755666ff2d874\",\"73ddb5b91f209ebd8e607c7110e82e4e866fdd43602944b97a2c900915d094fc\",\"e016069433a3a7ef66bc402b0802a78b1db43551d27d969ad96a98ef59009229\",\"d27b8499580bf152acdcd3215d28dd8895caf90bbf1930bab2eb83ecc0be98f2\",\"5a9be12e81c6e943039ad8926756c336c46a7f6385e6e1681653d98ca6e9ecdf\",\"13614d7b37602b94247b9d71af9a184a81379c7540284bc565adc3401c678c1a\",\"32cc74b12a449a10cacad92afeea429ee6bffe333496a2921a8e30e7461fc3fe\",\"b558378f590ffd360f03dbd162298a645d99850190f26945a74b3002fa25e9b0\",\"a9262f19d2ff5b783dff5aec3222441ddbf57077e26da2b46772aa8639352d0c\",\"3a2f466dfe77c931e72778168acd9c34bafd3dd8e83110217673d910144b7997\",\"54d2ffbbb04ef82511479676445427eba876c907f323ca550ea6ef03859a21f1\",\"83ce12e6340752307840233916ad61f93e28270270fd9c84be9307c53468617e\",\"d3b90209bbf04b7134544542d0dfbc339cb7d857ff59ff2ce3e2b6ab536b7173\",\"115dcd2f314b9683ac718fbaacabfd505cdc29196120feb410b638e459c64ec0\",\"bf5629662094af0c323f1ff02ad30c7a872e8758133eafced106af081d1d4712\",\"8e2e2d8d6caf74fbb5ab940749ea0ebdc33297d74a0ba79ccdfaf0f01a95f9eb\",\"883bfd1419c3a72468126952b2978c71759e3a2a8cf0d3bac77c01cc4fd79621\",\"b94c39415c3eac5d744a7471e3df22c30ba32cfd17f996a353496c343a156c06\",\"4d18c9040968c1faba8c1a3f6d28702b61f4bd02d7f02b45f9f3346f458f579d\",\"ae16d209563c9639dbab87c9112929200d67248f5c04810344008bdf63862d81\",\"c1bae9ec20efefc89f6820f1afeec158a4b2cef8a4b692627f03739718fb695e\",\"d73083a06161493be4fbf32a2994d61cfad7ae7e1d733a8dcc5f3e266fd94777\",\"fd3470fc12228c6d69e90dd22dcc6bca87e8b417b5d7f0ada23ba13c52f77479\",\"4ca2b0d8cc6406247716cc69fb5de14edc92da38ae5c14f91839bfecc0a3b1fa\",\"759d03394eb4617ac31aa568c36ca96011b1f98dcfa0c9f0653e857098d71102\",\"bc6ce5b05daa1eab4f6492d0fa0cdd86f8be634d69edc20765690e5643d24b2f\",\"d9fa2359aca0871fb0cb1fd61af085f874c8bafc954cb7456b6d815f040234d4\",\"b891ff1fcc3c2f3011ae9d40304ee4fbbae615fef513156a3ad05f8473417c96\",\"70a92a2f2186594f436b745e7d776869984eecf56f73283eaa0801074828fb56\",\"7f2a16c73e7a9708871247d54bd41b1014dd4bb73b5aac0ff59be651c4dbb92b\",\"a9d8441d30ef92d6e6db9cd2e616263d6949cb160b6d38ba147a72d79eca1bb8\",\"2dfe4e15d89adfb5df4c791fdda2c90de84e3a5c7cbd485f153c470aad588b4c\",\"dfb1ffa7ff04e1c3ed33176ef5ff10fea121f254358c951afe03c4b9406d74a9\",\"4f37124e6762e0920bc3ab67a307301190672c854db4e5fc4428d3c3075aec2d\",\"c5b6285d2eee42eb2117f456734d58c9ed861cdbf02cb741872a3b1b0270dd5f\",\"15829ebeb8e01487c4b92c25a4515d20f6c5aac18e9e1f04dd045c1f892e24c9\",\"41180c3ccbf080dd6b5cbccc3777418d838d0078f84a0cfbb0ea3016514cadd3\",\"4be2afe3d5bec0066a013f790e5e8b8697b296e0c37f57993a116a20e236e55e\",\"557b928223177a7d26bad4767b049aeab5ee38b8110db86fd1ee9b13d40969b7\",\"b0fd8d78dd9cee599600752d55fadc80843fbbb6c199ddad00d1c0321f7af556\",\"ec419ff55897a0fc079b95fcb471b45a8a8aab8b878609bd469fbb311a45afb9\",\"d82b35ebc73ff86472c4a9ec3d35dec65ad46d927ff475c38c0d72052a5fd015\",\"2d29f673be59d15e5a2863e93faac463433c52bcac2dd7d7bc49fdcc2b5af3ed\",\"12d1c83fb1c434f19ebdaa4919574104ff59364a2ae01117aa8200622c4a10a7\",\"a0fc899fe13538395a5135997a40eaf4451c13b38d22e9a1bad453a552dc8d07\",\"323eb64787f876ed22feb09786f00951b9930283dc116d393700886fc4938b11\",\"696f28aaa75c999228cd081080532d781f0209d8e8706ef372d755a644ed367c\",\"a45a5d78590853a6032714d2856b5aeadd687d0d793e705bbaecd974f1f5008b\",\"aac519038274e05cb130c5d2d484d8e2124fae7d1c6951ee2084ef39a9ce0232\",\"60873c69659c014a76e330ead6b33fe91e133590b0229ee8a00493cd953831f1\",\"f11c5e5cf4d8670a8113ff46e4052b6dd82e120f444c1be602a2c476e12509cc\",\"a756cb06def31dcb3e5b3ddebb030072f5a3a5235da6e4e6c2698ea121ddc856\",\"544437c3615a6a3bbb5876aa1ee43d808d788f37a2a7fa308cc5538332b2c217\",\"1bef20fca7f9ba80b7e66c0e4ed2d1570b1163c03c49b0f56bb5af25cbec2239\",\"a29007c4ea8a82376b4070970e065c9ba957a8a4192a033198741f518ac65bf5\",\"b7529fac6facfad45d9a7a53eb8a770fb2727c51e475186a9bbfa4b4e6aaecaf\",\"178854fc3f49df3690c4738518c9f48ee364dc211e1cf91411e3b50033314ee8\",\"f2d3282358f07d0b3003175bf5315fdb745d9ad5d17cbc3405379abfd35c713f\",\"b1fd1b8575c46c8bad7c786a7a347e13cdfbc51e22250e0cf6c712d0a904d002\",\"29432ce42bb836a4b6171727af742a6ca9786d36511580d14e07930c2bf2055f\",\"1499810ee387f2fe0192a210f114c2628ef1d24527d20e83c94490bbd1f91240\",\"a2d626cc59b48513de21b8a3a2069333308a4228006f2b44e3fd0d85a8404f4f\",\"544764f5fe823a250219907e79d685d351acefb42473316505d10c22aa387a50\",\"3254afc8ad643f291fc22d42c159769cd4e14112fd6df0cb5188b01dd13c33c6\",\"d505cb930712dc68453c20ff2aaab8f527eab3afb26d9fc23c10da60689c1352\",\"2db1071c1bab6b6685e75676bb91b8600c9235a5233e6d04c8ead16979ee0218\",\"304ee5d3493f215d823137bf8de7a22dde8676a43c5118905e84b137dab26742\",\"6efe442012307aab90074de3163c83f2e9fc9573590403f6e16d95514f4b5fda\",\"74eb17bfea2ae96d6eb14415f3f02c4132694624d21562e0f4b9d0bd6488f49a\",\"e7545fba3a5b2ccd4bc516d3b24f68a9b14ceb3268f9364ec2bbc12701429b78\",\"a625b5f8df60f72a6931dfecea1c998de995dc31e12412678e14c297ea6ea27a\",\"b999f2e4161bfc02401e1b30d9ce37e5284b15ae01c60a94477c835a524eb04d\",\"b40773418e664949984a06b27f010be15ae6c543a8146d688dbc2e9f9a6e9f11\",\"c15838f1fec7c7437d7a947fd3ac2aa61996051f7814bb09730a828fb20f9c0b\",\"ef02d6b72ca8de4a67a8fb9836c436fedceaba5fb44ec3eeec79b016fa50afe3\",\"1b6f916179d0ab657e7a150061c178706341e482baa5aab73225ea8c396b0293\",\"6f02ed1f3c103301f56cb28446e1e8238339c8d70fb72cc69b110383763067a5\",\"a05b8bd1817a5be3da18df5f0d583b4c8559c14f966a442b0987c956089dc8ae\",\"dd951beb36b88044a1686b8d2c30c7a8b3e57e10d2212c7fe13acfda6827ff6b\",\"c5fb5750f3763a43625bc664e9c7a60a83b0d431e37c2f3922b69c38fc4c9750\",\"f2ab6e685166b7fe2c50dfeac9f5fbee2a417d6e131756c7ceb8b90b2e48ce75\",\"bf18085346bbe34351635e5494135d3fc02dd6da870a106e77d5bf3a892486dd\",\"07b540262139f84e818461fa12a0b796f69c8b8910c0f2c40064b522d4ac902b\",\"1c8bb8d6ba40c4e8d4ba6a31e94b37aa298d0bafb41317659c4a322157dc0530\",\"7d28db62be463044e07d920efd2472a1aa28f111917c222a34a40b39e655e1a3\",\"7c1f0834b523b9ffbf9bd1961b186c5e39abe03451c83b430ad548303a15ab9c\",\"27ba88113c30e5298fda7d64da953b529c8a303b5f8bfef5d7bbde9affde5f54\",\"b3d3342d3173213c9a01e5474a2469dddfee791afb190b8481ec6ff32a6ea354\",\"2503d8d443032bc8ef4932a4abb1d89153355361895fdd93293515060a809ef6\",\"6af5eb9032bfb53187b2900f4a70c11c369dace40406372bb8bf32cd97beffc3\",\"b280b2f6e4168506697ee2d238025720a364a6fe70f92b7986a04bd7fec2fcff\",\"3c43ae60ca925ce12703cc28552e832fcae2f1e4a163915d1bad25aca540ba3f\",\"7a0a1c914b59d0a6d66b13e0f3104373e8d4aa51163b6f0b6552dd365f441ca5\",\"62f1ea38620d73e4d32467ea9a47fc2794558ba08d0c71eea3b65d7215b1adc3\",\"7b2b328858f1053f0f5bd9fb49464b9739dfa316daea93e5593ba74b79ca3397\",\"89c7b24bed3aaa3bf21fcf0ab3ec2faea80f9ed963461f7ab66015759b9768e6\",\"127a5654bf3d578d2a4f624c91490e71146fd918a5de81a9f8e56f40ed8a0091\",\"76258c108dc00c7b7f4d46382280dca24ba1a96154795b8ad2a85bfadb8525c2\",\"157d00be5e544b723cadd732dfc468ec647236114366e1133a49bfde3c3b23c4\",\"a396e9c8f24a0f2a7289377b6fb81cf9a992f7c0366114331b558c0246f98faf\",\"75b42e76143d8ecd13d58afb2f4822cfe703e1de5406ae63daba82f4ff3eb893\",\"c96f3fb3fb36be728981e256a182b0d90afd07a28a9d3095b0c6a15d4fc8bfe0\",\"1bbe0f0d554a233a329289500b1f10ebe152420153a0c85776ae95d265b327e7\",\"deb668fbd10e682e8d531d6dc605e301cfd45987787a8eeecdbc8bd7afce2ba7\",\"e771fd7bea36a50c86ecbac3b7eea4ab745e60f94689da5877e0acc61c9dcbf4\",\"8128cda39a85bd9586f5e33825c8a21505e09ce13813941dfb9cccfbb1ec626b\",\"acd3e1e1e3ec6deb86b59d9179548204b0c81368017a44587b595f7a833ab95a\",\"28382d56e5adfd309aae077ad5f0bb403c8c33e84570d25dcf3e5addb5227d2d\",\"b5367fd29076031d1cee2ab1780eb71727a59d4c0fd95790ad6291672b0f7e27\",\"2574ec8319cf30d7c3d714efa87f8ab7bd589940ac5d362bfece370b7ddf4df8\",\"9258c7ddc5f28207330267f561230752e8ecf556afee5ccd4265d300c0cf469b\",\"d25c7fbd596cc72f02fc7bf0f639cf708d160bb9e5986db3e02b0fd72d90a6e2\",\"f2a8eb0e0d33f54f9580da771d89e5d71a609658c7694408bc46ec6b9ebc881b\",\"c53bd9dd91b776bdc5d3e3941d1b26613a5b6cbcbe61905f1788f4e77915d9ca\",\"3e25a5d5629949dff6546a24a20ef278bee642b402001c701de83acb6acb8e59\",\"6c20274bada1a770930a7e5de93501baa1267f098459f62673bcf50a6f767bb4\",\"d1569b7cc6f403c7e40a1c00f6ec850baf40b6ae7fd72cf80f43e1e457f75621\",\"ada0c534de0544dc2e2f2fa341564430680460e20a6b48b49eaba1fdf0150468\",\"c88739c5fdcc16b70e494c01d02465bf0bd01289c823f12228dac108acc54cf6\",\"87235308ecd3dc475f3e369e1049ede16c5a864e27d66d094fa98445a3c8a2af\",\"b56ff31aad24eebf73b9d5ae9004821cf80491410a09fc62de05b56032747c59\",\"6516afd2dcdac99cb8848e2aed716a1c42d0ec6a213ccab2bed29cee42330edb\",\"5a10805f6fa5759bd51905fe4ee39118db739f19851193daf5cee0027789d8a9\",\"28babbfa15c41ac54803bbb0ba1bb2edc96e58f7199a9baf6fee27765e95602a\",\"6ee570226a48e1e7026119050bd0fd769758519165206784c4978654fa9f864c\",\"9fff4c02ab8afdfd704956bddda0f79e059250860b165d1f506bcb0432cf71c1\",\"e394af8a9f2b293c3649d84007f1230542c291830e091e0a95bfba726bfea359\",\"50ffe0ef6f16b39062706f6472ab478e03564015230c01e9ea9d06973fa3c6d9\",\"9c219e38003fc85d2dd3749186c3ffe42148d06df55f46477acb85e3318c11e0\",\"f8cc4879200d09de0f7ab3948bd79f74fdab96335e56987b444439a2b9d0914b\",\"0d40f105539879c795ce7ac56a64acad57e2af0b5f032d79f659b8594aa19524\",\"20b61912d3022b0d5d6265527d5c0d2056969b76c54d1e9c5b3ddc76c6a451b7\",\"cc6ecf53cecae4ffec6fa30ab0370c84fa75b315911e651afef7447ce87802f8\",\"31c2631776d96252b64d06e213cb0e1e7da96fe8d5686c61d84a87a99ce1ee63\",\"ae3aae4a38be61c0f1f06c05ab99cc5369259b0be28ead97d7adfd2bca0ee3d8\",\"7e4d65b46cb5c2442bfe26e9b3033318e96a92c5cb60403a87190fb7de8e959b\",\"04a343a4ffc632a00a4f9d11a500eeb76886382e9f864d523c58f9ae7b5febec\",\"c991300e4618dade94d13577cd1da3ee47671490a5b0aaf1355ac7216770974a\",\"49345972e9f2c1e14e5d4e80c6c7f5d46a0f6fa6beb950f7e8d58baf5058309e\",\"9d3bb8b9edf6007020d2e31ef584376509481c905dfd9f0b9c8cb68e24e34ec0\",\"845ecbac2dc8f615bf7ca5881f8dd0a603d6240692094ab234ced0a6d02a6bd6\",\"5cd6c816c71c14b6cb06ed822cfee40bb4a53498f230cca1f65904678c37da79\",\"f7500d578b34f1e3243d7eaa453f2179bdd7a2b048bd8a55120b383d65381134\",\"46baf17d3f992b8184acfb5349a5b553c6323b88f751250b8225aae06fb4c81f\",\"98d36daa600a565210b7d6699a7f2d7108e44975ef23e62bb8701b4ee2f2e906\",\"f616e31a61c632c7ceae45f333aed17789b1ca200a221e797c2b48213840ffb3\",\"bd1e37ec9cf895e5280ec8af625f776b6db8a90b2dad29d6eb4030df256df050\",\"e987a3880df1f5e34d3fc03f976dde733831dfca21880230bf6305963ce6b481\",\"c967ff5fe8abd1aff1d380b4c5bb0a02a24bcbbd08011ca87ab51e1b1dfa19dd\",\"35842af838ceb00d18247f2080e7e2ef70499fde16261016b0090e214df56102\",\"5d3e0415282930226fbacecda918516c153caed2498d9454d72ebc9d335e8a57\",\"bd04fbec69e094d47247f14603cb1cd7febb07a1f0433d8ed7ea65d9305f79f8\",\"be1082dddaffcdf86eb8149a8086f0c099d0ad1ca6c3b342ed74075b8a2bb7d6\",\"89101a6612141fed87f71e32ea72f32ce25a7cf2bd98fedee15e1aee8fa75e21\",\"9b4b4139b37e279dba42350ba33a24ef3f0431b9499939d03d3dc332676dcef6\",\"f801888af56223863ff2e7e6e64512d03c0d810880d52219e8fdb20cb54c02cd\",\"bc8ce7a2d63cdf9d769f7fc9b5124aa6ee4366483a1aa6a2c8548f2af8f3a432\",\"0c2a02205624a28f4a9bd7f0232b50943f70d758cc157069f36d4b30026e2c90\",\"048861bb4b60aeed20dfe33112502fba3ac0ce4efe477fecbec57a2fafe31ae8\",\"b2ef4a56a2fb3d2a41643bf07672cc9dc793386e0b46cd40eaaab6e9c4f13b8a\",\"7c615c5a60604a12da2f4cdc0c9b92cdf93b0d524c69cf5b98cb91ddd6775104\",\"6a5a1765df162d5bbe06eae6e10ea85f6a22aa391039115ac361cedf13cee6fc\",\"fdafbd522ede2d34d04b3df2d219fb5bab167a4ae7f77b6ec733f8d3bac11fd1\",\"33f9903b7f4f5f1dae3ed943e865c3b0b00b652fff6eecd6381c4e8ac2d18089\",\"626deb9adafb50a101a0acdcf05f3ff9488ac44b85ca845fc8f700204d216213\",\"ec237dce63f08cc3e7924558581f6577d74a5d39154e9e3c385b833998d04568\",\"3ebbd349710c1812e6a29d2409d5be7891a723a9098b3cbbd2be040dfcad9600\",\"59eacd1b19c987b86d685d5f16a0d0b42c180910febf0b7cf656c7231c6dd76b\",\"ded6ffe665e665be4084b8e25e01cb06e19a14fbf1022299ba20224580dd9097\",\"7bb000193d768043625f81ab53e8054e17172cc8b1d81b6336dcc5386ce769d7\",\"d875152e5a4c653606eff3c7de9b939a83b7dc8e886edac8c9b667a1197658b8\",\"ce33c6c6ef800558c26f7181aa22b074d417c62aabdd9cd6052e3c36ef00235e\",\"4d6e25da6a8efc82641c4475551e43ea0d848fc31b2a80b4e76fbbc2cb608358\",\"0257c66ce8905b45f46e1982de4214cec783a9f9a8203a069af84d4e38ee6c8e\",\"7dd52a635ff35d133426fe92709cfb81ef28c9f2df9d4c61dd17479f2764f8f5\",\"85c72a936c4879c4fcdb6b71e0b068fd4098cff9039f6f92f70ed20949b750cb\",\"9c3e850284eb6f9a1ca91ff0175db59eb18b51d61d529daba50476153f80bb1a\",\"db31c6b9afccb68afa9949d337c1c2bf23644faa7c844f32a2c96dc34a1a1b59\",\"1a37544c3c874ebba55d51f79ada5779062f0c285f8211398d4def97dc2792ba\",\"16f2e686718a77755f5427d368ac92ba1c6d6cb7d97e6eeb7d26a7be188fbdb9\",\"77015e93293d6b18f96783f111a4d9c61231b3ab1e531c40e412f4d7f66a6780\",\"de13c2bccd0d82b1c488f82fff51659c73a1d7869d9517916c8058405d0d5d0b\",\"e434dc86d2d882a6212598351827d3f31c25e811e42e95902ddeb71d04c5232a\",\"fe459d134848a4fd258750ac8be140b34bd637b6230676d48d077146ca47a939\",\"383a01688e86331c64a6de7f8c09289f5f0bfe247a3e0be4ba6da6ffe3938ae2\",\"29ff67cd37c05e99275f49e7f3be3e1cbf23da0f22a08232f4372a88705e1ff0\",\"d3ab55d8bc57b16c9ece766c71cf630b0741e4f4cc3af7724f30f9792ca86ccd\",\"1e3acfe6627f0d5f5ecaf2a0364eae17ecfc80d4d402340a8e8fa6c08e31ae00\",\"12e445e9de167af3dd9e0b231a8335d6f7267ea95d9499d9de15f1eaea6cb240\",\"55155f8141e7c126bc2c5bda367748f1852a564963c44a533b8dc7aac476203c\",\"af108327bea91ffdcdc6fbfbb7362626f863fdb7e69fa09738c8ad2364367956\",\"e0e1fb6c99c3ed875ae26405c0abb4b80ad046bcdbfc6aad58ea3e455c9c3620\",\"2ba07e0e6dbadbf3e4bfa29466108e291c54c06ee8e4aa48497b18de89d840fc\",\"c50cc157f8f6e2fd2eb7c0f01108685879130b51462857380fffb5d0cfd17eb9\",\"d34ff403e2f98a3b600308e216a96404d16808c34e157c39f8a8784da7b00505\",\"c7187f1493f497a36e9f6596127c6a5dfd1bb7a8d62eb4893b3b93ec7981fe81\",\"8b2f467c6954db079469bb52e763fe8e53d5e3dc5883faffc5d6aa4670adab85\",\"3f38b12fb20ea63189f2c86c80766b0180ee519ad835def80787fbf8c08df28e\",\"d9d4a11c97bfbe97f089466ef6b9397c7aae5f33229131bc6fa6f20b8c3e0f12\",\"9253c2d8d98f31bdf5634f70cd21fe1ff0f89e84778ac475549d662cfec2233b\",\"50bf48cb548de4c3478fda3955995b116d3c22f434308324df62bd6063854540\",\"ec678de29605f813ac40a2c14e1f74a73f2c4157020f3cc55118b823b1306312\",\"eb6fce5538e8c5cf269d47ea142b0762ca58b45e4c959a17f32cea06823173e5\",\"0a04a2265ae053b9ae7a89da6e691a7b788a4787abd443aef72a9fb14f6e05ab\",\"681822ff659cfec81dddf720750a71c34f2cec206a6eb5d7e662996c80382804\",\"22d91c28694c88d42fb856f54421114561badc12142e0259fdfde2071fb2f003\",\"8713224485697cffd18cf051490657048b641e326ea9695a610ac2aea9c546c8\",\"88585e7363d2078dae7100cf7bada9e23a6ecc4a6d6d154c1d837da8ee19f0d9\",\"5c2f7724e8563afabecf01837bb640ebcddaf57b03d8f1bd85dcd8bacd718530\",\"6399ba6f5257ce6bd2bc593caaaad4461ae8518e89ff85f62aa1ae7719355942\",\"205669ad208ddb8f78e1bf040511f8de7f7662bff3a4c1f341b9390123da189d\",\"a62aa3adf6b6666dbdf858977d395b4e00c3669b910806a4734ff645b0594d24\",\"7ce6aafc3b6a49be3c4bbb222e43e0052f66986d8be5fc515ff44c841038d716\",\"12b3603e12acdb55053a4f9d71b3fb17ec202fbf8ad38fa8c081ca0d3cd71cc8\",\"9e62bd41dfc45f8cdc51d6648dc68ae8ade32fe4f62493cf32e5b26fd6e26e1b\",\"0f43a1dc0c29cb96d5315c42956dc306202973c0382e6f7cc26670a9716e2ac6\",\"9fcf3228cebb397097866771ed7d915d0325f97529e156b7f90fcf54dfa1ff89\",\"7f8286597d0a46943bdf1d9268a071cfbbba9f950efc0c80b9e4e3db1a3a09a5\",\"4ee8db2c243567691a3b4da4a8ba230953ab16507e70347a91c21b41cda6460f\",\"3b29e89b68d6c012db717efd6d1dbee25b3e25fce986730618493f648aa892a0\",\"4728a392d3c9b15a89cfb7b86cc33799618afea25ab4005e92390a8529930b98\",\"586520a51a1a684b3db4fbc052ba3784b2aab21774c2179d99b2aac77ac26878\",\"e061dedc030829d647bf008c66f51ff88bee4e280cd554175e5466525f3ccf51\",\"037eaf75614a1fa1940cb1fd676cbf1ef857c5de18c0f1e923f4b212f7a0c977\",\"cc6c0bdc3aeea675448c3e16111e90d7a4695337c53e0435627364258734df71\",\"4e6d09a5d67b5ae2b6596914c838fcfbd7aee1a7fc0c38edcaf22d47ffaad2a7\",\"f7f321c9261cf9bc1b17704182c19ac3f119c0881d0f2cceae3be405201fb1c7\",\"fca7dfed152928f510b190c64043150da6c6074c48bdaf6921c5286c8b787e29\",\"8284689257e2f1b887ff83ae5fba89b1af19e4168a4cedda9b343d3d25846f43\",\"f3d7bc0984ce79a3e1404189a1cd2d68dbadb9efd6ca5e03411509ede0a5d0d4\",\"ade74d05d1b111ca7503f9260d1964e6b60fc257d0a82b62eb5c359b9639fe6b\",\"4fa7310df7ee3744fda37c6a5e3fc9e71806a1ee881ddca7799d7417eb911b4d\",\"3d5d3aabbc9a59bddfb13164598463897043d2726ca4cc9e21cb8e7bf6a630dd\",\"72176edf341d6037ffda6ffc2384c2e9d91849c10ffe68aa2d0a2fa0aca6dada\",\"015a4b26fd1bc3bf3ec338b3c457989b149ecbb6ff9d544ac31be40a3cb3026f\",\"c9e0aed5af543ce730a542039f082a608f001e8890e691626c79fefe3222898a\",\"246d9cae9741b9e0f998fede292610be3d2cbb754861720d542bf649e1c2a43a\",\"a3a314b2621e88d6cbe4721d42763a77cdd4e9230ada4c01f7fab914a731ea92\",\"02c8a9a53b719142ec16ab84a473bab8e61ce0f178fb81a022d394e8ddf78ec7\",\"3fbc61dfac6243d128e750f0861e376e13e6f26cbbfa3d305497506e4584e4c9\",\"a7554ba57420d3db1e80e0eee6c3d7096c19c6d1cf0bde4821c1efd998af0cfe\",\"59da88e9a34fc9c2dbb79c0e0218980f0fe6adc02a6a0c940c13c529285c7c30\",\"8ff510a6baf0838a09a1d7586d503a8d751b31801be6343be0481e2f726ce2dc\",\"acec40f15a58735f679b992da8b9cae3727ae67dcd0276faf05b6c29c4c275e5\",\"064e78cfa28aff55b4d8a68ad42e495565144b2602f2a92350f0602e05614f62\",\"418dd02f6313bec0a045d01ff23adfbde0ff11be56d1662853babecc5b4811e9\",\"f70a8297298ff2c29f9507c0e295b8f8399030feb2a6f014c81956a365578607\",\"f669033c250e3d8327329fa1aa93c9dbdf071d29745808808d42f8e719b188cc\",\"650d2da58252adf2e6434b0c8e5053bd7f4726f7b312fefe0b4516dad5decb58\",\"f7e6441a5341660a497c0fee5cccb91e62f5bcb833853846613159783bd0723d\",\"371953491131effec8912fb2d375cf2d7c39300211773e2b9c71cda3898e69a2\",\"f7c4dfec4959b854fbca26948befe762c72230970f67330c31dffafae70f408c\",\"7b42af95736810d4d6fb508bf01cef5b49815bff30596ce4a49117cf7a1d812e\",\"d474554bceea9f44e193442ecac7743c84e2b6155d1fe5a7c0199c53f227dde8\",\"4c4aed6b6fe3a9b1b944ad2ff45cb93d5b8b2d5f45c4e74db87b28d10270d1ac\",\"3893c62fdc99f36865bf9d5e5ef4f2c9b70c7f6654e095f3833e87050b84d931\",\"19fe08264e2f72e8f74308a75f21690852fc9d583c89bab8cfca5930a936b657\",\"78d28b4d2e10ad31bac66b72a73994975f009caa5fce9d6c4ec52fbade5531d6\",\"9ef85d521d6484685c20dc6a7a87d234d180a829d07166c1c5c47b8b3608a4ac\",\"90ba4266273359de89a8d09bd197e132b5423a389a5837fb107684f7cd3e85ac\",\"f52b0fcd6a9b22bad51f3a3d1685e39f4ddde74d3e7942ac8d1dd196783c7e43\",\"afec7b017c6a44f4d5cba91eaf9c10843005fd6a86b56c819e238edb86ea1b6e\",\"01dc98fb6f2de30a8ebf2cff2dd70002d1b562b10858026e1e1ef17eb0680c3a\",\"d4f53060c3220267000d148203d35310e4d22c63b703c63746e6cd5f3d87383a\",\"314130293e9f4462b7c415100a6c27ae6413df015faa54800c2028b9cf86151f\",\"7d123c933ab74a9b77006f074716e8b9585b0e8be7fc5d8eb22e4c26cf9f4cdc\",\"f7b740d308a81e335b5cfccd2718c114c8b5d8f1211ac12dbeb992c608ab2c6c\",\"2b33aae4f65628552af5761f6856699c8266b1fe51ca1e5ff3c2c07bfd863b71\",\"a58c567d6a5d90ecb73628ea5ca314a016308acba94fb23dcf2bc5924e5b4181\",\"284e086df84275d837cb6346244acf6adf57fda2a12ee31cbe625389604c30a0\",\"d0dc83d54227f30013595245b83bed70d8e3f4ada2735b7606b5a0530c5a9ebc\",\"17eb8649ea9c7f84f49798772d179604d3cafe1a877530e912e07f67c7ea1b3f\",\"a43a07642c5f4f9c39981fb83bfa9839844214297a2d4bdbb856a3854f6f7efd\",\"fa3fc73b7a1c7413cbdeac0d20650b13a1dd78fa7cab0adcd0c1998e2e250915\",\"cea4a237f4486d3ea8d2419f4adc06001ca149f8cbfbf546df5f6df44764a168\",\"b0dfd88e74520057b62299b0f76706be688ae909777165921409a09f2aab8f00\",\"506dc00074925fe40adf91c0f976c918ad5564ac38b14b61376154a66148a298\",\"13fffc337e84580c88cd94c427ed3a4a834c29ecf8794bbdf189077a43088772\",\"7309968f3dbeef8c1112ce4ee31d22d0b54a727fb81686c4afa1b4794fe2d1fd\",\"3fb1d74d7b46a619e2a8f7c04b355a9b0d93d6ab8c4277a699adf5815023b802\",\"2ea9d2f1dc52f5f725fb8bd404f93e2da8bdc406fd036eb26b0ddf14c5c6d3cd\",\"06c86fde2ed8cc623df3614bc1071166a58fa41a4aecc78a760e847586c33c9b\",\"efb96ddc0e27f94e81475d8dcaf3850e3b99724e8fdd587e768953f059535883\",\"8cd834881d3408f012636f91c05e1716d26f2bce4c6eabdeaa9976973d763fde\",\"94192c7630f52d9c3da0e394bc613ad499ee5323c1667276ec3daad7906003ac\",\"4a23daa442920167b60c1f7ef0125b135dcc9ff2db050de67ef481d04ef8e0bd\",\"eee906bccf6fada09ddc6cf8a02c32c99fcf79640c89626678a659ad8997d4dd\",\"bb25f70561248b0af73d95a89cbebade3c746df75381fab06e0111524b14960b\",\"cae09a06a205f3e6f4d3d8493ecf65c827abd94f5567214a3cb0b78744f71e3c\",\"d6d7302a4f5ab30a3c9e0d09c91d7deee32218f30ed170e55816415883cedee3\",\"7c90d72b3eccafd0536b45f2a38fc0305d5bc2ff7bbc865e616a1bc58557e030\",\"1e4da73fe672db93f27977d708584f8135958fcf1c018330980233d64ef863a1\",\"e7ffe4b08c5816cd56ba4d2da80e1f4d7f58c2acfd13157d2947689ee0b78366\",\"c9ca295cadeaf56c9b8d41c6ad93ac9a59730aad7ad1b60381d79d3c01a4d7ca\",\"ec9db1b8d930b815aa945f8b968c8a6afefc1623805b7537666399771cda59ec\",\"a058948a4306caf4219ae06d86d436fe7e9c40c7740d82e061054426ac036bc6\",\"ddb4923625a0562fd50a2fdec749d51a530482639e8bf8548db1e84abbea14db\",\"cb1b103363e30db5629748bae1fdfc1043a25eb095cd918a9a4cff183aec319a\",\"55ab23c88e5c1da9a2a660601ed0c0912f9931850aeccfd3134ba6838fe16c7e\",\"2b61651469d32f5d93c795715220aa41894c0b44a4ea1b2ce57b199b8392ae27\",\"a8557198e5f7535d021c67beb0160ee32b30f1fec19d142ec8f741cf64f9a975\",\"163808ac2126fea018e7063dce0837878bfa9851dbf384586056371095d5cab1\",\"ac82e104aa76cc343db13893a5f6724d3807e763a54a8c0b4164992dc693ef12\",\"a9c1046d70993033b4ca9fc4f1b4603ad9f68df8c4b0147a3aa0e79a8f139e31\",\"8bb073d623ab41f722c0ca5f8f961685ccbb233a5ede4d80bf931e394059a580\",\"101e43805201d072802ab97d9ba3cd1b1564339c54db227b91a3401ec06c8c5f\",\"f600b4e6739fd14fa69499ea0ef0256e59b756ede7fe41c84cb32593ca00a2ea\",\"62a8871ca77d85a58214b88040bd24c2391e75767afd07a98799f16f2837366d\",\"6fb1e8fd54b0fc59f391782ed8f83577973cf6a4c5cec9be94889de1af5cab5c\",\"f9662b3def27db40a6dda339b8f76b462d2ec46b15af270bc3c468bed3565361\",\"45de82ff35aa91a993716c5abf6158db9a4357b52f3ddd5a1288704b99ef25b0\",\"5e763aba834d48f474e39980899b38aea8633ced9ab67ccc6084c528387f5d22\",\"5251abc64029f5239d21eace16bb97cc6009cab429b74c88a37dcffdc94ec0ce\",\"0002f05545c47b73382b52026c3a19199149916ccdf887cf9c1e6e3fbba3bc22\",\"ba0945291240487dffb774e4a437d1d15cb1ed8fa44dc4ea77fba2853420d8ce\",\"e6f8f97dc4758ebacf0d5e0979a13a62d16ad31979b52af3a105d1971c5f3ac6\",\"7e52705bfed0a73bf4fffc5b02b8182a272e617e61cadaf41e9992ddff9ac364\",\"9629cf8f49bf85c13bc3b2429f2b713e9e8e0e729c482a3a7121e579e1b3bcf1\",\"52ea6e438568b8554a8da7414f9997e75b92d1ad4854f646df285d656a10cafd\",\"2d01d5e7ef4e272ee5dca7af4bff78366e6f6d4c8bc76bb64e1b117a7880569b\",\"abcb65832abb1e00afad749d614297fab2d9e58a6ee70bedadef2fb5842fa6eb\",\"ebe28dfa8f2ea2c5581bf84f74bdbaf0f114d538e688d9cd30910e8c36cc7736\",\"82713a88e03c7ff171fb215228cf6a85c60e66e0afabb59cd3047685c0779e75\",\"77bad6ad99a3380388415571a1da8df9358150d2d7f308ff8803d06dc034746e\",\"0d6c9d69aa2351f6fd57ead97cd4cd5dc92a6de9b414666e141bfa0df8587d6e\",\"b96262306d6d4fd6ab321740028f45c8dd9c5deb475bbbd1795c9bed56d3f127\",\"a6175a444c85adb6c5f160cd1ee72a7c47aa4bddac3cb72901ed3b2c26ed71bf\",\"6e2b2aebd24ba3ad840c9586dd6d8b265b5debe6bc080ef8d76de35efb4a0b02\",\"a6688e4a8bd45d00a05298c5fc8c60948e763ea6abaace26bf9e46368bb69434\",\"eb78e78c1a7106cd063a5b91b7130c06ed9658f7eda0dfe3cd7619ea8e843ccf\",\"591516021fc56282126dbf0473bc7b710e2c3d2048b1c11718d5c73ba4732d85\",\"55756fcacdf1c9a9185b4f41cac491d6a38aa39946595a2baa0efa768cde5b67\",\"3d691d32010c30917c3cf891c2b0bff33ad73410483cfe53d33a5b1e1dd63ae2\",\"0bd749f17c7a7cfa39e1a0a793a24f3638baf62ec28c3f8e03079239c469a825\",\"49a856c83af6174cbea0fd526eb800fb11a0487b69aad4e1f233872dfccde3d7\",\"af432565102519ecc785caf857defc9424bce348817fa6f6291c81abcb9f4ac8\",\"c7f66ecf37cb15a49cbafb73960f9f5a980eb72b282e493ea87a70612d8aa254\",\"a182ace74fc59ebcc84421f6b2462c7c2b5a3b59e7784530250b6eb3b2dad4b4\",\"42e201fa5042dd3e569a22d3180145f464f7f357fe000bd39baf3e65e032e4ff\",\"0c5ac1d8588ed6cd6f36fd58335fbf0d78fb09fdb1ddc97f8d68c00ae99ff89b\",\"b1d8d3ee64a07f3d01e998ff8a446427b1e0f58b0447a320296c8e935f5f57a4\",\"084ac5043bbce0e70ad03335d75028714436aa26cca4871d0b1d165ca36a129e\",\"463ec00227ab1dee91d0a2fff37225ac6cab88660db20e60b6543a9ffeb4e83a\",\"b0596950534677874480a42d5fb503640c3d01ca33fed7d78baddd9d93be25b7\",\"28b2b19e1ef1de51498e0c86fc3a89b28b03e90868d43924039caff36717b0fa\",\"583e5f395048e483be2d3b15c1842ee501c094829f2ea8c54e233c87948b48c3\",\"2f43dda541e8550e4630d8c7e81079641f931f51ec499d28966953223b3be1ad\",\"d4eda9cc6f4a9365a2fe7baa63600ee947191474ab424ced9895d98dbed4f1c1\",\"d55fe2372b40214645ba20b26a0dcbde2c93f2ae4067e00bbb31e524fc370324\",\"8b32b830952e38ecee8e4317dc596cc0c067c092a075a78319ffe5115f528f44\",\"28a316b1a20b71eca9c840d619882ceb51c3d29a4138394f49164cb4aa40cfc0\",\"21f8544fbdb03fb22000041c0381dfc185fbd437327f7fe13e45d7cd00648888\",\"1fa853dd14206f7eeb066c91d05e539d8e8443c3e07cbb702a08ef7851660563\",\"48cbd86bcc54a4b8f53db9dd0b5c76670310f76d357b2f6cb6b2ee8a6c39cc37\",\"9614ec6826abf86667b9476c777728e5f4df7f62e8e15afad57bb79cb0e38c58\",\"88ee14c8faf6f66167a51b3bfb54a4f1b2306284d7c20b945190bf6702159368\",\"060ee3b88fbcb6009ada778756159e1e8a4064a32f9d3d9d95a1431a6db0e6ca\",\"dd8d82b7c5997a0335e0e9cd2d7be8189611b3a624e21de0f12013c222c8dda5\",\"6b19b0775fbb1fc9476363d259f795d91514e7faefe98a056a4ff07100cb91c1\",\"8222f23018cd806b853ff9335cee57dcef949f09160c7aa6d61b4b226984b5ff\",\"e65adc88051c082260f1bb0f2c771cc07a2a2ba835be31fb1314a2400bed07f9\",\"16509102aa40d845f302da7a15571e12c1424ffeb50ad81675614a5421dab0a4\",\"5b65f88ce6dbf3be0c0ca5b006c879aaba0e814506adbf44fab94fe4304b59af\",\"f01ed4c3e94d0626bb025b3a3347aefd2c7e64239546cb8602408df3426dc00c\",\"70322874a0cf632371e43b4f87b9de5c45452721337a0a512068f654b95ad464\",\"fb9110bce52e1228ccc1b0f3418222a6adb390615d09e470aa340c31cb6f5ad1\",\"17e88ac8a1fbfc3d0de4fc9b46f025b1cbd735a65ac17daeaeb975e32c76fadc\",\"bd4e1fb493f6bfebf23a909fd1256ba8161e5eefcd1ac1acdbd3f3adbb4eff6f\",\"ee2e9480e8caaaa62cda28dd9a54a94c7c2a2d357322f68f364e436d59b96e51\",\"956622719b483391d6c6f8402a71eefad9e6b97a75701afff62bcdecb39091a1\",\"a49c82d0ed31ef13f725c00e1f64628ac5f7663c5bd8c8dd4bbcc7dfc6ac311d\",\"28246c66c8e45e27efeb6e3b877869d0cbc570056a42b65b1541bf4417fa2ba4\",\"eb4af95678b4d1891434dfaf82da1ab38d7cba9ccd9e88ea8c2f1e771e656cc2\",\"de3227eead48f11d324cd47aa3075dcd172bc1041298d27bb9f41397a20948f8\",\"796043bc1d3a5d8d5bc0099cdea6e19c5b84458e1e731198657b7c5de6998d0e\",\"717d8c5e228ba60f653380c1d76808462cecb40902c12a70da40b917e94ef646\",\"3c4de6aeee1d0a7155c0b9ef5d84f7a98ba5dff6ef4a2cac285872d0f75a49d2\",\"c703974c202ebbff40574ec30d05c60288c5b985900a004db0cee81e32f51b71\",\"15fd543e41b0b94e48f638547cc20941b7ca6ed5ada7dbd9f485868090bc7454\",\"7e37a2a257bf0621dd469d46eaf091e90860973ddd8e8d86a1afc3f333301d83\",\"149ec1824f1890d685a9ff9a3990bc3ae0d1665695265f84140a624ba542d1b8\",\"279726fe80540c52ceb8052bbd679946b95f5c22ac542d17ffb09ffc6e2c2fbe\",\"bbb4ccb4fad8af3ec1084047be8786bf454d9ec50d4bf0f80b87c659943b690c\",\"2855c16c636860316d274be096877dd2dbe2cd1cb7a827756416652298b11479\",\"a30b12b4b7b2835f75926d5eb0ee11da7a61dcbd2ccb5dae0854d2e61ccb33d5\",\"3f8300caee5ca281bce17b0702a524baec61eefb570f2fcc361f054daa58a797\",\"fad579df17598e4d88dc5eabbed5218d9aa28930ced5abc56006421417ad45cc\",\"73cbb8bc130cc94600e98856737d5dd1ea71ba910cb38ffaf636742849cc95f1\",\"ecbe661d1d4c6916f6e6806404272135aae4b731896a1e1b088d7a1e562abfba\",\"cadefa9f37e4adbc121a1c00f31ef2310a34f38557459b4dc52831ea19cc0d69\",\"14f5fd371efe19fcd04fed69836c0f999f2f07e35bfe98aeed000d89c484f4c6\",\"cac6276f02e6fdec28f8564745d239f58d10c78af1bcc8f253c4d9495bc4d1d6\",\"13e6d08a099218a6e7cc2a7a11d0c5032f00ec247b136e852406edf3be81f132\",\"edecaceaf8b433d0cde4dd2254e139ed1ab4606ef02c8ac70d22d013a4895f9c\",\"95d57a82e0b64cf4c918c18e7a97eb2f1d8434e361bd2645c1b7850486f3a4ac\",\"b8076bdcb00bef34a5058c84511e94d3fdca76c4003baf9020d205741d9e46b4\",\"668e6e4d507d70f723da44b0915ffcf70d7db640f2abd980ff905065b0796a6a\",\"e8cbaca1ffbae3a98b6ea1db11fb9145ff7ec39d4a7ef962faa4b5415183f72b\",\"63ff06047671a2325e378c942332723e423a992aa4691ce64d1349dcda92c980\",\"c7304d74f5209107c4f35566be27b7121fc17abff53d924af18940ccdbb427b6\",\"9a0d39bace7b0b07702f4a83368221327b96bfbe6db9d241c865e8d54194c7a0\",\"999e2cb75e78f1ceaff443c3cc8814810bf3bfd40b36ca88cab2d1143efae639\",\"b265df16b1cacfec144272ff711b2c8ed0ec5a871c8d04e8c115b8e0ce349b68\",\"d8ec0834333a45ab0562a890332641aac1d2b5c4135b76859328edab1e8812ca\",\"030758eac2dd5419d79d10bbe4f9114b8e331b8d2ade8a1bddd62f82619cc0d3\",\"608b068a610ecbf74863660524d8887e6ef3e5902e1cdb71f3e1108c57dff1a1\",\"fd525583332b554249401bdd230c5d581aa1386cb8c1324e4fd3d139e7036c0c\",\"0dafb3561c8fd7f2b5dbbe6ae4995026c1f21218781efa82f646ff26287e9bf2\",\"ecc7026bd0bfc14adf6e296b273a3df8b60a65eb867c91d14889cf54520dfa54\",\"72296ece5c411f0bebd8cd42038f1d625fe114645e037eb79f6dd6fdd3632bb8\",\"0680f5248dc83a98d1a5d95cfa15135dcce63a9d2140cab70f4145b76250e5e6\",\"1afb591d7669659e6645efe508035bd6ccb7305326259ad25e87f1b2c373ace6\",\"717dd09ac4a14c9c00c6bba3b13e456ed5b2c58bef2a58d63b8280eead16b2ed\",\"96855a7187be2dfb4304f6d952cdabdbe3a092acb1270c076366f46abae16e61\",\"6eb37291d25083b59e24406718e069ae2a5ef8b702d09ba34d4a5dc39fe726c4\",\"771bad387c475a3dcd0de4df25f326ba8c3a116a6e091e4b38f8a3ae9d8d4954\",\"91b676c3b03b3aac24964b05f35a15a37f4e61477dba5afe533005b1241a0ada\",\"e62441763d82406e82dd6d0dcd4047efa2da9458e4bfcee329628b47e412bb30\",\"b5f9ae5b22524a28812cb94d08808e216b10a57419569d9bc880eef0b4089c9a\",\"996cb6c644a4a97075ca0fce0c629cab3da29963b9b1b680dc328eca0bdb125e\",\"ecae74af49ea316cd2f045151a519ad9459282fe505fe4b24181c385fc9e5d9f\",\"03566e66b627a4c47c9a28b9ebba98dda69385d685be6d7471c33166843df5e5\",\"50912ef5ce09cc49bff9e425e23547cd046f89dec8d89a04349f4c63e1168754\",\"deb733b17b208c8007396f0fdd4fd52ea7ab4ac058b6963511a84aa3a3c013ef\",\"e9370ab6ae0e02fe7291b585f36bdb52dde553f846eee6ec326ddb29371dada8\",\"802e0a2a025ce02837c2530ef7836b20e1a28e324363a788a29969197d5b0a4c\",\"59461375f3ab73fb18f1a7b1595fc4c26ee16c8626c2b768122ba08b9334e246\",\"98099828dc1181cf1f4d9e5b1a3f7237490d5af0fbc7e637eb972f7099f76d2c\",\"ee2231a8fcf50d747e16ce95f051e11992c1c6c5abf0f5c5d8021231375b0b30\",\"7f519718bfa215fc1bd156911d86dd52c71d63171040947c8e60c39e59316be8\",\"d959da463f3671c490e945c5a2d6c5d187c3e7ac42a2b9901ffc2d46b1d915ec\",\"3a8e16cf0b599336838b42dab0ab3cc0b49773e65eef143bc2f08dd5ceb6d5d4\",\"dae87eee94421eeb4a09a834993e2af8e4ac0a1bc5fdcc585d8cd45de768bcf9\",\"c1642a87ab7e768e7c2a1eaaa8d43c6e02ae11514588f1a9bf38d15d9f75265e\",\"f3649ddd0f6be3f72e09a8b1e6a11410314e249b04afdf58263760c7fa572f20\",\"b9204370b9d81b4d782b873a4b3e9e0c16b1895392dbf82d43d44b8ac0f99f6d\",\"36802b2be96d39a2f8b1e218caecb56bf43e54cc00e710f06448c5baefe4192b\",\"53432b9da771f4eca90ff18415d4ac11ee66eb2ff95062a21f94c900a47a828c\",\"30a9b40fd9450c0bd69cf8a5efb0dec5c35674d93fca8ba8f57fbdd2b28024b8\",\"6d87720805f0fe9466d224d82255a7157795d9ddbe9c51f5ff4ee69b6f653de4\",\"cb1f570735aae52341706a2916201afae6d70bb8dc1b68dc02cd935f3e7cf76f\",\"d3f9f7aaca2a0f226fc9580bd61e2a9b4c1441103ee32e763a20115955394179\",\"8b4b09a972b02f0e471611af33b1eecaa0091a816783e8e3b6a9d683a074e12f\",\"97fc3c00248a9f0b5e967df9213e61d1890bcdfb9871f59505454068661b97de\",\"28d33e8884f1367417b4c1d192aa061512a6a5e686f6125333717a1329619207\",\"5d11cf234e3854ae6a02dd1c3a05923ad7ca3296d4135553558da00e336adb62\",\"3dae21c3f029a0ca1fcd9b0359d69168ba605e7ef4328ae43386947d36a6cccc\",\"5443e2b35278ff9c9cf2f1318ee34f9dbcbb99203d21d16e51db62aadff5c9b4\",\"766753a58f0cbce25f705f3da12e601ebd12a019153a06346fc72a927db0b14a\",\"2bebe17b6d882edc4285e97b730c4c1812f69b09be65029ef0e664518b01e488\",\"be7b85c84c7b391f945f750188108fd7d4afca7a2195eaebc18b73acf950973d\",\"b7c1f043bc17e6cd5285010c8f44a2602f5c272fa56e2ee6f8d5dc17923a19d0\",\"8a54a9c4423a86914c3f4072dc9b3ef0cd5479c2ca08f0cb76d5990db68ff868\",\"3e40cc308679f57980fcee9d9e8558b0fa08da2e18b883fdf93aee675829e142\",\"4a6447b223f54b7d914af8aaf160357c4bda5f858751d997f70fca669242e624\",\"c82157fc83a93d7b8625c8d722950015b35bb72e438e547897fcf45fd41365d5\",\"87dfb9bc311272a6f89779801be6c58597bbe13391288a2340977537552f70e4\",\"e4d1a998d5bb8ffd28892bdbaaa4d156d3a9d52c4bab79830a1dd34b83f44567\",\"f07d5591faa25af36dc62293f05b8d1fef7e3c137bc72baf0f8954682604ddb3\",\"13599f4ea9c768e0c4260944020860b8d7c9c570e0acf492e931a780f5fd4029\",\"478d72e9ce3a0e9f21b062dee6fde2a4702b901d301aa53e69f0818630c848a3\",\"93e8b0e1ace84f788473b2bbb471c48656e67920b83fe0cb77196629013412e7\",\"0655ddecd40f9610dedf20110863b53761676e7bef38199eb8b15a7310de482d\",\"ae160ac8e7691b931d89dd13556ef178fe2b0f07ddaddeaca9b69c49e3ce2abd\",\"bf0f0c0460bea814eb159deb7c8150f405a9b833ccdd59497601db98d5233443\",\"0f25385489b3ff481c7df83d8c727a0f87f0f6d5d259ff4dbc980b9971a563c6\",\"caacee23fdece68758c7508872384fe22271f0b8e6b9dd595926bc92c03fccd3\",\"0e186a546ceb694179e4211100500bd36664df378f79f162ad0b787f6aebde5f\",\"3a465e7ff0b80d062d1300d87a2ab961a19bdfdb094f1ddcda68b404ba4dcda9\",\"8f0427f9bc4ee623d4e13f89757d9ceedd1fb192801de2f53c37a75071a58bf7\",\"b431bdbfbee595dd9928ea7cd9b291790fb61e4fce3ff272b80a9823ff910862\",\"4b11261ea9167a4fcd9a3fefeb090eab50bd259a16afbe02e23e76e6e70910d5\",\"6593f44dfbc23d17435831fd9156c6aa9de5722a1edf8748532f93756996ea16\",\"94fef70d175eb1b4028aefb5ce60d0ec8c968f33b3861d30f321c3f4b67bf3a8\",\"bd1e144594522042fe25072576ca248706b85357bb5204a39e13906916f4a6ec\",\"485db142c33e029e5f154da3424a1b44ff0e5a2223c7cb3739a6004ecdfefc43\",\"625775bcfe9760ffbab7ccdf703a6dccd9ef68ed2bd016fb06687458aa47ba21\",\"42f27f2c5e2fe94e274877218f111a6ed6ccb52302b80a9b3f613dd58e20e9b6\",\"221f9e91f3e93713f7ecacfa6ad76a891bb8ef7a4cb0d3603020a0bc42fae94d\",\"2ae3d5284beb0d31b23e70cf5917ae949138599a92d3ccda35793e9c940f2de8\",\"2710fc82a4d8358bba2055e8dc30a045d7b212862da60b50f4449529d32851dc\",\"51fef40b96252822c265082e4bc0bd614de12c4cbb62dc8de1aa4bb851b3bfad\",\"4eba9fab8c7cd3cc0150e4cca21d70705076844ab95f95b3ed513b4b404862f3\",\"74fb53ab47282f1163d83bd0417f95f5a19a46d4a91b462d6bc3312a9b4f1734\",\"d9c6870f9393d9c885d9c2cb2422dd0a464e84186d5e038772bb0f32db998301\",\"fcbcc4fdaba78030a9a0f6d6b61de59a208c64fc2c89cb6a588918d3a6f1ec34\",\"c5ed7970cf77faf07caebfdfcd5222854553ac9e020b992ff7ff0d5f4ff82cc8\",\"64eb432e23bf9d64a44f4aea5e27b5c967dac6a6242960c1bb6e271879b31cbf\",\"9814fb91976d5380c85d78d77466980d6d6946ca10aa07c36ab86d191d4cda44\",\"d7cdc9a34587c91e11874f018f3723939d657ff5334a7be60c3e0cd641d77886\",\"5642cd8fa458183382f5e5330d88c1ed2a2e253230520c70110d1efd3441275e\",\"322c32e4558de7258d3aaa0d5ec58d02356aa85cfe138fb1326ef65423a12f4b\",\"24372e56935c44c8d867b556daff6f7708a65f138065d6928dfc9d231a693f47\",\"a8b517633740f439f56c0c5f014dfa2930192d0e368017c44c4db67fedd002a7\",\"475ad3293038f7d41caaf1af13c4ac66e4fc8f4025f551dfc648f3ad2e0b2a6c\",\"18018c4ef0a3a603cd62e982760d4adce86f28dee43533e263656ac3d3640324\",\"a2bd347934c3cb0caa81400ff5de52db3591b948f79b22141f2eeff3fd4001e1\",\"1e809b4276f77d3cd0267b96a2900ac8e3737d8bff4faf1308473e80deb4fdbd\",\"1a78d49680d66de6506de2f831b73fc12e5145c397754327c8daefb7a874a4b9\",\"68cec8f453a333317ff1409e30f2b631e4ca1d6720ecb76500e5664b29f25418\",\"75d63fcd069b22f722d9e6c72ba6425d9de05ce5449f8664d89c423e2b683fad\",\"a7059dd3272749f7b4c0934d15de00b03f26d9fec196dc4724bf8776c0357087\",\"33d40e990864bcfda28ff4d4e608ff6eebcde72a7e3cd412a6e72201c88342ec\",\"5081095a66e4272a233cd4e46ce36b49d3bb2b19e5caa318b036c362e25afb06\",\"0b9c1e8cc0f0c1961832b61d83e626aea6a184651ecc308cf4384f19fdf04a62\",\"dacc78e54ee09bb5df6a25af6b274aae5e534b896838a383b6ae6d4b22f1a595\",\"f151dee146b9b0d844e0872e692a5fc135a8f2e652a6581dfce59a72d8dcf37c\",\"1781b99890386291a3baaba38dc650a528b175bfea1571524f77d11515a8de11\",\"3b6e71e2cdc430d0bdc0220cd644c946c8baf680f7c8219f2cb69a30a70618c1\",\"387b1ab7077dbe85a7da5b97920f0e8fbaec7a91d2e74a66c7d8b62d4603e06c\",\"6d6d96adc03344a8b58abaca0c2f5e536808199066a9520de63487fbe091e085\",\"9c83e3940d95b0656dac3d8fb0a28a05b50c0b6297932c1527d8e25aa5208c6e\",\"1439e43f094c2e49f77f8a80a4343a27a5944534ad98ac7db3c5c4a62df4765e\",\"40268f0fdc138f14502a1895f9dc577150da573feb89a631870f2b857529ae1d\",\"4ebc7e4a4f6813569a25c7d57f2db898083f01dc1f10cf329efd5299a2b3f016\",\"ec36ca6dea1841e6fe4a9d4dc672e713f558621a99531d83f644f4ebe2d3b0c5\",\"3338577f7b6b8561a447eaa4cebe112c6097eae7bfdb2594971264768a963154\",\"3818c40d758bd370976cf644185e8927148c1408e332efab2bd309066b66a137\",\"9664248f5702f6a4d7619da817d2fd2ec4e422c7afd259cc60345d7dc5d78391\",\"7633b8e8122114d55a991146c068392b0546b50b6e88a79c71bb6b8731325821\",\"335d90a0ee2b2a84e5ec686804049862d753ca560605664fbdeeb70996274fc4\",\"bd2dcbdf8f92cfc84e38f5c98e75b4c509552515c2f91e753da696a4c1e562de\",\"5ed53b4533b310df796b0b276247eef67147360a19d1ca39b849c054e101a9c9\",\"83afcd1339f940edab0d0c898de2061bf451503817dbdc316fa48e34227be62d\",\"7295c81c0e76dc6ceda3f063ef152c514d11fed4f85b76a6a6d64248cd09bfc1\",\"35507aec773a6e30cf99986f23f9abd78a2a35c9a1fb8b7de338bca90708a508\",\"ea21956c755609ff4f10411c6712641d7a9e29dd6e7c0782d86c2be5e402fd1f\",\"91d7e356c034765ec2d291740d9058048f81eb955e736175090645f9fdaf0e88\",\"e231fd542746a65150079a26572f2548b2a137a9568e322f2740d79b83f814f4\",\"06e8d6b00d94d799becb99fe71ac77e5aa5fbd5719beaca515a2a0236e418195\",\"4baedca11ef7030f5b58fd2d2a4dc6e19f7ae5b96d8b4a4f3192a2f0fa3c943a\",\"657844799075a651e66cf76fbd2de728c628160f78b929a6b0cb35340ee49e53\",\"7dd2ae613451306cc26abf4c50f52ceed38567428f2c20ecde25dd85ab641842\",\"bb4f7522313f58a165d14c80deaa9b4bec8637f29f3fa57969ac5e47b96bf49b\",\"9c938bdaf497c5ec59a98abf0048f543648eabc4e3a30690a72ed150a52ac9dd\",\"8e362da6d589b3efd622c627e0ac7cba071223e4a974962565c317e0df193aa5\",\"70b3aaa94761ceb9f23cc6633f6aea3af8cf93ffe2c692edf802d0d2b7807654\",\"e2528f096e7aaad5b899a6ff7f6cf260572f02c8ad1699b62f7dc61b1db67332\",\"6be6cc7fc56355b3e70018b52fe200fcef0d156a5bf04e745e3205d954955ec7\",\"cc9e78062d78e07ca7e95b76a25856ff41f821727d0e74b039163d84db77e7c6\",\"442466324567b5f422ce974e8e32eadda683a4d8d62fb37160b58fb386fe5380\",\"2592b95b17ca01004bce28f7037b0ed2911d530ca1e84549d1f0f0672959e42a\",\"83e462a213f726d34947481ca3faf6ea8c58c563c8afd4c1cd4c79b347fa1d45\",\"76d09525e00aa040ce243d21378a3be01ebc3a9d1dc129d53fb7effa44135c42\",\"e5c73ad93533e1cc3b658d50e67a5f6bd42fc711eed9d46e61cab1d369d6d501\",\"74ad38c4e0aaca9f70078ff40526673a199ae0ccd2cf248ff736d4ce9e734f13\",\"63f9e7afa8b8b671b7866172fe58415a8a3ba23310a305da936dbcc98e4911bf\",\"673513fc970097fbb0e831c0a0c15f88b26dd9a171ed9a7263cff843a47da401\",\"55fb1f329a85adb03861fd26bcefdeac9f56543e45245976d9fa694cf5ec5a4e\",\"af6df2046c9b2dca9d935b27ed98b430f4ae8247ea5ff4385d993111aa855e1c\",\"948cfe05bd8fc1d6755bcd7695d62765f98aa2837ee45e9d846e62a0a19400f7\",\"17baeb734abfc6c7f483a4715bfd82ac194552adc749a268cf1123551d702ff5\",\"4f40ec1a3f58a3e322f7bc8f880035a355d78132b965c9aa51b8b848031b5933\",\"6ea65300202d52d6f3937a186d2159d03b8c50f67424abd66225a5d1125d7d06\",\"271faa7bb9610586154bd6253122ca63144c944a733cd529a799d33dce06d217\",\"5c7580a08d04f39a0d4b8558c29ff154b352e0821e8a85e3b59abfdc005ca8b2\",\"a14643a929a3582bf2a00ea3c00b3f5094880044e81896f32cc8a809e73d0802\",\"f402febfad05d976fb11dad172a28d234b0aeef66791715c696e1ff5b0aab67c\",\"28f6bfd89b5eea1ca659716e4f94b32efc4e86b9dadccdd7c50630b194360eb9\",\"1defa8cb824f2ec491f7cd209a8a1f05753e7219e1f11554df8bc037a820a8ac\",\"6e4b6b9dc72fca84ba801f0f62ac4752ad5c52f894b232ece5bfbb157d3d6278\",\"caed6dcaa4671bf89e2a61f06ef1f791bb4303e666056f9d5c9e56858ef31adf\",\"6879b7cd247cb4f888b68988774336a261e9c4d8a97b1c86ae63aa7cedabf751\",\"e94b013362397c5e723d8d1ebf4ca6fd96b926482a580589d7b53190aa2c788a\",\"3eaf7110e6fd537bc62f438d0d723a1fff4fe3ed0775cc0928ab302f4493570e\",\"9b3cedff379141ea50b01ba137c905f5e391c9a033f7bc65bfceef4c3cffaf77\",\"b4f60eb4fbb641a48e014e3b601278a5ba7e7b2ccc424a80f0578444431b4002\",\"7b355b0a4d6b1f36a306b73ac249b7835bce6694381b2e6d1db036c96ed7fb0d\",\"2ff681a0a4d0c5ff2efbace4fc36c9f3c731c075b2ca61368c386bd7f3f1edba\",\"11e8489861b8e48a06764bc4255c7eb9ed4cfbfd0c19d14c0bdc5b955cd9613a\",\"4c36f3e096a1f7787eec0113d2b3c959f009f5224844d1d77575385a40cdc4ff\",\"6a0950ae633fb199524a7abaffe205202f81b16c0459d63689a8b1290f13313d\",\"c3974dd2e157d0cc940e05b7f370dcce0a45f375510080830393d92a114cc540\",\"3c9697bc3c3359cb0ab94becc13f7c81aa514a4fae69e4c483340a6b12e81764\",\"a9f44091b3161f1c2600f2d9717d00b64d94bf3255d8cc7d3231d9f20a769116\",\"8e3d3f34db055db781bd3a531b2e9af1a7b0d6a934019d473e02d3c3ba06c519\",\"346456b456ea1bcdbc71e41bb4ab0f9769304e4666a4dd1df4b22409e6a58628\",\"9f3bd80b820713777d1e97d281f0ff5db80f03af41ff167bc386d5c124c41dbc\",\"f2c07aa5fca7a0c8c9e6b1e11713dcea2754cb6f2ed83be6a33bd4125b01e353\",\"7f7141c6d0bdd64c17ac01943b3c58c63d0c6518780de899fdde29e17ef3b3d7\",\"aea2b3bb0bf2b9bbf940ccccb7a1938f9ca097ad3f19be674d10ada6872f61bb\",\"086abd6d498652cdf3a20cd9c618a5366aaa37b939a06d369d39b4a7e8955a0f\",\"ddd0d1aee352c3e9dd83dff933c15fb001a824c9cd5fc53dc5f101fc072a531e\",\"ff6cbc95b5c4ee9f81032c34be892aff7cb08aaa064caa158d9039a6f034d3f4\",\"c4b51654e5a4e2c4a513fdacad55cbc28d41475211cfbbf442a4ea39e7ec1a8b\",\"6b9cf7fd97a508e757be33bfdfda5f0b5e861b0ffebd07b06ec000d7c3a758e4\",\"5a41769c63182db98ef011d56d475dfdf2360b41317af73978e186e3455ac029\",\"f969929bedb4483166fbd4ab87ffd083ef950cdeedfe5417de1078d1015fc90b\",\"8325f0dc312f253714c92bfead69ea8d20ed9732e1db7f48aeb4d29b6633469f\",\"fb421e5a45e9608a6a6852c638c41da92bae557b0845665ad71cfcd0e62c044e\",\"c48c0b4bb174eb50ee38312514df17671e77bb1591a8fccf2d38c4eecd9928ae\",\"937cc36229122dee48f82d8853afa7eed63f2cc716837938932dc5136683d9c1\",\"3408f1f9bfa25d0042a3a16783649c24eb7adc6302e7d1dfce530964e46acebf\",\"bace1706e7147927cb92f0824a93ce4840366c3693bb145c9f0c4e47e4c8c455\",\"b4df6cf06b20c4239f219254b7f78383b8ac1e459b825971803de481e9c89c53\",\"702b6f2b92f0ebcb5be682613ae6882a8d34af951fc3e1135b1981abd57bc452\",\"ba2c0febb3eab510a310aaffa811339cbfddf974e22802fc0077d930170320ec\",\"a6e1edad1f21bba8366b767bda2bb436184c170bbfad95a64c01147df671126c\",\"a023f9045803c6386831f1780c9a6da50cdb04f824136b87cd0ee9c155bba797\",\"61e7bb626f05b8a78a0ceb0e1e88e5cacc374bf6e5328cd50fa9b16f85845ca1\",\"5a865183ef52704d954776877cdeca474a45a9946147da2756d47d65d0badc91\",\"4bc04e97282052f2b43422ca0517441e69ba071ce746c59679e0f50c636ab211\",\"82f7d5df2d32ed61b0cfa4d8a0bce33fad38c26bef5213aa06b236025c742549\",\"a783dff81837658c80680eed7c7aef96a19749d31bf6f8b611fe079ff7fe9f3d\",\"11347d541020fd28fa1068c535ec7b93e5ca698d6182167290cedacb4e813e2f\",\"91f2233d86fdeb9f0c579095318257543dc464494ba8e996f1e8e9e679c6e583\",\"e1e6ed59a60ae73d94cff0f6dae9c2b833262f3dfe3cc0f4f98c031855320a2e\",\"7852dc7810f04a02fea5623d2382869878a9f6f9f3e83504bcaa8e2633013dd0\",\"143ff8ea7d7f69f1fbc08fcfc0718845c868c57a61c80e26ad7313dd79772791\",\"62b9c30be5acb8dbebb8faf3d3cc99b1a33859d019c2de116e81f3fdab9222eb\",\"618f6a13352ffa3dc6ada52d80ad9b749503627d20e8fb6650d0a8acd9dccedf\",\"5767d3f017888e1b746086a64896ddc9dca882c29b4be23df5d87ab264f7549e\",\"b88b8774ef292b6990650bac6a4b5b7e364cb9ec8a2375fb690daa65ebac3c12\",\"964bdd6f19c354eaa525d2802ec82709caf53b6f28075eea458946bcc500e97a\",\"e4ecec4f358801cd2353181e413a7e64bc2f7e35f3692efce3ec9601ce45112e\",\"afb33d7b71e5fa69237abba317a0aa047822edef75e442ffa247b80ecccd36a9\",\"2654a8d7ac3db75a1bf5979d855a7d25226b093cbffba69995fdfaa51f311c07\",\"e0246ce1780a1b5c3c6c04ad827fe2638d95055ead5674f0d8ae51c6ac409b29\",\"fae593bf4dd6ee767d589420f7c3289b58e1a049a4da868aed121d2927b970dc\",\"8ba17e47f813ee3a3220f03751af62144f8be11ae66093b6d8712f58819cece3\",\"4ce489d63dc7c4271e42160eccf4e14e9cbca73fee455901d2f71faf75d4334c\",\"0e8d388682b1e6d4d15c85f341ead8c38c28fa79bd6aee9e3c46ae460a314858\",\"7da06f73e1fc2a893b0c54afab2be6d87c159c5de18b226e07f515803a3a6884\",\"1a25283dcf6bbadadce5d43ebd0dbcc3b5c6894520953b86e1941c9dc8c55ceb\",\"c526f3a80f566b7c2e352b1610aeeb601e3af677e2e383fa1f82f5e28c14ac66\",\"2b38a29f2aae8a29ebd2f3cef55f71a6299a4a6e15fc91077f6200bea02a0f65\",\"bd87243d9609d66727c65d7ec347d998593313c883f0e4801b2fe6e91d6662fe\",\"54efa887f976217b96dc2718abb1e652e871702c6ab8fd092e7101948c30ea30\",\"e42a90fb105bf52b4dfe851966973cd168c39c666e9a160f86feb5f86bb8bcdd\",\"90db9c5218c688a6ec83a0f53faff807c467559f51be6c68bafdf039ad48929b\",\"175df0b873102d298d2f135e1ea363aafd44a50475c82a6f577e6cfbfc5a2003\",\"343cfd75656e4da8b1fe3a5eed9369b7d1e72c609c9728c4c29fa75637326233\",\"1728b56d7e50f045a11c8399c588f07f6f39aff63587f1e195da3906cfacb742\",\"edbb6594d03e30f974a5e56d146c401f48ec091f5f5afb814e31e2434e21448a\",\"bd390beb58166cbf447e5881ae071830bc2ed5d910929050473114067eef08db\",\"1b3d7a2d5d953dfa7c24aa7d71974638f3e88c8a617035df5d419224f7063bd6\",\"08621db93a8c02e6cc77c66857dcdaf20f0c7d25e5ebd1d040cbf67e61ec0f38\",\"00f39c1badbc2c01147b74e928e13173fbf7ce7602c42291e77c09ebda952c40\",\"957bfe1e5c826ca749f2c32cbe4075c3ad9dbcc0efe6329ee370413b93efe909\",\"a14373ee1e1ace4ee01d65ea7399e026e7bd59164f10229c2bb524a02013209c\",\"e8375b5371b08358e5d782783c09a7abeb3da9055a6cb5271b03a7c573d35718\",\"093a674a08a1d59031842052139fa6fc094d21013933c039d94c3a28d9d651b2\",\"893731a037973de8739825f09d76552209cce38075da7ff9da8d78098f752324\",\"3c98c922fc45452bf0ba8668fcae7179d84b4868a1cca11f7b1c04f4a79e6c84\",\"7dfe054f8d0ff9bafad9a089280660c1fad893aa112304a340265161d9e85211\",\"2f7dfeb03e4e0ab93f69c9c569a999e1f03aa19c2bc6a8bcab83c5a6405452e0\",\"9b57d02b02c4c4e45ea74cc2a18f703b0ad5c604e5a7c54a8343e71efe52333a\",\"609acf468399aa2c4efd764fb7a3f81e24aaa170660244bfa1c7c52b83a375f8\",\"a96c3c6c15d31d91d91e8538cf754c41877f41047b13072fdc9ee453c0335965\",\"75666396cedc9c565b9d2623cc51d56ca4691710b02452ba86004e419bb8c63c\",\"991e2fd652abed6a606e223d8ea5bae4b115926396a97f0b1ec96347ab5d5a27\",\"25b79b0f108e5489edcbd00c3a49dbfb37e5abddc0ca4d02ba791e4207dbfc97\",\"7ccd1ee2019e09e82ac9620dbfff20fd54e256c1149915d94aa8cd03f74f40ad\",\"3fb342aea3f96ca40f7f0eb1318ca2370ead3a7e78b23141c4006bdfcba094ea\",\"6cd3c3e47cb39f2a007a952bb84d08938bea8cbff981fc0f49b8e53726659be9\",\"917ac04d83ce4ab1e9cef4e6e7c4de94c5b2da99b1d26696b49e350e1dce1fa2\",\"7da40206b2b113b0d17c12d8d275e03db1bd0d67c6c950478cb4b736adb383a9\",\"c659f8cf494972f2550ac1bc70f286523c208809ec57ad6995e2b6d3b19f21b9\",\"222e47410d92b316170892cd9ce35370e37847791aed40fc66366ece864d51a6\",\"3745b54f79a2ec35b057f748932323c3fb4e5fcfe7ac0b6dfee339ae53d1a49c\",\"84e821ea79df8137e6b4a433fbd2d6b86d15e64eb0a73144a9f52a60ed61d5ac\",\"0865b184e833e888ccadb70a239e73a4e9d2205e041896dd7a5023e7973c5015\",\"a3318370e36bbd5aef3e3b0474886591b1d529995092f81641ab4d7c1e587965\",\"d04f48f41f782f263fc3215f3f4345c8568e2ae86ae28d8a1bb7df67526f148b\",\"d3345687a945c0cd9ab01fbd32514d49c511dbb91fa057d98a0f3fe23d65c029\",\"4f304dd6f36b828d5c40e928ed846217fa450962133e68903d0de45a319f1cb1\",\"3b5a60c15527499534f416fae69156d6e7e352942853faae31d01574da10e47f\",\"047439c76b38df7fcc0562cc9413d7d29baaff3148e212049bd89d8b48cad0ab\",\"4f7529b70ff5238dab260b663a4d0542c5cc68bbe08738e7f05a21f53a049ea7\",\"56af2c41c231fd9ebe24b4162a3d23258e97d5d1e002aa5165ea6ab1ae04e5ac\",\"dee834456ed56405768abf1c6863b5b9f9614798325a805923b5cb3208f4aa0e\",\"47db7ccbac71d45c2c1217a4955335a90759c9c612ba80c4b2212893c9417cad\",\"3a8587580e6a8357e8c5c579fc8f6e1d948260ac190923f26871836afd5eff33\",\"425464adea0ac05720c30c653586657c1ec5af49cb9a2b6f48b3a773c0e266b7\",\"2478a4683f2798154bc99fc9e77d56d3160cc37bebfd1301cb419e15ea98385b\",\"4de18ecc54ccb71b0aedaf36b23226d87e9b621c6648b3f701d90afaefb6cb97\",\"e6b16d336487e0d6cf578b12e7556347e2ebf7c0594aad8fe977a739073c88d4\",\"e8d77692f9dc63a550420c1bd738e8903e340d93fd17bdcc64ad982f7bf92244\",\"158e412762d38c598ba71ebfe7663f0a3583a8881c87a4dc5f05ed67891b9fa6\",\"87642a27af160b1d59f3862392803affac3dc9bca0328c9be5bf937391428f8b\",\"53b95917db1dd1cb1db866feeb100674e102c72ae48115db7ab097086f26c651\",\"778b8a38baf5da64a59dc98ad4ac44c7f551a682c95905bc98effed904c9a5ac\",\"183fb8d9f885eb9dcc4e4cc25006d4b51d040ce8861ce78af109fc21617d9513\",\"aa3fef941344df2608e1c9cee19d05f7a897a456889c90255499cf0ff871761a\",\"067b8cd8c7c9e53054227c52840f1250998e4db699d56d5a2c61ddd4348e1a85\",\"bdcb7dbb4934360528c76db1a230c16d92cd7a4f5c324f73db705735ab1a726b\",\"bd3d6cbcd1b39e9e0301bd55dcf3fdc847eca14858c413f3b3238eca5be6ce63\",\"24957ca1e7c1afe8850089d2832d9b658a3682c288b86af4bd20e0020262d96c\",\"5aaad896bece2cbf1bfda7ff275cd12c550e661a494dd438af6c45ece6e29fca\",\"8713e5822271670fa16ba9d9a9a37bc0e68257daf7bf45545cf23edc589082b1\",\"a7b9d2c2660a493b498f6faaad23e88977a77ccf97ed052a81b532860c70e48f\",\"08ebb3ed45d1b36ed1867afcc1ca3847c411c683a934fac6646cd93798f2987b\",\"43a820f48056c2859c44f78646ace30a1209fdd5034d19b234391f8fd8eab3c8\",\"408d309e92407caf71429a8a59851d59d4b6d68740d7bc85325ac49591ac1e33\",\"17325c07f2ec944b951bfd805d968fa8bc808316048a34d28e99c513bc8e7056\",\"e9374dc42cbb4e6fa28e8cbe717f9d23e6e5ae902116ef2594044ab78288ec2f\",\"eb98bc9dbc773930efdad2ecd99023b2c083de3e01108e84c0e06b219bd37f51\",\"afdb34db7417a6b9562a933869d2cee3c7e1ed507419936b17841b4845da3b73\",\"d73c050402f931f6127dfbeb2a5acc01f50171025ea51e316aac4675d9870f3f\",\"5c0268680e8b41f2580164c456bbd976cc1b7e1b9050bcc4ea183bda07080c92\",\"8432b3583c633c9842daffd1fbc26af5fecf37b8add36dc2f34b59f6a0334d53\",\"31c82b2e809cd2572f8bd6915a2dea90ffbbfe96a99fbb372fd700dd6d8efcd9\",\"77246e401edfbac76c128b9746488357a018214415d9a67791b6e87bd59a64bb\",\"e770af0d882f8778fbcb9b5ff816d256f17dbaebe5355e72a6403fbc2b2e18e8\",\"b1c47e730e216d3cbadf5bd9941796f3d30d72962e7d206d25c52b1bb3b47bf0\",\"3866f1bb33e15c798abeccfd4882171eb050154290bb928b0d316846898c93a8\",\"32c63c159766031ded5514baf678ad83b1df2b00209919f79a5ab546df595244\",\"1d933d909b88a88e181734e2ad0d476bb309be078fe1388a0025042a62c4a810\",\"c4a61da3f7ee8f9d055200866f7f2d4fb26d87a3282e3b5b847708bbae378f1a\",\"6f2d0312591dad468d966ffcb86387129a6df2755a66229ba276c4a96c0eb90c\",\"ce8f86576cb0416edd7c2d8105915c6e4dff4db9b0f7a29fab04dc3d0f4b38ea\",\"73615ad71b8ef8649437ef3df52e4d878eaed5be6cd4df2ccfc4455c540803cd\",\"474e455588445410e916db5e32765a02b895a2f43d727e2d188495ea8e3a75b4\",\"2aef89a6aa11f5573bdb411fbdccaab80ac4a569dd8cdbcea3ec5371cb6dbea8\",\"fc5fe776fac146a8b7f4a8d12b8ee9910e5bb0ebcbb2eccc777d91737623b594\",\"a47a366cf55b8a8b6baac173df8eeb3ceb9f709a8ae41f6dc7f6c80c777fd6dd\",\"0bed644d0e11b951af99d4e815503aaff9d6dea8972e6d50be11ab49f55ba0e1\",\"33d551c7e4ec680c6703c5485c50e824dea4979509e01c3a092b10943f12a9f5\",\"60e6fe6dee047ce7ecc0df7f4fe9a6bd6703bf1ab4cb3a968390e3c0975697ed\",\"5062833a1580f8941fbce128bfc68cb2a429e8ba037100a0fc70de88a1bc8ff1\",\"6160e83e6258bf4755bbbef31829785cd154e87ae331351f6b61eb94f6875b97\",\"ba40738cde554e78d5fd7d903c7a6a6848737443b5774fea79b2807f8d38529e\",\"ded230315649d5d80f5ff3b777c2155d259777a3c0b2d4346862fcc26a6acb97\",\"d38d1864e527399f62d7a1b7d034f173d2e7a187dd917fafc5ae9b1db5554cd0\",\"3fb1901303c864627afa399a51a29d9b165405a118520b2ae3d25a1df73ab8aa\",\"d54dacaa67cc130dceb4d685267b71d95330984b41694d41a2a05573d18f7fb0\",\"20b8083b9e14c3b84ae3227899753a59f39363b1ae5eeb5c07f17d7f02531a00\",\"8671705d591fd5ce01a7209ab4549b0984d0ec306800b4807c32ea839e9620aa\",\"899040d2c02652924fdf052e88776e47f9e1972e2ec0f198bdcae4268c74c9b4\",\"16fe8cd5798e340d414c5daf2b179d3ba9fa48847a9db76227f38c26df899788\",\"0f2b9e7c380f98f1923bbb1cec3be9536a4b57764d7addc58982dd9ba2512525\",\"191966278d3efad2e7fce96acbf251b6a285ea596b9241e99879859f463525b0\",\"bc6cbaa786ba2ae0509fdd4e46365d31bc22678b6c0c47d7596fcb4787d60bf8\",\"53074f62e77f4cb7bf6d1c8bdd6dc0eebc9a0c8377e7dc8957d9833eb504aaf2\",\"59a71bcf89f9389297a2704d13d3a4140cb10d0e65deb08c64f6cc3251d04952\",\"3c3baeccd23a4d75ebb60386bee2deae0c1a7dd4fc14a65c7d03bac2a5747543\",\"52c3d636554c23a0e809019ef261c9f27a14ea1e3f50cf075c6028f2a635203a\",\"2a4c30d1e722da8ef373ef91db4f42d41a0da8fe42dcbece8151ea3bbd12878a\",\"2ae761087b45aa536dd5179b57fe4560ec892e8b515315e861c54867d523d552\",\"be9aef0469472bf587440a75e8e0e83358c3dd08df7b7a1b8e5f4ee78f3f02ba\",\"ef44afb1e06d20f304659637d043e32116463ee9fede8f2da6363fe6dbc4720e\",\"553b178298f497b16db0a24229a524cd78808c122f241d9123fa3c478e23bc5f\",\"b932517993023390430f592fa19d023128fa3135e8bc52eb115e8855feac775e\",\"6edaa197607009dd4909d26b88bb9ed463cd7b8a5ee24cdf9891830518bbd007\",\"9418833d7174c5a0c39ad56cde487c7ace951acd343ee4031edbe2c16894116e\",\"025cf5c8a70558eeee1951b34a7735d35c477bd28206bc4b678dd85a12fd4b02\",\"5f08958dfa47a8b88fcff58469c599d3f92090ccd0dc1eb2f1a6aba306a5cf80\",\"d7fe6a0c63229780fa85cfab4480daeef95d921c17148978771136fb01a06356\",\"bc0dd3f38a85d99020bf589587170c313ed275b3479a2c61fff4dd45120c2329\",\"7fdd04c963a7dfdb498fa2309e046310a06f592404a357b09315da216153e443\",\"3eac53691788bb1f7cde78752db2d1b2b35081cc52363b90edb0b2afcbf7947f\",\"aa4c3752724d4c57cdc3f595fd394f9ad0d535a3544f85fa6d783cf8232d90bb\",\"8ba63b3f16425f1ea9a15912aa6e2c8b52f64e028ca4ce5b23d8c8adcb8ea866\",\"10c9e19ec67c88c832ff66a4df23cad06c1c65f2831aa1ed19e097c31cab0b0d\",\"4dbc32542c022bffda5a55f0ffa696b2a89b7dc42e926d55c49fd9c89d038e55\",\"f7e109c98b3629be25c71b29ecd68486a61a857adb82e24cc1f1004fa7c990dc\",\"d95f2a01251a666a07c44f7d350bfb16497147d2359ef30c7c2d06e8177a4376\",\"e235b799aa6218ae5f52853fc614ba0bc765ce9deaf540fa11b4746ebda6f85c\",\"6cde2d6e90a004cbf577df1d90641dbad04892e540e2b9fb1a649aa27f84e2f0\",\"0d4fa5f59e25a35b46a4b4b96ff35285d5c1223c275a0c315d7beb6b1f8b5bc5\",\"dc0b2f0761ad38c8411b3a7179eb0ed1561c489d38cca2c711325db93eaffd83\",\"6fbcb52591a09374488a128377ce8b01d4ac65fc3ef9428c398c16613c0cfb93\",\"73d86d71e475894c6eb3edee2d8f95f9b5cc915f4120680cb708120d96625ec4\",\"4d5be2d3da9968f954947674024a6e12851bb81acdee4f82d8bc71b4ccd13290\",\"f6cc40b3e0905717ba589982c5d394c444ea8c5f15d61e9ddad7cd1f678cabb5\",\"ad5947489357f04fb4c8f6354bfea68de13556de803c9f836a521e02861c7e08\",\"a526af00eb483ce4e7647f32f98232d5f342b52b982d25296edc29ee4cb4caf3\",\"1b687c0bd0f9532985c0b37ef5614970e55df4b1cb9d9bd192e749c42c9a4411\",\"60a2047a92bcd2bbfbf121e1dc512d57a21425fbfc33db9aa1526f789d4ea9bb\",\"cb08245f79124db74da1e97c5f3dc16fa3a5b5aa114c48442cbffd523ad6c6f5\",\"10afddf91f13729cbd1fd5252f261af0f8d0befcb89c0860ca9907c128ce56aa\",\"4ae721b6de91e2fc5892fd101cfcaadc49a1512e7dd82d16e44cc136a41d4a0d\",\"37d887f10e479f37f6b0f4a81da14422c837e2085ee44d40aa8fd42efe69e58e\",\"14d820cb603bedb910177a653d44779ed8bb1b76cba34c4feb72720ea7b5e1cb\",\"73d37c417f07636d8d9cfca7ca16a0d76dae3ddfb1ef8353398f6568fda0b215\",\"6f9bbb688e4b852624ebac913092634bb67ac0ba99e914e603584d13012ca292\",\"79e72e79f49ea1da039741306c8012f1772d74a8bef203375473b71fe9a3b04a\",\"b37cde0e8eb6a0a258b0bc4753831b72542bc478b4a8079db463caa564987bc9\",\"a84217048df09c5f2b77e6bb515fa829725af0ac478eaf925fefdd7206fccfb2\",\"05dc2ddc7d613bb42ba0b9c0855aaeb561be06fe3713f2e8f2f593a46fa09bfd\",\"631378474cd7797df7785fa1bf791a6bf642c6f85d87056215c48a90f5e2fb39\",\"359bc871e10b96fd1ad38e6cfed1ea74aa06237cde12575c094751adb39842e0\",\"7de7fe33aa9f1a09225cd107bb55e646d14f74d18e18ff925743c23776d91336\",\"5c3021ff653882e8a29b1b6380ea44f2f07d20c2a2ca2ba9c88b29008baf45de\",\"fda22a4fe939002af2126e224aad0e7acda384c17bc034b7367c8e2b4168bf74\",\"c204c8e4f26f3832fe6a7ff1b1efbe6ea9e0a10f81e32053374707fcdd57e924\",\"9f4d5b8380d7f32e9f684c5a4c8d3c811b2cf23a031870025d258ee27396d3f3\",\"8952c06f1e2c1f756379253e86289d630fcfbbc21a7ea592530a01191f03c575\",\"b0a75d1d84f3e261b67633bb0aaf00b0d474ab3e04d0cbbfed5ba2af9156fb86\",\"8357017b52e0f4a861e2b3527d26e81bd2d05b75e582203f311e57af0bd7eca7\",\"98dbbac30d3864e6017b19ac4afb4a687a2dfd251c12b452d760cb01f3074dc5\",\"a6acca28a71bee1d088a3c27304ed7b43831e75718fceef4eeb3571f5f3b44fb\",\"39992887d6df02240afea38829cfbc1b132cbb1976f22de7ae09e5066e7b1250\",\"804f3a9942fe15a5d602d43b4e26947a21b1b8cd61a2245b70b80bce0b320c4d\",\"8191c8de4ca22fa8967fb0a0b39a781709cda416de939baed297551f9d143836\",\"9f6bb332cef9dd12d4d0cb4beff3edd47b0b40c53dd14dc395817718c39145ae\",\"4e47228ede663705cc9108928d7caa118d44e38d97952a35a5ca63fb9064c766\",\"2be301508f771d5d3a23060893bca720a5411e158f290feb975a96a91691e7a8\",\"0f4f3f8c33300a141508664ede40b51bd263df90fa95ceba8b97b9a153827cd0\",\"16731d6ab2915d8111e9e4c43a47d50a8e1951a0fa36a543a41b107353fd5dd3\",\"8397a41f610825ef9ed2e454df85cdb1b897cc6d4aae7ce1edd0ecb732fabe28\",\"14c64cc379accad74ea114aa80d5cde2db72bb54fb1245822af3ee79f0d75e9f\",\"875a9b5418ef293794720f101587dc312c1cd9fb3444a30a2e9ad931ca7a427f\",\"e4cc19fddb618cc5e8775004f6aa59a5e1dbe51275d610b4d1ee1d6afc2f0727\",\"85bc8a6861411c6371b39935ee4860a568f9cd6bbefc5bfbd190fcb8c9b8f4e6\",\"116b45a6bc18ca3e28b91e23871c7e7cfdcbfdb20e1b65a5f2611299fff6c61b\",\"cc5177b0ff1d14234324d88eb5a387eb7d21224b5d41cf783c0bda4e42285bd2\",\"138f4530dd3dbe8bd0c1cc3fd08c823176ea7b1c53ce5a0f77d759a1030832b6\",\"618055b76ef1de714fb0ffcffb86cffdf5d3ce6a40c4bd3695ff2b617286bf1f\",\"bbded097376b795b762d61abc7bbc7527e9139435f1027e57fd843924e4c95b9\",\"7c4334902a6e1d523032404b85581153bc532987af0b3bc784cc9e8f2012dde4\",\"bf88d55d6a5139239000707791552c3e11e6cf06cba166cce06c785278acfd88\",\"b80c244c1d59f36a61e654c96c1a5fb7c00efde8ac304fecda3ed46f02213714\",\"1ddd08bc009a1c8e23d52c900600851a7ea9e1c807364949e4cadbdf8c39d40d\",\"846902220026ea32d07c5f600b6dc234d80152ce94d5afc02d7f6faba5ca15d6\",\"453febd52f07baa41170b15dcfe6c48041ea6d6721e43fbd76bcbf27c7d8a477\",\"91634941208b5177e186f2730c6d82baeb463f575e80da6f4957293840e1c949\",\"cab176f9f134e1d8106502d44bca84fece6f5cb3fb48c634dd11bd34ea1ab1af\",\"f2c38676da5d3de930037b50db5eac95178e4a09d506dc258e83fc62ccfa1f84\",\"087c722094019edcbb268ed107318d73161e961a4d80b8eae1700d133314b593\",\"cd91400c509ac146f213963ed1705adb4f0615433aa301b66a3a472e70150bcc\",\"f7b573542def3a9c8549a91a067dc14cecb69ce4f7a7b6ad4182a664f863f19a\",\"b07775e839fb0fe8fa5ed93359571ad29647318776a4e97de40a943fb7447226\",\"f8e06f65783932694499b6862ffec1c008e151d58cf8ec273222227432d8275e\",\"d4baac7c9d716d60d58f37daf6a889ee80cd18d1afa9374275fbc6623af963f8\",\"2380b91383c27ed040a5691fcdcd906d2036723941a4473c40c235dce49c42c2\",\"f91b6c2ba3e88150fe2ed911c695516f9588730aaffb851a85ee38e18c555420\",\"56d5d3ee71f8a01ac0296aad9cf37676d3b6f59d95c4d3bb21d2e0d4b19cf1ce\",\"eea641f20a8eeb02eefdc484ebbc2095bbfed67e3c5c884583b84cc965c36d3c\",\"ea291dae3f4295eb8142e2e322baeb11fec746293809156e584ed15d02c5be2d\",\"2274cd145353bbd7c922f462032e260a48e5ffb9d93a2ed7d3ddadd1ee37f6c5\",\"3e207c5f7b631ac652403e1ff55b4c126ed4d4e4888be6c29fafac7f60cbb196\",\"a3a8b91ae5112ff4c34893333056f1ea76b640195fc5440efdf4392f766cef51\",\"966ee5bf7a30b362f7469f0ace735ec49ccaba3f997586c2fe37d3089d6c32c6\",\"71e228f1b2ee61d44404933ae0fc89fd70bcc8400f982047d77a5247365b1a42\",\"c79bbf5ff309bfce7ea6c5389a9863caa802330419bb6c3608293cc5eaafdc9b\",\"f029f51d51522d19d52fbcb0ebea18b2f16c6baf7fb34078cd602d0dd30a23b0\",\"3af0dd60ce25a23ebaebf63d057f8be0d5bd0c72c2b1fe297982cebc8c09728c\",\"8c20256fb714dd312417f6deee71dfd413c21d11dba35a0f250d0ffdff0cd5db\",\"6361a2e5b8793460d17bf4c96d52062810cfc807e20ef78c286c019a85ac4786\",\"51ea171f0ac2b9d723b5c187d513a3f997dcb3f2b6814b1156baa4217a86afe9\",\"4bb12fc783ea33b0585b046e2abcf2f2d03f0fa5b877065af6fc8fa5f199ce0e\",\"1eec9a6024def50c4f5c128a93dc694d430019a924f15a52189c84f5bb7e4c62\",\"0a87b513939efca46c565078c99d27512a49dcb5efad731416a135dea108bdce\",\"e8cd614b3efcd225fe5b221b84be7eab8d80970da52e7cba2c648ec824a3331f\",\"5b2fa0354285e18de9b223a57de5401b1814a6e3e87e4972d38d10ca17618233\",\"adcf0aa00726a0f9869b547b8c6b214af096e331f99848a34a5c368a63ba456a\",\"0eae4f33dca169233a8abb4e438f13f8329caac3c74f6c07343d81b917588f8c\",\"8c8fef6cdc60f348c20dd5be4c709ed7a04d1288a0d013dcbe7a81fb29e6025b\",\"16ca4c55f789bf8dece595655a6ddf529e67ec249f9e2c8b3e3da5dd395e6ca0\",\"85f9442ee59f986f92bca174b8f92ebd019fe8efd7085fa43f94b39427c890f6\",\"b3fd8c6063fe6141d58729f50e67da2c8116dcdd071142885b0012254cca890d\",\"9378107190670c4749a4411770694665e89199792fd17dd0dd59786e6ff23c61\",\"22340006d76873b8b13dda2275059df594369a43bce4e01eee5b0734babf5777\",\"b50dc3852c46c73573e8a0afd6d4f99bd2254b74d37a4817f4db87954e852d8d\",\"f076210bd3072288e40201d87e163a2a8497d8c7adde6fa3c55d5c6a768bf896\",\"4895738c37bf6244f297a7ff7522325e3bc509ccafcf041b2336c418a0f207b0\",\"ee86bc177d0e04e9f70f74ce6e3277eddd028159c6ea8b517b27ce89fdeaa498\",\"7a926dfd5be1578d970e08ab80d8d0275b6e806659d2738605b6fcd99f81d183\",\"28ee46ab900a794963fb58162287b65139b932839851faab155ec3ab1cbf13a5\",\"30963841d910e489932a39362b3ef327bd7712cc6f8a95918822de1e7e55194d\",\"9fda96b3a7f28198e23d253a2ff926998cd1b28f52ed99a8dea2f9037fc5d596\",\"14c53579a78486dca3294231cc5ce13a2b922bea8575d984f76d731f283222e5\",\"d991f6bbb07295c775a5a0d5ca64542f64511a62183d9d53e7d06fb35b73275a\",\"798e2fc8b24d605a98a2a1051072693e2bda9986398a7dfa65e93895117200b9\",\"5fef3f804098372f7f6ffd9044805dea2b7b3ecd731252e51fb7f68aecf15fe7\",\"0dbabd4c0810b4bbe51543f389d8a20c062c9c1db6e8f9bff5bf3bfca6748822\",\"c17a5062472c5121a3904061650baf7a52274c272ad44dad4205156b5bf89c84\",\"da53c7e7d0b866d32ff109b1392a54018a6ef59c781c3a04f4355068594b8cd0\",\"850202dbe99234f69f4c0ad480fcc661f5a560e7047caa5f780d5211acfaa33e\",\"0ad6093682534d28f6b0fd9a1ed044a33685641f887f7cc75906a84ad2dcc082\",\"91782d11fc87df531f4ecef45bfeccd6027eb5d71f2e1153b50e7480143dca68\",\"10ba58f0f7b3687968d220edc6b444b666ee8e9c8412b99c368c33bd3b0205e4\",\"7e41e7312bc8bac653354870a54a94a843d4d44fe52135e23951821e850ae2ac\",\"d30e9e76988f335e4bae53ce939cdaf415da3cf209495c7b6e191dd227a2cf1a\",\"18d2320361d8e4c9507cf45ba30b356c84d491dd667f7c99499d08b61457e6da\",\"4ebd856a7e35ca8a076a0bcc78f814ddade3bef384b68527fb826b17dfb2d773\",\"d73bac8723c91f70c822169cbc5077359c9aa7e8431ad112e83b48083f4dd15b\",\"623a9b0366c04762c14aa693a5a452c3f6376fa1884ff5350cb62f708030e566\",\"4050675a01089ed8aaab54c8dedc474cce0b510ab55ce86fa3e1c67ec2f3f9e5\",\"3e8a4ad4314b13303fa068bd941c250b0f757eee9c256aae540d4ef5187484e7\",\"72dc83ad5201bf46707e7a94ff70da729332bb3bcbfe1a992f67841f7500c340\",\"3ebc0d87d4dda5a1db8673f225bebc544fade618434e7298c779bff80ed9f26e\",\"286612e828f51a78fea145379ecdc7a5c1982a10db26904c2543e2a961f38bf4\",\"37d55f012b82b4dd1ca8eae5814da42b6109471a7954b0da2bdc15416209de9d\",\"8190fb29f77b0b25b9ba7f85b00f3f39754f1b0855eeed08f91ccf027b3fadc7\",\"86be8febf9d4568d9282cb6bfd5d1f6843c0fed29b280f43ce4c81ca8dd501c0\",\"1f4f2df786c1c2a78811e3dab80ffe7cf46b7bd9b86561ad3e834ae3a924c61e\",\"775eca87fdd3f2f73664a6120a1c389302f4f615febe7129b9a4b1b88951efb4\",\"915f93d09dc48a1af0c9f59bd3ad177361942eb8de23310d65c615d1d40e0a91\",\"e0c281417d161b37727f112d51ce3ac9c9a36701b9a9ddbddceb5fd90b8dbc45\",\"420f268384bf9080e787f5ce6efbd7e83ca4b87e37370fb436bd89b17d4a5694\",\"b6524d453a55e1f2d444a26367cecf4eb70d2376d3400a2fda8329248a564ae0\",\"aa8cd3d59327d07e47c434f14d4a7073f2823e42044dc7cbffa2c6dd8f875273\",\"a4dd3cb4fb7e0090fca39ca59f62c17569ac3a57d3ad14694a996459548372c1\",\"1d2eef91e40e28e92eedc6d2d152f73b1bdfe3e521d42f5e84eeb0f437f63039\",\"5c88a9993e7fd74b0a776ea45cdec080aa8930dd8ac0c7b794067d3b662bb3dd\",\"560980322c8e5bf963a7a0cb1da00493eccc2a9a7a1d2933ef11064660e12598\",\"89af6a51eca1eb373190d00b0a792a2f50bc8a35b083d02066f1e355b34eaf4c\",\"0d0d4a8c87f7b4419e28c5b288ac437fcd2fa4a0f6b3a061fc47e90587f37e3f\",\"b1b3cce9940802f5ea074dfde824a9de81415d20bd77a0b69bca826393d157a1\",\"fa739f298163e05ce22b33c207c4cf121278fbbd04d3b627c9ea25bae9ad0ed7\",\"94467e06ab71f9e16249888fd63380a549334c9f1f44a3ca2b6a35cc13949c76\",\"7c3ef8e6431d3ec1504f5f0258c63af74b52db0f1313a455b3b5332920cae039\",\"6deb4d858517a7183f963cd3bdbfbdb313d5bffb64c828be5394652847694750\",\"955a4d3bf253c05125cf4d532a2a7aa2863a452421ef4ed996021e51f3c2384d\",\"07c452c4e70422c484a7cdbb4de9e394dbe3d00e0a9465051c7fcca2e345846b\",\"f50125bc9255ac105abef30131e229c885a9c97294d2c5ab069e4b54a41a395f\",\"eef43117ab3855b783efed9c252c08fb0339ed538602b61d91db4c3d718b6a49\",\"6a5cd8bfa7cabf06e04050dea3baaaf89409119543d22d1393de713c8f4b9035\",\"33545aa0d03520b889f56a4d47cb43dd7736bf59e8fb800e5c940e8cfa6dcfbb\",\"c5eed492b5d3af512832ad240d8e4c60117b5edd0fac4213d556e03a404f7fe1\",\"36ff121206a853a25109ffe12fdc83b64ac6a8ee6b42b0f32ffade910d8df169\",\"790d376ecde940f863c1f0593dce72e83e751d5915bca3493f33445b49db0724\",\"7d7c7ca6bf3cb7013392c19e564807517dc31f1193f1316f49af985a2e6a9cb1\",\"e42069ecd10c7f04bd650dd680c21f222befec4e0b5c1daad217184def94b734\",\"3867d199258ddad8663ddc155a71f3d967f6ca3e96ad8014e29fd039ec614468\",\"2613f61b98a3d2686c01fb0c77b82735760ee33c530dd90cf72968a1449674d2\",\"476b7ac6d1d23e94a5b0a33509a72be71780a03754d29ec4683bc46946c3e7e3\",\"710bd1aaa50fea492a79fcbd70abbaf5a7ab7776078ddb8159eabed9af3e39d3\",\"463c20836cc531a2d1a1531b89dff4dfd1d35e03e6c4b1112203198297ecb4ca\",\"b2d034c788cef95272bd0fe0037087353b1bcf1894911e2c4205ea3b0e9d5378\",\"eb527cba8207d7639e3fb886ba3c5080665cbceb2c7aa5f18dca2853882a33b1\",\"8165aade9e796c29250615ba8233aaa2dd36567c1e0c91c1f8472ac9531d3f7a\",\"053d791777cffab693bb0561c89c590eaa4e0b409fb7168b54c237fcfbb609d5\",\"cdec9793a4a11070545505bb792942ab66045ec37e1559c2ef275d2b589ec5d4\",\"bf603a5114e7b39107cc308c7ccc9334d6af8d737adf4dc161ff7ba4ceffec08\",\"9f3a6a5e6a8db037f6114cfd1cb169c6e64c22a7975c8af73ce77bc58941233a\",\"732a859de01a09681ef7d7ab53748c7d222e49b992edc42c11f6ddc9075296b6\",\"4dfe0dfed22e58329b3ed58188c36dcaf000dc74dc714f689abc2af400eefc54\",\"ad002180e7c13fd0034bc5594080cca49388d5bc605f39a694fb473af3b416ef\",\"cc8774242d057a359af034dc49631bbfa484ea2905be456707e9a02b7d6a8576\",\"ff20892c6ee93535310fe04eb7d28d5c58606fe462acf660d8780aa63fcd9000\",\"cc7711b241121f2486deca3a077a1371aec6aa091066f40a79e5ba92fb386b87\",\"be93d0993823c49c09db4ec90304f47171e5935de6cf9085c7688329d50d4dd8\",\"a12d1a02a5a97a906d346caf487857644dca1dd55edac53d8f64f97ecf318f76\",\"b70ac135c32ba845dc901ec89a2b0b344e05eba0fa625f367f7786dd44eec67a\",\"3809d592a4251673bd398324587c2f09a3a972c4afcc6fad28476a2b700de80b\",\"421edc181121e0162dd7975a5329745a4e0e448337863b0515b09119c6929015\",\"22570c6b45bb8b6949aa45b6ae7bfee793ca43e61177df2a283d4d8f5c509d0b\",\"3c50dd245d3fb59b33c357a27a0ae7744f6f2b4b8132b1ba8ce8c28fbd426d75\",\"78182d77528a9c494758a677e395b4701c00d24f05cdc0d782e3843283c18724\",\"2ab9f8651159a2e6e593c15075b444b2ba4629a80733e754202833792a8de073\",\"44f825b3f65e23671b253953110d66084fdb00ec5e1e4695628aa63aedc0bd4d\",\"a34a67213e3ebe24e88472929c5e4118067b5af5e410a087f4781ee6834e11f6\",\"ca85d235308f8d09adad1b71bfefa7a27827e84d7a7007063523fd5e34ee3e79\",\"1ac018269410ec61d616ff75382138bd39d8eba344da46cb75d4704fcc320905\",\"4375d0ad02537c6e1a5adaaa17043203e42578c82c2bb2fa5cea5b80054d95e6\",\"6fdbaef658b0379ac567777d15384091266d1a652133e11b2ebc6659bc028ef2\",\"0517c3d64f5571067fa6d891e64f3c6c849c19956f2453f3a5fdd493139351a7\",\"6252f8d7d563034a03cecca5a5a6e9fb424914856b5b1b383689900b9b480657\",\"44e91eb7def714f00c86a696aee9999badf233919ed764009ea331f796e66894\",\"7d337ceae9669fc5cc6748f92dd9f17b99b7716350ce75e459b78ed677507b57\",\"1911be060612f3e0557b16e84d0bb5244a74c8abbc3de26ad89495ed24e40f47\",\"a5c8a57bfd29cbbd84090ccf13d1b312cddc3186ab322834a198275eb74c671a\",\"b32d3f9968b634de80173f8329e156bc6d7835202fb434ced268c81f89135728\",\"fa86ea7df0e57bf7da2f380a578b16d7126e781f68e228da22caf109aeb5d696\",\"42cb94cd0f43e1286befa13b5dff07a055af25d2b8c0ca373b5f589b28e8bc43\",\"364970578e929a8e1bad45286fdf5d4240e75e64845a39ed3d6fcf8b05316582\",\"4a8209d4b11112a7232df1d77ceb7c837fc0b6c2a8eda921f7c617a09eb79a39\",\"1a142be0917e5afc9c99eefa15f0eb73f6e2b0acb460ebff21d99e6f223376ac\",\"701e675409df184207d74b38261ccb0c62d28427ce7fa70274ce8e373f3de3fc\",\"a549c9dcfce8c6441d6c2ad6263d1a66572cab367f5abdd2810e5779ca1db57d\",\"1e2235ae2639dc4e6ab347946003de064f6d7dfad2c98b8fcb9b4bcfd1903482\",\"4fd9c9398fdede0598c70dd7b84970b2f2d52dd357dbf872d7c18d422621753d\",\"4270d2794554bbe9fc07b7756c1492982d3adc0c22efa3d33d42545828cb0e10\",\"efa44c7e10fcd4c3034bb3fe74421d1765d6e6607d46695b7369743bf5fdaeb6\",\"570f22bca017551ff45d577f1ebaf4bc2f2b2261fcda3f0a080ffa67356a25d8\",\"9bd50e8abc84315b1aa5d6e58308a1d36efcc153ab6d23a5a13171b8c244d3bc\",\"533a745267a33b39f55afc788e0030732350dd5a98c85ad1b706d6d9ae306690\",\"77dd8be0883e74356a766340aae1ac3da06cbc2bb45b88104c0793d836c1ba85\",\"85460e4432095e0157f4ab5acdc46795a1faa1764799d024dcfb3dab1f812584\",\"5e76146133977bff29bf5797abf981f9885b7d7583aac7a273852c0f435f64bc\",\"c1215f703bc702212eba308bf0125384df7bb5d7b83c569979e59e5f20ba41ce\",\"d5ef65272912bbe844a714a9d5db7c388dd0c503b2227490fa3b38968778b139\",\"0d2c064ae82ff6fc685b5808094b88d62aa1a238114527ac7a28d6f9b9083625\",\"1980beb82a9f5ae994a22768b4863dfc5c109ebb4981410b72725144ff9a65e0\",\"fdae8d7858325966a3c1eb1cdc2adeac04377f363567729970d1b7d8c1b02835\",\"401d5c3f85f4ae193482cdefaa5d193d0faf7ef8365a9767e5de980a69aa69a3\",\"3e564075a06096988ea9aa166c4b602732d456bf7260696a98807c54dfd875f9\",\"7874e5a184c3413f63cc5d0168546697287144a2f207b269855067ca7b0db994\",\"eb17737a357ac1e042bd06fc07840378e7ef1961e7a843200c3014445bb1c98e\",\"5f48b719115221702a5ba367d38697b27f6ab8c9a8fa05f87df1e00d84452034\",\"910fdb011b89fdf7274d3c7033e80e74360053e7773771e13158fcf67cffe00e\",\"2c55c2681a315d18a4920f87ba46db1bc94c3cb6a2901a14c3ec0dd0e13b1639\",\"5488db367a3fe40555bfcab398fe3a58b170996535c2ff0a2f92b93e6f48067d\",\"cf2abd6eaf35741d13aa34bf06d0dc811bc341bb9074df282ae1bfb53a838838\",\"37b18f5ed8f796a685a4fb703d11d2d87f8a262d1d37c420fb2b731e744bc517\",\"03b7f765ab57abb54843a70ee4c7b7e52777931c82f6193fa6381eae70239277\",\"131cdd75dc091ef67a630ead9715296b7c92231b46a9f03b5ff59d48ec0b9707\",\"fbf0b3aef5f8be1801048f72bb643ac2754738b8b2e628ba2c8c156fec734885\",\"e5c5fbe0e6827685d43744dcff56653ff882a5c97797d21a4aeff24407a9ec72\",\"3e683c4fd3af12be892bedaad2a29f11714ec31b32216be067cd047f32a44a9c\",\"79cbfafb198d661b74f2dfe967711673d4cffe78be3752c43871d93c51fe7ad4\",\"b0c07f176b5a01292e527d19868cb8fe9033496c20c48c05f55c72dfda28efb9\",\"35ab2c491cabf43cc49d8e61dd75aeac840613916ac8a477399122d05837d653\",\"d49d0659a9f0b8c664d82d89dbb60fb18be40e56dcffbb71b0aa9f2d23345cd8\",\"1a847bd142e0290ce3a935b2cc2eeee512401ba52bf46e9d3d08abc29547e930\",\"7080b0ec812a6e4200ee92e5331801e080a38655cca4dbec58a5ef296faf32ea\",\"6cd217707df795772b4f598569e967076788c33dd8f45aa6c492e9b79c4b5907\",\"df1d39578b29b5a48e5f7e430625a164fe28afe99512a3b9d3aeed5f71f78377\",\"47e6bdd06262a3821e0fb0bc40a3d7ba3490917c3ff0d847634832f08c58020e\",\"dba1ea9c9661dfed406e25d96d067618926249bd41694e5a99bce977dec278e2\",\"fb5b42c3f2582dfb8289561229b3f2b02f3196ea33fe20824fe1e9cc1f5cb4c4\",\"a1c57b773381cde00beb1f60c4ac2fb7f1442f1240d41ba697686a512dddbeb0\",\"1d426971db2c732d13986f1ddb5ff2ba3b0aba95d856e5cd3476e81cb9c2793b\",\"680cc8a2cff912bb7a359eefb6cffbd4addf6feecf07649190bfbe487582feae\",\"6909789eef8bfa4b39bd2c8a8bcec3ecc3cf8b89785f5f994794a4e6bb3ea329\",\"ed36529673bafe45c932393e5102b99c09af1a4bd895b0097b4b6422288b87c5\",\"6220780d08700eb8c47f425f88669fe36d0fa523fb3c76f08617ee0ccfd74957\",\"6f90c4bfe3bbf2d9c766f135f43aa83da42f9977497b0e1b9ce35becf5a734c0\",\"b47aa4e37c8bfc2edfa0e67c2f5e9bc3dbfb3b641482fbe7a6850fd42f389a5e\",\"c3bf22784226711e7eb3675f9967490c94b8d3f3c8e56bfea7501acfeb46ff66\",\"aaaccbcb00957a13f718495b7e8da066d750c4756ae68f3d3e125484f77b33ea\",\"d35db7ff0a6a4ec426b56f5e4082f155d23d0868d6f682580b22203aea0c5d50\",\"b0f3eae334f591a1beb6872cef74eb5744e1406f21286f2489f43eeb099ffaba\",\"7cf319c42872575b3231b921d2034fd4e34b8069dd53002f2c504d2e61ee03eb\",\"8b5febe169bba9e99a0c7fd5fa9399ecae0ef3e9ad377787ece4450fa8841271\",\"e181c44813cd2e240ccef1a4f0e01bad2ad759c9cd995752c11c8af5dbeabc04\",\"738767f301f0489df499e1d96ce7f06824ab59240182d61439e243962fa88f07\",\"e026294e225ed5ab686821b8bec0657845ceb38bf4086256d2bedf78cf6e20d4\",\"251df9bf603c91ef29a622bf16f9a0e7cc6b7869c566b5dd6bfcc77ffbed6075\",\"e0c38f7e45582cda1c380d7cbab2313d4d396e1caa4510ff46c58348cea056f8\",\"c2613a74ead69ddc8244e29c83e4c86575067ad90c92ceec825e57b1d6fc9ae1\",\"8c352ebdbe6df41a0aed70017394f93640d39e270737acd27d2c041d67deb835\",\"e8a85db24d2ee36a35de5a528116926e91d227ef3f8a217a44ac17a5e28fa301\",\"02dc802ad5088236c35eff3b263341319467e688d855d00aa3629e2104fdefa4\",\"a577132ca32e13b2d900b03efd1d9f00c92b032828fdd345a9430c57e786d607\",\"40d022a29121ded9b09f3451f0bfd20b821584d4dcec11eebdecbb3dbbbd54dc\",\"35d940360f941ba881277156037a14de0dd2c07f5a589404a080392536e9e7a9\",\"455a98ff3659f8dc4e0d7550bc3aa4359fec42230b00eba32d4170b95ed5b6e6\",\"ada5fd63d43301a26f35a2ef27d808d914ecee2b7b95c8b96a59ad35bda0c883\",\"e4df6474970f81bc0e8cfcf5f622408cc68b4aecec993f3e4218569fcecd2687\",\"73075e8cc191bb7a1ebd133c6fdf0219be3b67a7fb7ae827e8af9f5f9fc434da\",\"3f3e41892389bb4ac5aa394c2ec1d9cbd6a3e7fe627f9711a0f7105220e03c56\",\"68913889967db4ebe035fbe8d2fcdb9917932444532ea55af99e5a410b95308f\",\"09d0dc1fbd252c35d44622f606ad1dc608f17bb673f9d9882f66055776d37d61\",\"04e6a5bd69022dc74805683035c7bd1111dd362e0d97f2b3f6e75821e64ca897\",\"ead81595aa417e6e130ba34aea9d6c7214082175fdca852becb89768fe0cb46d\",\"886d4ffda1e3ecc5c5f6f100b0ce5abf14d6d9ecbd87417f3d70dea22412c55f\",\"4a66fcd383b76721bd070639a8eb850dddddef46745a3b0754cbbb94c47187ea\",\"97a3139879e3f4c9d52c48c481ec9ebb925d4f20cec983e21cd6672f33ec8dbd\",\"f36036bfa33aef2b51e9647105c971a7455f349b4279e841fb836d5ffcf878d8\",\"5b90e686a840759bff6ae416aa8e6cceef66b631ca03f5d38d064e90da1fa17f\",\"81cabcc05d6e4cd73e5641f0a6f6399e9c0a20ebda2bfecaf69f762d258e2041\",\"f5baf4dd8c278055e6a5187bd26506c642d67c546199e8e5b1a457c90ffc9164\",\"38ab7758b858867f290cdba40e93992f0fdd0f9c2896b90cc4a1616fe1f8a469\",\"65b8ee3b2c611650cd201cb2a305342bde9471c43defd452a4801a35cfbfa683\",\"1dee843debab64d97deb0e913e4e110a071b0ac28856e8f1cc1f513791e743ae\",\"872c968a8550826e13ce389b19afc5551892088cee52a618a15733ab57003e30\",\"ea19ccd49da9f14bde7275b0ecab6e068ed1f5eb4e27b048a42e5ccb13b15d20\",\"385723f1cebe8f31506959b85bc05846873c8ab98f729beb74a295feefe12244\",\"ea66ba6c233cb1a115d3aee33e6c40576ef00fc1c86aae9496818fb8d3d50f6e\",\"b7814982f77484dafa4672451f2033ebf74c7f339272403ba50414b4bf88f7d1\",\"9eca6e441349570e117fea738b19619cc8f84828e0319bfbd0af9d8e139aae4e\",\"ba05a2a85ccabf8224193bb5bded98dc64d4b7c34f95a2cf8e0e152e09c5674d\",\"47b440d9b12a81243bd052e2fd3efd465743a037cf448bad65435d5554f971c2\",\"84d0456d143ba5df9bdc1cdfdb7b4ebb3a6fa600c21e3baf8acbfee0a70c34cb\",\"a651068602cbc75605312625c3ef19f488211af35793197180c5983089d0779b\",\"050a08dc1452f61cf822b81c3a8ad8beb067bf3e12a9c1a51f62526a53a431a3\",\"855d90009e4c7044ef59a1d76735b24acc837de55aec5f3cec4e88c84d1674d5\",\"d01dec54b61c7e58e155af28c798eb6e73aafc07fe2634e36f66afb4aec66859\",\"80c7ef0c3794de202eaf749177ba9ed8d293d70e3827020d293e51ddb5811ad2\",\"1a5d724de2f21a421ac55c552247ee9dd748fe0b8ae5df7ab48842a07f3a80fd\",\"0f86d144f128faf64701c3e775c7b42904d7c4153abcd1e0f6f51a77c8186a1e\",\"41ff0e124b85217bb6ffd160426d3a2de7b8fd2ea7ddb9d3e18b647d3696bf85\",\"07bfe9fb63799c3df2023f7400e00d7fcac644ba413f920810de1b25a4b62505\",\"d0187c99cc3b45a55aedf06e89511c438e6e41e04232b987888474fd0ee6be6f\",\"e1dcc007885287cb7a55ca783fdf0b050c80c2cf55eae7166e2718c60cf48728\",\"0229524878c17435735008abe505433919258b7f0fc1f9bc0e960ba1ee7b9695\",\"5a8b0b7d15908026e3cf43a130817d8d8c32b3cf726ab8bdf58f6a0c4c841818\",\"cd2b0debae75051f6a17c6d144ff3f85faa4b036fa57e905dcd50c65aeee6847\",\"4d8870ae089c57af5a79b565b41fcc1f4117c0cc5d19eecb6ebb2013e41f7699\",\"ac1f41967573aba71a6fc20489e92c2f4a3235f571a4e19bc0b1eaaa8eac2390\",\"7e40cf163872c8bb3c1a618df5274a484e24cc9b52bdaf912485cfe132f5d2b2\",\"bd7f6f14dcd4ee964026c95e01a7dcbd73ec23722a2d9a4417f59ba90b318f20\",\"9894cc71ac976603706bded406d3cc403e6673c57d967fb905f2d29fd1757c8d\",\"da3cdb9cf9fab92614bcf4a821fa270567ffc57f756ae97cb8a1cce4db975843\",\"3fad0c14aa1177180859fee13375faf14384cd404d8c015a7f8bf9a71173e79f\",\"3986d615d71b2cc8b7f115bc67988bdb582e2eeb5cdb829e11df81992b713095\",\"4314b2da1db368ed3beb2549853c620691f5af91325fc2aa63a1b7b6c936cacd\",\"cac1e8557477b663c220b4d363e57201d78222dfdefb592fb715e1a6d25ee966\",\"383cfcef0a993eb6ae6ce28ff5225bfb90695c7d13f805a7b5916fa0b3371c3f\",\"d09b5c9254ffda9a559ef2bbdbc931a10f715ad3929067ee7f851f7975f27d65\",\"6fbc0b7fb04d0690998cf90e1b479d5430672c079f5c49f9d02e60a8b599b394\",\"55f65e7f6ef60e01630246249887a17a61572e17446f21d5c62cc8a827174f7f\",\"95cb93667bd89ef142f9f3549908e1d74094b9b626400990d6442919518804f7\",\"ac78cb8855736197c72da307421e75cf0b7768e933c1d90919efe53fb879ade8\",\"bc217861543bb4022aad3b0f3c5c3ecdd8dfc45356261ccb3d90bdfa36ac96c7\",\"49370d4fa5869c81ef327cdb854c3cd2bbc52d899ed42da3c4b5c1ad78aeb315\",\"5cf8fb880f83ba9fba0fd5d0fc7a6beb0f57b95b3ccfeef05d8187b1c3735531\",\"efe5a4da04e8d777ce11280f360fbb7877fb4fd7aa8b463f62627df00059c569\",\"a0a8cfbaea9c7506da59d43659bdfc214fa9c35cfae528775f7446cf76d0e873\",\"c42a4dda6e4b662250d1e69d56d1b5b665cecddde197ae20357ec095d95fbce7\",\"1b35693b6eb409ddde4339e80aff07257bc3a1c182c9ae9921d4d3ad5d55bc9a\",\"a37f4ba565748dc6636209e4ad5eb8ea55e0e0f7b8fa0f9355a5c76dc05b1d15\",\"7495a950b6331853ec67433dfa123806d5eb421f69bda41fadd964520bac58c5\",\"566cb96b978f3bfdff36a42de707700f93ff0e0b4c03f56b70ba71e9e32cbbd5\",\"5ed746aa45ad919034226e9c264b3eba51370f7174c8c7bd9d4a94ad276c8e8a\",\"dc3a2a58d954fa2734b59048ebe7fa45d3edf53c6038342dab6f67f4aed2eed2\",\"760479b9bf40fb99f9262ff79f4f8b39702f4862de4ba2d67d0097f979fe329d\",\"07cfa0d0174026b12da99c0999185ed50d6128054c2e192cef3d8d044d77d2d8\",\"0193a10a67fee5ede1c836240966c4badbb4892168a028e6491d9a83b762c1b3\",\"7a80d215747912cae76ec8ad4ae1b8c153c934ec4f1168e1fa36a00fc98dd4c1\",\"e48c372abaf4ae5f1ba5fbc65c5a0e997bc153f36c733b432fe76f0d9ebe5388\",\"18be1ba96d4cbbadae4128c09361a4b0cf81fa32d1280ba449e12aa8227517c5\",\"da5c3b84f8d02f0b46aa2a80de6b79f117ca5d73a3ba6695f6e2f270c04a676b\",\"48d89c96b10dd155ee6585c54b4b671429d19f0293fe7fc9783d51e05cf5257d\",\"352847b9c25f496b9c00306b207cb294b89e5cf1bfa941749af457d1872eae05\",\"88663d151ff6df99d93b04236ae5a05b44de1d97212a1019a151986602702001\",\"6f2290f8cd8d7fa46adbe53abd8f0edbfe8b9ea1d3c76366de223a3915f49ff9\",\"f71b0a8f70d5bae60868e3aba6cf42f73e0440327cc3c2e50b5fbc7e2cf14469\",\"1ed6d62f1bcaed4c5d5cc7597aa3de9f6a25a6fff9e06d3c40957708606790fb\",\"13a8585708292a5f9ab08fbeb6a0666ea69ad9f24fbe9c7ae4a89274e16493e5\",\"fa3e16ebb0ced825b1a0fa98e591460984c75ba26cdb250f654a8c75047b71d4\",\"a3ee9a34447e83477668b953c24e7064242813afbf48c8b4a2931412f7556bd6\",\"6840a5bfba50a94d3c2e7e69527839df1542753c964d2bb6bfbb18202330a1b2\",\"af5e06ef2bf7eb6c4903d3af928dba92897917f6df3ba9c726aa24986f17543a\",\"17b3a8e0a326da3b902f74271072c33f8e4efddb9e06e1f5aa70e8e4edaeb1dd\",\"ec6c6cb89c8076f56ad4875b497c500a2312b0a1573473c5dbec4d334a83f922\",\"00323ebeba111a307e91fa1207357bdbcb054d9ba73f29efc1015782844e7bd0\",\"39621c560cc6ccc6bd6357d8ff831297b96adbb420d98412f75068de7c340a47\",\"76444bab3ffd62fb4b6970203ff664bcad7e3c6fad29b270a6fa42c53392b7e3\",\"a76e3c450ff6160061b7c7ffc6bec6858b1d53a8cbf00e23e81eec20a990fec3\",\"d19d70daf707ada07ac86a9ebd292c7211fc908c758a54e4dab8a4ef7b8e3e79\",\"73f5b7cd8a413f79f0cd6dcf4fc134056eebe50305a0ac86913d7ba96b59a2da\",\"aefdc3ad0847b443028d19ff7bce6718b6b02e0dbbbdd332a2bdc7b96ae81aa9\",\"350e6c6abcca891a70d03dec7260a57904fbe1486b244e0cd358453090485188\",\"0ab0442a7893386f2d25a4746c75876a6eb61ca239f23d4ae2105f01e330929d\",\"d09f4344545c0848ad6c903322237e5b19ee7065f124d6085e6e90680560f520\",\"083a0a22188e529c79b5ab834ed3a45126bfdc7738d4ae6d3aa533e39b5a6561\",\"1fcd2684232c17c52da121d0dfafa08b378cf76531c9dcb9659b6434aec8813b\",\"5e1996503eec27d408534eca034278240ff61577f9561b283ff9b95b36a31040\",\"c9ec91270d9f0220b851b19bc942f5917fd0acdf6417fbd9b206a3ab65fcd444\",\"070b6b74f8721a116ba0ab81ec28bf7a85f8bbf57f2bda134e0327af5e6698d3\",\"3d95c40a9441acc7d1fc3fd7194de6a95c3317f47c6afff3af5763cbb743555b\",\"2db502b227335819ea01ab42575cef9053b6e279a176e6945f2e7ddb61e67a37\",\"8446a4c74569a9223b9c7ef334f3ca6c0db263a25c61673bfaecfe477f33026d\",\"57f6e2b1c2d3e9621f4ffae6a80775e8ff039d4c7ca3968c23c52e58a87a013a\",\"33c06347a1de0dfa3ec40ad4b7bce213bdcf7bb8d0ad928a8669c51323f8dc55\",\"4e194f7ff641c1e7d114a18c1e601c4216d8758accc3394dec3f39917e2fbd7a\",\"61404a948a4f5690b2607ba1b45ba1772c6737743c62ee861def369fadbfd69d\",\"9bf1133e8da8d46b864bcaa074cfa9b74a6ed48edd590b3c2242f16dcff3fc23\",\"0130a0595cd0d79135dcb7af66df58a1990104b32c079716b9d56b0e38ae28ad\",\"549c30597b4f8439b4c75038f884efcc52ea132f70f07f7172575cca7f565016\",\"b1f156731c26142b7f0951085cab4ff5c70ed494d9e8f76b774f1b968eb5c094\",\"26374ec80ce535827ca31fa52f0ba76e808ea5ae14f309e4e88dbf842159aecd\",\"1f5f99141f6cde44ee37a17f1fa005e33b00130be4487a47c26be5f50ec38780\",\"0dafa76a4aa474295008b247bdb64ae2d80bf56a5565cf6ae7eba24b5d62159c\",\"a7ea03dfb213ea25dba625cf96ea66d55f68a5eb4feddee0276863638c992a04\",\"360107a173c86e8c7db58e99bc510013d731dac51962874fd7118eef1a4a3631\",\"cde063d5007944f9714132067ab1ef4b3f6ac07d9a2953101da3ff79b3fcf785\",\"b2213505e3db21311200c60da697755af46bea0db99da668e9ebf1f579b51a5a\",\"4a0874586d78d9d17f827d49eeef98f0dde9ce9c07083bb51a485ae06d724dfd\",\"7eb9fba3f1940a5dcd9ed92bb8c8cc222131f142aa2465eefdc9ba26c6f6630e\",\"7b34c48167a6d68ae21d6245269ae26741e95c3bf4b979dc5ac0c205288475b9\",\"3328c001fa1427fb21fe1bf303031f8add1852f6b9b006fb241a725325f01a4a\",\"5dfa2472dda825f6ee24fae5a3cd5cfd83d27e18a86c2b4f4019a38b0c6eda38\",\"00fd3f81dd0aac00ce71b743a996bb4c0070002dffbf8dfcc346d1fe624549e7\",\"cfd6467c4a001bd63acd578c242e09b93d254fb5891a38e74c9e6e0dbabebf64\",\"d74ec0500cff5b36d27cf1ae556553be6715dd5fff9509a00018b49557ccbce5\",\"5f23333aee78c4b9a4395b5772df33ebe2fc623ca12155f23976c9cbf96d09b2\",\"3c79e592d3bb1828a9e9638d24923134c3570a956dfa527155e92946b3cb1b46\",\"d4ec6cfa719a848c61694c926d0ecd9cb8cb6b69772078f69f692b40ac576e4f\",\"06aff297a334d7f07568a468da486d07780184f0f7054eb6f237134842beb332\",\"447a0aac6f482ab88bcc6415b378048d4436a383fcb8de886ce319e0d9708595\",\"70a7758725888e5360e8092c2e8ced72bac483a2879a4f803c7c578947b15a66\",\"711226a34144b8b16de343974fc83fd43bc6a3112a6b818afafd829c282e00a1\",\"e04ca28b20d5a291b1801180c27a52ab39aa2f1d8d600c25d6c502a778d9aa22\",\"319d04659cd9be13a76314c5f32441770c08baad935f253947d3049cedf8f5bb\",\"9f5cd71cdb650eb10b8ff279a8e20031024497a5f3d692fb4d33add4ea20eefc\",\"2eda0d79c15c6b4fcbdb226de6f5e75191922e4ac86ad588471f13113efb6c07\",\"002e0d6ebbc79c06199c3bde7a79ea27d3c30706174517dde07619d504bf4075\",\"2e042dba3d6136fda3a6857b3c0a78b78d45f8ae3f966044ed94fa1209d1f870\",\"918b5d9bdee715c24ea685fcf4398c8831aac4240ab10704711b3ceefb2e462f\",\"335035a71b132d79293bac8196a1f00bff7ee68fc3682fd3e25660ed861363b9\",\"5db4d23601ac6293f8f29590cf7fedc54723e010ac95285109d037a7dff5f700\",\"90fc0a8dabac31b1dadf9a55aca812613a13bbdd13470e577f14fea238ec0578\",\"1530df723106a367020b1a06a9159be151838112a8b1b3288fbab0c8e1a9f9c7\",\"5185b1e002708e8fa52f7a104fb03cfa9be3daf257385b014fbf0d28bdcc07a4\",\"2d2f816d168ad02c7e5900cf30e5eab50de98e1016c755838ce20d0f32b97dd1\",\"e791d19dd35836fd3695089e02946dc34975a1574e54aa684a81f2b4d8a4a37c\",\"4cd2272ca38983b2134dbbf1f4730391fe7a7cd7e4e78ecfc5e1cc515c8536dd\",\"5100aa1c9ae7aab4e285c3a6e9916dd7a6e3e4f5cf4b96bbd439c60166b37419\",\"0375ec5ab1b58e2cec30e4b68ceea8c73b67554ead54b6d2d82d2865ae688f62\",\"4b1b2e1b9d15caa30b1522e28a6f807eda99bffebb7f8408a70852c8e6919fd4\",\"bc1b5e379bbd7da7fe6543257419a49e591de17ed75f02b5ebc691c7f036470c\",\"9aff3f21b290dd4c4e39d56dfc77414e6f472455bbb977d375a0a33a73db7c37\",\"e2cdd1d0c7746a24c1e05d2fd8f20cbd5ab692ac305e56479e48402f1e99e544\",\"4ad9e5902a932663dd6a36f8b2b1b6eee20c1a743774073be3b43796ade18b50\",\"e52e57bf5845dcdc9e7adecccc893dae6b1afc25e6fc287a9e949b1aaaeb3bf6\",\"c0bc4dc4f63f9365225edf9fd4aaa5baa766a516bd2ac4a720de66efca225848\",\"ff5d9e1e0ba51512a9c40c42d47a4fb217e5d61e409b3eec2685b9a6fd356cfa\",\"1633f3ec7f67bfb33f795cc13c71abc033d7f51461f63eefcdc14bcdeae77387\",\"947a79687907ab33012da398d8f985083c9e31d33258bfefa6cac32a5e9d998d\",\"04c52bbd80c324807f060df7226f8a3a3e27c5b204b45eaa319d4493dd00fd0c\",\"74b894dbbcccbf4b6db4138c935b16a271ae504ed46e18bc76702de75a58f25a\",\"1223f20b46d9e40c1469a04beef80dd6966966b7712ce418ce881be3f31fd30a\",\"6b491e413020cc7c3ba0fdf2c2af7388da475d01cc17d284068a1f46486108da\",\"f20b3f892254841721ce395ff2d7f4e3d22d182829933aed098aed4e268d016e\",\"47e11aa508c83190e4858db1875e4df9a068d10f6ec6c1929a5b8757ba9db998\",\"581764dab13b91f1e85ed9fa91dd0c1ce3d5ef3df8322277dd9f4a97d820acbf\",\"3da93ae19bdcfe0341e6e5c237c36e02642de8caa5a2fe566b69c4793c372962\",\"5295ef0564afc01cbaf2684857ee3496cbf44b138b4becdaf77a9f136c5f0059\",\"41dc87a8aeb28a6db82c985d60aca862e9ebddaac4166bc3d0887b47eef5cac4\",\"354fb23fb4d417ffc2a317b8e7bcb869b64e7f698e465a20580bdc787df79d75\",\"8ec5e640dfef276660168011d1fa39f52a134b1df891a65cbf3b1d1e180ad146\",\"e282938018458cb424f0736c932ac9821cd7cc7cc371dfec17cd4d226618b7df\",\"3afd4e60aa4bfa2bd2bc072008d690910d3df44d0b736e73128e52ebfaa41893\",\"d78068bba948f2f423bb0de5818732dde683bbb1e7c8fc2f0ae7751499bfec45\",\"337cdd5aee596356ba8175f8241478665d36eebe2e505b51b07c5d4416da8d73\",\"4d998b3bdbbc7f8464305afbab3e1975566eac7062e8e955c24ab88560acc2f7\",\"1a4fc4a11efe27ed89f6d25aad6e89e9062da6a8c3cc5e7c20751d72f623ea42\",\"82b1cbea9572b4a3011313aa617ada763e2b8dafc575a64b62a3c8e63ee6d552\",\"8c19969c29c314aa6d70dd081b0ade73672f1537e3d16d001f3aea1f8c71e6eb\",\"3a78549f6d217ce8bb1bf363910d5caad53779a73042b01653d7619fdecefda7\",\"372c8aefd87b276b172f4de2c84e62debf66d31771bf3f7e5098c330c36e44b9\",\"e6bd5295200c83577434388a02958516fcad092d2c4ec3a763f7905e5b4622b1\",\"738dfd6e2f16b82b4522f896b675c06c3b11b36dfbc6b668f23e1db2ce06757e\",\"e8cb8388163746fa74cb675727261d082fb2c5cff28eb71acfa7e1c1d3868a28\",\"f4cd6ddc8108b0ad38022864b3b5326a348701fd5bff2ab03b27058a27490115\",\"d02df72e8fc79f6fc1681360fbee4a817179b866a3204e4807b5f37b0c81aee3\",\"4b00a7367f9cdb000471cb809680c45662d1bce98a373f9233395bffc27276ac\",\"11b981a1bcd86d7bf3f0103d4eaa65cee01fcd04beec1af7743bc7d62a49578f\",\"d3f3c78107d0d57db7f5580afa12ad19080f807d94596b5634c15a66d806dc03\",\"9e49c867d96dce53372f4431c2a4c5fc74638ef229de544298c52ac5e74ab4bd\",\"f38b7496dd7d946a6426ad5d78cd60d43e54e2b561e1c584d7d23a2ed8e5985b\",\"be02a8aff8fce2052720a03d7e313d791bc4c5e46905c06062028ef83e52f39f\",\"02502515bc9d96a2656f0e2e048c5e9c659972f0c0a617112ecc391d830f1c7c\",\"b901b2f3e33a990ae25c23b8babbbb13878a5613a20d55fd5acf2a997b8b0204\",\"0b19a1577ed35171dcfa6f3ed66c182829be15dddd3428e07a8b9cac0e7dd952\",\"7ab27fb3bc958a1b200dd3cb8df0d4e9ff71e6bfe7b7290feb49c662cf639579\",\"3a6bc4b0084193510dfa4d450f934b506476f2eac5853b0d925dad6e7f96ec7c\",\"710b06fc3901fed896b41d3347f7260270c59eb2c8400a28ace6688aa488fc69\",\"9fb45980254a80dc56770e1d741eecceacca6361c0940e1a17e504f55f9fb04b\",\"7600fb4a34ce43b79da2d5c80c54e571f39e6ef579a421a25f06968927386088\",\"d17cf2ec3f3a6a9e01e3ae7e0dc87cc0512fe01bf2f8183d49323cd2146e51e7\",\"2a1e096e25fbf1cf5948a22695157677e20d104e143b0ced0f0b602b82fe0101\",\"fa06f7905f8d21b0c42349d3a260220c3cd58ecea74e84cf35e1cbbc14178b4c\",\"34ee0be3677af8e1a534750be40862f713c775aa793afcb46f63ab0f1a646579\",\"403c7850737a2a2d5c5673aa388ad01f3e3ba812d75fe9a218c2e039e4c29121\",\"f9d738218852fad30f1f38909ab8b0d0c5ed89c6d6506f0cd3efe4abddbb96c0\",\"e4b915474928e765c0c9f3182e44f1a15e5e9a98a1b624ede5f0f7952af7e05b\",\"9d260ca1e20eba3bd47b841251f4b14fcd1b42034c3a8c69d241afcb2660a59b\",\"b01f76f382e67c1b6fa55dbc5e2361d1b83fda0ad15629d0ce1095c54116005d\",\"05b63fa8530a810a9f5d6ef5129584bf215ceb396f86b83000a8abce3aa605cc\",\"c5bf6cdc0a108ad774306357bd81441a42c335945301d77f462b723718aa21c7\",\"2e59709edbc538c2a83ba956dd63213f859039e42f2c16f5d4b6115fef3b4115\",\"e9b06622229cc55567f6d312064c286c001ead06304232ffb21a089c6fb1b6b7\",\"342f03487ad7d3e0ee68167f8f3b761377834962034f32ddbff5e8e1023d551c\",\"183388fa547d3c1467ffcb35e07bc156fd6b416a3e969f68247be6d560b22c19\",\"f3c372caa2452053661447f7c14f1bb01bafb8a35eccce7f244d9b49cec8d203\",\"9578f0e9aadfb48d6e9d29223aaf53f15972b57c47d83a37228d257ef377dcec\",\"277f4b7c9ed1f346dce8523a937f86a20ab6ff734c6447664a12941a05e1b080\",\"f1f2ff603ca3908feac837d88de99e44bba30f0fdaff620b70dc65b038c40663\",\"c5db9203fc67f81abd4166e69a953ecf4e624e3a6ddec6f7bfb9e3971d581305\",\"18d5f32c480bd60e9f9fe66d9a50ae60d532e5f29dc5c833b180b38ee5b71589\",\"e2ea9827cd115eb1729a656593bbde5ea80e048c49ee0b7284cc2d559147bb45\",\"9c2da344718ddcc0c587ebc4e7cda57b785ad43b7efef0f38795b8283c21fb28\",\"0729fd179213c4a7667dca520ec5f73587cc312301820e4b6c1747f16e802064\",\"1adf2a68b9af53895dc18782557faa05d516485eb63f3da046c3c0e971ec957a\",\"017b8390d68cbcf9257d7349702c8c02e5b58464eb4ef3d6146fcda0944a0d3c\",\"7c92e8ef16cd2f6d7b7063af8251f90b7f2070efbf77afd549902946ad79ab56\",\"029591ea4c17045fe3d9e3b807fd4fd8e8af76bbab2f8b24200231c5396b4f3d\",\"d006511903d2df94665b697916cff848a1324d367e41443800e1cfc184b22b8f\",\"125c5672fc96f8c4bc559f4942826d3424670aacced40873167dde61dd3b2afd\",\"9d51989dad15388991ac4db74d0456734eb41513ebf4ad4bea0c4e2d6d2d0df1\",\"8feadf478fb4b0a8394366279525d0c88d15f56bbe9eeecc3af514a6ea21df10\",\"42dc2a3e4985d0267a0c90eb4ec1905bcac1c6c332f2876c9aab757ef4a18a72\",\"ae00b03e1be11491fe412213be84072a757f6b56cf23e6672aaf563efa89bdb7\",\"244a0fd51b3d3fbb02fcf303736809925e6865a38bb85cce2dbd63d09730c61d\",\"92897dcf8006ece9a6dd0e9b6f80f3e260cb663dea3b3eb0f0293eb9a1c0b69a\",\"d86be115c8f040189670eaca37b5547a1d4e1b83d7b33527e3cfbe7084a72c3b\",\"52cc315b8dc4f91484037f541282be5ee41c5b7105dd650e581f8d7fece123a9\",\"04119170e4ba4bdbf8296e071bb3ba1a13961d27f85aaa44cc12f9ca8d12e112\",\"93e92732073ec3d0de7217c007fcc93f949e0f777226d4468fb00de8aaf91b84\",\"2b8fe382c55cea8c8bdceee0800b2358fc63064c64b982aa119383cca1411db4\",\"2a14770c0c4dfaa0cf4e4e2338171dc822aefe0a476da6255d26aff746f27625\",\"4f7c8306abe6ce2b13d7eb829905c36f201403093280911b24f84e714655426c\",\"de81ff4f7de35de9ad6d18d4ec2f2777a005c34d6d47a7bb648f8abb9706ba16\",\"dceff88ba41d0197c3f8e7f5d67a07e9530dcff9a58174c85add4b48c93b9daf\",\"cecc51fe80ece2121535b3b0d43c0e12eadc8a5d32464da0fe2355e4c1c2de5e\",\"66909bcfd2191feef9cd4e9ab91b676b4fd5a2d82d320f96e2177c74123a489d\",\"47093102e73268e849eee5f833872ef627012d04acf343d43e64a83b1c91a532\",\"a1951df8132042e061644427b0a967cd02fc9f8c7523a154719aad3a77bb4227\",\"933ee927df8d3f5ce53bdc97e5651310da4840f9f026c1a61637d6b5de84ff18\",\"5468cb55e492e942a835528bc208d238dd8fa9f2a196b11b8328579cf292d58c\",\"1345e852ecc23486b887f9a6371fe335e1f40b83ca9df31850c302bcd1ca4aa0\",\"7e05d5dc8ffad637e3fe8354f74528e95013011dfc7f0cfa5c1d4f0f54623479\",\"5da540bac6f7132fb82c356d1080bb7e073e4977f7276807d379fc3282d735de\",\"f5007b75f9cd549621cfbf1469520f524912c529ec506793ad3edd6c4d3e1cd5\",\"5326d7df5f56b26911806298c8c733f8fcd33722d75880e452319b066e1f7718\",\"90136e1fc44110258169051670ddc5fa8337a4b621a988987840bc09771d54d6\",\"302a487a35866bb715544173fb97f60b618688084f5a1e4df0cf3c8f8369c71b\",\"270d51640bf5c47bff126fae2ccd1fb291daf5ea350f35b632c690f80f67faef\",\"7ee89bd8d6175cc2dcfbfabea811dc8695accbf0c0045f71a35bdb237118eaa6\",\"78c48c1bcba47022498958fc01b275353dcb9e37bbb622faa0dd67ef2066ae8f\",\"3c2c1b6a255a60e2bc8fadc4d0b6dbe502aa20020ae022710e1927b4572f36a2\",\"9c6a7bfbace2835f204465c603ed3c6d9031a462fac52626d451f54ccfc574e4\",\"e733868b66d78c04e3b1dcc641473ebc4cbc86411d2f32fb6bf354fdbd478393\",\"2c4056efd171aed35e2b84da24d4d3f61ce180e2b0c1282c8456c7a6469c349d\",\"5ee024255157dd5e88a913d89f5c8092c19c48d1116da30ab4f90a861f31ccc2\",\"43eb6f6366e603dabfe42b0938cf251ccb8be56664c326e7e7067db6f5f632b5\",\"ec4e320ce81a2022b86e93bb56025b1b089ef9864a6c382278bd23b20127d364\",\"ffbcfe00fdf5d8a10a005ffeb8f7f3948643a148eaa97e9c304667ed03b7dcee\",\"a42cea58b52ff409d32cfe2980d5bfa4733a93877f0ca7fa84bb920f9a178d96\",\"9ba6de496ef838d4c310c0c8a440707044907fd9731597ac00ef3c4fd90a46d3\",\"54942c74ee1acf161547c167d837fa4115bb0f9f4fb854dcaeb87ddeb823a3c8\",\"34a594a2afed201a2d14a26968f6e06d83429d170ea13180ddeb4386df778d18\",\"ca4c0b90f06513e37d7d005f4445b67cab4d925bbb98012b4d54817a7e8f15c2\",\"e3642ab0265bc7c0eb31444b5c3c059144168abdaf61917b33537dc36dc5daa7\",\"08954dda8a1480a60bb92505a7e8c7abcc83821fdb877e7f0e1c9ec0ee39c505\",\"0a528e2db4c3a1bd36b2be51bd95b106bd398d126adc290231222f19281605e4\",\"05db506526c424fed84e1b5fefb5797bb73d86e5b687ca7f9c5f9c272d69dc6e\",\"e5216609b6274a74367eecdbf0969ae50f23cc40f7d0df1f9ddf412a564a027c\",\"fef54dc3dfcf0ab4ebd185bda3509721c60dd81578f8b8db5d60a34d5890e13c\",\"d068c407a15ed69e49920404bf6c3d28d7b5e95f0e68c9422cd07ab438906318\",\"fc2488693bf5a077529de9691b809eb74d704ef7fcb316be9e6b6d7bd7ff4213\",\"cf179ae9ab7f67eaaf9dd00e70ba5927826f088a14b65bafb605dd808e5d5073\",\"cc4184dfd71b186c02dbfc45e3267816e2259a9eddf2df9b99bc9a02b2817a86\",\"6678293dbe3440edc23dd2b1f61980863108234f222087b1d765a3d050bf3e3c\",\"2139bdf228521163796395310a7f91a286644d47da4e329b1daf235fe6deb32b\",\"c76610866f94bfac0f9cf745eda6cf82e85cd3b6b1135cf45472fbc36d35a874\",\"65834eae08067b31110c088af4dbdebad99a26a00a834fafb3ca771fe1f0ff2b\",\"7a08dd8ade9984720e01161b9d2925ffb1e589b7520edc77c8355c70f8a08e92\",\"cb634761f5bb3ee982e8c1b9f91d4c66794d260ba2aac2b2c7665c72893eead7\",\"fe9c293fdd15539d943f0304c5e621ecdd35f3c7e752302cf469cda29497e65d\",\"e5539bef82ef9bb8f3c63d2863d7e998821329b1db27a5ccb7f4e5656a907ecf\",\"7940d1d8f96c6042f588ed956694384f3312397485f0f2e8b72fdda08006e977\",\"84ed5b8902fb4f56f7bdab04dcf75902ecf4adf5952b010ad63ff62a48a746a3\",\"be893056a3728baaa236be58e9f5b44db227bf560a2d48edd9944ec5b9e0e360\",\"3386ea455a5abdf880b9708846cc025436911b71431a1ee8573181e40f780845\",\"9dc5978f3f5f4b4839610d03420d6195ec1d0547b6d58aab3d9f77f532f5476d\",\"c124f7ef9feeb27364999c66cb4d4dcb2c9a212cb64c827c53cc2932bbff27fb\",\"3a60a940bfd373bac1b3db54bb0587093eeb22f6658f356547c7b84e9b2e220b\",\"74fee859a8bb4dee57f7a6787975064123c2646076b8acb8926cd35137652d22\",\"e065bb596bd955db22168575629f17c03d567f990e7ee207a53068515efe45c5\",\"1adb4aead167ebb9167b669c03d08da2dab4492a6c493109a8054c11fba34812\",\"3a768d9ab52ac91314d16a5dbdf581dfd7a9303df3c7e5be4340f19aaca276ca\",\"0bbf35c2708555d00609fc4c1476a8ae662d1fad269d1645e2ce4eb3cd5e7052\",\"125c4e2a83255e8859e371ce2f689808e25f028445f5be7514aaed1667a02cbf\",\"c0da91a901ef6a7664d97eccfc80772ef445ff6befe775649e625e4185d08f11\",\"39e70780da5d2a8068b147860e4c2a4d041a664a37f8298715ffe12f52bc2781\",\"5293cb81d335d99182b319ff04811ffec86e01485df3374fd456f2eab20ae4ac\",\"29bbc3f341e920d64bc87d2b912a31f75449373feeac2d8b5b2e4424d16c4880\",\"6ac02551d7a0fb34a95a570558f6a99b5bc2381a955b91347b16a4407791f971\",\"60a94cc30b006bd14fe8685dd88f2006f67d322f424914ceb9d045993d046b22\",\"80b62819128deca8e5c2f82dbfebbcfc89bb7910634ac3ec0b974cb6da2e802b\",\"5142d1ea5a75ba46612254662baf90ee3f7ee1fd5f56dae38238fbaf0737c802\",\"d0dd2ce92c240c110289de3b0f108dad3d0db97e593d07d1f7d452742bc3ace0\",\"792178b1d04bb04eefa5dc380029b74562b4f6d20c4ed7dfa85425cc8d252df0\",\"beb52f892decbdce03e06204fc2437345a3502a6b93da5211076dca08de72d47\",\"969935fb8ec8139c2a9077d974bb08fb24f39e37fbded175d3fb00352f564588\",\"7a94ecd3d0371cade54232ed2c0759af7941f8c4a40ffba596a4627f8af295ed\",\"4a6f9a3daa01574bc0ca7626ff38201fe3f04d883cc129882b76b5430dbf8ad9\",\"0d8491fa68675ac44d2473383dc6a0d5181a6c29e2a440ea554c6e7f8a1f5385\",\"ff1e4acc3a5fd004152c4d311887e7b62490850cc7f3af70fd4985a67ed7559a\",\"5b8e3d3b2d343841aec02731ae690aac52b36bc5499273f89bd4ce3d1b35b19d\",\"edb8dc1d0fea9b249273495a8802cdb3c1c2635e8776f1e95a8c47c09c19c87e\",\"e18cb9eb835c6f1d3d1855e6c169a6201ff67ce50d962863a465b6aef8cbea65\",\"022cae4cffda99a8eb3443b979405b047e11f880874d5a0a134d3a153b428ded\",\"5245cba31a8abfc8d3b7e770eb9f889bf379bf453a1a5da5ae28209db610fad3\",\"90f57922c6e231625758f2a98de2e067d4429f748bf32906fefb79a4e38ff731\",\"c6da84fda0329f857665899efa468a5f2f057db580626ee883086de25103676c\",\"6502bf8210206fe05b26a0b386d6671cc901ad7c28bc51735bcdc025a3427775\",\"48fa60be3a94c3cdf887545bc019424aa95c2a8e832441ee20d42bae0be832ea\",\"0bf071e6b877ad3cae852e8fd4515ca2d3af4cb386d30890bcd50dbbcf5c752a\",\"bf7afcc39a0efc430250ba4c012761cf36ee64d9391f0c956c2a18e005c1440d\",\"0bb62c248f10e6f1753171d8676ee9a891a4eaa7b77e6bc53e44b3e9fb37cdb7\",\"5c0d2ffd526cb3161a63b013f6bc4c5b06659993e230ab0c3d678a2503035774\",\"6a1c37604f714620589b28668f07f3640fa244ae44a383078d32e61d2adfaf92\",\"507bfef6a16fddb176b4848383ecb7a79369749b66801da665d01e5b83b63b29\",\"6389478a0b8d4b7791b1cf5bdad7d240bad0b91af9905d29b408cfff6dcd8062\",\"0aa3a0f97c31d924194dc561f6a9e354d9ce63d71cc0cbedb6c9a2561ade4787\",\"3b0d7c043139f9fd39739c3772167da6a70f47a6c269a676b7c259b82ac08416\",\"29fb3dcab038ea783d6a791fd701397ee08786e9f5574502fd9b3fefbcdc0ad8\",\"c425f41c45e1fc9cef931e9ba76af9a14e115cf1226d9df4b1db517df6fe068d\",\"56b79e88cddf7ede7c109802dda25cf26bdcbd3c99d4b1c4b55275f2ac987a2f\",\"9e157e0dc2c993edf2b061110b2cecfcc30e798f2cb66ea926a74197c3dbbdf0\",\"3723952455a3e4eccd23b512b406c81f5c7f2139b99f0821ad687037b5a23860\",\"3fe712a1f4c6ac032b4e569080fa9c847ad4695ba37a85ca9ffc96033fd12b6a\",\"564e99278ec246ea15fb1f1c53e2dfc140595e50226e2be7159ad5aed57c73c3\",\"55079f66324d0fddc91ea58fa530ed13a4345efec3bbdde81c3ea4363961891f\",\"90ba7f24b88a01587b26fe7c82581ff9043291a257e6bef542c0542f4285eaae\",\"9237df465b835fe2ba9b1bdc8052c2da14871f69247f411c2e71b9274fcad18f\",\"6e7901c89e407767079eede45e04bd648372d0cd1aae74acb60707d8e1ea79f0\",\"2f950eaee6817d599d11ee48ee3a4b0a73916bb1d5cdfb31c103f47b1bf391dc\",\"0ec60dc6f581b4782a74e7659770c946973bb7c20e44bc25d1b7225815b016dc\",\"9f50cd0e7398842a52f075e22e7c88127ed2201c44f86ccbc7e5d009ccff05fd\",\"bb7d1f8ac12fc17322b5470590abd84907731129aeffce2f8ffdd631026aaf4d\",\"3a15db835acdb6a9d17b6a5b76bd2e81898ed78ab6f4e1348e1151f4b82dc8ee\",\"366012f7166a6a2a5c4e6854bea656fc8506e906d9e462eec73920264fb59972\",\"1a73669b31787ba64b498c2ec2180bc9d8fb24f8c707eccf7a2bc19d0e83a6a0\",\"3f44441e7e1f4ddf766e63d0e8eb7e4ae0534b5140f6b82e795b53ed990d98e1\",\"d6c0a82708aca8c1fdea422cd4606b7bc28b5f90dbae929913c126d702bd60c2\",\"f8ef907c6beb51da345ae660ff8c44cd27e9c005fc2f90320da450f4c43ae2a5\",\"7b1f24cbe262403fbcd73ed2d18271d06a4af7147764fb58467957e62b8fb7db\",\"8ba8b7cb54ffc3734f60ac0d4cdc174d0936ddfed8759f5c70ff7807eac00742\",\"17e4090f7d88c6295aa3a6b635661ab73ec80268fb227ec64234c1f92882960e\",\"e89fde669a12368df807b6dcd0896e33ce612986516830e2064e72fc3dab6a35\",\"2fa41caf3a481453b93a591bb6d99b3d3b0f9ffedf024e4d042a7d7bd2c30b5b\",\"cfc916b40f0c847c0105d941c67d878b4f72732d42cf3ea5236a962a7797f502\",\"d4cc2ee621bf84cf0539f088a9bff1c3d3469e6a7eec9a003921305e6edc9ad8\",\"8dc6c5ceceb644f473aa975d359ba41f6458f513f725182fea36b0b58eb5153f\",\"899bc00eb1fd849a9b52f60bcdfe0826c084fdb06c2748c9a579655eaad1b265\",\"0cbf9c0464685ed80052ecde0da7fa6d899a47e0552722a7a09e4bec6d47032c\",\"79b18ffe6093be66efca8a52bbde46171dc0bab7c6cead917b659f75637eaba9\",\"43ac8da1d0c65e07bbd4ee9da0e9bfa57a0cc2f03c0d04ed2fb1679018ecf8dc\",\"3dfbc6c64ed7b5fec1e9f380ec3a9a4122b605931afeea14f293f729e8d0b99d\",\"c01146ba8813845bd76359931bc822e79e270c808156190e54f996db21a38f6b\",\"60df863d9b1c30c71274f62e9a4e57f2176f4093d786daca91549f65938e5bc0\",\"eadcc3e783a19a64cf80fdd396cc6f02535ee86202f9e59697fcdd2b275dbb1f\",\"8936d771a638138ced42934194f7708531a339932b71524b2eb4b74ed11710dc\",\"c80d23eec88f3a1a1734357792f671fd7bbe0d70463cbd7252fb0f8314b7002a\",\"48e8c2053fb3989272e6872cee5e8ac09bf2ba89ded6cd8a66cf3ae7e0855684\",\"2a67ba42b05aabd5f1b1de59ee38e405eb2e7890e04a3d6503a7bfd7b5c3a83c\",\"f63c366695f8fdefec245438d99c339b67dd13d89887ac407364a10adefe6455\",\"530e82f3fcaa0ffcc7928d55ff476aebd2c764619d9b43f752909cf6bf66f84c\",\"a9ef2796ae21a6d2fd9a1f82a76f8691537af9ed58cf05263dddcdfc992add45\",\"4146e23ea508ea7258fc7dfe97e429c3b56b5c9c11eb327fea303cc0f17ac387\",\"096efb013ffa5b83ea2f4a6befc58bd0978b8283dae722ae143e85594e92b675\",\"be88fbcd7105698bb793568ec7dc732581a95a956d39c378c2a05de39f16dccf\",\"465eed98ddc26e2b00dfcc621fbed7a5859083a78537a442750e4aaa63761658\",\"700a587b45b88a75f1494d04ff7e1d3bd65fc53ada67385eaa15faa7f7bb1dfa\",\"99c23819eea6d06f003e5a14b3052341548f126fe8348ddffac3bbf61d853668\",\"4635adf12d42b7386f63f0089c7d601c066af19283629e9c02ef04e55e58d607\",\"3dcaf6f69cc17961d36e9009b1c64cb996ccc40f6bfef8434c7665216867bd24\",\"76a0d0dc9f6960c77d88df9c824979fbbb22bbfe6d4d31fd2e4d2077280b9079\",\"773c868c38a5766d349085061600e169e3bb3d00dc8e83d217e76527babcd8b4\",\"18851d1a71dcd348d3e43c43f1f53cf0125122d5d367f1be88117eaa1839c0ff\",\"a543f3541210edc764ebf6f2839b3b4fcbb49b87d7aada872541348993ceaee5\",\"1a4a9dc21fa31bf6468d3a97c7c455e280735a6e5c079b991e171237d544e622\",\"1c2734c95f93fa010dedebae2cf96141fb7e18a8054ecbd098cf748aae4beef4\",\"bac938b519378f38af3f5f3bbc5e867100647fa4fe88d6d518893fe257825be5\",\"08b3486128d667815b6278c720f827a0416ac82836d7c86cd79efd0ebfb09fb1\",\"f8159cb9c334f9df706607380942a4b1fcef6f4b05235d6a8f62d028dcfcbd49\",\"180d846c3521ffa69b94c9f250f562411a448a7a00d2ffa528fdd7bfe67b4db0\",\"acf3a5aa666ba07f1f1699c2d02d71844ab3e8c57297f21b6a366daca9f62d72\",\"5d64343e0526b1655160bdcd3cdaa9bc37f2870915bab338493b4ae174355b37\",\"6f91925aa5436b3a75da88d4f67a572e2cb141450ee5a9cb47907cd7d9ab0620\",\"5688ee895583963146cb061f3d249ec52d3af75b3a491c2f1fb449a77822d6de\",\"481d0669aa9fd4e1cf12fcf3e69bd2c279c746ca0700aa3216ad9777c9b6fe15\",\"0be028d1f2490a772a73cd0583fd7b4bb0c3407b5f967adb777a34b561f51d95\",\"650ee2a5978e6f8619d7c1ab14a58902420ea057f945655a4afb4e6d65d4dab8\",\"a3ff4d62c54f3a349777431ec9b2d11f6c62e42ec7958448cc9f7ab95f3d87d9\",\"4cbdfd7441214af7701c348d4663a48d4c371b5702d00daef240708cc401541e\",\"df3f971ad1ea9ef501fa7a3ccfe0feb10d08598717b7960a8812ec5c80f055ed\",\"3ff1cea389b3e8c1a4fdc623f9041be3fd587986635c1e4cbe3ca5074d7ae1f1\",\"e1b9a6ff0755cb74d7e831e8e1476adf6ef840af2d972dc09b8586859105f26d\",\"94a943882240d06a99a3dbfa1d76c11844b49d0a6413fbcaef9146ba24280667\",\"40ba2c9961fb9461a89d1bd617a7262f73d2afcedb6d0cb8cc612ec20caaf941\",\"e8b679e446e6288b80c0fdf47365c6eaff24192c732b099648bcf0ef0bbc4553\",\"49d46884f4446560f63e01c7888f59362d895e8abe0d32a6edb64571e1a05e44\",\"019b9a4569af61860d79f3d4d16c10ee31d42eadd15aeeb40649f8b070f80928\",\"3cf3855747cebca2bf854c2ff1be363b698559e8df246c369154435e74583a0c\",\"2c5cf7afbcf50aa9637122d42834ff526d01cadedbf4f9aeacfdce631e174dd9\",\"bb8321ecb18fe0035b395f83ccd276942bd5d58b09cf91c2f114a2205970c49b\",\"c9eca0a99b02162ea07c5914628a1890f6ba76fefadd1f949d70f7dd81be29df\",\"53760dd36e0fddeeb7f1621cd9946ff18fcd81847e53d977047f20112c5e29d5\",\"a9f4e79bf356ddc61fc659b0ae0bd21be0b07400f88bac2489de8fbf63f10b24\",\"5d2f4f219ce4d4e0ad05f568b7118a4be52518da7e596e8fd8dac07076209ebb\",\"49e827779fd798431b278f6134c574ce84cfec13180ea875b4d4b9bdc51e1689\",\"c67033c41ad06ccde9238aaf6eb95eb389f1acd65767d1543798de956226df0a\",\"4ebfd0f007c920cf03c6a2efc6af8891ff32c178c6d6bc1c64c47d0bb7841673\",\"97ecd23f73bf43cdc6f7386ddb70b5943ec26bd6cd7ed48d262e3ac8d387d50d\",\"4ec2abf7628d99f462b614e6bb5b6da945f3ff4169f252165d9da21d2d43d8fc\",\"9c08e8eb1d31b3fcca4fd54cc84b627614082feb68a684d662c8ced751b41a74\",\"9a0c893050e962ede4fb0061c301e7d1b7d3154ec211d5130eaa2d1056e20c5c\",\"c1fcbe2dd4f02cfca813d4562b6741144153b3da2b47366f87f67a0e5530bbc1\",\"1156036616b58e3a4b110b6642bb91cff38a3dd77e5bbcec1fdf0d9b4e5e5e9f\",\"d957e279abf966d05820c03bbb22b7afc96b30bcf8bfdeeaa099290de5741856\",\"062d7e55ea2108579553041b91566ec2b925be74c5a8f506216085afcea14478\",\"de07427058f274ac6167543fe38cbcb65a90d894d7db7d58098727c9e4efec69\",\"945ac29a279ee0d6224bdde7a204186f4ceab228cedfed5f4bcbc3c33126728c\",\"697ded040498cee174ae7bcc398e7fb404bc4f1c41177579b5e5498e2b1cc415\",\"a70f442ab93127ccef880029fd1da8d948efc3a3c274334759fed13458129bff\",\"0a0b4f1ba4c74007aa2aac96a64d32cc9d3acc5c464a8c6eac659d3543a8fe7c\",\"bad3dbc197b3b9541b7983cb4d6ed57ab4c05e14150e833a7b67fd3deb4ae9ea\",\"12a15d0e0a5b69fde50e31e7615220ce64b75cf77ea5ed3011d94df1403618ad\",\"cabdc60d6f9f8fe4f6c5838f6188e0219c598fbe9dca1363aa7ab9fe786826d4\",\"b1346fc8349aba1657c8d47b0ab4c03aea84530f8c36a61f5825d03c4bc3baf4\",\"14e832771f51416178324e3ae7aba4e780373071c6ebeb7712bed3749469d2b6\",\"223d55da766605da8c10ba259110b65c8d3187a69cf472aa98bd980408fc012b\",\"85128db5a7a74beb67c5aa1085d0018648fbca0358c8af8beb559fb1ad8907c2\",\"80417a6aa2cdef0bbc8c938dae66a4ac3f4916426be9f4c032fc736e55372c11\",\"976b01f188abb5aa43ea71f950accde656879fd979d688ea667f7101c7335918\",\"9337022842fbb809de8bac4634ac7b630a8f9df5ecf1db827df5abcf5101a7d8\",\"74001a386ded87f60db5260359cd746382dea961729d8f6859869c1fa5bceb02\",\"2485622cf96dedafc516e4f472d994c3b7b7ce97b95716358f73291c0734b093\",\"cbb7d2e25e8655d9bc39af3d42d1a852fbb57d7ad9bf2eebe87407af1c32110c\",\"8a91820df3947f2991d65cd6ad7b8dd1bbe0c9124f6eb305eec099678b66f763\",\"6316e1e1a750699a25be5902d50f9d9ccf76aabc2ee50955d6b8aadefc58b1f3\",\"326f8583f44419d66424a5dff2dd4acde11b8e47f6a049bc7b50c6ac1261a798\",\"e6067062cd7449ffdaee32c78a98b042051990370866fe6f576df1af69229157\",\"c1cde4052ef9041739a7569235ee5e26d14b060d27acf8194641c3d563189ab7\",\"4ed06c79236c364139e1d0347410299ba61bc686759e8e99e347de42c6283fb4\",\"6b85b80f0dfdb0d38cfc5e06961b255dcda906942b94367bbf823d0d88712238\",\"56e4ffd35083df03c2d95110ad56e113650ca1c18ead4cf79b565a28c073ec3c\",\"64edba79e398c22e835ae59c9a843275dc3756b1fe1729ebca0980053fd527fa\",\"a7f60b82e4d00d9352a7ea56ff182d83462b2185c164c6a04eaa7cb2b1a5a4b4\",\"245611f9ce892613c159d11943be8b9b9c7024eca263708da06724d5f7509e19\",\"f6676e36cd659fdac5ca60faee99fb7fcf5d5fbcad8d2680386be901237d2cf9\",\"b442f8fd9ae8889899c0f010eac269951e381d20535447fb1c258ca2244a718e\",\"801c3165887be3b281f1f4099b2901de729e88d11eed8144e6879d4d68e6a4f6\",\"11c37c336cadf21bc2ae3a96f9d6e4c142d65bd74962cd2643abbe69fcb1d88a\",\"d3df93c7e033e6cfa66dd481e802203bb2eb05f9c189dfce829f1fdf9c6c3ee7\",\"743714eb7d4824c6678c217fa64ea9ee5548ab2e453f0d8b5594e1ca0c7ecc4a\",\"af1e6ff39c07063abc04c3c75f5c8f3e475c26db794e6f716d3e710271494e7f\",\"030ce62786382802ed65140bd90088c4dbe0561bb864b09f6dd44d0285a43e87\",\"7e84babc50647291cde303be73242e6dc462ac057ae99ee5b20b87712f73ba28\",\"93717eb192ad33add7e39f06163af4766a9b903ea0e4abcb4a7cc1d5027080ca\",\"0488c3619b51c7e877689618a6fc4d929b853a04b7960ba71770e2db9b80dca6\",\"79730886852d3c39dd806d89265efe6ec8e2e1cafa72c409f1598f2edd10cfba\",\"1874905016cf3ddaa2f193ace51b9f6569475ec99226408c3416942fb47ee48a\",\"bdf1579ae165caa87783216ff0c8bc1d89cde68ce249ac568362d66d9bc68110\",\"898dc2cc4da5b9e2ddda3fe2f502a73cb8da2b5e85326802c1447887895267c6\",\"5a2a66261658dc85b8a1c673c62fad86b16f4fdbedb5d62d57acaa49fca1753f\",\"0c3f8a16cf9f3ac06f89240bd63e724a5c8ea3ab7cfdcebe748224b574f4abb7\",\"d05131c76c34540b64b1047b3400f128873353be0e0f70fe46549752941276ed\",\"4ab19e333eb8e3c830a097fa6392fe55ace710e3e99aa0b465d5e21c8232c3ec\",\"d01ec6573850b3fbb0fa586310238998827c6cfadf04498aa9e61ddbea9d46e9\",\"d649baf528fcdc85d24e0211d0f7addc2ab22d0ff700930fa6cc5c3fff194b9f\",\"ba5e5898740d30a102ec598e4e0582a7729aa8fe081fbfe1292cfefd9053b5ff\",\"3d84728d8e9b70b222ec6f2cb86f6551b1a70dcfd3fea2c140c63b9a4dca4fdf\",\"71fdfd99aacbc560cd94fcfd10a7e998d65c222e28c8a2c1a257ca5189a94a8f\",\"04f1737cfd2456a99b47e2bafc8743f07ec8b264e813d1a12b290f0fd146be8d\",\"d71e24f192e7001def0fc1f81847052ff8ceb0338c1756f5ca60c4d48f138081\",\"3156678550e02328a89157e258a61e7436ed177ce92c1cc1c765e5ac5bae2e45\",\"166941902b4a86f2793f5ce71c3ce6240bc053ccd78962b7bd0689e57f26aa23\",\"dd8641341aec32e8d01c0cc3baf5a58fd0c422addb8e5936203634b08f9b1c4e\",\"28e745104e9e8f6edd41c764cac09fb2251f048c20e5eae5806054218db6634e\",\"fc3a2e0bd602a8e048751c880aab5d2692603924a0c157d143e3503ecafadd8f\",\"1ef90f11099193c58a7b73702ca7f2db4e38888c5f965f13ce2eb112227f9dac\",\"ad11ea527338bd7b84be921b618534fc38f68e48cf286f39f76bb83806d36892\",\"1e3773990b3997fef77d82410baccda11a2a21d81dc9d667c634a1df03ef0c83\",\"e48d1ee7d0323803a6f1dd17970b0ddc24f06dcc626dc8a2576c8c299d233b5a\",\"cddc396ec3c8e5588b92922d1f39228c31755ab56e6871f1586e63810e588b81\",\"fa56eb773a9212f62e4e25c9bcd25dc9f5bd1b3a33a3ef9717f4c9410233a95d\",\"f016300be3d6394052eaf7a759e37c8ae10477eed5ad3845da1106070b05e8df\",\"5d479c52e7fe9bd8c3619977b4f54be4f3d5346e0c6973af13f4f667bdd59e36\",\"902ec26afabe3a35bb5f9c4f9cf2ea2c424d34a56e438166ca366286019ea00f\",\"b60079d90957435944f30098195c32a47dde3fdf67f405933512e0f12754a3f2\",\"1027b07927c8fae06c164a132616c09f925bb660321de0eba3079ece48f91f3c\",\"4fbf6987371f08140a8613be45dbb81e4a5cb75c268f21f49e489a294853a056\",\"8d2da1e430b53432e1a05afac476814a2197281a0a41ca69a8b7f23b6ee09285\",\"93da7f636b6ceeac98aef97c7575289196814463a8a00ee843f56bc665e074d9\",\"67f4b56c033149f3a31894bc2d99fe020cb8e4176b1461a88547e8ce84bdab33\",\"75874b39550a98d992e0573c424c865b35c58b10ee05b3ce912f1e6b06937032\",\"aee501a8a253fca95303d9b8ae2956a0ef86798f473fcf94461952b044af8ed8\",\"5116249efad68a9642770acc154a819f1ce0b325c3b1361afe59af759d55d8dd\",\"89c9528f7f8641e05f9c0d60312862946b218ac51716edcdce35428a12efb068\",\"61ea5b69a20099c9876d6abda7771c99e61ef57cf3b221d8bfa77ed2a0d025bc\",\"23f7d598ca324be18e61b564d3aabd4da3734e1c2f7997970b2a60f641f7cbc8\",\"043f0f4463976db659acedb486ec3dc625c1f29f6d15a595815975ae41a3310a\",\"7089b3ea8fdfbe5eb0943372e3d1758e396203ebce3ad264e04eb044c9990928\",\"a1585e6bf333491c3de5a2a0431f4ba21af68b5880fef327badad6c1dac3db53\",\"53f6aba0f5eb8ce5b64884631117495725184498d76dff64aebd95ac04a0b1df\",\"a317c9c5930af4886cf07fc329aca0105ada9040d706708f88aee4f5f6931303\",\"d2b027777a8023b0d33ee0f3a18e8c926fec22abde54847d02471d41f5a2ff9d\",\"a388a69edf8ae52c8a2e00fe2af3df8f8f8561618b4a5a2b448accc3d6d3bf54\",\"9bf06773c788f3888b5f551aadee05b136226236c56a4f4b1f90097c802908bd\",\"31d4d8a36494979d5e23eb32598dc9a0b2736a44c85495815e5f6478fef3b318\",\"b622fa9bcfbfbf6df30e05cf9bb850e2945378853d916dd89573b6b059c9c41a\",\"9efdbce98e412bc3c7b7caf0369690c63632d0b2dafa3beb3421eaa41d4c1504\",\"26876b0deec671a327316760b3d28d42e53f29a7581367e81ca795c1f7e7ec90\",\"e1a783e64844802afb3be68cf59b5cc2df1c63e0dfbd1851c4de71a2a556617b\",\"88f41d33dba6a8ca5c45a39f654a55c499736a6b0577aaf61b19353e3f7c14a7\",\"fc847670de0ab2bd1413e1a1f08c6fc5ee6b4ec07fee704fd9daf17e50936305\",\"be438389692342f283c7e700ebd88bd7c007a49cf036ecdc0763158a1e7c3fc1\",\"777ac52140935ddee0b41d450dc4b2a7a1b606a7e9ef33990544fdcae8cef45a\",\"a5a4dae1c21668f954ed07d5ac9ddbd509989c32005517c8703b7d3983572758\",\"2f95355ae29079eb161863fb9aa84907e293c4814c6c85a2f37fb53c313c0701\",\"66a0a136767de9d4738b81dd9bee0def2e266e68b7c34b51c4438255875bbcf6\",\"4146206da35b41225d64a4eac4ff5a96bf186f0ecc0064f3d5932b161a4d6f5e\",\"08591fa6dc88b627d65eb7a59fe572c5e4131c5f3e5b9a67a37c405f20da5693\",\"4da3e9cfe5422d6e9ab643b6c3446695ec52058793a9abfbb3ba0dbc63222e02\",\"449fb207c621886c040d5cac81e194f5de3dd3d4e22fd0d365974f4774341316\",\"48479ea6eb73217e72470beba60b84b8cf39544396c8f6af0732efeccd069d50\",\"59d5aead424ff0e087158da748211223b4464027b6e6a8bbc3b108d9f0f9439d\",\"819f2e0d7f5ed2507ea701bac635c50ac192a03f207dd90e828647f844ccdcb9\",\"eb69f5c025bd8283c11fd4c005f8c294fd9c5c4c8d650a5b338f9474e315727e\",\"50cfe23482fc12ec818eba0b3932d5eee34ef6e28da39e1a59014a8895cb56f4\",\"e7f23be5cbf3858ef6ad469661b51279140577f00a2d31607d1a106128e6cdda\",\"a24d4636bb7e7698b337aafb757b4c1db202eb6c695f79bb0c4518548a82499f\",\"c991aa8ff4c90d035ef54154a8129b22a2e0dc46ad63060094baeb19d22b6d88\",\"ac7e67bf9d38c703e558263a7a0f09228f3f00e244d3d2e22275d395a2bd7c84\",\"bd929b9c92b46421a21b55316cd2ef57c3efc78767e3c8a842fe481fd3f61885\",\"14061028610e09f8afd664c1fbe050f305c9915919f8a1c6dc33af953239109a\",\"17cdb613d80a4879159ff07e85ce63df15babdf3ed28f3af6a04e036b8ec8f05\",\"95b8b4b4686dc0a9623d921a6b70942a47e4e82e0d5344fc7c9d52d0f2fe1c64\",\"595fe704655920a57ad7b00a2fc9478f16b6ed2f930955bc60f7e0f67a403ed1\",\"91b4742e5898aa6162282af9e58b66e04f3920bbad21fc967defc1a90c32a765\",\"cd5594fa691c76a5ef031430e6237c146e646af06188245871740075ceb821b5\",\"57209654ad366c24f76aa29610038142a73580fc1c1929d8e56820f21356d130\",\"91b79fb6b43f9c9ec61f8566b4d923f9c63c8baa13d3639abee6155b6ee017ee\",\"fbf24553701df6bdd54d51715c35e304a6ade159c9518044ee8260d434b91b97\",\"57cd12e142601b88bc04200be411a8bd196cf08f902dbe8053811fc4a9723b52\",\"575d1e277ed9a81c4bf346721459a293ff3fe66c0e7245278adbc23a51054021\",\"7ef79563edced810bc311cb83a670fba630d809219b37f28a6d312cfa6f8f615\",\"84f1a5b27210fc7aa2409f9f1498a3ca8ec0e38e632a911312241bea5079324c\",\"5d763ab34737e4d358e9cf5baf6159618f18f7d80fd6554a51234dc6dd45460b\",\"1910589e3ed828f68f0a2a560f18bcbddcc069221122fa085ce2bae5ac6d2719\",\"ece41afd4e08b704234361804079204225078460a62c74092dcd9a4ddca6206f\",\"d2151dbb31c758f08703f4e654ae9e845e8aa6297d2b2d5e0fca36d0f7591865\",\"baa8aaa3205cc250ac4a71b1654641adc7c7d186116be31a841ade31a7ad5915\",\"c8983c28003d7b82dfc6230c7deac2395a64fe4c46dd2b976b005b4e6af9dd8b\",\"0ff920978b43deb5ace6ef7e07f95a41d356d2e16f43f4f145cd3a962d8705cc\",\"415b001caf177be8cce32eec6d9ff5cfec4935f334ab897f110d3eaca4cb2dc0\",\"f5882b61112840c96d35e8708b9b27b1d542a12ff4fa7db83b775dd13c16f580\",\"ba38745f469f5081dab4c93fd56f0fe0fe98f03351aaaea6b98e702f160700a7\",\"d2b0cb32ec5bb5adb04cfd5f2c3bd806c4cd34f508053b97a7e59775e80499d9\",\"19cd04190e5438298f0ec465db7874dccccee7063265744100869038e8cc330e\",\"2ca8a08e2fb2b90200ac790059e2704dcc1fdf7dd2bae305a5af4d3ecd57af43\",\"141893fb670bb51cb3b9550a4776e97b433c890ad5be953f89b842079cab0f57\",\"569ad6aea4b7ee88360ede0b05596663eb24924456d44df769870f343e2b3f31\",\"14c664a2ed2da7411357768ef6c65e36b6fa413af0a51f72f4b7914b332825cf\",\"3850994458ed738e14113417ab4502084236f415fcf7708ab3d6397d153a6c5c\",\"1962cd12dd6e384a329c9a98321c31e76d0a0095c185cb4718c48718d7e1c5b4\",\"3b7cc9bc9c3f0a11565af53fd43a75dfcd345d4345a559e33de5a7ee66fb0b62\",\"6ebec9ca214631f182d4518ab96e17ba157945f462cc82e25a3248ec7d2d4c46\",\"4c818e1c7286e143aab510b83b4ba352547294ebec6112866ab9b67328a45075\",\"021f2bf59ca28d62d0dff57e75396dc85fc813f8e8ff2bd34bf83ed57e7e6fc5\",\"68d511fe4f790232bf4b1f8dd768a2e0ecfc88c499665252ca766e19296f34c2\",\"17a72e061ec114deb17ea7a0d362c44b754e939c4ea7feb7d97c0a785834634f\",\"419d4bd554b7d3178973ef112699d82873ce0b51846add2d0fae5bdbd6666d97\",\"7bd6c97483360c433936425dbb7e59ca22983f6975d6454e6954751282a4ccb6\",\"6195d819e4080739befe6950d9171d76989c689b0454f0897cefdee6ab10a198\",\"b29a756b378ae5c1c3646271ed7c97875a3e99e3b472ee4c872fd6f3768f6c24\",\"7594a86af0da9cd5d254f9f5cca93d82785ea10e7281f30895c00ac19b23cfff\",\"e82514ba0a5022c9aa2d5f2bb2742c932cc137f2fae5d7fe491a0da1278f5903\",\"9325f40a14e35889fedf80795e953642c5883779d4943d9ce24cc9d27284cd14\",\"1fe41054e2e57e78dea0e77bfb033083309abab91e37ac638d216018feabb20b\",\"8ebe06aaff17132e1025628a85be4fabb7f6d0e3ada49464261280ca7d1cf985\",\"2a207700e66950b4cfbde293a12210a4484dcf98fdaa3f16173d058b204dc330\",\"6b8d9dc1a2ab82c1a936120fec44ebfd134fff0c700b7c91d57354b0943d9469\",\"24ec8fc9547aca4204e4889be2e59ce06eeb0565373177a1b544ec2b53e7bc80\",\"f17183efc985fc25e882914bd0b206a17e390d819ff327c1f5e7cf814762f24d\",\"2ef6e7b73d43940c48010e22320dcd75838d381874f98d10f76175c7041e67bf\",\"a57ba88bcc3d46a764d39cdeb193d20a8f0f20119dac357283e9e0f6dcb2e82b\",\"751c14f3299739ee6b936ce2d1407b668b797862cf8eda4ec9b81f87a4394375\",\"ae45957503595e91983a824a0cb55c65482425e9e2cc71c19ae861b93d8ce17e\",\"e0d21b80bee5eb2edc0847d4e264586b1e0c50050009b2b291f20a51b47989cf\",\"9fb4b98c560b1689d49a7690c54eb58a9218494f30a41704bc1744088d1f774b\",\"87e29dbe01271ad676c2c56496085c7c0de25756e4b138cf31d8326b164e9a87\",\"b4acaef26355d750f765a0a36a14f9481c521d87ace83a0f0a1ecbe8fac57870\",\"d3cff182b49715e15813a99c8f8c57e36c6151ced7a9144a75bdce960af0bd9d\",\"b78391aad0db0459b3c19a3d8a4b3e7cadd7c534b9a5f9da5ed9aaaebb3fa65a\",\"3de3806c78a68b595446714ac061cce3496a592f8fd3cd8654038daa29535a74\",\"a9447b99dc092fc18d875dc18cc31ee135fdb230c5f0b46a8ef8ab29c39ef474\",\"a1596dd523a43aef5e8c3e4a35285996ee2f9f92f42649c34e58be33c98340b4\",\"f9de05d9c053a66c913709ba2699cbceb053107ee5ce66a3bda53b1cdfcaca73\",\"51880886b393ed254e0fc8e27ba471c1a860960b7a1c03400ba281b8fd4a653a\",\"50cbb5e6d1146a1db74e091387ab82a185a5dc2b55173dec748672a60603fe86\",\"b4229841740e981d98c3507b7b1a9ae60951c973e6bc1e9dd6901b15ced011df\",\"4b8f3a02cd17956e26ed71d95fe9a09e150dacf027564d855c59582ae3f64302\",\"b1563c10fffb5d670dfdc45b298e1c2f73a54c1bc12dffb8b32346d009bcc8be\",\"84a04cc3bd045529505209c6591196d67ca4cc56557584d894a67bf6aeae5c23\",\"52269c3f02fbafb9568303e788688570766cc1bc5c5ca02364a1e0161ec49d81\",\"73f6a142a285c5813c02f0c7f72138376623c44b67dd95737fe7eef222a8908b\",\"75f72b2742a62db02ca36ec46efb07660f9d86793b6623385eac3ea28514fa63\",\"e5cc5c193c5139a562004d696bde21efecca653f7e54bf629f325aa7f73e26fc\",\"e85bb4684d6991286da0392d1a22840a28c99bf5ba7ec335db76d9e8e9aa83b6\",\"c6ba6b95167050e23586fe802b3071d62e8f85699d7fbb1a267fdeddfdc0a90c\",\"3a7f0bfad86b87f3946e33660df1383dec2a5fa540f775182045ca72f16f45a7\",\"f95bb081af6f66e3434547f0adee304047414e9e0653456e96c29468caa78e4f\",\"b091b535988131b9f40b86c21cf77465a406f755ddf1db0e7cd9bbfefef2fba6\",\"1b2b3a71006b0f27821b96fc60ef4c63506c66e42f296d984eeda50955f18e60\",\"c0b016307fda686c302162ecbc0c427894bfbccd483bfe4f0184039f8a8f8309\",\"2b6532255ae2dd43faf6d6b536d7875f91a7fe887fe63756400a1d41f1f7fd7b\",\"ab63a961bda5569826dc92407cfd5737cbf334d30ff64f1487025c0a7ab98716\",\"87cdf7b7472d9f367f954dbc030684ec6ddc4a689b8467dea8097687e65d6bb9\",\"8546041a79054929d3e8e004184e0e3f39a6967df1115d20a1b4ef2efbc4a345\",\"f7dbf6b454169f9e1f6aa628493ad84ea36efa2387284bb170d4c687a2fc58bf\",\"9cb3a650d132bd8b48e4fda6099c53418dd534ac8ca6fb0aebf4a6c8550ffeff\",\"89b2acebac970df180cb7bb7dc865ae5f2943e08e3f0b10b01de61e3ba48dcda\",\"9d031243e66fe3b5a2ac3a3c1e23048f8cdc141bd94cec1f19d6f6e1f97077ca\",\"f4004bc500124dd645efe1ca89ea174a5af9c9b668605c8fa3b0aff3f0f9c791\",\"6efe8799b3226a14b0a6891e25176b9318d90de611767231b651a4c8520de9f7\",\"3366f0ece56d517bccef5fd5f36ff38fbada692e785569867a9bcf42730937b2\",\"08c0f96486043b932b225bbc462525cf2400626a1a322ddb612f36a42f2c8472\",\"fdfb430da069a3d59f6ee4986b58d2d37832b0cb612470a73d1e7498a1cd0798\",\"51d07b2a47d141de0b7d953e0ee38ff6ea124e05e6aa196fb0a6ef845b2fae6b\",\"946a04115da3c8951ec743ccc8dd0a4ebda4386274bbf7616b31b04b14c6165d\",\"4d3b80c455d508cbc8c5c469aadc5b5fc82a9dbaf86ea3eb53b3ee08a4a4a36f\",\"63b13f37f253350293aad0c69fd7586f8921992b4822d0e08498bc85b43dec01\",\"fc5db46e47b09d7e9e2bf6e84523829032558eac95a806bb2fcc8d5c24a4f1ca\",\"7caedfd960c0bfa4c179b132df7c4b13361d97d09a44263b7d21e490d44788e5\",\"1ba4f6e405d761b53b9a33ba27dd457a7516c05ea3802f8580c9a93a8ed2266a\",\"820ecc12a51515a1fd5176029d3ef18101208eaab801b018ae7e1cd3f7514fdd\",\"404dcfc3f7f783fabde3f0ec4572094d79bfefe32cb4019a56133383f980f9cc\",\"948aa7f6e23034773802f89ee0cb3cc116debc78d9be8bb88f0838490a043542\",\"8e8e9edbea46ff07818a5e7ac5ec19cac5da2717120f4efa64782891e1bf8ca6\",\"755ce68dca64fd969cc07dd245974f93fb8bc11982ac1289b385c0df786568c1\",\"a6d155c2b84d889caf2d94ac01f9951c1eb7c5c4f953450212e170dca6b88c7b\",\"6ab39b9dadfb1fd3902ea38f933c5e96bed3646e140532fbd27b473aebd100d5\",\"1a4665e4a6bff7387ee5f8bc90c02e95685eb1bca806599c3803587ac0e24bc5\",\"433dcdd2635cbc67578a236bc43ca077e0e59381174e90eb5f11607915de3a88\",\"cfe2439b96191a15f48b50e820bacf9f97f07e1668d5e43e57bb92e5414adc78\",\"be5b97b201fff3589ff66511c55884773526c5133fc435dcd65f7059bf7ea9e7\",\"f765c707867848a550ca9926d350b3fd07864fd4c163dd891e7dc8c191f8383b\",\"30fe7867bc5b6ed35d1b56f568c5753f6586047990de05ba0dfced0dbc2699a4\",\"1a81b50da4893187960a364b3d7db90ca0d1c0ba2915023c7befea6e45a9cbbf\",\"b5314a286405e9ee3261251290a3ea6a8a0add408bb9d70163ad9c23667f0c21\",\"8822261726341601928b914da69f8ac79ddc7787eadfb6cdddadfd1a8847d698\",\"bfe85ecd29c8fe281fc0990e2f5ce21a1605e2e7a76c5a414c8358b4c504959e\",\"d297da56d31a268bd746abf9d21564122e73d240c471dac3f8326fb4f929ab9c\",\"ad0c7fcf5255d126a4d119785a9b3a63d4f9735dfd92ecf10c4a879aff93451e\",\"c12c6844a68001fd7254535b237af08c643badd7056078431564f7445a702384\",\"655e4eacf635da30f5d85665cb18d48c06e5492c85e66047b3452a85812c9821\",\"0c2518b8170c439cf40d02200e7692e0a865d6993afeb425c9839650c72f9700\",\"cdd379759394d0ab348647dfc7d0f410795b91e2792f6bd0f3bc253d3586d8f4\",\"4da4cb07790659082e0690087a4cedcbc8bc2c3f75f1c27f786c3c905cf12d71\",\"be1839caa058de7d889414cf4a7aa40b256e82604f57a0c74372834887d3707c\",\"783dae47e81179b2b6757d4b9e2305c239a7fdde693747fe5186b16e52cf13ab\",\"0189a5d3b1f31bf2e3877c3842f98a224b424c6184551e1af5e35e9dc3d2f4b8\",\"c1496e3ffcb0fdf8f4e6de5cfe2c9cabb2b44282b05cd4f2060c4f007d909efa\",\"1a751ccb9059e1a0a92bf1ca59b6c62c895130cb1aaf73d35e047b8d3b75c909\",\"b5254fd892e1619efb8f58ccf9709bf3adeec114bfdbe62b92048b22ac6dce15\",\"871a6350db1a8ca903effd97c46c379c0169b4707e19332514ef7c1bbdd5d4cd\",\"4d44d8867167ec1f0b4eab245a7b2322c1b4b8365135d7967044ce6765281f08\",\"111c04fc76d9a820b32c39130703cf4f8abedd2063c3a2d435dc11141dfd913d\",\"2a51a5be2a59ebffe584596a27e87530e72dbb4ece1320c530751535592917f6\",\"964d07d826cc8926a540ebecf6e3340f1c79382248d10a0727f6fd77cbbbf552\",\"febd4e4ad3548995b5b3c6b1b65af9b59c9cc5d7de3327c650885f8765f7fd00\",\"8ad1579749d3f0d315bb99fd13df7ca166e683d58944acf4bfda9551e325553a\",\"658be6b593dff80eaf4bdbb352bd0390d8f6f8f9fb60fe2e732cfff2fca9fc10\",\"e6aa66bcc2d0c30cccff142fa8017f700c5a5782ca8fad519be462908450356a\",\"99ec257acd3f3bb53721f8898b315b6a831eeaa2c736e4da29057767664c55b2\",\"fdf56350d1b99682481c02163ac6fbba1db9e2910aa9131005763771291376d6\",\"8f415d0541cab8dda138f464a39318e5f4ebcc511a2507b5c199f7cf0a889d8f\",\"b97dcd48fd7724badc167d7567063527a8f7d02355c33616f3b71e82abdf069e\",\"cb51a4479a7d8779cabdfe01f263bcba5df3f632e7bd564e8d5f2d9f1f3058af\",\"b4ca18f997e9c7324313d8e65dc1c41ba8e894d39c5c3ee46757075e0407d77f\",\"121274cad994b537dfa075e5e8bb36842f29ee9c13c70bfe07c241d5967f1035\",\"6b343676bde9a152e7ffe3f30773aa6f342d5e66162312ea35f460280a6a6b81\",\"3f9083b17d9ea8d6e9ef2c8c2a9468ea7c769075c879fdc6260e46ad107fd1dd\",\"883cc748948cca54ba435d05542f561a43fa16e8b6e22f3605d521459da1a348\",\"303d8492640da7c967f460d2faa2537c06fcbd1becfe65f63add8bf00f5011b6\",\"ac2c85778ce4c15864164da103338f08c9d7f854cb89924f094dfe96e2e7e1d4\",\"4cfc772710cab29b792c0fe6cf357de06c1228c381fed62ef3461154c2f01e60\",\"634edf32babff839bb2f94443deca2d5324ae0088b737937f30499cbceb98e06\",\"5ddc54c4c53efeaf5971245c34d088b9831675cfcdc1c3113429f50823b7f68d\",\"ceebdc45db2b67ba1a688a515e5142961f23312b29af368c1a19e1f2be003eb7\",\"883ddd7795086758b9379bd1169c78484a6d64bb855c6b107f9a56cdacfee05c\",\"e4d375ad10b19b4cb4784b7c4383c5ca5677059dc441b972c4b62a9fa1e9c77e\",\"f397f7142f70968b8239e8803d9f0eabfb955d6cbc02f7ec1547e9a8d10eb31c\",\"f8229371c2741252e54f93aed39d771455d6c307d5dc410ade4d3f4791270c98\",\"06c8e4cc6cfffb76a027a6d577015105f26d7c88368b036098978c385cd7bbfb\",\"9aeb2b2a731be4dae937fb6e4efb9708c964222f724087d7e278115e6175c5d8\",\"e51ca6b84fd58a3c3832528ea9d2adeef63dbf1bd62d654437b758fa85cd94ef\",\"9e2810b1adf463950bbc3d4f6d1e9e2e364949836d7eac6f2db20dfa43210446\",\"e87164689fe42d4ac56184de249ee17b288a0e8ad18922edd0243800a93501b8\",\"d8b769e75fcdca1bd8522560165ac3a4302ad82ad5211320d7604bf7831f48cd\",\"fda7df7ac6f4582303c2b3d8f3a331f6c048802b7dc3e882997e1ca1ed1d6c80\",\"374b21bc3e583181f48f43fd4374469ad0ded484cb325645dc38e1c71ada4054\",\"1a9729c9b91298e57388dc473740c84a6c5ef690e9f5d1152c344ce331e8de75\",\"4a89e48bf38678f64dba34912a8b8a3f592aa0a3a2689be75f2439a6194dddb0\",\"cbbd50ac27524b79eedafc512f07800cb200916d9f947ea0c92791162f923b00\",\"4332cad9e5f2992209aaf277b6fc5b4a7b543c4ba56352bd4137abc31c6aeaff\",\"c9bec830049ff741aed36e0dbc5277df6372a67be912fe2c8ec036337421d926\",\"7c6410e51b057652b31035985ee7144361166dbb34bf1270fb475f7b729d17b8\",\"f6b2562913bf1555d7f3f636b0082e753dbb8a5395990b75087ccb578a45e711\",\"efd1d76419f2c385551d9ee2e955698682c09e8a2695f8743f12ebced6853e17\",\"9e1dca140b979a656d5c533a2370b31f746773a78509785436e4916f907367ac\",\"a2bc3820b5c161bed8d61d1de99ef435fd88e7edd3ba166aae6495e6ee0605e3\",\"a22ea49748b4e7608d02ece36aec79545ff2b7c68a7c8f26fdc665e3bd7a5f7e\",\"00821a1154b2af7d8492ca4ce8cb7b8e7c2c671daada576728c639a6d0490bee\",\"a7a9f2691e16168cc5aee3aede838490b8c8eb4c631e1f3b2d501fdd05e102d0\",\"fdf9caa09b966309b0f7273f94539718631fa6f426a60b241f0767ed0cb32c25\",\"4fac8155ae65e6206e1febb5a1fce31ff5a4dbd4ddfe6b3dbb70ceaa03e1f05a\",\"1fff90f7609486cf2476c47c118c751cea19d9303dd8ae612cc84fd726a83a2b\",\"dc117aa6cb0a3127cdf5349c2430c42d7e156b3105d31146344c5864527659e1\",\"d3ffefd07f4dbf471762a302dd98ff682da1f11808bcc81f78593e497aeca69e\",\"2d2390bafe9e961629f2a7b2572d14ccb79bf230872828001bf00e646e8f5f07\",\"58b6ea7c14b3a79cb425e5dc63dc16972d251f19cb43ff67b9b92b13be993d47\",\"20f4466a2296e2602a51e9f38db2be22568a5e00ca66f1438f0a3352dcad61bb\",\"0e07a081a8ca0a4ada397b21eb82e7af1309d7972179d4ef0999b5924a3f689c\",\"5afa708e0d70baff4e154b6a3e99db22594597e305024f2dc1aba5206e5a9a78\",\"f86fd429d1cb0b5d0757d9c22aff8d7fddf0f4de7799652d695b5be3a68faf68\",\"4cac1c8734a5ba8a7b87a723f1d473bb753a8c853ffc9c6c68b1dbbf0e39438d\",\"ad2a937faec66e649f1d8b8ad954b286305e6a94dbbb65fdde9fbe13217cae5a\",\"5f191d519d6f370f6145dbbb45093d32e07f617b4c20c197c739e030fd8bf1ee\",\"8d80e06c4f8d0722b357b06de7f768d12c2b0c0f7e8f6aba4831ed1e70c11165\",\"0ec3fd9db450f589159934452601e7b66c416646aa62cb1815381e44840a53a7\",\"7649cc4ef96fcbd328ad53a01bf2b82e0f20817dc58e5dfa9d10912765af0f71\",\"7e7e440d5ee9ab3cf6aa4499f7dac1427f8ce9f43806f50fff62154cbc47f0e7\",\"2d7891f29fb514a3d15f37af0676b1b13cf324151c1ad9a03ab1374af081e951\",\"4c51f2a652e459f6f35e35c6d755af6fc96faa47becb4e2a2710c3d1bca62cb4\",\"aaad6e8a55d08190a68c13095f07c9d9dc4e1cd024fe8fd3bb70985909ff8e62\",\"090de96fa54417dfd3a89b5a57d35d26c13eed21927561e3d0131f11853d92fe\",\"11460045c761f48e86834ac301c0b6ee9e4b31b1f1ce0389094e2a7b50f7c99c\",\"b63e128ef7085e41aa7660c0df9e88f017276cc22b8a0392d699d7d15537b53a\",\"030e1382f5a1c31790017f9b51808778674d6c7c7abc8ba4dd248f465051e661\",\"bfdacdfb4c3e4f0359e70623b79f39e54479bfe6deae6cae62012bbe9cf07742\",\"07220c856c99ecb1ffb621346d35cfc9c0ea541401a323d8ea973c2291310136\",\"c00a72e915d8631edd0da74459385fc2e74ff408c55410fb685bf7346d299ce4\",\"51d1558f2e553ef1e75bfb95abadd20bac032fb3f0068ca67f0eddb96ac885cc\",\"65f83a32e8fcaaf118f9ac45f984c4135dc75977921e370df3731e7944ff22a6\",\"9175b7804eced2cd67ea0efbffc11744566e16a813c2de8cbb8e44f2b5029ac3\",\"82e75d84ee6b7e7974be4a796680e6f0581f7443a3f8381900f8763041b5a0dc\",\"be4b79f253415b2202e3029c15176d05fc166ddbedbc6ca0537500d860c004a9\",\"ef4bae8e3da8982f9113b527eb8ec614c10fec464435ba6fd1a8a7bced323970\",\"fc868163deaae000b95d25eb5638cbbd025b68793a552a186d3efe1bc26b5d29\",\"84d6fcbc8cc63d3311e503c808bd6c2c2215699868d5571350b141b89ea4601b\",\"da579837637699e06d0ecf2023fd4f56c30adfec71995e4b4c075dbfa4d2ba97\",\"28ee56c91487ff624908f2560550708c30439481f6b9879815ffab7dcbd0298e\",\"ad6372262c8976b09f092f49de78d995264672d3ec2d3a0ef1c5dfd827ab8755\",\"165dff940c3301e7a99e193299286d01e69fb5fff045f3eaa9bdd6f480371dfa\",\"a9e9ee02198842c842a7c6d876791fc8a12a783091d1c04bf55cced7c4728f08\",\"7333e2539cc0207b2aad7b85f515e5e34d6d2414694102e2d6a3fcd1f739f529\",\"2fde35dae0be6e1ff379a17096ed472ab4dfb7e4db4c78e5d435497a2b6f369e\",\"0de65aa333fc3ffd7db646a535eb3e437ced19a371cbc438df8b875fcc7f30de\",\"7ad7d7029db5e180b31baad57f62555bf565b1cb620f705e3d4b1cafdefdc873\",\"e4973e68053d8e8ee0a1e1cd87c8f9909ff5a855e657875bac1dd54cefb5adf8\",\"fd5f091021f63d25d529fc316aae0b43df0bf23e6d57cf23a1e43ddef3ad9550\",\"0b8eaab055f0ca54045e4262526630d4a365f722b867062c27cd50cb8a668f6a\",\"f65779d4afb9dfdedcb65de0d8a7d184ad7def3b994c8da2846a3dbd45177dbe\",\"5e6f3afd969faaaca3325d885c3d4488b0af47a03f7fa9e519bc7f1a4f833b6a\",\"15581e46ea17c06e365f12576e639f54d240167235ee9530a301e5d237368e9e\",\"4f7e7042bd59ac7742a1bea506d5ccfa9cecc27d70ff80f975f91433e385294a\",\"090834b2c1f210dbd34536e372be1d06c35168ed176f238d056a850187733f18\",\"31e4ac88dcc854c36afeadd751808db4fba6cbe63b50f8c2050142fecfc61c99\",\"957af783f6e69638dae637989500d4bc04137083b5474a628e9f46653128cbc2\",\"8dbfad0c9460c32afa2ab62194613cafd17c4147639abc35347fe41519a72def\",\"fc8a79d699faf09786d3c11f082763de5a43aeb47b180e7eef46931ed8c2a97d\",\"4748ed880872c90e39078155080f4930282330345e5af21ba45b1438f0b613ea\",\"9421095d47bfd133b8dcb7d377d15c731ab496abb05e73e1de1a532ee62cef16\",\"bb644b8a003f200275541469d6d4376d234fecc3dc87fa9534cb369c718618fd\",\"5acb5e44811d47fd652b3c878508a1d0275f0c344dbb1e0135bff49f7a3226b5\",\"30ea51fbdc6414b3661a35b9b7aba897fb3a23e1bb24bb173a2439350d3314e8\",\"79c087b12a11bab392cdb3f979cdfa25412f1613c134f64623dd2cb93f722bef\",\"19f66e817983a622fbdac0a6d58582267b274f99279f6e37ec00b32cd6cc9b27\",\"6c31d9d167394978778bb3aef8e6f93648426bc63d43bfba75313eaa04ce4769\",\"9696b94da943e8e5b58b8b58c45ea99455a0da42ead67c2fba892cf59011eb2e\",\"0433c3167044bd656d1698c025e5470bd9f10ddc4c8020a10322203704ceb3ce\",\"c55095ab319b4b4bc9d74e6e56318387b2a569f49641987a4539d0aebffb4c5c\",\"ea010a658765cb67f2278b7665c9d9a214f36ee00b0105e8d1535431f0f74ce0\",\"717de5e3301e9538af87947c4eaf036c53bca11c3f70b4a614b0e4a316c54fbe\",\"97f0f767eee44264717c40ec68be5c743f563b97d3f5b2c10770fbfdf8b361ab\",\"de42b864161d3b190a887663842b3fdcce6e9a5106c73ba33219bf05020aabc0\",\"07a17806694d31122f2dfdf64bdde04cfabb65805a87f3c8dee249a38712c109\",\"9dc831e39a4f45364ef014528a12043c5fb11f98b6a7c95724a20e20b3e0203e\",\"2197026b066b5656335da3ab7039a352a9899033afd32f57d596acaa1f3177d1\",\"bfcec3c7191055066ddbdecb0a5282bf4625366c174e00c6c6d6dcca72145d09\",\"399e961c9c5c3cf2bf4810b0fd15ed28ec27f71894a0238c10cc0f6c0bd92697\",\"e383e181b0476329e45424fec062d3d28282bc23539a4baefab70a1ecf7f7afd\",\"538e05925396a1e4fedbd4f6be1cb39121f48f4c2ac6c726a86ffc8d2b84d9b7\",\"f76a977201c38f62f7c63fdff963473129e5565d28879f3579eb66d744ebf468\",\"563b84aa06a1f788dac5d788778e80b5b26d0821bda3e60d332bfdfd07719920\",\"30d0df5ff8d037f94b3f675dfac41c2b58d3ad9a017256beccbd0e6f30985da7\",\"e21abb9a0bd78fe991cbcfb6da6f423335a3c371ccc224668b500c8d75563b68\",\"6017157c9f0a4d8a862c811b652d37f7cc38b27159385a4bc5b293e384f0744b\",\"f8ced658d8881211174b2e2504c4516b503f3a3f0c082f68b0ec4bb88fe59428\",\"cb82205df483f8400be6e5ee6924680ec936269823cccae3d3186269ffb01460\",\"a6776575220db2b0937f05b462f7fe44665471e91db4cc183834337bf6e0c449\",\"0f968e4c06f2d78c9a8ea684383a1b8bafe38ebdadf26c789dd5b06505dc2475\",\"7a19c8be931a923c6fce26ff767abe5ddb070dd9c3f3fc7861f94fb3b863de12\",\"e9fc05aaf227e9b7238a573ebb33fdbe730590cfaf9925180b3f1766d48bc8a0\",\"e7cffd7265b3fabf35209277141abbec56ff3bc08bd169c147a616883b1c20dc\",\"e7ab8b32e94a85028b70902457f58ebd93f4cfb2945784a6469b0db03d710e20\",\"96506b27a8a1e125719fd08079c1822c1682d51a49b4cbe0f11f25b4aaa0af0d\",\"a4e94c835ac5de3a73db94a380f827816ac92620480244acdd591ec1ea6ab0b0\",\"4cde2a29906ffc734387d1da30fc7a10f2a3b82f3744ab95d1add2e7505ac531\",\"d474130fa53ca2fa698548c8c23b4e4b68de2dd8c549725b323c376eaaec9319\",\"6ce5370cdf405dfc73e59f8ff858f9b6eb329e64a1d893e68ea026699208884d\",\"8afaf7fdf0c4d9183529894066b77192459da7af5f6edac44fb178d9dca9e7cc\",\"1b6930551416962058759abf68f0898bfeff1ce7582d06fa7fe0dbe89e5995f6\",\"5944ea9f9d37e46349c130d1561116b7c02d839e18ffa149d7ef74618029efe5\",\"e935dff41524dcc01f7fc70dcf72e8e07f2b7d0f2382fce99cd0cae15debbfb3\",\"47d7cda49066aa3bcf7afffcecf52a64e74ebc55935144a64f3b8f163055255d\",\"01cf33582f11048ade0f69778900054a410cdb21559635b46119f905c80e60d1\",\"152f48711b9ac9e170856bfe97cefa231f024e322a528285cb65cc9ba46c3518\",\"589788d432ff6a0a7ecc3dd564c68bc7574d592ae0e27acf730f3aa9b1d1e504\",\"0eb66361334beaa9fc32ac85be141a0023c9db2aefcca95c445c9759da284350\",\"276b0f2c76e8193592c37c081e11a5f800cd8561c63cb3281ed139d453a29d58\",\"12d7fb903eb7036c7d13de690a7369adcfc9f4e28685e38ebba64100b8992372\",\"85348ae09fbf1e92f5c1fb39c36d3b050795255344eeffee6adda8be2db5f8db\",\"4cdd7a82a4ef659824a4989e61494a0fd039559861780f2fa0669e6af4280bb2\",\"0a4c90e792522f005d500f0c4aeff18a608f58ae9f15ea222e98395dd4dd03b5\",\"d4a5ca3d915d094e2fcaaceae5f09dc0188cb60673ed14f7df8e70be8051357c\",\"002a337c5ee2efec9909239949b91d239d60aeb8b03c69ce546e3f52bf09bcd0\",\"6fb3b0e354606844f5a8b24d9df4bb28d7acf785f25060134b9639dba79ed61c\",\"6fef66c81e716ac8d0b26fe8ed67c96eadf8b1334b415a391a016d3861d7cf10\",\"3655ad7de036497701f9a608aa0b8b8b9de7784dab7ab9dddc944cece68f77a7\",\"504a8eaa8cb78805650cbc8fa061d8d33e36d1597e2857a5c9548bf9c080794c\",\"87c411c597b60d56f504c146757affa57483aba4773e511e9526fdd4d6ce46d1\",\"3e9e7acff3b9e8c6ed33cc686994781dee8e6232407d3b9f0806aec6e6f15d38\",\"f64ccd7213621ac92476235cbabd342178bd1eaf955433757fb20aeffbb4ea2f\",\"b4ceeeae26e439791ed2e3aedd13bbe6e901539015a64a1a83a32dd8695253ad\",\"cf6ceffc3019fcea9fdbcf124157628d1f69ab5a2f869afe5c6f09a6b975def3\",\"cc0e19f2469947671a519269f8fd783a8a1b38bb148c4de0511fefe29503029e\",\"08477a5dbac18e67bd58ba232e141dcce57383381f70fc5378257403c0025bf9\",\"48b0e0024f539b31adb130b168f11e52182376f25a9bd72faa69854f3c44178e\",\"c77c3b09701adf24fb8058998849ef086ce7219cb41b74116d4c13edd202d833\",\"f5456c7169a14ccd3bbcdacc42e1ea536d655cc1670e40dcb54410af578d1a60\",\"ba90066b59d3fd13d05cc1338e1896f31bfa8a15e04a0c6cbfc4ff76fd9d998d\",\"4f49c3b50e30333d858871a035f9794cbd8c046658b255d0898295b72f8ae33b\",\"d7143fb10cbb791ff840b7b6ac8b8a600d7ae07fa29eb3ba5b8981482d86ee6b\",\"60058297f98ae05859ad569a66be4b88714aba6d8ec0e99c3e1dafc9253fe158\",\"a7a7af598298f00fd7eb23401efaa210ac1d7005cfcaa9fd1164b56b3e564172\",\"3dddf32c2b3349b751174baf0eeec502154b539957ae3668c8f5d9e4535cd6a4\",\"2756ff7d722f277ff472f8630a51d2bf7219fe90e9e86c01c4cd55e65668fcba\",\"c2940fed5beef958a0881df042b2b49d657d5cd52b73c7c29669c7b442c15ef7\",\"8a58f1a79788454b3120b4a515bd871510e4f4217c1ed70de245357e9171bfe4\",\"2c7cb376ee794a7e490396d4dc70c057f7f1f53fd033f2f43605d45ee85dfe5a\",\"4253fe95f65335e23282e9f02c6d6119a791d9faa9b993767a2b307d2c710d76\",\"a4ad1958145ca88458ebe1fdac4e79f6a8e83fefbb82b8cd24bc018862f9df42\",\"b93085758663877a0971f79efa6fa0d3fe8c1e4fc15b2629a21c24b88dde6c7a\",\"da791a3630415f61fd260af38dbcbfb2db1ebdb5a08029f33a831f781920f9ae\",\"a43dee50276a60fb92da81ff327791c013b55284b855916994d3398f310b26dd\",\"6fe95d2ebee184c7cb1cb729b219b630362bbeb1517c30f7b3a03e0d5bad206b\",\"a879134752b09e1c9334d1b64ce503226e5a90e9655bb7fa075ca22930ac9035\",\"612defd86efb38d031ca2335187230c9b7abb6bfb18a268a36b363ac0b058e8b\",\"efc4c4bb9da0d9885f67f685818d4ff8fd39284bde571551ce2ba5369c99fb2d\",\"9d6dd81639ae23607cc7edd5834bdd7989359310250b3d3e2ff930bb4f085db2\",\"c9dece6e79b59e9e65b2cf9eab7260b8331d4083066d4db92b73f0f7b8e3188a\",\"522d74b60adb5b4f2aa68986cb3f4d69971f561042e828ddc91f3d92e7ea7939\",\"62348079a27305f5c86135ecfbbef0d3875757e13ab5130cb278f2f3fc4cbd4e\",\"fbf27e3ded6a585ba1bc0bfb167bf1b23fa82b7b3ce546aadba77c98011569b4\",\"2e790e4c0488c0e76da3838f77bb2101330200a5f3634d78562cad3199576f80\",\"0fb72b109780a99e5ad3351d6194abcdffe556acd2dc48ac9cacfa42cce3e0c2\",\"5fba5ccbe3687c658690d576641631c97b9415c7c87f9631f274031d8ff3511e\",\"62caa2f08ed31d3490ef4fbaa3300ea54b4fe54618a3a4be28bebd30963acff6\",\"18a09a043276c7364ddc2f20859c13694e0530803c60710e980347c2c20f9de4\",\"f562c84154659f6aa3280559baec9d2b64943c53fbd113e566c6e763f561a8d6\",\"7e44b6383a18f11969b9aee563321e8afe2dc963e0a3ec911535effaf79dd7a2\",\"a8023e8bcaf905f28298ec7b32d1083cf26e26a14f9c70e854ecf8c3e2b8172b\",\"d5b8f416ca1264ac21a33262bca81c5f70374a60f3afc5e080625488f76ec3df\",\"e8c4336293a4025d296d5c6596a30559b8c3dcf8afa879068c73acd4d079965b\",\"bd5bfae41cfda7a5f2dc6d6e74732dda5db9f1c1cdec2b217657bacffc22f983\",\"9d9370699792af49664f933244a9ff1c275db1915d0b2421e977140dba930b9e\",\"e78946170614d2e5e10d1ecadb399ea4119508a1fe0f9e1d61a771a1158952ab\",\"dcaef942189397dc87aa00e8b505a7430e31ef18acde8ca4392b98d638fe6970\",\"baedff5a9c195b220616ade4ecbf1b140ac4719ebd2e179d5d9d6315fbef921b\",\"a2374a38eaaaa556e990cd898e08dfe8eee6d101fd1593fdf54cfbcd903fea21\",\"e963c7adbf5b73e91134b1b8ea7a6c2e179764593ac7c75616fbc2aa1628d132\",\"9b083cf31740d4d4b9c875fa6482af7af96753ff379ecfb6a38ff290eee22fdd\",\"ea2105f37e95a6f4dfc6e429dfb812db7ac99f4641dde11ab8e24423e5203a2b\",\"24ab7851c061ac96ecb76df9c34a42ff444340a04fc91ec5ac7628156d958be4\",\"af86fe8a56f291259f40d42f1e52ba459f5c9c415b301c622ae36c5b4a47f5b8\",\"88d88277055298a380109a999062ad5c3e4a38fedc5c00459b9f5e302dad0d00\",\"23dabe8100c58273aa926e4b8b175f57afb7843401aa4fa748eda8b035f6a73d\",\"d475b185bf15b274c573a11168b0e4e5ace7ff1252e6fa81544950f514c157d2\",\"980df824745d47256ebd0c34da7a079b3118d162cc5a52517c5b558401eb2a4b\",\"3e7ea346a631446b23a35b89721519f3657eb5bff8ddd8686e0fd4dd0c1f65e6\",\"24206a1d7ef923bd7733886f694311966231ccf9eb98ee96465e7a58ee7b3f0e\",\"90bc36da0177b06d6d5e30f98b5e03dea72a7f805152f4c80afd52278a4474c8\",\"46453351bd73662fdd561fc5f3f0d3db9da6c0d3635f299ee1cdfe7bc432039f\",\"ad2e8ca78e4abc11ce770158b701bb9b85683daedb1005482fa7b58253963ba7\",\"2d83d635fa68569ec14e72138e06bd7583df76513875d234b0c6730abdf51c68\",\"1e77e55d0ea84b8f8e8eba44bdbe06fef3735e9385a2c5d12707df46bb1d1972\",\"4c6cfb151c1253c34777d77c2156dd2bdc52ee803bc28db29fbe32d05870f876\",\"0e097555ec16d266a88b95368b86c203288abc3d05023789419d5292bf8cb2d0\",\"b6e832a454805528989913a15cdcba316a910e01bc456d33261b78e5116c12e4\",\"48d79b42e5f8804cbb55a79492876681f3753c7f9976046bd2c449b9f2623ce5\",\"7d059001bdee60fc795394b37dc87bef770fd9b6181674c425c1424f96a353a1\",\"fba01004413be88ea4ff865aaf253bf84b34d4d24fbf15bc6fbd57d13f38e433\",\"c67bab5be52f7c1ad442ad7536f2ab56bfe5cd0fe2db8a0e9601818b93d985ad\",\"3f0feb082a78788c62978768d58a9b52a2aba1305e92a425f6afeef4b655c5e1\",\"8e8db1fed3b2e8bb7847d85c83accabad3e458242452db6750b2f4a354bc81c2\",\"ed575d6268ed87fd473669af439d6ec0c8f12924f3b649ea2ce27543d3ebb53a\",\"b7b8106f5d038a606e38d65cff2936c1e29bda9399d73aa1ff0a2fa09242a6ed\",\"45b8ee3bbf28ebb19131fdf5955d7c87c3f5474fba9bca292533007ce98b180e\",\"6e0c2c84c7809ef4fa791bdf624ff3f2b1c963437999d828d5908b22cb7ddafd\",\"13926027f11fe7e8c4be9efaecabe0266c74807f261f9692598095e43902146a\",\"b538e7e2006517fbae7a0744ed0f85f82788cf34eebd2a16727120b1f1ef0e52\",\"467a609c9d6e3540ce2bcc8c497ff45dce93351a94984019f223509f4d43e6ec\",\"8553b7cfffec323bd41a18214c237255e89eb03d285063840af63ec8a54c2a7e\",\"93d13726f3262dc9044098a53e6594f4a7a2a28e4de7ce54cc34068b6d6e9926\",\"c1fca336b540c1babae0e7e35968c0f79f4dfc3d9eae91aa219ac5a2d5524c25\",\"0a23a17024e18823f405759aad965dd60cf27714349b64b73023df3d6dbd904a\",\"3ce61ba0131da965a1a94101d7ca72bcfdf425c09af87157e4195e309996a096\",\"e26f949a70b24ffbeda51edf14dc1957d7fbcb017852a7161bce01f99e10ba3c\",\"6f90844f4738ab9a1da9aeeaf4e7bbb9ce302b99ba5d15a922a2770d316ffeb6\",\"79082b48a961082d9ed53dea933253fa38873a399757b772eb98a1832292924a\",\"ad8f7489656a2fa13955d8f7dd53fbbf9b85af4b6bd06e74b0f6085461e1a2bb\",\"7b1d06d7932c7508e69458b995f64f612e50d8cd481a25918992fd7ff5921603\",\"a2c265fdb68e552b88fe2d732e574486cb93e1e4fe78a3bf6ff93c5ecfc31cee\",\"5c6590e85300e81837fa4d6792799c8c4a4dcdf69610ca679c5ebf671c173d4e\",\"c01323e4dbeae9eaa6366c28842caf8e34325f224a57b4ccfe2c2ad33e6be814\",\"85409254ad6491e56e31ffa0b09b8457e27fbdd61bb67fba456a0ec8346b634f\",\"75b423d033f79651b579c88f540ed3953239f7f188f1aaf02abc5fe0a9e929b6\",\"e927a6524c84e03f66ee771c6c2b8391790ace29ce843e2f8a5dba03e1271f85\",\"e29e91adcf66f27c208215dbd6d9cab22ef8cead113db1135b5b4eab9ad8b181\",\"95efd8312a1bcadfa08d874c9e0718f7b42cc184599913e8065fb0359d78348a\",\"bc91e7b6dbf4d58d0b3fe4b19e954ef365927be24196e34191bee7f7e4e265a6\",\"66051b9aee24013bf3d19ec3b95d84516e6290e4dfa0f6257a4514eba775e282\",\"9998a19d3e510846d8fcd555e8bcafb3940b8e12315108e2b822a120748ed093\",\"40175c35b4cc00c60d4f0643c83bb78ed6764ebe2bf28cafbdc75f2219fc1288\",\"d68fe9c7addd4541f4b1b7caef49cadfdda6f32c4d9918f7a2487e6151e8b558\",\"02b705ad42e1df9238f84568a4d0f240e634b785f369284ae7a093a5d51fa810\",\"9633e3d45f798b1a3790ab0987c100e1debce1cbc0ee87e522f0a77bc58fd84c\",\"4360b9dc2472ff1e2192ec939b6d443194a3ff401e61773bd34c3bd8277d23f4\",\"b4f98db68ffedcedff1830c41dc45a2a56bffaf3c370b99d9d67fb2b2ff52ce0\",\"8964752273f0a5b68be7fccb21c3bebe2c147b43ea183d83105f441a70307f38\",\"fdbdcdbb4fe4ce3a14a09f38d91fccaf774370ca328d866fae8207eaa0296e13\",\"fe4669e557dbc0e31c634ac08e59f9f81f63f3fa00699c34a9bbc2e731a7de97\",\"af82ad4acf3c06c6f2bf8515c110e110275ad4f575250909582059a84c72cfda\",\"eb31879c98422eb5047adc9a11bff95632f41fc836f22ad9f9681c0627279fb3\",\"6b00004524f841370846c003e90045dbb8c2d3a685a05b3dc3f1ea471255e19a\",\"9cc2bb6dab55a2204dcf27b1c1299bbcf7317e8abd1aa0ec0ea8141e24491c72\",\"ac51f9d6ed0952f2f837fc19a5fbc7db71f619b5a60d9bfe90724624914a7da9\",\"7812a59e288e893ed87ebfa78458c24dbaed4937bb79511056f21e8233ec7307\",\"a84f15792a8096a34ca62d2f276857e94e1bb56d0dafc435e37a4b501fc547e8\",\"9ba859ba61a5768db7f34762161d12810f96130f6e5a5bde443506417545d22a\",\"2b25f4e2a4481b00c3f3eee6a676111bfd14056d6a90ac597d54104c5cef3115\",\"f4655ee448081b4ca694062013119c54f6f09d2b3775e6b7c844535d8105d23c\",\"952c63361a757caf164bd043e7ca5cc42a1e04f2271b963317d07dd351fa0c05\",\"d95dd88c7cbc3c73ea7b8e089b35b7b46b4c40fa132417dc1483c9779bd3dbdc\",\"310c4824e5e0f1e3c59c9a582543c2977c40cb4db4ff639c57fa8fd8f036b0ff\",\"9e2e307a941998856bd3445dd3f889ef97bf3847abb3ef01a534af2686746399\",\"fc805bb11a14b1c92c62fa9a7476bcb3e58d338f4e382abf6922ec1bd9e0945c\",\"353425cc2bc48300e6aafa81da29973ca9f4f5493ed6980969069f69ddc091e8\",\"7e7b752ef3ab8d90b5cf1eadd45017e03fc2b0b37b23a1906e98842dbb9c4cc7\",\"53b8859cad2aad38499fb401c451f1abdf1be0703916477016a863612dcca3e6\",\"33c420fca33f9efeb531ece276efab5658ce0a30cff515ca84f435c856beacd7\",\"db3af8f0078bccbdab3213c5c8b372cf2def526e6cec8343820fb3e5447981b0\",\"b1c76cf72549f20d78f98a254b183fb462e97c7ad1e3d172e41352012d99aea6\",\"85dc3cb9849cf49a312e53e7a4c39f64d46b59846355a0b57a6198034e15d968\",\"db90a28334f1add059ebf81064f19329e3c00554081c665ad0afba51b124f244\",\"5378915a738394b721217019c0397636aafabbc39da7da16e64311d3065804ed\",\"8974831c771cc55f829f9325235778f316b338b183e9e10b94d7a8bf50d64e24\",\"2f4ebf9cbc2fbf804ddff6827a2e47c4f5fd044bf55f714f2a16f22fb92e7d3a\",\"28fcea7a033f9141031c15637a3adc3a4db637fa42fa560d1d4b917d78b9c1d0\",\"fdcef12248e6a57ea013fb8469669141176bb18bfd292c36ba9b9194cd4936fe\",\"2434355465a190bfaaf301aa47d5fd7ea5e1305fcf45a5d732f3588d83a62530\",\"dd3a5ac73ca25355bc4cf8ccadb817677b8a2c1a418fa4c071867e8aa033cf5e\",\"2f7206f1bfb4c1b707dac8485118a6c4288943f91d262b1c76f370f6c59c08c2\",\"a11353f0abbcbcfb762c035c76dcc12180198a0019f15450f94012947ae4193d\",\"e95cf117ecf3ac4503019ffb3ee5c252eaca966d725bea8320b5875f39fdebe1\",\"37d1f478c4552db1af5064180bc1bad3cf44d65afbe71e90128c5a841f3d5679\",\"aa80ae1c866d98b6c75c06aeb851c3d35fc19151408682d1e48e529e7720adbb\",\"5c1e55ba02dc19acad92a6ec3e73fbe26f71fb1af99e577c5a66b2228a7ae91a\",\"b706f57eeff14c7b2f425dada52b21dfd61cc5ed59a0b7c8164a88b06153f925\",\"27d85f234b176f297b520006b6e9e23aa39ab36995c16c446d2bef2b328e8690\",\"5376d59ce0d568b078b8ee1ceda3c018cced77e372158320eb1746dccdccf14e\",\"ec4dd23e0d197fd3c737ef77fed41219a5771028375afcb97e56610dfefa9615\",\"bc3606e25a5aa059b62a840e5ca4c4d5a8858dde68e9d5e97489eca8d1d70eeb\",\"ea273f9457184b85fa4f8141b077c434239942282e3226cbb01f7da8b9dab458\",\"2e591589904a363f3468c5cb1e4e586c938b9d3178f969ba6d36cfec45f16bac\",\"61bb04a53ef5d8f00bab797d20bef344ee549b93d6e0a841a44d1888deba5a8e\",\"07a79b4522c93cdc1eac31bac4857cc9d46149a649742d9362043c10d53bd9df\",\"924692a7241ffc63777aef12435121a67d7c484533e832570532bf0704b8206a\",\"5d3bc923c30f2e8acf8fa16b43ba2cb063fcb462c6f43ea245bd495aba9f9db1\",\"0377652cadbb683df3e40391e6ca679f7c0ac575f249b08bd99226103e7d3251\",\"e5d445219db907a4d9deddbab711e38016ddaee5071b11ba29e7b0fc7235d8a4\",\"cf1b24ce021364e99ef1da6e83270a3e03c1504dce0c74e8742fa213edd49314\",\"6b3e9a3f1902117ef0f37acbeec97e970e6eff6bec319e7f27edcc62f13bccda\",\"7d2e14a6874b22c0bf03829e1d20c5187ba2df3874f1a622988e55760e4fcde2\",\"397bfc2cd1b0b2457f6aaf866de7bfc7fbde9de3ee33f08e3bfff362da1e21f6\",\"c1ac50cd018c4014f88125d85e311af481d11c96d627fb98887ce0550f28a6f1\",\"af8e66a3cc090b3a08a3679b1381284cda23a2f60617d01516c3936a7a6ff8a5\",\"dfe32feb5aeecf0f1ae551db6212a2b54f2e97c5d28c5c0270521c6ec4b434f0\",\"ade6a03ee337ffc253a90f31c50a0f7104cb5c0bb66bcec6b68f98902368bf0d\",\"10d9d45326f353822e202e9d0048ea7fe25889a38f77e40288e783bbda471b64\",\"583514c6679ab020cc60b7c24800b631c13bd9613fb2ec2f8bd56fd2062bb3e3\",\"6ce2d1209a57bcd23a2ff807b68cd7880dbc22542ef80b1fce145a1758335677\",\"b0f65a39a8d6035342162bb56eea78c63a5b3037226d742e8e21c60f24083991\",\"1b3d9d2786afa5d84fb908e0bcd8b657c7a96b0e0fdf71d8cf440809609fc8f8\",\"6488d813a3373879255392c275bf8574b9bcc35b831b0c4ccb111930cceb5188\",\"4340e682dc67655d95838f43e66156c49f942de0c3af177aeebeacb65c6b22a5\",\"67ecabbdad662457b2180bb6d36911514ac5dfede037c8bfab650c22236fa859\",\"e41ba64a34c303826aa954576cdc77a2c539d203ec05e54f89655da58ff24446\",\"8afd35cffd7cb76133f001ecfcc88b5e4eadd4e59a7e4de1b6fde692819ab1b9\",\"cafe9c9fc4a6ae48e3c95b1c5764ddc578cd4ac87302b67f28f8a54ea247e32a\",\"10cb02dbde0c81bb6954b906f3f71d3f5d8dbc8cc961d9cc9c403f1b58abb2fe\",\"dc07bdc64c1d5b14a4b0a45dfa136c21b716a003021910c04e750ffaf168f571\",\"0c536726e9617b40d4ee6cbe9d7a734d29ff381c47529cc8e2db9a8dbd8300c7\",\"fa63531e744f73caa9a04fcb8ddc133489a86e4453733dfe79e910318577c5bb\",\"4314438aee20cb27a692b1b40f83a901cb7baa686866cb4b7b2c06f6763b578b\",\"5b764ddcf97ce5e607b99799f13fbb06fa6742b95ae2480985763e2e9544605c\",\"daad6d8d916a5e6f3925dd713e1d7028f9f46f16337c7b74e79d7b12bc1c1e6c\",\"c3a59f31666e8a19d53ef1a762cb6ba4527b05c9fb489078a570f4bb306b9844\",\"367a676e19e8f5289abf451b877a0bdbefee938daa926e0f832693720aa232ae\",\"e8e0e345ee9af44fefa383a336002065be589696843ebf04536ea8049417b47d\",\"bf1d7c1bcc26520b50d34b761c593392663ef0e3783953b9d40353f072903fb8\",\"bda17dec7735d52b565142a51971f71306f3b29a6c99781447efd0910203dcc0\",\"2478f0fd36618d8106780327e2e8cd48dfe69d3dda4a0d8489cee905d8574710\",\"f4f1be4cd54eb09dbcadb55af7a35c0d754625bf1160007fcdc9154d8085ed00\",\"82f2efedf52d4e64a96aae07a4136afa9f91455012ada59489e592eadb2e6d67\",\"7b5133e6e33f305dfa5ee8dcf04e9bbe6dc8f02532757d1b629f5f1dba425809\",\"1d286310e4d917d1627b56a36a92e5c523808c5c052b72bc545cc72b4ed7d8c6\",\"addc792028167cbcd3cfb53b5dc52c5fc4548ec726201ba64938958394316ce8\",\"34961010de29102bdda98de8f2c20dcf8f62083b9644af2ec7e27ae9031536f3\",\"74886e74f05d5bfc41bad1c36c43f43b4c1a5053f799f3a78f8733037f09dd79\",\"d7ca0a2dd736a2afc12b9343223c5d19685a09c37978beac0279f50f74cc39c8\",\"b8fdad4d97f35afaf3828372fdef8bfaa8bbc5a890f35fd805e17952e981f7e8\",\"fd69a71be41284bdf4f1eb168ba89b7eab0a95238557c0b066a6e63c8dcb67af\",\"07e364082926fe5c830f09027ef3e5fa0df58981cbb58d76f66850ad98eb3182\",\"99167505483830c01b86d1af049cc5fb814b7e8c504ce5f9064cdf9e4fa742a0\",\"bf0e84db15c5e27f163f51d750e2fe5d709a78ff933555aefd5a82c0a7c42099\",\"e16092ae0b22c8846b3fc4650f6145ea0a8c0ae42b5cc287844d7ff1c87940b5\",\"b4311657e8744bb8c65fdd766d127061c5cf345e52cbcbeff828d0b64fcfa68c\",\"9d40094bf1556db465b32fce42555e7529e2b8b25819af611a6a92cce60277f9\",\"8974672ac0482c5c22572643c73a9be64efa7d8d3c0a7f4916c9667a7885dcb8\",\"f4b02ea41c24e978c9ef74d80140b73ba8c378eee31f0387d9ae2761e64d7eb8\",\"eab355b2b71ea6cd4b559d332aecfb19d7cb250139fa18ca322e5ae8e3b347bd\",\"36d57590b8065f3777ad291dda7b8b07a8820f6c23806dc69f1fe0716211c888\",\"6f57658f24bc34d3932fa8cd43f2b8d23aa52a66031749f4398f41537cf3d6cb\",\"406288c912a90e5c3a087e21f9a52bdeca82b29ed99d11a35a133b0008bd06fa\",\"dd7559c72c09862aacee7222ef31d6288ea05f367cfd1d5eebe0f6523d1b6b49\",\"e7476c8f75521d56682879f91bbe29ad825091c2ab293fcea6c5376c985d25e3\",\"ca8dca33d24216133949a94aea3c91ed17231f5bf1f584157f0da9f06d1f5b41\",\"921c867aed1f7e3c151d16396b66999c95845f6d86958d12069e71cc839d1db0\",\"f3453ae53db417ceae7587c35b906581ac80adca09a4ce9241e9576127f943e9\",\"adc371cdae23166f3399a0f98eae18defef0d645fbd134d81990b6451f0283d1\",\"b8b37bf0eecfe4511ecbbb5fb099db5783aaba0cd7bf75fef8461c3b598964d4\",\"8c865505915fc10d4dda50bb169fc91ca82a9cb8441f5fada6175e5a2d92ab9c\",\"e8967957d6c1939305f4e4c6d6e3e8800dc9cd19b670a8f0000c56deffe81562\",\"accb8c36a9adb301bd2adac33a323a05ec4906edc588f94a0e0e7d00dc30f554\",\"71b4773c548bb58e67201ceabc977b3c86bebd75c69429931575bb86e8db597d\",\"a0a18ff25bdc563ff0a16615dcafdb04372e39bb56717083833eeac3e77556a5\",\"7e1343e09a510ad7109c309617dbfcb29042a17302e2421f1a8b6645aefad8d9\",\"0dc9e32bb57adf1fa5bf0f6c7dba35d5eb77b4601a5f90172f18d83478a4b7a0\",\"3fca528aef7c33d111a185edffe96f675f4ed98f01c765a1af84128fefa8ab27\",\"26bd17120983526827b9a7aa460be3676df60ca56b23a02747dc1bc7e0740883\",\"8f22bf8474035189971059cad5018ba1e87b9087874582e97c0cf4962524c522\",\"b577c37a785c1ab5a73ece5499e8268783806ffd0bb98e07e4fbdcf21eb80ca9\",\"4f77a3ef85e3f473cacd3e39cac26ffdc97ae1e52d6c65b7b0cdf3824a4a47be\",\"77409d91d606d02d38bbb54eff22807d7cb7d386b9a24ca03d1d68af0b9729f6\",\"e6bca5bcef9c215def5977b09c236a6208a0c5e4f6400957c103b89d4a4c3cad\",\"609863e45b0a7c028656c44641491621f49595184c333eed4a3018d8f3061b7d\",\"824e2ab3769c191cee2b16a734dd49d324027c3688bbf571ed0b11a1fe07cece\",\"757e6fe2fc9c2c94058c25023d7c9f681917baf4e7b5cd1657d3b0c3c672139e\",\"cbc40174fa067e9977c3a5668ec54a6587026ee14b2700286507a3bdb372d56f\",\"0d1cb8b0ac7a50ca3754789ff7bada3a300a66e45a4ff858844506b5a7d36075\",\"7004787d454e5f2464b8245398374d39b89d16084acc2d4f1ca20edd257dec9e\",\"bf298911241bf7642a31c8a105f9a497f5c65d16b23fe5e93e5d100836b54036\",\"5dbdb1cecb13fafcc2558d51edef434301ad1c75c04e4f610dbdb59fd5d03d30\",\"cda50f649bd9db2c1b96ca1230ae81d5ca7d826de34057f2fb4513db88d167d9\",\"c2be9474b362a3585bd17aa9665d7c4149c4aa6eeac0d05a6e98e5d042da5783\",\"04b22682036daaa41862e73ea91ae7ee2f9504952b7915fb3aac12d5f567442c\",\"6ebb18c2f2a4d5a0d0391ba50c159cb53d871d9d3c21dc487c18758382bb34b0\",\"37c6f810742accfcbfa278882864334ec459aed60db2e93b77cd201043b88272\",\"77789beae662646ca19730483e032a4bbe2392c8b0af2c34766824752b6de7aa\",\"af1a964c9e03e104f90a0c92ead6fae54eef3099a5b355a3809a967357e96847\",\"2364e85f6aac7256703e35a059d6bb2ae81e40dbacf7e1b83c86b6b3fe92c642\",\"021feab500e48993e1adb0a5a92484bb52dd44dcda2045b0a4fee71bd64f9ba1\",\"f2146677c451e0a088a683e2423c8eaa5607872a554b168b5c6d281d08187361\",\"3b1398f51ba295e5931b95d634bc8c52e40a7174305808e62b43799a6aa2d41b\",\"6b8cd6491eca139cdcc7eb57d7d28108dbd34fc5602cfe25310926b76909b86c\",\"b547c12220c70ba1cf29a85b4f650b746da191328e8f6a6d4a7759ae365faa76\",\"b1e9c778a780f52fb159c4897c43185bbe321d6803348e94b3600847eda4a2bd\",\"72c674e7919471ca1e4f32154b84cc86c135553290e58cd98c4b60a0f61c6a91\",\"0e08ad6269bd93409072a42c433d7035a6d4f8f380eddfce1336875d010053e4\",\"7fff035165a3d04baa8c71b00ea0aec33b165b275cd3930afdc9a2e0eccd3555\",\"958fc9b948f97b3ccf999067eefea590e7d1c033a47ecb9798c96f48b334af3d\",\"2717ee688142e8f9a8a399ba24fd6019c2d6300d2c60fcd805cf7294775038c4\",\"cf203500bfef3a82f9bef2cacfcd5d95da9d7647804299151a88c6c7e2ecdd45\",\"1f3e9dbba9b4b6400fa0141bcd84f8f0c4f6672f281532e47b4f3e11d57f6fd9\",\"98cb27a35918ff5a1c04e685a26156eb393f911c5b10d1c83513f51da6165963\",\"7006bb0599102454599456e7ee4bb04f52386efa7f00fda57c42e71bd0f425d6\",\"d55977b6b99d7b0a85d313eea204c1fb4752436d66f1034aeeb6f5253c9c18de\",\"41e6ee7c239fa37238cbee933a600bf7b2405b1ac224220214f5c304a97b0b6e\",\"93a0577b48709e63ea4b1ee37eb2d87123984f90868233c749a8a3334734f485\",\"7ba204878ad9a0b6a71117685d05cb4c0138ffb9718c6370e3206099a91dc691\",\"b920004c289b92f4de25f07dc729fcde941818ba2880911bde64ee3f2b798a73\",\"1dbe47a48bcbd17c995d72e1696ac86b0bb2a901013ee56b33b26aef6d0953c8\",\"ab9b0cd638fb15f046f824ca520c35d1e57db540d8094f6a367d4ba61a418242\",\"07b65535061443d35ac21180681256370aa72c8e6271f0100052164b6c38eeb8\",\"bc83573fd45bbb54f7365fbc840b0422b78cceae4d7d373e702dab0bd76500db\",\"784f17b9968c1463fd8fca2c6495b0f006a30aafbef3203785488fea7943b511\",\"02e8d1d39ce77fc1651a68f1b03d29b113171d78f904a4382a75178b906ceb37\",\"918211931b3951c655366da28521bed07f00675e8ddafae02b61398efce5ff24\",\"0014383e2840f55cc33aac2b4da1dd4cf34e21cd54662e7c49684dc57ae0e5f9\",\"5440850fd18615741d1cc48e1cc8d8225b1ecbd5456c6faf0f4d30c21832a478\",\"0fc3e2852695323fa4c61066614689bed4c24f198fe4ae47c09a6b99c1472856\",\"ee717be7a0f0a391c02dadca9f3e5b296f08de985324a14ae64ed2034000cd6b\",\"0194cca858d5fcdaf5bc73ea8b8f47567ed746fd51f8e90f4404fc9b75b53c50\",\"afc2ede3c8ffedba293ce0681f3984ee8d996c24ec3464823a5aefcea0e0cba9\",\"6e4ff68882f0c9c656f13e8aeeadc135030086706b19eff2b64cff921ac2402d\",\"7c2a0ff8eb59423ec2c5a442a4786322a336cca6e001e3f8185d826c3d757e5c\",\"f1ac98104492208a5a8ea8048c998f904c5a22da9e03aed41d2eee104b4341fe\",\"cfec1603035a75708e2b3619a1da0482308dcbf575b8da9fd430627f3128c5d6\",\"099085978f44f976625984de7cb429f37ad8ed3328ff4b3e392da54250f231aa\",\"3079151f15f0cdd55a7ad50912965b3c5471f483e904cf35715ec66502104532\",\"8df7f4e92a5e4533c066ad5e9ad0791ac5ffa727e4c343182a9ec262c3876c7c\",\"de3da0f8a5e5a863319d8a84c72b26cfabe89bca575fa88a6cca088569f2cbb8\",\"466eff81caf2b841d18b31758677b29d1f1fc4d4f5fedfb55f8e4aa2a558df39\",\"ca5389f57924ea88d0005461f791b9059be42e482c8adf51dd05463f667fb376\",\"630bffbfbb4e6c8288061674efa3a3289746d5ad2f6ed5043c79d087ce25338d\",\"2cdbfc89b7c3881a05e8b2503de9eb6b1e595e5c8b05a9950669b5086e402f27\",\"10ce3d6b43dd990a7b9637865df9c2dd5326d6b501580e1bb67cebf3e0895239\",\"d6ca31b26b5dced76c5622ffa8db64a45cd4d5f98138f0a0349d31497849f2b4\",\"781bf8d7ab392c61ceee8bc40a2c646387c75bce6cd42e809f3c2aaaf06278db\",\"0bff1294cf388cc3f67c31221a23e9e5e025b8b52dcfdb9947db8cce2d191d08\",\"504e5a697058181c5e70f6ef3e80478158b02a5c540f6732fc75b29f6a1e0ac7\",\"0077cf4c335700de949786ff414c45161d8be0b92f4acdcd0c4b81100bb3c854\",\"dfdd4adba158ad45beb397810a3c0899615175585377171288964a3dc531458b\",\"f11e335432163d9311ffadd81ad51ebf61e6feb5aed2d5eac890cb685fcfa676\",\"cbae16c08117175fc285634a850cb08040e8943e46109221b27a2561dc9419a1\",\"b9206810fe6bb35706d9060a083ae460e5c5b03c347f6ed2532a55d79ecfdfbd\",\"5af90aee0ada1429f3ebc7c8b19cdf68f416bd527fa5ecd3cd19c601244e9978\",\"29f58b9de0c7a842f95f494fc3db3b5404911fc8a2baee4237656a1e8e417280\",\"87daa03f55b54c4525fc9f921641f99061e428c88cb2f8e03e37996873d0cdb3\",\"b01199a14e7d292e824c4fb16a3dcc52ec6c8ae4402dbfb119e5c1e9e64dcebe\",\"ca8b67b55ebba58fbd5343e3c888c2e87d5dcb6952c8c56cae1912493019fd0c\",\"ba236f1026bde5aa9bd5ff24ac08574b6f1eb089caa49222568269368a49c838\",\"930d476887affc409ed26db24a7f6ae22a8aa81f3b3f25f583fdc0a9d79e29c8\",\"05812a78c822395f563b175dc47a50018e467e6da3b2fa0c5dccba2ff6e93918\",\"6b0c25b04cb1b0334ddcdbf750c0edfc41d0baf57c213ba9dd3ad98fff708251\",\"9a05cf8c6cefe8c986edabce129637e890a9662309cbeed9990427b04711149d\",\"948d5f7f7ea7975e2eb8ff2cd407ff3a75bd43b905dc64f537b3eb3c4f371015\",\"411740735daf4a1e6596a682e85f7d7a68f534550fa387a7e5a18b3740315050\",\"8d84ea9c3ebcce1437db881bf75d61ebadbf5ff5b8b370a18ffa37f4b53e2943\",\"a101dc49130c299467fe463fb7a563305503e9fcd48a821280c4f3668777b4b5\",\"8db185b49fe3f2222945bdca029f220b3615d934dcdc762fdf95ff1391d277ce\",\"a2ea0f721f2725b8e0deaa6b88f667ad39f315a61352096cfff7d421109e3128\",\"2dbee6833c6d2e474f85547503cbc1fc8547bf642b3b121c6cf02052feb5806d\",\"e0b82f9262fb5334330c1e244bfa0844f8a35f1aa00fc71510336c17e1b70f04\",\"feef0a8984a3ff455326eb7cf8c18fdb9905ef3f83f87a6f4e5ada051efbe159\",\"93ac45010514f37a1afc2e1f3fd8adef0396bd4813ad4a1b5013e93e82f1b2d9\",\"191d1d55eeb4f2285a115e567ebfbb1cfcac96201679cef2fd78a8ffdcd6da58\",\"85b71a1fc38d0e96d5222b8f57d71edb87f9f2a5c5ab4b00ca21c9ebddfbdb3d\",\"3033b5d814c5e72075fdf591f5b38176ca3d6f4ffb43d555764cff60321d8879\",\"5159d74e332789fcdc5de51d347e08a69a279862a752d99d287c43d493f54945\",\"2b242e85970924cfde88ec9e9e7d325376956231ce1bfc55032401e396917f9a\",\"3a1d1dd6e0b13fa2027b94e010966aaa48d5bc07923a2b8f41f4983caff2835e\",\"d3c0bc27e5bc036e5422beb609fbee7dc515d361092bab5a3b9bd53f9a67f8a7\",\"b2c317c8d82dc0912244c7b28ed7e036bf0109bfaf9eeb8e99b4d25a73a208f2\",\"39cf3c2e07f40851c85d9f40a65963b5c48f5bc01c78a73053b5e9665433788b\",\"274572e00aae4d2034c3931be241b7cfc1556010c6df3f5fbb00a813a4468395\",\"430ed475cf404763524a7acb83ad7913bbff9fb58c4bff2d7c5d75cc6be1b0c6\",\"d9d9cf6592a110d8bd331dc80b83008f1df058b256cce2dee10b1a45aa75fd8c\",\"9e84a87a8d9b8141c6e2360ae94b379c6066ba47c2133dce9141c395f464609c\",\"4cd8d1a462825026b313d22fee1ca323ed99259afd1a3d8077e57260b33ae516\",\"9d79e1ca7700088b80f8ff82b4b19b76c1129722a9214275dae841e2167b9df8\",\"a9d8fd38e7d5cbe28009765402cfad2a8d4ef4660a457e91995ec3c88ddba85e\",\"fa0e3011987b3deb5216f9c3eecce0a4223fc80270a70b430b9c407dc766f082\",\"b3b7d876a53e7acbc67a639c1e09a6423c5523e7ae5276ba71e2d2702816cf5b\",\"982d624770ea041e86ef2f98100b062b4834ddc949cee844c73ce42665505443\",\"dadb0d7f07666bee34d829b86677c03ed7e0e431e9fe405f0f5d18a5a82c050e\",\"25ff416ec63e2c9e622a226a4ff37f7c54bbd50a065407042b374601926300dc\",\"a78b88dfccdf9243cbf4d008af0649763a1ef04f0241e42430c6e426f58cbe33\",\"45925eab593c6d345383d8f77bc5e72217360b17e45e8beb50cbb5ff2f6f5eba\",\"c0078d38862109d149609fc68c7137893991d946221c625d358f2a7643d256a9\",\"ccbf599bc612524cb87f3b30a0261f0c7e6cec86cf9575630ddcc9068ed74ee1\",\"3de1c6e1be858ad8b24c8cb74b18d516c09d37a724bdce79d8fe67a0795770b3\",\"4347bdb7a5968c6914bdbcc8fe14514d8ffb729ef94830f5b2a3541632d06b6a\",\"36b5fd831c9c6f9a86b0693e5351f91d275c43dfa844e578d70f3c9c11758be6\",\"b6930a0748d46a526f5fccb2f404d5c3daf84e6cd32a098b4096cf528b1d9f92\",\"f18d85114bca91d156a4fbda6ec978a160eef33c2fdc3899c9a61487ee0ed591\",\"adcdf61adfd3299518d0b90ff2b6274b00d6e55cbbf9db24b009b2d6fd3669fc\",\"ba1ada4d3a5b640f2faae802e585518cfc16ca829974df9505f6342248f5ab46\",\"0b3d8e24cf2f8c34ff24b5b3b32f44ebadcaef72a850b85b21378b4f32726c62\",\"a81021a396c237da3229417af745045283641dfde103a36b0c316b7397a595b3\",\"013a1f4ce8ad7a80efba975a39fc65da5ee880f69bafd89d58d87616a55c8e43\",\"843f2f7f8e929d185892cb20728fa7b076b2dcb336d6ad7a896d0579a9ef3bfb\",\"14b4a4a7f6971610c380716f45536b9b0f4fb8903e5df01588bee0d423f86f33\",\"ee875535212c321db19cbbae15bb60f333f389392fc37f5c3dc66eacd0b7b0fb\",\"789b3f7298240fd8c057e68d0bc70e5f2b7e89d558ef515e7740392e8f951613\",\"0e74811096e204ddbdcd1e5098181fb3ddbc686d4dcc985c8e9a485b4a5d012b\",\"e44f643f2d3c375070c286970c5be0d3771fe5b02fc5c75b313834c72ac0f0b2\",\"93d473b66719e425a5c8c1bb3453262bce77b2de561c23db7ba2d0cb12773799\",\"a0461e7722faa2e8d8d9cd930b6a1d3c36d91489cfcf6e583bee0212469cf195\",\"68cb6bfda6961da489de4b06745dcdaffa1898b06d2c073a53323e2812d2050a\",\"a9935b0d045c93689552e49b15ba161e37444af650c040f723b49dbdb16e6784\",\"61ae5a8a2cf1e84ffb9a9854f55f8f8a605e7e7de6dfee54d9b44a2a89a6ab9a\",\"daec586feb2f8d3187c5ff45bf08bb3f540fd5778427ae8c470de3463ba0fa15\",\"5f1b8a7d59514d435591d138c8cc82399b1a65b93cf0ed631fb453dfb60efc1b\",\"b5beb07c71da32b62f7e166773e50033cfb89e8618341e81df9b378227b90190\",\"262f71606989b99c9819697fa40a663456c60a947d42fa13ee3b73ada6f1db29\",\"990b1dc0f35d64c31642bb4961a46493bc8be09eb1408fb06de114998abdd7a4\",\"14d542ed3476cf393cec81f911866afcbfcadeac6d2425d4673e56a4bf9b6697\",\"f17dea94f528a1f66756f74bf92c78cc4688b5b521cfa1bb587857d3af155be2\",\"9ee460ce86b0621d2816175224f4d3e6fb895e0b716cd9285d50d69d9779ebe9\",\"eb849092f38ac818099c6d33efe4247465e5cc1446b7676fd21ffe33f3b5307f\",\"03f6c1dfbc8f81e5b0432143ea3b8cd0f71f0507840a22a5f88ab003db35b34a\",\"429ded44fe004740c61221812c4da8232a84cc315b021d7e8f6afdca37c0045c\",\"4d18084b564abc345576bcae3065532274ff88e7e16359afc9411dd159f78a4b\",\"0fc43b59ef9adddd66c2df7af480f77b403830f9b2908615c4549ffd8c71b9f0\",\"470a479d1f375142765933ebaf7dcab50a7b9435446667b58c6e0041a790c41a\",\"b4b32a9abe39ac171a40fb3fb06eb46c3513ddb13c563225450612398356a2a1\",\"dd1e41606c3c4ffb5cb35f302143639c7148838d2a8cabc89687b24bacff7710\",\"3165b66bf6491186757adac43c3d1dacc07f9fbe0bd94935c4c6eeba62dd7ac5\",\"b22feebfe9e715fddeb67092443375e0818f2585d9eca8b1aafb6e8fa7787a96\",\"c13726ea12744b000fbf6937e524f1ae652724d104f9a17525f257ffe88a4225\",\"0aff92678aacf9ae148c1489d77fc73b8ac2ef4ed8244a9db945a8e109c8ce59\",\"9330cef3aa114f64c2df62cc5172cbae8d67e63239d7983366b56d79b1ff15e9\",\"a8a4f5bc37f38aa08730f5eaab16b555699d97ccf35fb2f3f2f9383b9365fc24\",\"6c3f229404cbf91ad90e9284f5ad97e25e5ba867fb254ce962aabaac3e090912\",\"0dcb1c9d952135a10ea9cdc50f4306a6d47db1dbc01b40ce5d9a5b16f512e7fe\",\"a3047160ffb7dc10d8683d7e303f50e0c71760045962b41dda7d05374c084907\",\"bbb6f6750bcbaa3ae88149f94d2c6ef364218063dd52336cc5ca63430761a332\",\"5457ccae3d04948d30d0896a845fc3dd158f2772145f39da04b0b97efa293031\",\"0784c6ae56540839ed266215aca53080d69006173247957369a1c0f51b34057a\",\"d5b8ccef68739e46c9f4e6b4d9b961e8df79a718b77edd251e7befb2cfbae98c\",\"8868ee8e541c7913f0dfa9af6eb8f2013dea3b96654ec1e637bae71aca2d26de\",\"e1eb3cd9c41f2ba449e26b474871c07e4cb3d070a0b9bf69943c94858a9367c9\",\"a3402a033b3915fc088bb4c0dfd570f88e8a3e9385f93958f944cb3c87d856c6\",\"8304cb33730f1d24473f7d9d664f7c0343eb164b847e7ba7bbd2ab88563d14d5\",\"9ad7717c3074c8c1a85d052156e945eb5f7dace8bde102e81786749f192774cf\",\"841eda52acfbb37c4fc3c157e233ad4db0bd0aab61358479cf5572a3a9f74c98\",\"a471199555282c4f47b7362a86d72be3206ff34dff4d8f53462022bdf0e54807\",\"b1b236ac2a73d340d9726e61f27e9400158185e8499f813e7d03f1c5eddfc1c2\",\"0d840a4c860cbd637133a59683605af0cf1abcfdd87a244afa6c2c95aed6ff16\",\"4a60c9e8e9c4064c79074deee1d961d5bf2e698b03e4d4c9b96925a12f01f299\",\"c7b0605c9dbc7522540223f4e2be594bfc100f1ecd2c07a766bd6c70a57dc909\",\"0309b9f2aa2e5ef0dd07b977476214c76bae223d54ae0cccf830b191fd9bf46d\",\"7c07c39d1186baa1b89f9323bf72e5489058c260b6d014687c57a0f25c09d836\",\"e7c7b1f433a08603d3493f6f005a49c1d0ec38909d44a31e9c9493d67924a4fe\",\"86a7fb03a6b8c0e1901de08fd7cf2ed958115691dc6af20a4182328dc21c1d24\",\"7ef855a4533ff97bd95f58026033aa7c19eeaac73b803283000f40c6c6bc1eb2\",\"12296c287c0fc4238a14025ff3be3c9168cacf263e59b1c4b05dff127e1f71c9\",\"da4baa1448df604aab648ca0ba3b87e6ed437c937b29d4089a7df8f5273e9837\",\"4f3417b61ba441c3af68f4f2c3cd8835a94cd64dc5778e1036a1758bf1cd8db3\",\"925ecf49c850339d9a031ff1c32e01373a1e24bde51ed02742c994a757ced592\",\"db038bda4c56073fa7241c91d82dc24dda62a8590a90f5c341d1825b80c913cb\",\"042966c5151173444ed0c3c3ed0c7adc5f543b8570ef065c2d6958ed8108d8bb\",\"96d441e34f761ae044880cb2bf37949efd37cf1ffe4ea9fdb7ea2eee94c0e42e\",\"6ff8bd5f9c62202762efd8bf6140198d44e776bdba599eec872c2ed02b54cbaa\",\"96aab05bc9763034e940466e9d4c6c07cff6348eeafd4e17f301c349aa3aa2f6\",\"445fa73398599b9399367ef0f436ba8ad435b2d7077c49510ef67d3692c85c8d\",\"b81925855a95a3df0e4c961d5158f3065fc9778a9c730040522bd109b0313c0f\",\"7798caa3cd909f4b79883ac38c52611b71c9470c226e4513d0c76d508bbbe2c3\",\"ed1621e1f4e81b15b92da9ca6796f769dadb7be93263998072de84e85f31f553\",\"636548fa3bc6926f1f0d88e9d2fe83c81ab0a1a3fe3d6fb08beb7d73acd0f1e2\",\"94fea5b1530c5f3faf3497893cd0011c4fcc9050200aabe472c733f033bf4ea5\",\"74a5e7e579a5f2cfc41ea2ea9da4bd8baacec49262837dcbe116079817e3bce1\",\"c84b45b244712486f0bd7cc7c049aff51ae89ccb39335f348c06e943021d8291\",\"d9b5f4a83bc057e9fd59ff2f76f81ccb90f0fb928a939bf3a74dbae211ee8751\",\"38998cb27e295681a73100116cb231c5977f854f181a66fac1e51790d41622be\",\"47732c17025cb24c50d28bdcfb43d3633372ef385a0a682b4f989afd2950d01c\",\"ce0281859d922dd27d2823960ab20c9412cff36a3c8f5a3d277692f875b9e5ee\",\"b62b38b1161fe769d9cd8b66c6005cf35267495d44c091bee16abbb5af0aa9ce\",\"a72e05a90d730a4c3b3139fb35ef47e67780e61cdb85c6d2e6ddb7a62269b534\",\"77f4cdd5a2ae1839fabe3d1cc38763f2e1db80134fdcfd6cd3f857db06a92689\",\"ed6b33686ca7a0c8dbebb6aee5e8774224e3ec893d01d66e6356bb56356d84db\",\"b481e48f0ba0f2c7b73962bb2af8ce2c9c4c7eb24a2958716e744ff933138909\",\"3ac54df658673b35efad7701d282b8b68d0992e47897cff11e4bb0bed687e4c6\",\"91430803e6d6f0695f074abb48f111d3d4f97cb26a38f10c342e68d263735b5a\",\"a9db744f283e733926178bb33ccb9fd44438ea9b20993073bb17d9096fccae20\",\"3db88333e6d737229c8afb3eba5743c8792379ad109c805b67ef1fe3b0f3df36\",\"b68d091e46f5aae90e8b9bd598e8bd7cbdfe2dab9e9114c634f434af287de62a\",\"953cfc0e754388b8d707c5be64a3514910e57b67a8461f92e9db602f6e38bb06\",\"bb9fa61018d4e91e1308cf1be022ab428e04f7bcf68a85b82287bce72fc66bfa\",\"aed48a3160f24a04285d32b9c3bbce867d0c4d0601e72ec21fe40d48b2f1fb9a\",\"08be95f6b63b8a2af47086136cf12c7ad10850507d48b4a2248932ffc7f65eba\",\"3439ea5864f3d55a14da76b8c044a0681531100d12196bde8e4084520c7fbcc1\",\"ec5837d46475b49c3e3d68eae14f2b5673f017ba022a1e6d127739f7fd36babb\",\"da1cf1f217a280d3429f948e448b18764a2745986ad57969be0dd16bf9387236\",\"debb492d4e77f7c89cb81e040fbafddc0aeb8019a64a52b6a399518a454c81a4\",\"683c3aa85d71b7e2eabb59e27e8dded7e6a774a2be598b23b5cd7feeb49f9300\",\"0cc6ee63a9f2084a8b738e0dfa4ed164350929dec6ba4aeaf3d0d02d2296c1cc\",\"bb462c637734ddedc0a04248227cfb7b76a4113e83ba650d9c08236b562f418d\",\"b293d7f8306783829ba5a065134d342722d98ae0e40d248b176640990233af0d\",\"db9f02b6edf2c9c107f1f3a1cf22424fc048a122fdd728f866cab5dd5489ad56\",\"b6744f0d1b92c875f6efa4e374b52b1518612cbe70859b7f6409e0289cb8cf1c\",\"2c7cfc835c38c692c76cbd527818b994b871404918f30cc675dbbb493b206300\",\"984d9438c8dc49a58d606fe1903d1dd10e821d48acbcd167f62b33b3e3322659\",\"937352aad975bf5eb935b127fcf023da9f29cd4dd52377aed3ed28470112defa\",\"77887d4faacf854829493aa59c99e6df7bebde2185c3da0cc73043a1c73d1af8\",\"9eba7954e4528fb31408f2754d4da7df206cd73f101261cd4bb22d0a1add7c7f\",\"5be7a3d7dd87ef3d35875b7e736fa6a4b421f4e7746860990ade647944cc9d14\",\"359f2bc52019f2d234a1dbc224e79b5770c81b7e6455737fd2d7551c68569f4a\",\"812ce239946e02cf4da5101e77bf71d910b5e276bc9768746fe546d589e527c1\",\"823cca751e1000af46b98e618b9a4b4cee882ca87936af5eddd665cc5e9577f8\",\"815b21032a7221840a098d497d160fae928cf0760d7a425a28bf266540cac38d\",\"2e591fe5fde1e18ad55accb251bae5b6dc160ff7a8d0913d9ca61f7ecc95cb5c\",\"019051448e2c0719479f7c0741b422616e91f07e353f2de9e8fbadc917f574e8\",\"8e1d156843ed9068268829eecf8b487b5572baab64ab92c5d1ee0408152c225c\",\"0deb123d5b32dbd93bf7f6132b269144ac79fb05ec2aa8cf5fd7ca32a8ff2d8c\",\"eb9c4a61a2bc88689d1248fb7a35d1b9e8461c22c85bfbec9a5dfe4c6e3ba4d4\",\"895e01c03426dc945483774a8f5ee88a2dfcbc0c0f8ebe673978840f95444364\",\"ea78b95fcd78e5c05139b6de6d30671b02749b87a0935751c1b3c157302fbdda\",\"73f0f4581622e3c5c327336c7bb285ab0c41eb9202f3a0e64218be8dd0615065\",\"f88dfa777bf82cf54a1c6a9b7779be75de37633076db32b7fc20ed42d0d2ece5\",\"fcc8e4ad535630cb4557a296b370cee71a08888e791ba0aa48bbca577693330d\",\"64eb038eb39c87ceb401b0787345c9696b8bf03e1e5db6e5f79d471e1b8c5c33\",\"f8a3095f04f41c08dd35455501d9acf0bae32c9441f636ffcd4ddf30fa6eb339\",\"79ac8e27eb2123894f48840fe7917c5d7f52dd87ad6c17042ead8bffe145cbba\",\"6292ac8023e4a4da1644b872b8aae16c6864562af04b05946b31688f97540eaf\",\"e2abd1bc1285d05ed90d84f7a50ccbcc4ba8b7519a7d7fed759c5b78dbed4f59\",\"a3d74ff49a9198fa315595b1021a40849f625872733495b658ce38159c125489\",\"23d2e5a846060ce34681e52007e4df9e2553b0ec95705570120b56ad1652f465\",\"6e950f931eea248a34da449b201ecd3f6cb4fcc010443a2cd48fc648d65f1086\",\"00d9a54a73a35ee72ef8647c6ba874e9dec635a956d186a688358db47abcba74\",\"a657bbf2589eb539b9195d85359393b28b749cde9070ef9a801f6b7d8a576387\",\"321c8e412fbe77d3047e1898e5cab1474152a2e895eb09a4156bb33fc33c9871\",\"0e3121b72f495d82ed14160f6f80f3ada224eac9a1843ce756e8f44232c83db4\",\"a216f333012602e35243f83935fcfeb044a6c36e4c095d24ed153282acc82880\",\"7ac2a5988bbfcb4831cb33bbc647c868d1d6e2b4daac9176e02f780871e156c9\",\"dc69bae9715f79df8903fe025c828b9f107ad6e379a0de90a2846858f0777296\",\"2e9b3c9e7cae5409bbf000c2d16c2670914d5139dc9a7a0ef8ba74f359b23879\",\"0a7227d8dd16fd61a105783cf14fedb06045654082220c588dc7de92adc8a00b\",\"22355ec4dd98ad7fbfefabce824d88784dd91da00c53d71631576f5ab0d61762\",\"34ee93be984c81ce4e5f56800fd39fc0530cae29d50219feba2f72930473b47e\",\"4fe7358ce56f980b3d5ca42661813ce89a821270719476424ff68586791d849d\",\"5ae264d873b1fceff70c1e054101f07cb665e68c56671c81e4fad21a376f2a01\",\"8534ecac6cd2f859c3ff203d433c96122333fc2b77892c61df8d4fb2ae2e9ab6\",\"97517ba187bc4949a943d62d03c6ff40b07ef55a5e9f1a643069e3126b7c1564\",\"c6f3510923afa07c7d51e4d34861ec1fa4fcc39e8fa4efebcac470edcaad5eea\",\"c5c4b892959ce9f95f4d0068663f7f853b2ac6b3e726b17079ed306f6facc1cd\",\"accf9ebd5d4edb53df537d779d9cd69f0a3de632178f9d0e872a06f23a811c7f\",\"ad7ffbede88d5fec0974e94905537424d865a81a25f0430844790086cbe6ab64\",\"c7b312ae151a233b0c8d7d411889cea3e49646048b79176ea2a6f6f42eda64ac\",\"1aa4cf283e3a21adae5dc902807237fc6ee8f1055e58001783cae77aaadef285\",\"8852fba24b10899d384ae6b970d85c4e41a6c81fcf579e0d89b718d12ab36383\",\"7ca619d080d7e8175cc52b48f19b5eb993ea40c1d0692c8230544966558f0881\",\"5fa6dd2bc9eb08c219c07d5a5c3e6a0f95059ce166031746d404f0c804f42c4c\",\"37990b09df2fbd68d1d60d963b7d53a325fd1a69bd5d686995eb4717e280c249\",\"754960b7abfccc0dc0bd04e492b4e9829a0b9f9f1f45d369e432ea364d97b985\",\"7b8d390a9c7f708dfea392ee45f8e49ca960f0ae3e570e2aa0fdf5c3794f240c\",\"b85615903ce9402291c5558e666515f39597d4fda8f9e2c0e6bc07e400505737\",\"54ace052a5f08d9a9f18edc199cc3ab9c421a9c180fddefce6284bf66b6dc1cb\",\"79450252532bc2692c38f64dc7be6874639dcae9415075ff64e5a8f5fea9da6b\",\"5b595156937034cec3496000e837d51fb7b07c98809b12b2fd1113c3ae0eb9d5\",\"257364c8cce60055a99b1585865b1d3a7d25ada6f25f3406f84c712981b6f93b\",\"1d2da507d3b5fc96df03e63fdd971ba71adc4c6cc1dc9918c834aeb26dbd44b4\",\"c8b49a7b25a5b7d62079d40a0c7f16d35883faeab881bcee1d9c2c917e4faff1\",\"538c8961f1fcc028bd52802572fcbc5c00021b9f154f2f1275871d1e4ea27ece\",\"883f43c198b6a66ffe4a08a7db7ff38bf7d9c7b237aff16a317ea4be61dcb083\",\"4acafe43605ecb780537fcaa7a92402c9cf6fda3a7475c118e1663d1c149d282\",\"fd63d3816965dc5a10ab7ecc01763a389bd2844c2d31ecf447aa4df6b7dd23c3\",\"b621f6602302d07577b3be96f08fdb5a5f360fd37974abb42af01f4f166a6faf\",\"53583cb8a097eb1a0fd25e0f109ae1aac9d30c2128dce20102fbddc10e46cacd\",\"7043b2f3f4c07d026d8b78fa288f9034f9aecbb18abd97e8a0beb0c98e0b9eda\",\"13489b761a910bc9c0af4bfdff70da0ed5be76f8667937a4b48e3d57b9128133\",\"bee4f52f09f80d4b0b0b5b8658e76ea4f17936c9c04b55fafaef5fb18cdcabc3\",\"ef8bb3e1b93e3059853ef661308887378b4fbcfc799facee81e17000830712dc\",\"6182a0ae235ae460a8ded408fab95ab528c4f9f1c1f2b601aaf8d36d846634ab\",\"0279553f683546a6d54327b2b99922eca836fcb52ee924f9c8a8ba67b2899d78\",\"517e432f6813d83c408ed8cd77927997ed196bb44ce29a770da8a27492ea09ef\",\"d5cc23a48460bf6b7a55bda5280d51243965c3614c69fb186fc0c6dec3856281\",\"9e1853192be40bfb0597c6b5324fe7303f0bf3e333652a398ca7540b1a20440c\",\"67dddba886f8d6f0295db06763bd7ed03877a75ed30d784ece6374e4e553aa97\",\"7728ff2b1caeb1e6cf9473e733411f762610eabaad662cf4ca2b3328953a9553\",\"3235f5bbb4eb1baedfd09c9be20cbc88608b49996d473aaf1375bc44f7c9df6a\",\"5cc94f55e6c9329894e15cb0a38984533d674138218b9e779b3641c5529b2508\",\"48ed7d209884adea3e68ade3617a4a870fa3e1f26d1341e358613418e8567c28\",\"b29ba079539312d16b91a2186b24fb1c40541c2cfadf9520a09ad0b2bd849c09\",\"d61bb0f97f2cf12bb44a75b21ddabe0b1dddb95f016c0ed2f897fd9590c52429\",\"c33848fa2de119c8cac88e50d8e2a2c0dde924432d92f7828d7c2e5630adba69\",\"d9d2d6cb5cb5e97b96fe439978eaf2c721561969274cc8888d5e511c69db1134\",\"3f41421c31f67a48f3f9c55cd4375d6c50d5c4fcc9e82ea5c463b9ca032c09ab\",\"971112b9edecaee8aba380cf3bada61b852baeaee3f95644ce5a98f23dea984b\",\"eb8239b42a8c4f9dc2c0aa2514725b40cfd69a7f5bc255ae2607e4f42ae7d1e6\",\"9c91d9025bd358e19901c201df9bacc800775372f595c62acacd0f6f7ea88c3c\",\"d75df63a921e111f22a70a33662b703fe10e8f83214be9bdb3056301f36c87b9\",\"51148c6ead88a45f2e3b7e4a2f427bb9cd25fce7a0c03fcf94b019d94713cf3f\",\"70f0c254cfe34579f11c13b0abcc0058c0cbf1280521099e241729e28b8eb994\",\"e5f075af5829ed87263664e91a193abeadb3b27d49caf78b3a27042d6dea5185\",\"82a21a4849d43cbebe8533b398b613ac2e347050ff0f09b21e7126d4cddb5a1a\",\"016f7034721c0b7ec41112ccd0d30f12346b1e74f27e3cd2fc67a723118ff8c2\",\"19eaec330ec8c90ae67caa68e19e03a7d43d1cfbe5bfb1a4381d9479f1adb815\",\"8d28c969c96b4a0c9badb0b1ae1f4608e1137f0a031739bdecd9d3d39ee63b90\",\"c0a4343c18940479c1adebcf85b6db3d10585beed1a5a88d8619dd1145f31eb6\",\"18c74e8e6253b046a50b49de8737daa12bbc5a8d4f05d85ac4195e6e0a797e59\",\"0335002f96ef76bbe20353483e0a8c21f6a719c600218f9edc52067f17d22b3a\",\"3e5bef102ae0c79c883a60373a424aac0202d52e898d2aeaeb42fd9682b374d3\",\"b5f7c01e1d4261854ab4bf1c26553ede6a54b521d2b6a7cefc2322e17369eb53\",\"d4a65f749b40eaf94420ce9789da22d038ead3a80df080eda5037f3ab5f7154d\",\"6496692c23e450c22ba6e52c1fd5f21f92def558ad8c2aafc8047be6d95c066e\",\"3efb7ee6d2445e83964c2844cba5a52db68f5d8b971be88ac129fc3adb1439f8\",\"c14c9b26c54db226288212cba1c82f47460b79f8cec33b84963a09f66f743ec2\",\"8176ed7e949940074c89eed0e655159135c43e1a91fdb63206b222cf5198e5c5\",\"002a8a75903c1ed1fabd88b6c6b05df935ff8143ed55eb3deeb07aa5fd700c4a\",\"b8b5d20f2fdc264377bbd91aa577c5354d1e05a713e1419dade103adeeab96fd\",\"82808a9d691be567910798a65a099da8bf4d007b994763038fe0be02f41515e2\",\"13a57b4e384bf4b202864d90efa214401681a07074ea21f12da2941aa1c56909\",\"c4df4de57dbf626648bc54463ef6b2be9365528c74d99b0d7af9f54048a62b08\",\"8aaba908f7f25f11560192f861b8be8567bc7e01e39869a01ddf44d222b05c30\",\"42ffe7ed83daeeb3563a57a82665a634067681f622d3106dc0ede301b458d1e8\",\"f5fbaa18adc0269f88ac69e71e399cbe4ee61d77ceb81f9753761b8b46a0d290\",\"3a78a259b3731cd0a62690aa9a1f3de5e0a557d2f56da490e1c5979b86f17c34\",\"b398a2b7d2363fd7ccf72590d5762c18a96cf60314a49778667ffbd54fe7402f\",\"992a572799fe231fb8053cdf5f9ba260b52018a5d762fa5e6441a4ec1edd7aaa\",\"b1a5bee24433e79a5e9cd138d3a8e6de9a4f1af52a90f8748a91750f763c54b4\",\"e8e812c3f9367399e59c91ea0dc77734e81a68450426f37f659238196574eb33\",\"609ad190e956eab5c0d190278a3301ec3c89d42242e1bacd94ea0c2da9e46af2\",\"5581ae9340d5f416132c3020243fe667a25eb9b6cb0f9ed344d90a2cdbfdfa88\",\"589e1a42734993671f1cf55a877c73edcf2f454f3e6c7441ba6bc94c902e21c3\",\"b5aecbae0c16fb1313d540f221d2814edd4a8c0649335160fdc933e46df26c5d\",\"26e1a65674ac50c2f508ea63e89a5c381ced9246a145c80ecb36bad1b66b58dd\",\"c744c3a00ea4918e875a379ca7b058bb983b90714fe85c79ca225de7acba3df3\",\"2cbfb751a44c49eb757c08cbc50e3351e4ff92b5127ba91ba6b6adf880575789\",\"32a7b562e3324f3e6a4204fc41b9b64bdb25e8936a5d8294b8ec847681ab3485\",\"a53982e84254fd0c3c57452b30381f47febdcd696020153d27956d56eaee1347\",\"56e8dce1b7b756a2a883e6b989e1173427f5ac4aeb050a4834fca708e9b40e1a\",\"053c6ded576005c01c323d2706cb21cd20498d11dab21a79d0a7b748e83da4c1\",\"fff52c5c77bbac16bdf03eab27cc75bf612536893e67bcd6bc13d48b6d64c028\",\"8bf84276439fd97d276ba795d64f50a7fbb851625f692f00c2c4732c390802d8\",\"983ca7c1a794ac0eb65febf2a7e1293b8d603365d1baa4d47434fadede9d765a\",\"70d3fea6e357ade4123770ee947f7693604b892356012a8a24cd444e5fd145ff\",\"1692cd2cb7cd1417225855d62da90acb9eaab15aa1c6c3f302920f6d00673822\",\"9eee88ec15e2651d9fc6d721b8e03f0d628c0d17a6c12e08f2b27b77fe8f930a\",\"b0a46454ecf6f04a87f39deab4b8e9a50be876517710860cd88ae1da62c3b520\",\"297421cf2e3a636b618aef07305bbda298779b800b8ffb946451244dcf4100a1\",\"14d9908462bf3b25f04a72d4c9fed4cd0496f3fabc89f03f4310cc2bffe7ecb8\",\"246a1c7e6b67a867afba529c446e23bc71c075f970eab6a20b64e1755b6d36b6\",\"8f9b6293837002545d7d1c9a10b7c50d5f63d3325cd45fd3bd94987ea24b8c9a\",\"e3cd6785fe337e2419c1071712fd692783133517e7ceab3fc3b7e9f6b207a331\",\"522fe958a6943284cf7e7604c69b5905fff9df25b2968e29941f276e5ec7e3d5\",\"3e56a18399292ea7727bd699b1412d94ca473c230ec0494541536b8feb607f9a\",\"9add6e95c0d244191e438abb2fe32215968b8e12b36766917ca0e2a987026222\",\"10197b8e241bd4e66f668c25f545328a1e93cef10d648e9788571086710e1691\",\"a60a3c54d73bd07315d6dd0192ddb443e566aaaeaba0d6632f653fde0c1b1d0a\",\"7e1700f0272dffc9a3151c5f51e5c4e4bd57afb6c8ce8967fc90a3c56a49f284\",\"a315b48a4122e6826226bbfc2a3d6cbda282cedc0b579c4c98484bac37a663f6\",\"af68bc3a9e4055e2a9c96e6b7133382daa5c3ca6218f301cda6c74caeb9b27e9\",\"bc2992b526f4d98774e8d9393fd58dd023c1b543968a7abafd5302a81a705423\",\"88805b44dd98b6127bedc7ef4329966bbbcafb6126d70a95fc16d9a0d05a06e2\",\"85d960977ecb43f6ee62d4b1fe7ee15feb5d33deb847f259429b75f90be06e0b\",\"b5ff094658664d97fc01de462f0ad7cd81b109263105883e870a1adb1c0ba790\",\"e5e745167049736b5d5961989d8ce04c5cd3ae9f89a2ebe1441fdd8dba168279\",\"ff52930b3b73c950b04e6e8bc654e870ce95939dda49cc35cd57e9d6a875f762\",\"f42971c3e1240e07f0c2bb315d0c374997a00b337a86ee14faea5527ada9b671\",\"67763f778bc1f3098543d9678f7bfd942a1fef41bec29eccd914bca1ad236aab\",\"e12a810d9cd266b574b7c82950ee15d05d903682b931268ee1148c23c813ba42\",\"e13d056abef391890a954da57ac48a587a5bf388229bd9807a6d2db92ae7e86c\",\"b165818edad3be28ff4794869895ffb079bc93c880e302663aec6e84fcd1769f\",\"76766157ab383469586b900ad6ca35165f5d5483ec21d35ca0e2c6b8dd5a42ef\",\"7c4e42799b5f3f81036bad1d934031bfe879c6fc4bd2f30e20ca5c5f06fbf251\",\"1fc41bdbedf31408d264a10bb9d031a5a06ee6b6bf8e81d8ea9c04ac395f5b0a\",\"66126b0062b9ee502ac7ac392bbdd26703c9dbc925282e95c4b41fb393a4758d\",\"2f92f7f52a949c2173dd1bd5227b86d3680dd02a04ed98d5618789d3fd71c695\",\"c16cca4a9a091320f32992d7ce78cc50df84c8406d2d035b2ddcb32736c05380\",\"c93ff994c61aea36425456f1a73eb5606d323933b5bb73416c5b3ccbfe3b81d1\",\"77845c65b97ee214dfb1447f920019694e0e6e4f7f19c956c5d91e14e7738234\",\"31803b6fe938f184750c94dbdebf5b305b99c0ee68e6c492ea43665e608c34a9\",\"ad1a4b4af9c620ba53c7f74835e6070d8e45cad28ee4c60298b5924c2f35f1aa\",\"1caf2314b399ef6832ef0a0b4020935be393378c2c9d37c0e0dd095d161b490b\",\"4bac07fbdadcc4510f3ca47aabe2ed7263070af9b4fcd842ca9be0645220ee4f\",\"ecf1852e4ed900440d5c2f13b46f7021ac253c63b382ede782511b958d2e4f59\",\"921ef32c8202787f37892119e31dbecd8a1e38ee95f0cb4a99d97003b3705185\",\"ba6f2569df69e328f44f767e99ef52bdc371635a9e2102efc327f563291971a5\",\"8b3741151472fce1f65c42521160c9607ba1e8b6cb3bdbaa0c25d547f7df96a2\",\"7cd7667670581a5e1af3a5b557e9114f6ce3e05f219887d3295e70fb4bd763aa\",\"efe92e12d33b59a4e46795679c8d124dbccd0706c3e6d9cbbbfe0adba07cc5f5\",\"1998240e738ebdb6bca41e6e07f65a437c59ca37c3049f07d7f439f5b520b139\",\"6b83780da86d29f6900ed08784aafae652420faf3baafde710a1363d660f3daf\",\"eab5741ad5a47293179304f3d5defb3beb7002bab41f720373734b830a2f1264\",\"bed3bc3c516a25b58fabb1431ce351500f62f3969a0cf6a5ae429ec16655f677\",\"159c03b3594f03795ef5f68f62d48ae4ffe380d9a6c8a4dea4563286f6c9abd4\",\"f580f976b6a775e7c79c36478dc703d7ff02438ca1701affaebc83ad2af8b5f3\",\"f4862e729721e88c7042d1155547b03e72a2a58ab9bda1ecd47659d6e30eea04\",\"968b90661bbadb091f77bee5f10a3d28ee3b8fd26a17a2c420441e67fb456434\",\"694a8e024ac0d34193434dbca81b45d85e35c67418fadb6cbaa80535438c5630\",\"50e06bd3db38267eb67070377b50a9bafa7df32ee7ee8319cf6652d2b14f95f7\",\"5b6ca7dcd6a586245535b351934fa197a60e37baa919106114b3b62dc5879bf1\",\"16c78d4f3334a175f57ea7f78c89f3b303e183bde0866bfdf0f2154d1055d5f5\",\"2352070855507870010ad400ebfe2ec337f37647a2882b24cff43e11f4864fb5\",\"b49c349f7918796ad9ee89b380809d096a6e63f490e65efa92160b2284a4a741\",\"b840773f0004ea877f7c6f68f932b53041832912ddb29fc8229062b06388acc4\",\"f8218e438c7cce4a5e6986e99552c0630968fa6ec0029f1e12f82bc8e5009d46\",\"ccad3def03b29d127fc5b7d3e2a4cc6fa0118393fbc39cdb66833f3c5a512348\",\"efe82e03157e18940ce8b3d8993215d0c87f0b88cb179c8630edb18914f76245\",\"424a7fc832103437e12040f933d54fdc3851a37722df9c7555e78ea2d7ac3790\",\"956ae5164d36cc7febc73f081a7d4796e8301d19d93bdf75ca0d5abfa3c7fc2e\",\"88649976b70c37569add221ea4fac89606dc7b2e646ac49da2022838a66372a5\",\"a9f6ec00d7557e5c8abc8ee3f28d3b0a0d864aac888455db6da6f808575e3be7\",\"f378013152ca1aa24c655306067425919a218f2ac0241da9792e10e36c91bb87\",\"184f9886a7326f85d24cbb8334170371450ad2b0db2340dbafc21c229f58e52e\",\"6e85007108d4bc5d4e173ae0389af86464834fa9d960ed459c18d6c74fa30032\",\"ca312abe86d4c8f31ccb2bef8aeb94664727df88254f1c68b2d52bf00d974bfb\",\"a569e19a45ce9aa0c6dde7af1504214b341db497e1b0f20e9e28e68d24467f1e\",\"31be33351f74f414b1eb1415de98f67c0ac08f689659eb58b11de7b0b4f7a6d5\",\"8e4bab8303b371d0a46f89a1ae571320331f193a3a632dc157b196df0cac16e2\",\"1187e47ec95684a50d7d9c12622ffec172ba5d4837eb4268ba37a181f80f47df\",\"59ca349d888091e24bed8788798f0c036217317fd4ed12962960ee207ab82232\",\"0575cbfb0c67b4874c66cf375888209a36d3c4f332f5c846ee3f20e19b82e9cc\",\"6780b9006dadf22fa6576f66bbfeb175d303d3135c1feb2d4949dbcd14d2f75f\",\"9b7c6e9b73b6c8a8fc18308efbc68314d9959cc31957e280114ac754c56b1665\",\"66de58f982a1188b8d123648e795fab4082bb62e2dfc971f97ab832015e1e511\",\"039e12c8b82b4f3329b1e5a618bba034cb5b5b4e22e91ec17edf0aab9d2321e6\",\"052f1d8d8eee78a5893c3fa9419785a901dfeae0bc34d7a52299594637bcf194\",\"dafdc7c838b6fce5f825bcd2088040845fe2ee250cfd5d7b04e0a0377f1ccc13\",\"91e78c5ce85572222f4f6821dd13e28231e79def5a2b39020424bb7bf457e3bd\",\"b1ea295e3680ab2ece5b72c19a58672ceb1f4fbded63f86074526bd56cb61c1f\",\"4387d71c881edb65b48275b9223459298053bde2790a480effc2da2bc8f56ad8\",\"5a92d24c7df1325721c8fe04620baedcf16f9ad974bae0a837cb7f80333843ae\",\"7d8b5b7716e7af8703a82c0f0adc4c4d5fade4af052b8381770d4a7ef76875ec\",\"90e8eb275f65288a2612df2b5e50feef19f1dc86c00a967ea280a0090ac3f178\",\"c87c29d0a6d9f43cb74cf7f28d4cbf86bd81ee5465d44c640658297e5dff48ea\",\"d080b0f5dcf724c4d9d4d13eed90205d1519fb5a04483fe32d0bcdecd120447e\",\"38f13235afc3f8a36fc42cf0e6c8452606688f23650a56883dd79ba8e35e22c0\",\"b71b4737d27bce581109281e4bc32327771f3cf8d0be6251ac329038b18b2278\",\"fd1ad5a590eb50efe41d7e4b28b8d994e53b7d97deff113db075925af8d11778\",\"800d097091459f5e4fc987af55211434d62a781fcb7f51a82fb288938be099ac\",\"ad6f1aa71a2372b76be4b0d1e0b99db60101ddafd1b506128b5510e5f781f2bc\",\"ef9404d8fb576ed915e2be15b7c9b36369c903afe491931851def295a97154b8\",\"ad1e79057508828d1f4affe7b6af35dacd204cca236db6190fb3db74c017d2ca\",\"6f35860f693df124ec00965ff44b57e39b94ef8962be16c28bbb51cd869f73c3\",\"b23f0129f558409387ea25bef57447b1db828cdc3a529792e6963c87abbbebeb\",\"c45512220076fb7d73d1b44ecfaf63a64a5dcfc449b71fbc9b67015224a905d6\",\"435a3016c2a3e40f6c51a83ecec8f51fded352a7e4087049582693aecb86fb52\",\"cc7ecf1f9292d6b4668137519b6a144754302d890fcaba58fa791af0cc0c5b3b\",\"5f85239cc8ce8d11d9bbc04f45a5a3981089c8ead1411d5c65d2574183c0dc5b\",\"197ef2c2c5f5ca6ad741b3ba3a2b2ec06d656cc9311b1261635a2bbada138edc\",\"ddf486794669f9db03f6ed4b36c87b72bf775f60476f5dd5da8134b30909ed94\",\"edf50c5eb7143dbd3b292dac0f619021f9f438c2c93774163f005e12982e35a9\",\"c6a5f742bb005413614eb3a9b1beb398cb03b23d385895294649977fba5a854d\",\"dacf318d227953c8218e213ae2c8b5651ef0a8c957a2faa34a67217760d59462\",\"32efad57d05c30cf9cbf1a957a048295c2350e07f41fd2abc1baeeed258250ca\",\"6e1c1a8f42f96132bf235fc6ed41ec2e82606c00a63fe430da88a8ec142b51b6\",\"2369e8fae827c6cd78b3207ebccd20166db31f1898becf79a2e654cf1b2df8a9\",\"a4807d750194efff8749952eb8a1f0fcb7d9ac22829934800fbec29c03b5ce4f\",\"48c03e35a9ce64cc8d35d07c170a275f343c8d5738e81fa0e48de07de5a4dab1\",\"43d1c5146c3c5ac8b3d78e96a1d0837b46963bfa1984ac200782ba1d49325cd2\",\"e895a7069e19eb1dc6b07fe9daa3d76457c7fc706aa68b652e45b6dde733f145\",\"6e5c29c573e35b9e093bceb0807255c9cfc77c48523caa23d16ff0d729c1853b\",\"41f9d92a4f7ce441a5934c7fcfed32ec12cfd51969702c8925df50aedfa2111c\",\"4441fe0906adc7c9b684fc4536f45a6fe19340b41810b432c0747a50852fd894\",\"c8ba34610d5cfebd12fcadeaadf93fdd6d66426ace18d48f841465c4c656e33b\",\"ecd317be2735f80359bdbab0807465b80f4175a9748f7471f2741359c933f866\",\"8587967e1f34c41612c91f1811b935d8003cb0701c7979852215b6b844847bc3\",\"9deb24a588fda49ea82e82877606797036fb50cc80d8fbaf76b1cbc2bdd6e45e\",\"a6dd69986d5a751f2652747a9bba88ad0cfd7c918e19e7481401084dc5dc0ccb\",\"4b76fcc52fef5b0f1728b13cfbc40582327ace36a0db3eb8806aed7e09036205\",\"d2414eb4703308ec990b59f584670cb38faa295f240f18344478b7f5d1d9af73\",\"a903e6ab0b8cd15debe9471e6481e0781db3605aba61d670989a9f7e4b9769d2\",\"18d371f3c6f66d016920adea0675043a19f3528bbff672964818c7e77e7a8a90\",\"bab908b9b4665b4b36bb95f6b8b66a4e31f75ce952cedf61e0d3815842d6d442\",\"514d6352144dfb7b169c34766406eddd4ee1cc18e40aed4bf99b5383eb913570\",\"5c3d3a566cade52dd9a6768d50a1c20bd95b9ac8ec17431a8f4ff12ce8823c3d\",\"c93847d6a2fe494eb48762e810c2e4f8c3070f8fffa4495dee4aeeb72ac20da6\",\"610d3e4f22aba55efda064457544cc5255094cfc5810d558d0a3701cb8557a39\",\"5235fa10d33fa3e1cf850babb3090e01221b04a86bdc9bc6680365425b0cb5be\",\"43e3b13a278fd352a91e2268f65bd03c5fceb94522bdf0421e3dc91cd7166084\",\"2bd6fac4f27f0da2be1d988aa4d789bb7504626c067c52cec519dc839e62b423\",\"6cb5019115cbb1c131d9a26dd294fe42050145c52e7bcd4d96fdf2ed428dd7c6\",\"ac60097680b3cca90b35e6511e03d2fd678508113b593b72f4a5bf485627700c\",\"c280e634e5b3eebc37f51c03c0b384b189ed6decf1fb6391d0b0e0ad2e8cfa57\",\"8d3b19f71639e14d30786c7f28be24960d08760bbd36c19e17c0f5132cafd06a\",\"6b1795117432f4bcf3d3583e16dcc01f4be7c120069a38099ae4026213986ef6\",\"da018229576d44a43fb7205a11b5f8b355a582f32a59b201c5b79944bc2e9d76\",\"956643e73cc43fa4e08c82b0f2a495f15d10246f9c301fbe8c8de1d57742a75d\",\"5d45c643882361f3fcfc0c2ed4232438bee45354acdfab7bcedcefb45b786b36\",\"c3316f075257fdfa1a070d39a3e8d513b10f32dccd24724122c481f0a26df0b6\",\"8378096511271f9a4fbf613b9db2b57002f6846d6dc3c18bf94d83c7dff25474\",\"91a5155cdcf6a4570bf52c9cca70461314099a138e0a513d9012a08e098e758c\",\"2f66a018a14f8bb9f0dfa91464cd49093cc3a2e93807d59337da81b3907cbb26\",\"eb7d096cea43d15103994edc424a2acbe4aeee59a9bfe528f1122aa998e10456\",\"ca29513a7e4dad66f9ede774368c4b81b344d060ce47bbb30ad1c9d3eada454a\",\"cc61e31a434510008e49e0abb21c75f15c0e51772dbee7da45545fe61bb596ba\",\"2e48cc6874cdd6b5eea63adaf87ba79f1202dbb7452869d2d4f03e46e0d3402f\",\"3baadffaa1187bcd1eae387421f02c38ed4cca8f1d18a4fd1e2224bc881d34d1\",\"e33d1e0caeff8645b05ea2787e8fd23227954708c1dbbe6190942f155097ef1b\",\"e7644094b9c7a2418b4948cdee86833ed7781151bc15c5d5a0a4d8b92c2ee3b8\",\"a1738e7c3c89eed77f55be008348e5c880cba9a57781c4bdb0f6a09a3e1a633a\",\"a6af33c7477198886e6eca81c0c0d125a8be302b41dbb9428dce580ac011f96b\",\"d679c334bd745cbe29a00d09269fe01c6b5fb05ad0c976832f848ef399bf376c\",\"c948087fd4fce077daaa13db10d5944f5cbaaf49b632d8805ee96f3a9e50ceed\",\"eeb0db6227c27b0ca328839805d2ae75379a5b1af9bcced7697ba5c23c240c62\",\"fa6751114d092f645e5dcf17679383e73460f19800a7738d59c42a6ef56b3955\",\"ed22a05864eb72303f520b691e04bd7357880463f08f76bec7dbb6e9e289d86b\",\"f77de7d7f0e63b4ef7523bafb6a8639232ebe51a75966104c0d29b778c2ff14a\",\"e5fec11b366b1c2ee150558bbce9d8d50a697228ac10b6a7c39fe0e3469ad2f4\",\"89d77998630e64352fd0df76ed9eeb425269c4944b1db16c9c900900244611ac\",\"addfa54d2dced55878981279b99be82ba80bae9fc22d86ee848754484f22cbe5\",\"772bb26f3248ad5ad8d386d830a221a4bc8befb26246759cfec7272ec86df4e8\",\"753ea2ef87c262b4bbb48832f83dcd38e2564edc05993c4ff7022077976709bc\",\"6f71c9225f7176f2fad2df44db2c4759f90a0c5468635dc257b15cb320ff48c4\",\"e3bb127051f769fa6e11e17aaf298bbab0159c12ba0fe1c0abaacac6ada92325\",\"70d3ad1125605fbad031560be2a1d26192ae9761a03aa3124822d9bc2abbc3cb\",\"8feb7ccbd5ac9e190c73c8561043192fdba62933be786279488415fde859ce13\",\"2482eddd04cc799ba6708bc293fbf4936b14d997a3fe3df883ebe0deb961f96a\",\"44113287f2bbf714a073d2eaa187d0cc8487855cdb3c1ca437c7e679fa510947\",\"611d8b96e05d671202c25162a35b548676623c0632e31436994ff2d43c1c9017\",\"b3255fe3d11e6e47006a395b2744a6bd454519fb4c8764a3c0febe574b5a6e11\",\"58c3e30f420b3e269c54f73f3a2f400c0dd9ed28a88c553027f2c49e8a40e0ab\",\"6f63e20aa59bca9f7189beaed15a96c0e2273d346eb99218accb9dd29f5f713c\",\"8803db79c57190f9f8d5d257aaa8004caaace3f3cf55744019925bc49c7a5bd5\",\"0e3d741ef22ddcb444a148f5948bf5faaa19e62e5b31a2e4b6ad2ea82a6b1e39\",\"fd1d4d097067e9a8709c254707d14181b1bd04b51a8fac9d293eb8dd42503491\",\"e40966cd805b19766f297b4f7e4abfa6f72d17b82f0e0f7e15ada71ee637ffee\",\"80f42c30fe549566a17f7df3ef02cc6a030ae22521c15eda69df4fc85e518923\",\"2e1d09a4b93524908d3e5b3eefc0ab03753f22092a4378b69715eaec5d55b83d\",\"ce8a3bd104339b22b2da080ae3f695db0a512452dd85d7802b026a0aabce4875\",\"501fd7cd7a6f45fd2c4210de89076502be8f720d5900bc3dbd1f32687cda531f\",\"dcb4ed24c7aeb4e867c0ebe86ec6bb81bced425488709da8d382c5f827360df1\",\"9e0aa4c1292cb702d0c50f59262a76e1487f5fc3c2cef9d4a66388559998388a\",\"6bed3732c86b16a439c52c3ef839ad9cacfaee266aa45f5d6e21adfc9cfecab2\",\"ebc9d605cff48f6ab8cf2982faf66836b74132096c11b8ac8e21c6cb7e069293\",\"81ddea34274d5b22f32818d1cdac7645102449a1651c3962f16140694af83624\",\"8a45a11f231a6e3fbabda1cce795b7829c101e90df7428675f9b59c0971eb641\",\"c35f1ac1feb73d1cee83b0b95be940237c565a65a8a3f26e785438f3f2172784\",\"b6735d21585ecd866efa0b98953403f0bccee82fea092092d615e271caf3331d\",\"99df3f52b91dc98982ab2380893063243164fd6e2ddcbdce097932af607f0fe3\",\"427cc3eae0c4e05bb2489cc4212ecca3d8cc94b76e7c3d285bbd2b09a85210f9\",\"a1bc0829695011b10440d3b999974bad7032827941ccd6bf05fc34b96f832016\",\"00540cd748e10705e9aacad294f4f04a40ee4d27a0892c41a1440a352bc8bc5e\",\"12647e58e32249d1da98a3dbb2c0ff9cad76fa74132e59d67948ba07611c742a\",\"fa7b73aec605ac2fc19ccf242518a4ffed1970cae20d24e62391f04b81cbe706\",\"eebeee8277d9e63ccf76a0710302e671fb66f72ed8fbf510b27942fe621f7d09\",\"2963368a5d8f4898a160fd1716d7a4ed0fc95491ef274ceeaef59b884ef5b003\",\"e27d4aaddc00127c7d939c00681b079a58a6fafb339789aba4f23ed9ed409f9a\",\"86617c2b2452b03a935a86d03584dffabad4d997b66d7d78c99518a47194fba8\",\"a3d92170613f7c7c50784864f92159cc79135c1af77cd9f7e6b20e713d68d7ad\",\"8df390d03b0966b40f1ee5f3e34b0e3b1a67b451197796a88d38b30519c6a996\",\"c91e44fa8b444bbe8a4974c2afffabb3f3ad028709b6ff9a2bdd7f32ebe61807\",\"dab619308288a3055b2d3d24860cfd5ac96855323ef8a99a71fa2ee6cb26a0e2\",\"d6feab3628d302be0278cc4bc4f881c29f2fbd5fec6b71970d4d0a8fa2c9c4d0\",\"fb09e801dd8431330a679358d8859e56e9cab3dd897284ab8f0f170eaf2008fc\",\"0ef2c98a608c90bf4ffdd63107a03a15edb317b642a30d61c85c5b40dbd640ef\",\"23a039e7b0834f8218b56b1e07b3cbf76bf6ce4d032ffeed1d5eeece2305f26f\",\"fe33454c5d51dc8c0872ba98bb1e726a832d3e64f475cbe7720d7ce57e46696a\",\"8de02a563355ba3cda1fa1ad8d774cc338342599f5cb2bb02fedc74c6be3db58\",\"8074a98ca938a8f31d39f27d639c7ae5cd701de386126be033e98ce10561905e\",\"05874bb590018bec9135ed64e054c9d0301aeccb16fb63d4a48500391337bab9\",\"0564c4e02206dce1b2794535866b1004ae749d1fffcbf0bec0c074529f698646\",\"2fcc304035f686f248e20fa54474ddb488e3e1f38e7dcadef76b74edc142691d\",\"38ec9612ea08f71c6fd707900ca5d1a0f6bb187575fa3adb5c35fec873327541\",\"90a3794c442628d8159a916d9b22d84bc5d2c97c9215ecfaf0154df34e151322\",\"96c5f3a1279560a5a21fbc0f11da9640f7e066061efcdc6f6314f206a4bc6369\",\"e977d85b4722437e31c1714af91426b92c6de07281a1f44f9334c7a9df6c20ac\",\"597d7e2b74e20f3b098ab83ebb016cc9c3f2623023463c694d924a4fa2e5ed68\",\"17df191a22d66d584264d382930e047279dde52f06a9632acc20c52a9739ee86\",\"94f4e53de3d439c17659484811f248919fe7376952c8a4829f0b4dac82ec1433\",\"2252b6fa5bb3c22bb76e5b12a7ae8654d2b8c467d6a1e1d6dba9d814ca0cb831\",\"4b69252261bc9e180112c3f681d61b8d61f9f87c86efd9539d7af7c1d4ed2b29\",\"19f282b4d6d6477c43e079202f36111f5bbe7d187192a63cc7d312b57d8d291e\",\"ce61cb30c4fc22d342e61c688b4a5ec5a0fbd06f7f93b9e93878b26b2e105baa\",\"857f420d4906063de26c9aceac8474d3c4c77f64dc68da2e5a28cd6cfcacf871\",\"70bdb87ffb9ad284a26d0b159608b2ca3fb19ef57a27a78927c5926680f13748\",\"951a23e0f6e0fb3b12d8d4c0d76d38312adbb05d1bd1aabb15382886f9fcbe3f\",\"db90f9aaa991e52ee87d1338465db8e3306f0f651983a51bc6d074054164d7b2\",\"a6e1070cd5978bc7474e8af950e3f0afc41a8d1d8e2b52b745fbb407a8f40a75\",\"dfe1e290b88d931cbaaf6fd29d84f28b1a4afe7995f9e18b739a202c6b16ba77\",\"7f3e044b08498ea0a3e55310574035c29dbb9e51f627a26d7a4eece86350649e\",\"8b0a638f28e3ff1107341cf0b792d8334fca879cd62e1286cdab1c3e2cbc2c9f\",\"efb46e28b803964507b9fcaecca86684e4542646069f6827f2d322a810990e0f\",\"80f4b2f8f4b40d45d849c8e77a7c4ce742217fe384edd25035dc687352dbe9c6\",\"088f3e4dd32e9a8b9e2518ee4fe7ce824e85cd8545ef85e0b96393a5efabb1a6\",\"808cdbc5d38cbb309e88245ea56cb490a525d0423db841b84a13080432cb30e1\",\"38829d307195440d658b0c99c913bbd2b134df4f92db39b579170c63db67c716\",\"e21e462fecbeb4b14fbd5459c1f1cb1b655c0674814ef7cc070e8b28dd547be6\",\"6c6b0039cf03d0cac81b2ee43d3c20bc6813112851b62cdfed427bd541d63ef7\",\"cf9f3ae813eea405d20d2b6d94df37e6db3fb0c05b23104285a78deb671dda27\",\"9dd9bc60913abb04af967e8ba5d8fd0b47b227db3a1094ebe462792412887499\",\"4d0ff92ae60c0dbdc7f99d9191145aa027b2062a2b8201de6faf6bcae4142f29\",\"abe61822f61e03bd47aac63ab841e17749fece2ec5ebe20931dd9c64b9d5d3f4\",\"937f5bb6ccece09accd9d1f7bf921ddbc2cffd74608e1d37ef7ae6dc08c40276\",\"6410d33ece846f79bbac615b466948ce3ed4a834b0f714b6c73569b3df65cd29\",\"694433fc04e85b7d5b7fbb051176d3996664afb044efb8d20b699005d6548f87\",\"ab5f84e77bf4cb209dbb1ea3873f2ade2930bf3476d296c74a8a1cd91d49112d\",\"a40cfac42e981447679640ab4aac10c3cc69bd687f0fef5928c33226f3c79953\",\"aa830e63cf3c57d7ac4ad170e907d28e32fb64d4e9a5e0f2ab66bb05a419c918\",\"517112a1480908560b476e94907c324f1467e7c67afe89bec4c74857c05bbfb9\",\"d0d43be17f41959106afda0dc11bc50f18e2510b32570a89422c29a22147dea3\",\"3030c5eb72b7ad5174a2e5b25fb301ec657502a1382ab7f821a93071aca2c05e\",\"49770ca502e92eb520122bd077ae8f4db743ea7fa2f0542084c86996e9383363\",\"3a962f3e7e2ae0ca16be69a8468e215f1fdb5f25302588bddfad64f585ed66bb\",\"fa3f9ff86a4bf2973a4b79c7d69c1c460826d572b0bade673bc31671ce786185\",\"9692314c3344009ee743b1f899cacc15c24ed0249fae84311972522ce36f47c6\",\"6140cb1b506934da8e444070d958e7e052addacaac65af1a2bec228210f91c3e\",\"4f9425e95b9e452a0b66d486fa88b8c6e9250362e5db7de1a7d243118f7f2788\",\"43ea0ff71a6dda8871c1637f63c2dbdc5f177257f342304ce36a057870bb2ca6\",\"f7b3e3524b60c63573a5b58c7131b0e3404c768fbd8e6f1dc880f2c67dbd0953\",\"44669dfe6c31829b4749dfedd9d2cab3d50fe87995749d8966b29ed536b8d55c\",\"342b764e83fc67d5dfd54efcdd92704dddf1532a7dfa51f70396150baaf4bf3e\",\"b9d147b15ffef78e2f0648824db339b942066824c0215126b8e008f8dffc2de1\",\"1285c07842de73426f41b19b482b03206885406b83f9cdd0f8a9509df964d894\",\"0fceab96f853495427100b424dfbd4c70dc002c919b67cd6832a71225f46301b\",\"20c1730fbd8069dc047c6c655930b005a5295ea542ae0179cfdac503b7c65c48\",\"c01d004c9efb7bf8cfc88121085d7f7b9cfb4cd64d61ee47238569565acbb2a9\",\"c517b63a32fc48891009211c68daf1b09f126b137fa1effcd85664915f6a8211\",\"a201bb3a532ae3005e3a0e523177e281f1f979f944f2f18bc3258501bbc3feac\",\"30050ce6f9ee8c12938115b4b4a6a562114a40d92014e80b75c56277a261250f\",\"73440025a29c002dc347f4b09ed6478be3e8e6f95c8258681484447080f5b786\",\"76b6919688849a97d0210bbea40c2b6e312dec14b99f936944fd2a9580fa4cd5\",\"ef9a5866a9c03a04e3acf9ffd0d7e116d013bfa9b892e0c82c30010e33012a9d\",\"fcf97b6108e6b7caf90b4560c33a9786137832ffce8ef148f16c054b6fce5813\",\"cac0a1691b3ea8c36e37b47a5934e2ad6e1aac4af7c2ce1d3337c01b3ad0218c\",\"64ace71bcd8f7552c0ea6fc4630c9e73b1df769116f49df3997fb2ca470da2b9\",\"901840a092dbe2fd39334d59a05c6b9021dae6385a6c4802127c6cd50b2249fa\",\"ba1475ce54c6616529a9788f81bf2c5e3fa6faa8b9542390121c1947af714cbd\",\"d9ec460675fd1e79851d1ab315f6e371ee36a967df24f191154d6623c5218571\",\"ecb223780295198086d15580106556f18474d988458bf93308c1b6020ffa8b01\",\"eab31a2e0421ea984532172b18a033488d240143b4cb69608ea1bfb452a25af7\",\"a7104b813c675c2f00f5b8121f99d0b6414880b587e335011efc3359f3c35481\",\"095bd160704af9d21ffc6b9f99f6dd4915bb6432e8daa81378d230ebc9d9eadf\",\"cc2a3076648e6aba7e8d2c598f371954b5bf5cb7b5920d52ec1f2755f0a2c990\",\"9d6acdd717d2069d421befc40674efe57ba3f5c9d0637c80657f2aeb4bc1ac86\",\"95511babb163100ba2e0998e4d2d40582f0652e692135abd66700d9ae410468a\",\"f58f19e7613f77644c26edad01b9c3099000c191fa9aa590f9e8dbf47d08963d\",\"f2db31cdd3536a21a247b090413e2fa47db18bfa29f93d9d35ca6861dbb9fecb\",\"aa3b5bdfdea8c99b9502b699c9776adebfe0407aba0215cf2aa7d1c37b6889ad\",\"7349dcf5dd1f3ab6ac8689fdf35921d8294c5639a703a40fa8654e67c1f94f2e\",\"ef497a0e1074092ede8bac69d22263345042feefa292994bb17e421eee1dc9a8\",\"8cb78df0c8b4f6d1d77e97ca760a18f463b3aaa8f53080099a93d537a0e2955f\",\"3999084b27ddb09c8a536829022a268c9b0a87665e704626194b5dcc91e074de\",\"3e2424dc799fc32f5b7313eabe374b483f67d0a2bcb84558b83c740fdf8dd49d\",\"45573a2fc69544c0e958d3613689e48a6a1501a664df63f4a3e8d44a1aa96487\",\"243a32ba6b89bd35036c4861a1fd91577582b26941a9737ca95a57d16f37fd85\",\"d2c39b05b53620e5eb0e71eb79f8b6ed9c4b06d01af7ea39863a822c3319888e\",\"6fc729a7595c80bb1a59382f21f09387c12778cc59955f4ca8f2bad1d78602bc\",\"07c66f292b93d9d11c0d67b2947cd9ae2eb02bb1fd43c63b805bc165c4e333ce\",\"79ba2080e4273f06803aa851be91ffcbb558d9f4c393b84b0ab64221a21bb714\",\"f0a5c6e280f002ffb7717f99f001bb8ba97d0828486e466e2404b2bc64be4955\",\"8e92bf6659e37878e0c305fe043e85589f917c7b1cb69493391ce90746670ff1\",\"1612dfc94ae55b404dea87576b2ab01440d2cf4b1af8b390c9e61f3cbab22520\",\"5c91dc2859d91ce1a8bac32d9394b58aa6f81e7c63f9dca019efa3ab3c99b109\",\"5012d49c0d8dd7108112bb57aecae5c5ff8fecd2530106f62f43d52a03762217\",\"daddc2dd91c3a4ee1b9cc284e943e2429c0b1223542bfa4dc68e7886669ce711\",\"50b4d4de1548073fd6bfb4846fcff4f127f7dafd012737579cbff43559e629dd\",\"6f07f998655bfac987b27338da4fa1e83cffd7b6f683caced74fd2b6f03e8cd5\",\"894d1d5cbdb0bd8b9f813f80c97ca258554fb6d72b8c89b4b5b9633693a9b8c6\",\"94dfbaf9435fa187dcfb161150979adf88f963a84037f72459a7dd8efdd06ceb\",\"95b1cd00b4932812c5589cafb5617873090ef14deee69b3bccdf5159b00f4046\",\"f9c0ce0f0db1c8a4635b498f2d47b451328c763e4c5770b3c66c96208bd683db\",\"2ad4058c0e7054d72592e0aabc8b2952217d92d2d8c7fcb6894ec58da49edb77\",\"ae33e4e1e6dd80418c14175de1882cce6ee262dfcbcba93928f801476df9be3c\",\"3a69cc774a10f1dea74d8e197e5851bce8af6c778b8f2542efda4651c143592c\",\"ff1762cc352e7a8122479305b72a71aaa09c00b4037483a15b74cedfe1302394\",\"a1cf9e0f8c1478ffc939e8cd7a7f99a5506504fc5a0cca9de137ef9577a049d0\",\"c9c89d64202aa1a6b160a324f8e9f2efaa06bc7fcbf50b2493089ca26125220b\",\"415e33bea5a4d4440595bbddec3a1182ee8bc58edab79d984d7bd19bf5664c5c\",\"68c0afe98bcbd2d2654c254109b4bc662b062a4fc4610d9391c2a36afb74b8f8\",\"8feaf5a95d9e2d7b839245baa15773e7ea5be908b3993407c9572ecce0a47416\",\"602b5ced535aae70ababa8006f34cec44b64e5b215c658d7c16c8b47527df8f2\",\"fb11f84f5069ca1c6ff165912e6f782339d0339274ad79f4e5a1efc6eda9c8a5\",\"648e42f4d0ba3063fc38987433efc6054935d095364179db40aee2b28b6b889f\",\"16814bf816de041dcef8002f834e6f8fc1d9de892b1501b438ffbc2db772a0ef\",\"7b6b35c10fd12278aa61afd8bb3993ea3b708c069b790e13b7399d9ba95773ca\",\"21879be79532fa985468f945e968b9947b99d7f50369eb7f2f5aa89ac959151c\",\"5f1055f7cb9f34d4d3a4a74b1e606d49b138a37d65126a4a88c4848dcfcb2a0e\",\"c9897864f23e353a9169909b7924b62841a1a62d146b80fef20bf429346d3d88\",\"0f27d39f79db0cd43c46e1292fb46f5a4effa1f092695b45a076b7a18f5139a3\",\"65e7b38a249f6f9d4635b2da5fefe52977f3e45ade60a4025c9aa8c2b059f4f3\",\"0a4438e528be54cf58bcc03179194c5407454da42df935f98bca8cba3e3fc87e\",\"2b6a9ead8954115354f6199dbc58c50a0496a98c94e3d05b30c4dec6eb62393d\",\"43daff0b9801b9054a07d15b19d89804aca50f465ad36076292894f17340858e\",\"9223436d544827248440cf75d69ca593e8999130fea04f0617c16a5a68515a6d\",\"5c7fa3dfa14b18f777ace5b4dda3a48abda6edcffa2150a0ee23ae9939b95a79\",\"0eaf76bd0ddf7aefd23ced4ec34ce5e5e9bcddb328afd2404e54dbf02eceb3dd\",\"21155fe65983a4bb3ba6294b4142d2c06ed24f0b6978d7bdfba9dc26e22dbc61\",\"e7923791638575dc0f5ff98ee7a5000f2942bd7d2890717c84b848620778cd0e\",\"1a35a4eeea01ac81ea927b85fe6c1d669c19675233b442561761de6161d38921\",\"bf9884644ce6848eb8a0256973ba563facc77d92021e2e879f08c32e48becf75\",\"11aa559da5cbc7c25d1980bedce55bda836a8c66cc1376e514460b66e1c05b7e\",\"622c22a33e997ff17656045ce061b11a86eaa7f4016028de4c6bff86c14b3e79\",\"5424d46f4d08b710291d5adab826a0df0d652711329887be2331d9bc346d363b\",\"ac9933c6340e3081de929cc8d0d73061eb6d16876fedca2562dd677608c92c1b\",\"c1d1e0df8f0925375cad5cfc965b189904751cb44ed19f7b66b4c46410f54b5b\",\"2c1251c2d0e78e22357bd2634704ddec616c9f2f1734732b0d08a942a6f8f4a6\",\"3d66f492c9c4b8f5086179e2dcb3a152145f5f888c63d2bf188344b961a0d418\",\"7d7a2cc53479ecad1e2ad199248a1cc8c789d8134942ba2cecdbd54aea055d9f\",\"2da1a7e7599fea86b80a1ee29b4f77ff60574ead889739e7b18534e2887a4e00\",\"230ad924ca6b859c7199baa3a2fbef0db3257785851d8a7c560b7fb3b9f13ced\",\"e42f0b8d508de68701a98f338188be15a137301c0fb8f96ad09dcba1d8bd2e51\",\"613197787ccc9740405ba735c7fae3ef535039356dac4b537cfad15775fb1e8e\",\"4c1b94ae21de281e21d80d768990036c6270b3caf43f5a0344d7af801bbc2a8c\",\"ed35a1827852e6b211edd4486c8a56cf8643dd5c1471535654b85a47dc6e0d73\",\"ce2c815a2defdc38b4821fe8cb135a5d9253bdac7a8826a87ae6becf63c63eb5\",\"0702d85b881c1687fcbde386245dcdb2e2a64ddb2f4fd88eda902c768b94856c\",\"ed667d81dba596d8f9cfa855951f9fa9c9352f57daf2013b8c7001815a508093\",\"55a65c0188449561ce63153782aabca9329acf464fdc7cf5d910c51bf90e4a12\",\"8f87d65906bbb885426d0aa07cbc4e4c245ac8bc30c07f5b6ff13a5f8bb868f5\",\"5cb5c7baa2f1b515352e7ca60ede7a3fa10eabb6e5fdefe625b9edfd60ae78de\",\"ea9deb0922dc6f7c4b36020b45ca42c8b5705497344be3d1cfa57ec8295236fc\",\"5441eacf514b4958c7fb7f01e9e683f180cd5f6bee698b10762afebad957f05a\",\"9d83c4a64ee1d461cc4a52c9bd1e2fc70ebb970d687b44ae16ff496b657a3fe8\",\"4012317d1bd1b4b420ff364b5c8e42b2c1e2f46bfe150463f89eb7ccdf4100b8\",\"4be12fed621bda3705b5c4eac1f501de857b40050cbc1fd7a4ba65498a0d9dcc\",\"70b14723184d1943adffacda2f656b196c3700bdaaaa30bfba3275e4fa764f12\",\"7be622ef3abef2e05bac2f2be168d64c76d5eeb012ef5de1fd8f3a2a608c5642\",\"f19cf036231571646a9ea3e5c562d639fe754ce6300d74f2aa3a04334934f9d8\",\"f8425aef43b7223f9a900acaea7f9d2669736300108688eae7d7353d32f1660c\",\"c5abd2860ec932089f6e57766ed394825be02ae3684d795a5b6a09fc57ce5557\",\"e13bc51c03f48ccf9a7b7606a4f75800158a6c120d979e920669856a97a0cc7f\",\"50e14b807da81bb7f015572284ff83b0dc5680a772f968de2b95d5b8ca3b07b3\",\"1f139dde7bd8444831b746713271c6c7005e014e2ae1d449a3791f136daefec8\",\"b34aebc79da281b259768cecac9894d843cf489875608cebe4ab8c5342f671ff\",\"463dcc28288f6f9c5336a1a7a2812986d3a54b34d52c241663b99c4ee04b2f30\",\"9f60d38af7745555eb011c23f7b3ff2368116d43127ea83f19fc05afb86a8ee2\",\"71e33834fbf7e3e794e7dcdfc86937ecf2f6c2171e5ae08e483f71d023504528\",\"fe5c63362ab136eff1884f46f878708f58592ccbb4d736f3b91142fe6900abc5\",\"ed57421cf71af3ee1cdfb24ff2a99bd1af3295805059c4488218662084a40a53\",\"37e5e9fa4de17227c95acd2142ef5568e5b591cf7d2f7828bbf856ffbec28047\",\"317a61aa05f51f7cf73d2d3d68c875e1c209dc652a57237bf130ca469ebe5636\",\"f14cc3fe303bbabc132e6e9e6c99730309b4f75343cb685f7f182df72a275769\",\"553f8c20160fe6f04b108f6e8c29bc41168a0a16227e0666696b67749ded5682\",\"a91d49e53dadc69ec6fa15582a36d033a209c6c6079d0c48b6914d3cc4e0b35b\",\"2435cc45d896228b790cc37a98858034d11138ad0afdadf3e752e36ac4521989\",\"b58cfba6f80a67930c78b44748b0801124f971a89d9e9ee4008c87fff8a17605\",\"eda06338316795293ea29c81456d807f39ba985c05fdfc624aa7b5af0aefb682\",\"6b2688d66dd9e2a77c2ae9273cc0f2dfc977fb8465c54861ce9df10d08fad376\",\"d5118947ecfa1eeb2dd372bbbe032abd7b717f69a0dce237307d96c350cc429e\",\"b78e560cd3506472085beabd77c223e4accc3c2ef7f04e83c461d815b75ce3c9\",\"0285604758623692e67051f7e664d6766ba3ec9d3fd2e96591a52415f1a927dc\",\"975f3aa80fecd56eaad784c9247bf96dd65fe83b92e62fa08dc9047d2206971c\",\"25d83961b5cc1322d828bd59aa38202d5e695158acac9cf7bd54c3a5f0911dab\",\"e7eb04237bb7a9c596c74b2ef5ec4a01c03798d664ceaf2b453d0b964c134335\",\"72a0c63228b9e7743bd19b0a66c892c2ed4364e77ce75fad98f9c7014ef355d9\",\"7565514a2208442978711a50760ea8f4b2fe650301adfae981149d2259d84166\",\"18eb7c48b80686152d4ed36fca0502533dcef94f397c7cf5203c2a46e49593ac\",\"e63c126eee68b231e66ddb2f313e16105b0d10a4a27112a266cc9e95738bb84e\",\"3c9d1d35bef7b4739c649769fb0b96e3978a74344714d8cf7b3748d1b4d5e2dc\",\"c1c4c25dbac5939149c70d80e41c8be45c08ba5efb0cbb973ec7fe202869dca4\",\"5a612a4b8c08bb07303c33b22572ae521c66c59fa8a2875e2058044a94658780\",\"92da7c03aace652e49303eb39380e9c43c91fa07783bd2917e8d5cae6d6766f8\",\"592ead46e2a1460c19bd4f30d1b1301ff276d0da9bc686ece98a198bcadf3095\",\"6aba2d17a5b6b8980ae54e6af0810b069343000a25fc274bcb887741d6ad7a25\",\"b0000365219929ed15fd186c436c7cc1d51c38733d3a6320225c9f9aeee93d19\",\"e5c4d008a7c5638b32f5a9934fe4072613fa9c989052225bc77ab58482b39c0a\",\"d64fe2ec105c5f77184f2cf68621c32df7cd29851d093b01254f449bf07dc803\",\"4b977290e3b559f21ddfc9eb1d2bc2236a7f51b258db7e12cb6458207964b098\",\"6860dd3c6774e5fc28bacda6d90aee2eb682c5dc0c26b1b0068a1a3957f63f00\",\"e6629f0781e20b7e7af64ae2e292c1266cd5b5914065ef20a66fca503a613b00\",\"9f2456df4a956d90f5fc99e2e0f38326dcae602fc7104981155397e3e4a18962\",\"bdca70a369ee9c12b09ca95e72b90d5a31cc7de254629cb9b5639424c2c9cb3d\",\"8b17e8960419be3c8a8b38eb9a1b533c7973d60bbce4e15a9987c3c7fb8af220\",\"3995cc2ea2cb14a5fa99ecaeb8faafead5d6ff6a74d2eddace8c471bd4a5b3ee\",\"a1a4262ab584bb1e8e96fc65c53634b00293bf7026923a7ed732fcf3eaa8fe33\",\"9aa7db0d684c20645e35e5144c2d45df09d85293027f89bb797b612a94d51bd2\",\"201bee45718ee0dc7d514e86d5d35269ebcd444c43f0e40f8f3071b83966c35f\",\"0cf8e8dcc01d96cb7835a96749dad124376dd1f0c8160036c46dac51544b3ee0\",\"d96e0b5d00610a6b34290589f29e7c9c160b100711d69fb212b8d152d187076d\",\"e79a81586231f7717f65f9639ab394a53d061a64397f488eba69d8279bb130f0\",\"9f3c0e7187188f534f2a3dcf28475837eda6a4421b3dce6a22ebffedbf70a9ff\",\"e6bf8e32c1babfe26dd7b9aecfc7f4d7893f9f329a45bd4fb23417ec9d2e1732\",\"dc89c6118376046ccd42df6e888fe493eb991514031819c092b27f474226adeb\",\"e32c7a815f1f2f520fe41bd5142c5cf1cdabd032382b23887ceb4cff3abb8c19\",\"e2d9d36e88e3b36e55e406ac3ccdab6a4d0b96733f252bcc3736359a5d7aef31\",\"497915828f69764ab05c617c6ceb6528d687e670aa0e246d7dc6360a0774b50c\",\"00f94b17206b83169ece6ae6733331f480fb9a14d08e62c4cc610a8a911e87b4\",\"4e497d19e3044f9f7d790392665fc9aec2f4711334dc3ac4c4888261e353f3e3\",\"83d0b396f70748917e2cd29611708d547bec100cb3add95da3e0889bf30a4310\",\"b93ae884559e44b24f849f22e696e510ba9b70a1dc8c7391a725283d9bb37ff0\",\"e48bb5b251525d44a63ea85d626e424458c45b3caf8f721cc664031effc24727\",\"ca85418e7bc15e130855bfcfbdf521803c7fa1917fad1161f040e139ad68aa1b\",\"c219129cd9718afa9a391d97d28c92e2076865a622b2acdb11187970943319d8\",\"e91b8c18e1662f59e7753efcd8f20a99fd09577a2f43ded01c5575046d6e17b3\",\"41c80b39ab750c89b9c1e9a5201502c5d1ca43b36650bdbad2c1f8a4f812e4df\",\"cd1604ad76f2c2d8a2b4bea20390c9120d174113a7b010fa280519825b6272ae\",\"d9ae216ac5c5615591bab3d32bb082d4e53d2ea3dbf5466b45f5eb98e2d39c9d\",\"014a40b0edff26aabd390bd0e88da70420a26260f3a8dbf7d06f0fcac9ecc488\",\"64e1507f6de1e274385d93c824b7ba5170ec240d21ee720b9be12a542251c06b\",\"0f7a227cd897b95f9f1ac4e27954918dbaa62ec031a3aeef8e27c99c0a4e2a8d\",\"6b78e072ecc7b99f75eab18447b7fcb1b81853ede72300c69295ac9d599dd249\",\"a374d17240a6942983395515899dd21a80f7f3fd82f2446b45c7177f2111d215\",\"daccbfba51d5db36f2b1ebb849cc874d7496c9a004ea9d80088258980a329f56\",\"c5c0c63dc1343407678444f05406242c60fde973c966162a2065a2bf150ba600\",\"3b8f7c68f9c22a486a1b1a8838cee90a94f26d57c67a00c924128527b4b6b57e\",\"feb2209941b0e62c7cbb2221152ee9ab8dcfdde230093b512b73968bcfdca345\",\"34ba17b2fed2c05998c1f9375a80c3c1f3bae09379dd5d7902cbaadac7d7ad12\",\"a4eb27f41dd5834b7bb98df22be9a553e260cbfbcdde6a10466c596f957de835\",\"1f44cf14de8d065a10e1b5d78baca06f3a89941d96e6ac4975fa03c97540bddb\",\"4ff8f0977e0a88a2fde7542ac6caed589c0288457dc4a2507c460aa57319863c\",\"f5007051df44417b16f160954e2d8fe8a3a85d2fca4b50bd9d639bc8de9d0be9\",\"643bef391ef2c11053ef17d4efa275c1473298888ce85eb39751238c6fc7f9fe\",\"dc9228667f177426024fce88b293f662071d42fc168aae47fccad9c247f0c37e\",\"2c0838ec9bfc7fd8b562d944304ed75754001064987bf6956f104f2d88c0d1a7\",\"d463a8fa87c1acbc49aa6d74b554901d6889ba4cadfa51385b042ed32cb5ffce\",\"a41bc0f3212b77cb931f6218cd1bcf2693aae5290c37e444122c1d10459a0bb7\",\"16e6bcac0a5366f78da438374a1fb94904125642b5a26484038a4c63b3c63a84\",\"5429e00814b7d006fd5cfe9dde08bfcc95610ad4dc226c006eaa30deb895f985\",\"1dc125c5b42a7a66897eb2630c23acfc49ad02dbc9c3631b288b5fcf330ee57b\",\"61626accf4b3e7efb26784a5258fe80d64e277ddca468938599e2c30ec13d1c1\",\"f2ea472df480feaf2439fd31df578c39bea667417fabac9d6a147a70e8a31fc5\",\"e002d46eb908ea8b696318f7676787bd3d3d2219646b56ecbef76d2b09550d38\",\"ccdff1cd8e208896e66ce58c5a420a731b2638297bd93e9f5e1e028384c6542f\",\"bc6c75e7c9a0700f0f9d434095bb5d929efd81a3451c2df568380327207b49ba\",\"3c525e63b06bf6f126193ff0557a9b34b720d35eff1737f94a7bae559da6a464\",\"ca06f95f5b04377124b9447db49f51859081dc168a1ff0289aa5fd1955ff92c0\",\"96717d8af32b16e9d878862ad025035621d47bb4b03d87f8b06fe076380552c0\",\"77886d293036f079e051d003ce67acd4c784a64923694faf45f267f742d12024\",\"bf2c3e35bfcdc2ccef023ce60dd22d3909639f971c5d150bef6e0e28d3a2b02e\",\"ff6f1002b6be669338a27ba30fde9b1031dd670c60d2195613500371fccd5a5a\",\"328394a9a147aa23474e78a844366af97e2426bca61c3ef51ba10b23a34f9206\",\"7eaa7fac59c4689cde8ebb741d6c996a5bfccafdc70c281bdce15b0c0797cace\",\"318b2552c7c6c97298ea5880b122452cae481f193d2060e453dca6b4db2fc309\",\"0c7a7a1e84292b9cd988d1a063ded4772a8b5cbb98e00648f14a9771a023cd80\",\"ced2142358055c777dbfc6e85528705423b4b5d23d55e30501554bcd0d15ed65\",\"6ef571eedbf3593bfb818b892d8108c18372f28b14c3cad31e08a6eb919a1dd5\",\"4391788a904d61b7f16d8dcb27887ec26ce03e076aa9132da24f8e4d0ba1b2ff\",\"b61b8065267efc816b6bd8a19e2384276204e99d7cc5a1341101109c43c32848\",\"9ed1b925e2fef95d400f294d4859cd3350ec772f10024dbe2a3acef6d52f0ace\",\"222b4be55b70376d6f5a768122a705f173117462ff61a80457170f95e4dbc0a4\",\"651accb2706afefe644bfa68722920d08a79764e62cc7e4bfcf5578701764983\",\"0d3745eb1c01053018d472bce7d882a9335423d7b946c0b0fffb6eb19679a5d1\",\"56e600ae53254ed8fac79dd93aa4aab908dd08fa022ec998273300b4544034a5\",\"6c09d6c47b43abfe2ba956f872ceb90057d18d1742f0eb22e90635463776feb7\",\"191d4817c8fbed1bc9e4dd9eb003796789939b03e788adf50cc7efa35ae8e8e2\",\"53b6679d7f62c6b60ac535ddfc5451009a3736173d75e0bfd0d46dd906783aaf\",\"45c33e3b4f26323c948ea6d0ee8a2d0ef1451107ed797a3cdf4cc89ecb3995ac\",\"8ebf5ec527fbf2cf8391ebf9c6a2fdc69233d343e64f7b060348134f4b351260\",\"8ec42c976bcd9b08b179eb5522a923bbc41cbe35a6e3f4d6b860656ab09b5513\",\"b2302b203f8ab9fafa7dc039da898eead2e756acf8d2629f1708c4e36475ad16\",\"db9a640ca0d5c907247c0d9943dc2a6cf314cd5a79de4e30708a9fc93de2ece9\",\"a0afc184809c86dba9a9d7c5baaf920b71da18d5d51e93bbd7c8b150f934010d\",\"942547a950112ba3b8fffeedada76b5a1bf3287c616f2e6890ee5149f65c7b48\",\"7f8a7b94bf11c8dd823cc5ef804600bcb6ad79fca1d93c734482b7334900b196\",\"d21f016341d777bc6dcd7203963a49fed7de12eb11151b9857f9c2d94ce983ee\",\"c8e0564f77b2b4582f284369ac66c3f973d67a05809a78d5e51a9226d5b7fb12\",\"415d23aa40592afaf08f3c94d1e7e25e7e4fe780e846bd28f4c13c7670ac3f64\",\"34634d2f6b232122f2fe392a79306d89b716e38b4872a39d0c7ccb843de5e663\",\"89ace01853d4cd82c9ede553a423cc68b8fbab1fb492893875d98e9499251df7\",\"0a64a62abf2e7b345ef832d7c220138682a7c20bbf3ac954efbde3540bff1a72\",\"56626eecb9f63611f6e7d5b36f52d1f487ec4bb45db66602914483e7afd32b0b\",\"f91d3a79ec4f6b6c584af230a90c900d4afb72f248c82ab3f35af01515168b6b\",\"e11bd035ef7670638d976df1e8b944feac55bd8bceedadd01162707d1a480c55\",\"9f0626853d5895121045267489429bc0c0545b2346cdf966e1bb25f0f098f314\",\"e84b758b2464b285a7ff9c2fc9f6c00fcf0074027961536398f6258bcbd1d453\",\"bf545acf15030ae3a437aeb7b15e7d41bfc94fd5fb85aed718c69c58cce68485\",\"6922fa68976d904554dfd77d131f26690eeb3273c8c522bc2330a39a1543b1f6\",\"505e23f91e6a960f2628406a4cdf8a67cbe2085df45ec421336296474683d3e7\",\"f1619a30e740fa0d9547fc77c5a14a3c8c2b4107da32cbdebf300efa7118c6e1\",\"9a9537ebb686e9854c971c8c27edab567ea9e59e773bad02cc2debf1be39829d\",\"0efb3b23b9df26b33a0c66e4f5cbaf4909243a338adfc508ac7d343ae6487faa\",\"aecc3e4cee53146656c7b324653ef41e2dd30615230df2bd40f0ecf846c7009b\",\"1959ea3a1e9b61633b44b970501e2481580946522f34843d031013f0eef623e3\",\"17264bc6f470577d72700f3115def0c378f50dd7842c27958ead4a415ae799c5\",\"7895a22fbd95bd1cdade0b3046bc55e25210869b62b3f579c6cdf4f0d6440e76\",\"7be49c294aac379e788406371d3caca876bcfd621b23de3502cd95a9ae5d95ad\",\"d0c2a48f3ec9f493877a57826a58cc18893caf2731f8ee688c8518b0108fe847\",\"7d9fc980ecd4212aa13dd1d2b66616dc5c99f984e455388a55170600b93c37ce\",\"dad640d4d1a576604da0326b03d520cb6fa1541c946ccdc2b7865746b791366d\",\"857b4d4c5398ae7ede4440cbe1f70426f7fb0c50bae7a0ca0d2149da6d7d3c68\",\"14d0e43f38341ee187d9dd03c0e84d57b96b3cbd5eebd41ad94a3e13f54b1e8f\",\"15c89eaeccfda379617134c41d664923c3cf5565c8baa7587d06f6da505d32bd\",\"da1fb377972ac549f10e334e83223235383d868198b0b8b083250288c2efaef1\",\"b2cc686a96774858b8cd39f40c2c54455923bdafefbd4157287f15f44dabbf4e\",\"867cc792a5de92ae3325615947af86f7e0dc25432ef28550533588b7723ec507\",\"396a6ef84c0d0c523534366d618395312ff4c10088d8a1f59479596df28fb90b\",\"a4a59305fb6c93bd38a6dde5f06e892d9f011a9e3e67117a2035938d6227aa3f\",\"68544f16f2155fffc1fcb877c87382c15176cb68564537ee7613dfe9a1b963d6\",\"e84cfc536c1e6e7c314cc6963be58635f2ea9278131dd0f3a26304caf6976c77\",\"dfd4339203f335b2314fe4d5dd35adb435f313534a97f0eacebaf201176a7270\",\"33ccab2fd47b10464c3c3ea06872501b8833a78d3f937f04f1896356dac9f939\",\"11c1f893f2bc0da018ed503a2bcede8fbc482b8ceeedec26425824b2f8c8f004\",\"8811f997a716ca3a06264616bb14a29724aa5abc475c89923cee0d9aa85c1f99\",\"9ab37edccaa7a42240683e4c44ece5d0b1a4c8d9a79039fa9049e59ca2307973\",\"6b2cb0848143f0d2ca9d7894276b2940ec1fe0007881ad7cef39234effaef967\",\"a5d52382c5653c1a72323574bfcec54963792a757c6be3f3cce7eed744d5f019\",\"072aaeac6747dca88388ba1f8ea564cfbb98b1869a7e5c3df75bbe27de7f6714\",\"9899d8c1800ae4eb5579d317daedd120ff9e6b6ba9b3aa791863588fe89b40b8\",\"7ac4cadd80f1e2ead0f8ac963cd5ee0841c3ea5b3b8c000ed3a82778f2f17781\",\"f93b5b61d4132c520b377042da8c6160b04a18d9caac25a8c78bc5b394b2a746\",\"b5d5f1630016d07a8dd558bf0af47f18db5c148b89ecffa2548d09a50f562a9f\",\"39b239dbc8d122962573b3e346bc3de58530824d519e719a87ae5a99c9fbd114\",\"913a14fe929f6665cc18804a15af1263768bbaf14643f4570514c6b3c2d0920f\",\"e27a531117810dcd1c0aabc4ab3f8725962867d2f6a5673dc90124a7af421baa\",\"a2410bee1485df92e4202b674f0f984f67074b3751b7a91ee55a26018eaa7bc1\",\"51098284e8b2b5621bd69156729009a5e85bc643420814914143e22d531749f9\",\"d642f33d3350c1bbc0d85aa48fd23725071a9749b8a901186ca58462081134a1\",\"a3429401c877bfbb27892b08a2e6f2ff29216fa8f5c5dd1e1ccfa9dcd9fd9c1b\",\"3094496ab4571ea9e97565b4a9ae2e73f154d4bb13a4a2623b2644669ea6b499\",\"5cd7dc4d4cf016d637d6bef9d9da6641fd8edaa24a2ec6f7c8ac73be46aff23b\",\"7f675dc63b74096322d27972deefaccdb1d7d124635dc9a3db129cc6ab1e473e\",\"7122b7156e73e64581da3192c7bc617f3a85776c30323ac35c6e1a77f85a5992\",\"fb68eec4f74bf7665d93901e8dbe2d8ee4abad6a6e0e4b385e81a78f76380e5f\",\"07a7a26964855fed2e9af17451277a52c0d6e51a9d2e13dbc59c0a9a346f239f\",\"3067facbab1e681d99d897d8417b35c8fec90464e75c8a9329cc546eb458fb38\",\"bfd3466ad97721a34d7b4c3fe3a2164683deb63abfbee5680e3c85e2e6bb843a\",\"78f2d05ed0e31813a6b8223618a9044f807452bc4a6204604fa1205990f440d5\",\"ef60a0869f395cf75a5c702abdd40f8e17a8fe513d2e589a8db6004affda0892\",\"8620b482c2047d8a36e09cdbc50c852841ad1a4fe9da262952bfa7e5dc92a962\",\"0a776685e9b1d77a2d0c8481995792cd9a4fb63eddb68f4703169a25c299357b\",\"69a1732a8b879dd1bb550cd63d14607b99cc3ca05402321f702ddfa4d9ceb7d5\",\"73b11f1c5ff8a4544834ea6797f0f64f543758e3c0f66680b4cf6c09a33e28f0\",\"1a31f214e266edf8eac0fe32c73f9e966bd12551aca83c88e98f848513ca3979\",\"b96408696bdad6431e46e112401fccd43883ced55b66d401fbf4d5616adb611c\",\"76fc18009e5731af32b31a5ba98e449788d9bc46d7c458a582ff99fb8617ef99\",\"d57c68ce296d4a75171141dc08113ddd17bcf4f1401aeb25ede01f0558d46404\",\"01d5019aa4f9f6db87a82021bc3b96ae69acd973ba66fc0a0bdf5600f23052c2\",\"2d8b997159156195e41e410b42dded78f509cf254bd5d1e26f0901dbb97d9433\",\"aa1c563dab2e3f28aadc06c9d250aa1a7a28ec1eb4cdd124afa23c8224f21703\",\"dee312aa98c269cee7481119ac72447603db3271f61cde39cd62646d3a095a51\",\"13699096ab941b41ba7583f01484184dc3ef2034de5fe3d8a6491feca1b0dee2\",\"43486cbe177dff463f8cf75d6312794f8d559da2e226f578482d1c2c0a0c4bef\",\"01753cfb57e82158c40d2e5b4628c3eb64edd28155900c6c99432b6a7bee9f73\",\"b0659375d768aa5411fdc7e1a6c0943cbd931f1bcd6c18d8936b7afaee4185f0\",\"12b94e4ff6a10f1971043b19b947e65e0dd904d46b944a0c8ce4e6da92d0eab4\",\"43d6c03ce064873fed127bb98ae863f9493cfa777b0dfd5704f7165657857660\",\"faf88cfcf255ed999686bcf28506639c04d10bf613325c64568e49908ee4d23b\",\"cb3e26c69b7f988a1ab019527628f73194f0af191d957996b9dd2a0573afd71d\",\"751619a7906b2b8cdd2357ce628c6fb4228072cf1da65cea8e4a727fd74dddcc\",\"1d548128284ac6b6b964fc753c22a1417f8bfd344562d2ce55a2bfa19c45e2ca\",\"6f53ca1e351c57b2abca0ff2d04d02b04c0cc47a7f043c2a85afd0191fbf940d\",\"5c1f911435e4c37db4c1baa6a6a880d6c7934c256650c2086293cb43299377cc\",\"02bab6dc25eb45ed72d412ce62b85c63f1bbd2e75f13130482145a91ff95936d\",\"c6ba0490358872fe540a64fce0fab86e1497007634013648feb2a3b5dee66131\",\"2bd68cb80c44caf215105f10b734fe9c3a1ce97e4b45e1b8c75667f033d13e09\",\"dd4f32df5a3724034ece22abdb00286ac135b77bceb32376e9a1f11a5efd4191\",\"5e62ecf0fc71503079783e8723702fa259d14c56465c055fef1b467b7440bc7f\",\"60cdb0c9ee5e3aafc2b283d60d1e395b520482203da44d165fbecedc8f3dfe99\",\"163d8f9699231af5ef93d49833b0dc9defb0c0656ce3769bf16c67f474388b46\",\"eb42ca9a1328ddab5805484e9d52eb0ceae4bddeae60eb0a0a4fd5d83f8ab7d8\",\"44ad6712ed8109187d6387e8422d6d66e3bba7028f52b072642be4a013c02c7f\",\"2f9210b7f63b5384552cbdf93ee884e975c44e1b7de9d271574b782adbe3f307\",\"2e2b778ba946019c443e9b0e059d795b7a9926bf6ffb470f3166e50feeb5d8bf\",\"8c1bcbc49b84680101c973295d0866f76f1e7f2d593888abe959ea2c0c3e189f\",\"5ddbed419a50ace5949951ed85c689eeaca20d858919daa51ea8f6e38cb816a2\",\"74d64a48eda75411882487bd1a3b0e3096c8161437ddc071a5f5dbdb86e82720\",\"bded508bc1f93d8079c509eb435bec18be86f286c9ff6555cf4182fa2d097af1\",\"a88457bb229c77df5b3bc02505e738e11cd79871954fc3fdb216b6961790e6fc\",\"97d81427e756dd46b4a4d08e24a6a98bb56ed3cd9d64d7a6b28a65a94b7296bd\",\"d6ac1896be2aa1d43e9220c23c30bbb2eea1c34f37fa0e43c95a50980c8e6b88\",\"cc8a61bd9c5e707cefb238787f3886c9190e73e956efcdda6d50cdf7ca7393a6\",\"25b19c7d09b21db107fec5648d775f8f6bc6fc09726e026aa9f07ff951e40741\",\"14d0bc0ca72a4675331760e46092182bab3e9594d90e70fc5ade1eb818ddd7d3\",\"0d5da56f3fde248c4a0d78fca2b432dc2cf4ca7315ed5d6cc28c882416699d2d\",\"adf8fee0b589aa352f6fae70b7c00fc0bb62c6e3349f57d46bc29a0e0211cd98\",\"428fcae1192028c976ddd026394c1f21ea5e2fcff64bdf0b83eb8e00e8642a58\",\"def7de0628123c20af2b26b4183e3421f326553601f8da9ea301d652a90c258a\",\"5ea19f43a738c7670495baa65f468c54b816b39ba277a7cd1213a0b828c76be8\",\"0a282bdd8a2184dc31b11eb6c645e8cb02fc2d29b3615822a0cb319468761cfd\",\"2ff14a1688d47f0a26a81db11d4336e33a00c0c9cab1f1631d3e24738afdf053\",\"571ea7b471be93d78c85ee76ef6557df3d7e9faf416883aca1f0217b60849a7c\",\"5a64b6d463332c038dbd052109ad9b595927881bbe0b0d1eac361ccaa8482029\",\"9c4cfe471e8714e9850e1bcd6d4d2e7ab1767857e488452c07005eac5024521b\",\"1d957e9516e9cb9dd722a22c8f50cd165208174e18fd192354931566a1ec2147\",\"a5e7a0cf61f890881e5d1bc86f1ac8c497f5e734be59b68e0f9eb9c7874b381d\",\"2e72e652781a0eb82ecd91486d00ab0c64b51eb24e41bad3622b232d5ee7759b\",\"f36b0464699d2ce1b3054afc840185efa052cd050553d695aa5d8381c99fd20c\",\"d8f01eb7d39b210200bf65250ab4ca28d7bc25cc2e3dfa2112339e5960b20709\",\"a36ac9b1ad890963c000391a19b81b80585e2c1830ee96d2f01e6e83f39ab157\",\"029458532c3e6166bc6415774a541c4121d81fe9f0401884781da2a54bb773bd\",\"5c24e0ad5c0f33056e2642a98da697d199ce8f9d3882642732680667bae1d22d\",\"e2c73ab60edc30d61bafec83130f347a87366feb12b77caa7acb9f323d677253\",\"384ddc8c8c1451f18f2fbee94de3a84441d26718ba49f4b1169b4f4b46ab1c47\",\"b1ad3b1d3f6f09bf9abd165e84e06db87a64db1b55c18f45482c15005dbc29a1\",\"2d43008c41ba200d2209dc4eeacb7035a2e0e1045543aa96ef5faa402028971b\",\"34e127c6bb67a97536fb48c1055bb54ebb453ca7d8494d4584f524df4d257aa6\",\"9d0d11f09ab44f0f883989f15e100e641c3e4cbb2b339d2d2c4d212244290ebb\",\"6eb3219896b3ceb59c2260ab297543c207ca83a33bef12ef8cc7f3ad010e47f3\",\"1b5e27e4d42f0729e3f79a1e721cb1a38d11f4604acf7260b331ef882b4d71e5\",\"bb07f7fa24a6fdba724d4351887569fb0a7427b8c558cdc6e9d606354059532e\",\"9fe3c56b742a889bd14aa4ba8c7cbe2ff3f410582785944e2905be5f6c1e080e\",\"4319be21f8ede5a615f90e8b7d8b59a2d29feba483047850685724b287da9523\",\"6683dc1fb8e83e433dab1eaa1dd042510a9bd95e9a45ba75ce41416948a43f81\",\"3acc24a47e207d3ef967ef405ded3dd7e1dcb74be10f3e0ee66cfed890e21175\",\"060f3f62cc25fe3a3c9eedf2fbcd82f56033a4dc502b7b884dcc3ef832568ff8\",\"9a156962fd60d0c902be5b5d74617c7494c63e59df1c0fd22d92ce9ce3acf904\",\"6d3dc47d25e44ddb6cc11a8b73f6543a5fbe934b23f53d8f9211c63bf3d0b0d2\",\"e18fe1a0520002407fd572a29b064ca922e64da125f7455438af3b35608393de\",\"8fd39d4ce393a160013ac54a716de3545e071e1340d67208e728530eafa96c11\",\"4ba3f273052e24e4884aeda243d4b2deaf330843e29698a3f065568c3587ee03\",\"6a3ddf3ee095beea7bb260823f7bf354b1f26805453baefec3db98f745a3038f\",\"0ecc3a49496e19514fa048e964b83e76a9ce897efd887570eedee04ebed213f9\",\"83a562143ac0002215e2707122abffa6e73babffbb7c8fe2a8bbb61a3e891e36\",\"3769676edfe68643e1c24f7c230c1082f552efa0a730f54ce56dd05f3e7daec1\",\"bff874946b08dd54fb35ddd8659a3f6244b2d99c53c18882bd9ebaba6c8e00b4\",\"c6c2b16fd2a09f898f5944bb409b84161be8cbfae4946a3d1a389dbf3baf4f7e\",\"3a3a6a0122c365f62bbe3cf153258fcf1b0543efe9aa68493da3572b3be03063\",\"4c81cc66fd107e443649ec17bb997f257553fa57403f91efac41d15a5efd3e62\",\"91725d813737198931b313cb454551752707422cdaddd3e89d3beea41d4441d6\",\"bdbc95acd9d23393bc69be8c28f1fef94dc806d9e4abdca658c2a67aa9c5d8d6\",\"e3beeeb09a2ca8e2eb24ce442860325588cf16ec91001a8abe792184ac924341\",\"a2dbd460b7089ac7baf8de9374e8d9d25844f834e30fef2ddd3a41c415d47fad\",\"2e35f843af6038320bd0a50319ce2ce88d7cc3c8df000a7c9b9f3a613f981fea\",\"8637513480ffb09a5e59a871e11846afc27adf1de65f385debba48c2640f9c7c\",\"d1021e562f95a187bfe0c88bc5f632a08279f47de84fbb8948eb1fac87c26af5\",\"3af06196c18840cb270d7da2504a6370bd242be2be8946d90eb7c474c3d88d97\",\"4ee03f49bde70c7d3791fb9a0bf58738b91cc8f927763e12c03d7130eaedb023\",\"8bdfaa077685e4b2e40bdffcfe5c7e34087604ba07c46e664661061744630da5\",\"c45435d033a7ea406b70a29a6b3de0e7f37a9a18e00bdf8cb11b3b29355fd7f6\",\"24365193af6e3a466c352e156f0b92a9b25428522a656753846dd67fd6ccd816\",\"7ff45aaeea2a85e31b6bcfd212d7c9c9bb593403b02fc17b64f45c58357f24b9\",\"0ec3ab459a6b502613437bb8f9aa92dc8c97d47983eac2b1cb1851ce71e9cd82\",\"4465646274cdc5c36d8c31f7b744b153b5c184c50ec30d70dc2858dd89f3b3bd\",\"05ee0fbc8cc0751995ee231a635f794e55377bd968c5a7e09dc34bfb7ca31ad3\",\"f90e55921825563a20bf6bd69af66cda0e07b3a990a387afedef5b9498cb4294\",\"cb94448551bc56eca2c0b676b2956899f53b4e0be98349518b4d51fce1f0da31\",\"61c211b30665b6a08cd24b8fc2d440cdba2b43089ce60442c1220e86cce7184c\",\"640c3a15ebce025ef95209f90f12d2d425c4f65f4bb5b6e03d7f8b05c787ab79\",\"8c607683947c2ca0bdcd8294594977e264139f8233aba71164b6c365a8515e2c\",\"cb7e2c947196d4e0bbfaabf1d5f25ac1e71622e2ade31e6614a18c54559404de\",\"0701e79caa6ebd6ad3743f520e501e1ceb6a54e4c65e97f0efebadf1b97c0cd7\",\"4ba45c486c200a25b2ed323faa689af8e9028bb51568b4b8dc21e95e4d010e7c\",\"188fa418625af2a0d48d636ee5b3dafb84dda39c619cdc3b576d839e7fb42f3a\",\"da14430732a05dece4a2579ddb6728c975dabb21288ae77f3ba4fdc39fb81b21\",\"b7254f3fe219dec5d93e4159b6f3a16addda53c3bd19c4ab065480aec702935d\",\"cc7127df31d010bcb0b449a705fa344d78973caaaa95274dfa42759163d02a7a\",\"d3576a8ffad4158d8f8169143b526293c6dd6fe78ee3b3b9fff93fb3943deafd\",\"10cbc031fe78d0eed45cc10cf7f7419d4524307ef720dbcf9b3a9700838ac419\",\"9043dedbc6c4149c3fb026d0a34da265ac05776352638c9f123589391faa2224\",\"8b7683ed7c82ec6f264cda31306926d23ee4f89580d5ca01b295e436636f7290\",\"79a05107f1ec4c9001f729651af36effd903daf59620fa3f75ed307e6445aee8\",\"70cde63d853af0e18d0390e922338c3b5fa3a73260a8401d4c96ca71b90875c2\",\"23bac51bc80be54fe44ba17205ad1f39d867539cdac28e7cb0ab57d74e444879\",\"0293d1bac678d7946096f561e63ee385571407a695a6a8eb15e48bbb4e8f1037\",\"c8c0c09ef0755ccd211a91b75361c0829a4dcaeb15521b0f9db4c34416cb3fe4\",\"c120c7c75b26c6b6fac6e2556d77254a967d39dd8134113dc64ea3c7f4785033\",\"662872b761103a376a414cf5931eb3fd5d724a946474c319a4347f88eae689ff\",\"39b3d8fb201bf8679f62f5af017b8f76474a328dfab3d2c35486106629a8602a\",\"5e060d2518ee008a6d17de0560b382c86f231e8dde33c149116d01aa3ee7dd8a\",\"4f8d44d19b491c8790ac6155e4ca937399b2fb3122f4ab93dd2aade8d97d23ef\",\"8ced7c3efde47411c96d5103e51155a5b328e074938d7097e30d498c20872d8e\",\"ff29ad3dd8ccafef115db4e5cc4d7bba0701f5b6a070a25c1f437356866b9d37\",\"cf39acb2e423d8371b2edc316f0e10bcf446127bdb2f80786d4db28613ab804c\",\"68523bdee0ca010e0e71171e32ba855d7b1382438fa987aea0c9878d3ce22115\",\"68221acf172cd3012576763de34973f1dcd511aa5f8ff39ab03b0fbffef99b33\",\"63d31583317983375695c3c6afbe7f08f67e1184ebb4ba45f82465ee232f7a0b\",\"cc7c7e349b46d08425ac8a3d72443514fa4808cd74bbd2f427bfa2a4abcfb1fc\",\"57d10bda1840866f9702b5ff17a907b815bf80daed0a430717b74e477d1d4fed\",\"9f47fd27e4aa0c5fe9611d1e4486ddf75aef04dac0f64f164caae74002576b93\",\"c9a405cd540a83d3cf984351c03228e5a6a10ecb4d7e66138343eedf90accfd1\",\"14a2b6af8c3241eef52d4f9195c23498aeb7cff0d5164577fa92ec93d3e0ef38\",\"1266eaec1abdb7443b05acd3371a3539aa957cd0b486a2f7f4cbfbe491169cc4\",\"6e22dea39571dc2c1646a4bac06109828cb7f5e4f5afbd2ba5ac5ce6e581cbc5\",\"209b7adfc024eba8db3d0f00e17420136627739a3eee8dd0b71eac5d277c9c9a\",\"b5cfe2a73723e6e9f535272ac805ef3a62d24c0bcd3cf966d49fd627f04df2db\",\"83baec53eb7b3b676f3fda17fea5d636e5bf2d82b9b8b343da235a2392082a4c\",\"4fd0798710e9711a38296f45ee2611d446c32c2998b637274869b671e334cd81\",\"6ef5ca5893059c46264ccfd16c5c971b8ce1e029a07bfb577691912817bfde6f\",\"b2c6600c612845e2be1ec5af3aaf11f4ed15168c5955a701a24a6e86d8839a15\",\"4d3dc3e3f1e8fe3c51782fcb4ca42dea23d779770fab6e0215840194b2c165f3\",\"190b02f0d2777285b16bf8eb6b9c4cfe39c618af080477c459c61dddc7c5f522\",\"05e977b526d43663e71ddb39873c1e87d9a58ded55d5852eb6270c59e4520306\",\"00fb47388cede97e5f7039f2189f9a27f119dadfbced9c6b9bcaf5d2c95ec62e\",\"e1f337119b21e8dcb54e49fae640604b588330b307582caca13975ddc6f776a9\",\"de77d1c94a9f5503def5e04cfef7b477096e9fd80456cd0c6aca89d65cd638c2\",\"fd64ba4a071504f0bf095df02e35502e5a3b6c501ddc12e64593b595e49d6f82\",\"5b95cb0d36c35eca5c71171de39f514ab699e976a745f3a947fdac411fe81fd6\",\"892263d23cea0e7498fcdbfb8a2e3946b4fcd5a862b2885e802cefd607ab5f84\",\"d64cc1f553a4ce8c563d872057238874be94e9c506bb1348c69030c17443c19b\",\"bd1d45b176db06cce17a8d9c3a1f6367953fa5cf8bae9d65ec00a24694981e8f\",\"2964b2bd494a6cd2227479398682ee28b6063e852cc70c70e305aa287f043ef2\",\"cc76e738775766d91173aa1ff33d88048ba78b748b5e36eae4d73197b18c61c8\",\"86d04be8846acd3bcc75e25537a0c7488ebc8fc6a8a604603e7f039ad505495d\",\"feb261041b28941a6a76977466864aa059e56b4c54ae5ac0fcfcdf58ff3435a8\",\"21257be6eee5944628f79580867873295517d332b6c563544fb898317815536c\",\"99072061905e48afc4beff230fdf0b989a40d240b84563c6d5f507c2702b57a3\",\"e9ac9fdf6f648525796faf8922f3c468c3ebe80c52151b562f5c4d8b4f458b3d\",\"c457f43f8152e411b083c6d1fdb15ffd19d8477b1f6d24c08f86710a893b4481\",\"f1e5c0db553d5cf69423129974ba757ebae78007c0b26a9b169cfd9efcf8e290\",\"cbb42e13650dd5b1c4eefb743b8a2fba271cb80b152114fbabaeb79005f6d487\",\"45d42000086832c6ba47fe1bd5d774c1be726ca3fed84531696e5b12927c6ae1\",\"6279583a9e1a17a9a474ac4334f2521a99a2a76bd49f9981d3099d933a68c06d\",\"71ffa50d1b11f32be37bc7d76ce2776382ceda661941e731c213bf17ae06c72e\",\"6ca3d21116c911cecf88db5b6ddb32d1e9ee19af60ae18aac2422cd9931005b0\",\"34c0b3fa92f17b6cf7499385f4337670d7bfa4fcf1e48f1b33d8a8f8b96fde72\",\"18837175c5d840243071910d2051916b542a9015681ebd251de07aa861470aaa\",\"d30346ca1d090766f0d28d8b53c5f1b0c3f3bc75831d6513a4ac5e9c955e505a\",\"4aaad87ed9af9dcb63b41ca2f4c2b9b11d595999c4e132db568f141313b42847\",\"91499ebd76e7042fb1887d4c96a26cfc786e594df3bc2cfa6f36a545ed6334a8\",\"27dd8eae0c1af1ebd54e15cae4f5e42d15c81a1c210c9592546df2da12f3d2dc\",\"3e9f1d3826f6421291959aef161f205b604bb5ec877d5fd95ca233586c319c25\",\"97d1440090690444a231bcb4927aa32ef80b6b4a466c4e49f40a55973c473afc\",\"3eba21db1b6c33b43a1c5288380f5b076c986018ae29ab76dc8f2c1ffbb8ea60\",\"badd9ffebc77497c4f828c21b3861dafae662d71a61236c72a687d116e8ce10d\",\"16d6d82f9d6f0f8532bcf11437cb11df82fd8e68c1ba1653e458ed2948b45805\",\"339262d4f525fa03a70aa42f32dd2902163ba2d61c0717fa975a608e5dd63c94\",\"24bcea0c75513d16949649be489da8493c31257719e864bb64a13d07718c702a\",\"398f317de702294e85183e6b17641c4d652fa83e7e13cf733c155410721eb208\",\"2528cc51ae306dbb89adb13aa1afce30425d3394e080a443a3310bdbdb659d15\",\"56dbe3a0b9fe0436b9c136732b47d601b3f95848292a2477be39363fa64715a0\",\"e8b5b5e95bfee179c4d6e415b4132d6d8c37b726069c3352feae708e6dfb0597\",\"363ccd31d4994003f193b0c6a02300a6efd52b9fd4b5f075a1d0b73fb4e4c8b3\",\"713c837015e792f513958e47cc2c1dc0ab70d703c795c428fdddefb9e2ea0d7e\",\"114407133ca8328bc7f0a772df86ffefdbe92d3c04b158a6e2315830a1026e53\",\"89797e4061acff42cec264692a6bd0fe29cf7cbbd837979b44735c1069179437\",\"fb361cb88bde40f5843f9e62656a866d04e79320f12ad021e045d4dd550eb7c4\",\"67bb995ef03772406eb383f71d02238f7c10215fb3b0d86f9feb91a037cec0a4\",\"edfdfc5213d118ee6a0314d02c9fd2cef7818f3cbe18397a576ce2966e8fb7a9\",\"d9c6b8ff79de6135178b3708e893a07144e33ac94519e7f931152309a81f37b3\",\"37236503f26ef7f82c8b5899bfd16b83f345701aa0e233e8ba706cd6369eb035\",\"78260613ec965750146648403b0d6e7475858a1bf574ea6a4550b8291144f843\",\"e79c5f8ec72ff6a044887c727766165a5b32c5b07ad851e05e49b92104b8ecbd\",\"28358400e2ba88311820ee286c5b900e0ed62d9469628f792b0a596e47cffc0e\",\"9d39887752b874ffcf6ca8727e6c1227fa44b01586f7be2c4aaa1adc3de82259\",\"5791fe0974bc63ade9e39dc0175ea5b0082348b5c249ae19946775e3718cdf2f\",\"c719ca62d5cd8e505c2af342d73e90c5ab9b22deb1acd671bbe12713ce10b1ad\",\"cc5652c5c0cefd22ea100819a3448894d4a243223fb84abd734e218224a0bfb8\",\"4b457042d30b8023f044f174da8a72ca95c675bd8007bccc4eb8193ee4af4ffd\",\"f758ce3cc98b06753e2afae1aa34b51d66c37e6993b4828eaa47dce404c61d0b\",\"2044b4a80878dc05998e582f70ee72cff26c1f6db0da24840a63fb86f43a8317\",\"f7ff7479e14b533f185b02b06bed04c7e88c9ac3cccb6355ec9321b6aa98b193\",\"79d759996d15c621124276c4b83349a11e45391f24f0d9ced46256802e5cb953\",\"b0901dd6cc87026a730883ac1334e743629490635a9b584f457900505520a5ba\",\"f477f2a07a1aeca62e4a9b4f6349a618bdad480c9be8a3b050ea40ffd06b7f86\",\"3d31e220648b277cf1c9876a12cf0512546681a8d13aa839b9851c1468898285\",\"e8b5315039a67e218a4ee6dfda175e2bb5a865d0641a780a7cd5ef4381d06f79\",\"7f6d5f6774eac67a4f52a7a379635e92fd08bfde73a08b1cab901b8fce107784\",\"e95910b053d6a301c01e292cc121bf9a7bf1cb4e29a83cb8daa8426b4301eb76\",\"41d75acd9a40b5ae8f11b7dedaab2a76d5f524332568a1c6afb5b1d6db69d067\",\"54800e1d49c779b3e9bd8744a67049d7cdd2c87c197bcd2e740d873fd0d3cefb\",\"1c038448697d98e7ec71b16728e5226803b4727973f1cd2f5fe57108956941fa\",\"78c184d40b861311de8c3755522c0a2bd609dda3f083110305da942f2c52fe85\",\"56f86b217481a9f56517e756b2da2b90108c6d7cf78331a90cca9f49efa631ed\",\"124a7c8e992997a64da468c938f3a01922f24b76a9910adf6cee276c420673c1\",\"0e9b3c12bdd2ce7444d71c44decef66cb1dc15c225aa666a9ebd1842c26bcd9e\",\"d455948692299e88d39cca9e2eb711f98a96de3aa8890669dcf4b9a72b7a92b0\",\"1c5c4f8aa7c385cc0a2698853b200a5b01818af90c83bfa3bdedf9b7fd7ead50\",\"9011100fee780e6d5d6f1ab864ab30a8515e1547154001155f33a172b47b951e\",\"aab3c732ba9d67cef5fd92e5fb72f09aac3dcafb63f96e1da5f972f22f925edc\",\"eeb42bfaa8fcd0950007ae009561a06c3db2b9a3467187f2a3723cb86ee827bd\",\"19fd38dd5f75314946c6db2bfd24899570d7492210ef266be9348e0470284e45\",\"e2f18542fb1dadf8a79cbc9a86848eee0dc595f0d53bde69f8e661db45937c1c\",\"876a68f11922d1fdcfe0f2f1129bdc668a7936b0c843de48b0e4fb43114c2b11\",\"93455b601cbf8b009d08e5b983fd721e56c852c411be47027d97676cd41e4d9d\",\"44a7f70431148fe7f05e4e1d85627104a571e9532c6bb2d7d01a10b31b4b5b78\",\"ac03b5f0ba5b2c392b997100eca12bd038cb08a2cf572626cafc9dfbab502f3c\",\"a4e58ae12e6ba4ff70db904abb8251ffcb1dd9bb018619909bdf8961eca8cb9d\",\"4f2bd9b9bea8088501695b191fe44a579edf542020cfdf83a81bf642a4633b1e\",\"b0d3787852e0b0f3b2c4838a2c8b12a2ab318151ec952edc046195feb987e5f6\",\"20ebf0c4cab0a5045502f4d206541a1db7881db6e1f5818517ef80c0a1c6ce04\",\"20f43a4fbe52c49194ea176cee9d9fb142fb1f2f694c07449164bc265e1c7464\",\"5aaa7c63b502395c538a53d02c98c17e9c737d594b94e23bbc7a08d61ead86aa\",\"3615159cbfe203bb9db2e6294eb8aeeb85a7ab4cc996e20a3b8fa677203079d4\",\"b8087cc96602330a3e1765c29b5e025cc00243c6ce88cc5361ba31d7a6608d3d\",\"3b7f0cf9653aeb08b579b762b2201b20cf2a338ccb55e759a72e5dd31b8a4b2c\",\"5282f4be02e703eb1e7a499eaadd65b08ef49141c322be3a86bc63bcfedcb7a0\",\"eb22f4014959f5ed7a382f2f5ee6cbcff88b7f477071c6cea97e302e241d4c0d\",\"beee2a8507714a9157b4160ce24809eb3fafb960dfded49f8af72812fcfcdfad\",\"4d251f1a0fe81f67ae3ca6461a9a22e54fe8d02790e91aa23f795ff1431e9b7a\",\"efc075d8d559b44b121e39e91c66e2c4624a2badfd783fdb20ba18a16e1cb6c8\",\"f0dd15bd237e0e8197ac0d831884dd30295ffca48bb90e0922aa2b59495436e0\",\"f089626f18d2ad095deb70d3da8a3e557af08dd2d2bc945412b4fd43c8a32d09\",\"962ef49ca4b010f0247d061ce116ca1e5280ca641075226400cb4eddbe685432\",\"e672260a34820cdd8b4ef25356f965665460b625dedbb874eed31afa2770222d\",\"7319b95e7408a1f91a115d63c20fef104cad17d6766894f0af16274342975e54\",\"d10dece76fb1a27e4d75d8e6bb77e0a23ea1cd5f2d1ae39b9261ee3269241012\",\"0e486c05977a36b03246bd4037f340cb3b7c5844f0411cedf861fe68152138d3\",\"5c205ce676bc1da0521492df78c2f849ba09eb534792aca27bdd4a940f8b3d5d\",\"e3681fb2520be435709d4bd2d50e5bdc7d5214b3ba0fd513f71e61bf3d46a656\",\"01de44c16d7c241d5a86049e12974a7ad7c1e118c39ec2ad28d0bf1718734281\",\"f897cc99f623060202f68139ca19d80a816c5c8e2ce5d1e37cec73d14178ba19\",\"535fe6f256307916add9629bfe2bb392816af7579dd0b021768a3e8497643650\",\"e08b144536d47ed8dfce3b29e9782d95367257b9488cc83d36c77e320cca91ed\",\"81bf373bd261e19ca721768eccf85c959aa9937e655e01f9fab81225f6b1ee53\",\"a39b7eaf884950eb162fffd4dc7ee1987b229949764739cb881f37ab3755d2a5\",\"bac2e17177689473f8f2ca8b3ecfed3c4837238308b309b0222aa8e2d3332732\",\"fd8177d921ae015457fec748d7ba0ed3bad4853ca812916e9356ff7399607a32\",\"ca9398e8582ebda2379eb5557567b661759eb563532c64153a1680ce7dec13ec\",\"8da4d9824c64bb83e01b7b294d4c9584e54bc06001dcf14a4687acad7469fdbb\",\"9702926240f4d00705fce913efaa54130066c97db830056da2a0607b9d77fccc\",\"e7d558c649f4ab74f76f3429fc249015945f3b231ef1fdb4498134dd0311c6ec\",\"dac635525c925ebf3c4afa525320bc7514e86f8c820a155e5405c677dc569850\",\"fdd1bb91ccc48022146c6fed2114be7af37d7fc665544856ddf0a252e521da76\",\"d5b5d15ccc14050564a779ba21c68104344db54a0e647ce3fa32f91733431e36\",\"622952f60c545824dd8e1dc2df2c2d472959fcf7a39a8bdcd4dbefe52559fcde\",\"d80146db8b73815501843367b57e9d27a92b970bfd480764f52a83bd5b75fe20\",\"5641b733e0a6d5b2d7f8f9e4158f8cc4248526436a20ff6ccc0ea08ed67c2e9b\",\"a8c751003791528315df1b6ad132b3f82fd782b1fef9cbaa7360183857475dfb\",\"4371151aa672a48b90a9c6ce97d8a3db3a9cd3d2e19d94ce6747abf96d8722f6\",\"0bb839c14fdf90a9151796e80c8696538adef50ac59aafe5198f81c3bd5fa8b5\",\"b4b0b095f4d199790ce2254914292b0532bd60e1c533a1b9db0c436869b3fe22\",\"84b126e3f7e063bd0056905d61e39cc9fb5037293aa579b66309ebc3af2bc904\",\"68c379aa312cbc8ec86fd173dde05cf2a43c69fc956072c5b5b98d67eafaa483\",\"9fdfe356aa20990c69993ad4f09d921ab1d9eeaf6c9ccd246da93a66975566f5\",\"db9c0be909ce5a6d75bc1f4b1a6683b139ce4dbdad028174d95777adf89b3c71\",\"eea498255a7c757cb5dbd35a6e017ebd4afe16f75c25fd790c378465ee0db5c5\",\"c04ed18c5035ed15607d1be2642ba5b85f090dd007e83bf2bab4e38114af4470\",\"31a874b1f0a7e9af24da2d5d3344528caf82d4e6d6eb41c8a1a6d894846c9efd\",\"941687a30f67396cc42a71f1a70269ef0cd160616985da99eb7485df84e1379d\",\"33ee1fe319856432c11323cf80b5c3754ee51b5af4ec3d4d0eebaf79fab44a77\",\"f27f340c190a209283bcc95409c31843cb681ca08fbaa27509b1b15ab69efe32\",\"d307cf0779517c04d614c679bbcdbef42e945b00f85e89dc088adcd004a38863\",\"20432fbafa60b0cbba900290516ef4319325992406ef6938de56179d257b7cb5\",\"0ce11eb663ba66560e479461e6a768bef7085cf39a061370b0f27d31a1e0c8d0\",\"0bec90350766fec2c60864dad98cf8bdebb2a0dce5a449170eaed238418661fa\",\"ce891d78614cb98fdc4da78edba1e1a8e5d88b1dbda3cbf8074d85919e3993ae\",\"9b28d1f3daec637d50bf9473a3f9be17460d46bc6dba4e260224209c74ee9d61\",\"455dc785d54f96199e7714cc26b6ce6f3ef475c65e976536da4f954fc3351ea6\",\"86a2bef811ea13a1095a92975dc98574ac18a070ab5805240b4ba96e2b57548b\",\"1951853ae8ac1d1f74c09d4af9bd44871c7ced3e460ca4d6b6ea4af23738ec96\",\"4a8a7bc9b86fa0b7abb214289a55613638bff38133ba239632509986dfd36689\",\"0aa49b012ced25b420e84570150b884c06e7ba5384b3d873f7ad1bf4b25a1fac\",\"5ba13891d97245448d43de81862802f474e6e7f001ed3571978eedd6540e064e\",\"eeb1474a312bf84430bb24e104e34a671b0f2faa0a7f41465ffd89892c8f4309\",\"5b2a9e3f1d6ae9366c979e5e34792847c9ff7effa680977f9bed2e4a3f0e7557\",\"78e810ae9b824889d1f74891d29a6552af8e9d8c09dd11bde3489d0e6e2463de\",\"4efaf6feeeaac06020c7843f0cf97c907700198286b1e1d0e43f9df10e743558\",\"f9db48538e3b80513a9d59ba59090df1d91e990acd404aa8ae0ccf25c1188322\",\"709fb86002b360a7c897c5aa887f853f9f71f30af0946b600897fabd08efaac9\",\"d87182de88d2d53df5bedb55fb89c3c5b24b3bf5fb188a36dc45f53ca4c9fdac\",\"3c3663b9a7ac4a5445d13ca31f7de866fca2a21fe132223f0256b95ec9c52467\",\"e8d7db422edf722afd1b0df1687d0400240a5929585fffcdce21a5a89b496715\",\"b36ace389d00d4e7344b9f09c7e3c31dc67b1a8541906f3b5ba2fa57e9ff168e\",\"5b59ee9c2842a7ce1b916cc94fb93eb51d0ae94163f1ccdcefffa420a17f7346\",\"58d328a4e7a7ad8d4049af42eac360eec7578ada578f1fe26a69f4c4c6a16edd\",\"04be0a0ef39ad74250d6f4398117666cb03eecb041ad808efe92720218b45f93\",\"181cecd72dfce89f94c436ff7f49f5207ef4259dd4a2ed5a3459c0b93f2bc0a5\",\"7a188174188f581728aef7a72fb0b2c1f66b4f263bcd388b2db081886e594bf7\",\"612f9b29244caa14527a669c883e6bc2fc04aef064f58180f34bfdf5f4285e89\",\"06db8f78f93c6c0216209cfa070a027577a67cec273af2f3faca1935d34fd133\",\"a7313459b3e4aece3b0daf5fbb142eab9a267bd739f91e07655ee85d2d9c54b6\",\"5f36a20e6661c32414b78bfe40ed86a827260245f06383bb7ed49c38544969ef\",\"77ee7736b6f01053c9170bdc0b1ea15660fd68087cefbe40fe5468e5552051d1\",\"f25fdd7c09d4b91a3935df777f01334bb9b1451f06a1422cc0c65ee7ddf6a651\",\"7f3d76aa281c71d21fda490cd382a172fa7985d43ea3300769aa470540b0b3c1\",\"68e7b775f0088e02e9300bc576c4016db6d349a2d17759ff39f777732ba99ca4\",\"2e4ab71deabdcf4580be89811225ccdc7810d49032fec708257bb5e978f09c76\",\"add7a5e5069f92ab2309f3490f1043463a02921aea47c7ed20547540ebdbe433\",\"3ccd36e5c9455dea97fc7193a8665153a6d2ab34c80c10bf47bf675c74314f89\",\"83420848918ff610aba07149ee0d11bcaccf485bfc6723947f94803ff8890697\",\"a2d5d59df4bf5061b4ada154b90a9e792911dd04b95b8418668e40a026ff9235\",\"56dc8b650e07ea2986a3ce9cd569530b65129355a83350416752f40f754ae020\",\"cecbdfcc54fe4e1080a8a356198b24e0be45f2af065ebffad59c3e808da1ba9d\",\"8c29772f281c5db118c7e97eeecbfdbed52f5e00095ade890b0cf4366db3cb25\",\"f9a147e1b2d5e3428468edaeef913b52c59436371e38d6dd17ac84528b8755dd\",\"026ad14c9464c24e6c88e906213bf66bdda4b9cd09b7db8747c4ee8deaf05fe6\",\"82762ac691675cdbae4d84f44ee335571c47e6db6a52f6ff7c6aa732df511ab2\",\"52a04f077d3310c60ecf3f95ab74d916521bde67ce57244afe42b7064ea290f8\",\"f6c37461e7caf5f75f14d5c820fb00edab5acd2bc07f90557e2be6b9d7cf97d3\",\"118cc26400b4c15f0b48fe22aedc200ec0b1f4d17add400d7d746361bd113e62\",\"53bd852598c2902d75c63adf3ee350602e3a4b48de84f9d38955570c564170f8\",\"70a348682eb5daa9dfd20c55603584f11eed52b99da26fc0b4bd89a4352d66ae\",\"1786f6b375e6fd41cb13288d786e8aca8d0b9cee46e15cea6b73dffd5d841c4d\",\"4f965afdabd4e4096e97379b57a54a6e4b46d75c645783a428375befa17db2db\",\"2130f0825844e46e3415dfe7218e4dd494a2441b75b43ebf438699e58a6c28f3\",\"a9747d49b08b29286bbda90c8f6df9c2a06cbca461296ea1ab800287502783d1\",\"d70bd47b5fbf59db1bad8e82f55c156315c2d56ca9a38ad1a7bad2fe11351d5d\",\"3912a7b919f4b7f3ad435ed485c5d8ee8510d88ff8f537a9a8483f0aacfdbee3\",\"784cf7e2af08b210b87c2423a244b8bda70da5a0722ddf97dc9cb5d19b1237b6\",\"d1ade0e09e67da87e99b1d043b0eb270c6ede10a9b600b0c8237f82c149bc592\",\"f17660c1fe86c12ca5fda6d16f8fed42aaf2d357adc07273e9e6847eb7cd0711\",\"5cfe39caa2c21c84d1ad7ecf187db146a26743f74241d3586adb5ff83e76df0c\",\"9c8cc3f538acf24716865c1104c49428ad316f8968ef1691ee6c470281f44d78\",\"68d4b2b780f67bffd5ff3653633e4613eebae1f4152c4b8980d711845e49fad1\",\"4a7ac6c1ee0f88e5dcb4a73bbec36dcc25e1a7ad99d6d083ba954058b380ba70\",\"bf559b1319a4191207829f9b878ae01902a079c318e800d7e10e31fabba6e204\",\"ceb824cebf2c9fc66f907cf322d6cacbdab389f3477b03c0490bd8078eb2a597\",\"fdcea3f7eb7e4b337e84523f6da00480145fa6f423427f73199f293bd265d6f1\",\"ec60316ec52004f06719c118ca30b76fae269d342a4e7a441c1bf3c7f97536c5\",\"39dddc9abc6eff265a98e5b7a07a21640478d5b9f09226dffa3d2b46986f8b6f\",\"7173c6e9f1f18387d45c77146b98e1ddd74dda8c2fad9b0762b58f94f6a2ff74\",\"51a6398259b35f4b95f2b87d1572fa83d14f395abea511dee8b2192652666fa7\",\"57337c093febe564ae9223a26205abf2f2e583535a457ffce7ebe54d1c48c061\",\"6a9249e4c1b5f60d081aedf03fec41cd1be44e30aafddb7b27d868f41a840b2c\",\"17e0a960df16f2c2a455ad1e8ff51e176ec42e80a9557e4bd33022d2ea9fe6e1\",\"b11adcca55dc083534d8d2e3e5bafee9ee28d99a88120974cbb92c2a968da2cf\",\"09a5cc33a28ada9e46b0b7121bc1ec2912b84a527277d97d70bbb1e9f57c91a2\",\"9cb2789a404602ef156a059fbe689341421aefba404a34692b295dc2e74d94d8\",\"af80688166b5d1bb4106e121203097c9e54eba281f16f90c9f451b7935b27e77\",\"d310e08686d391585310e4c6b3b24d88eda533b06e1c2191ed6e774dd15c9371\",\"724989f85488108f8e4538191cc2b460c28451c68c67e552dcaa5df9552f2af9\",\"58d9ab1b5f570aa144f31b17e6dfdc6550ab338af36b8cbf4dafb0d888ab1cfc\",\"3432adb18b58bce58123d80a60ceeded4d5a632138edf6822a2973eca0c95f87\",\"44c796074f08ce842277d067b1c5e5d4fb3a90de2a427a7ec96b1c4bafc367e1\",\"7d8142e1b96cc96eae526307b3bf85b9462e4a6f59c9dc1db791777f7cd1595c\",\"eeee228234431d2d0f8a039357089a03c9a91ed22db8e92008ff82a56659936b\",\"6d841b8f169a03daddfe769080d724819642e5c9762dae0a91fe77dfd9171415\",\"73aef15f990a6336fd8d829bc1c437e521dccfd587ddb0ed45c1ccb215ad08b2\",\"1a990998ad2a047fb7c1be9cced590bd5bfb5e339750f3f71196322cf3157aca\",\"8c9445dfec0bbb38072f75ddb4a1e5fb7e4d0dd3abd38891d4c837d1c515c0ce\",\"86af6adce93858ae32d8afb39dd75160f263e316430a7db646e62877d5cf1bd1\",\"18e771b98f3e568ebd2146fa88a01ddc4bcea9c4062f27bf26ff56d78923a25e\",\"21b42a8733b6e422d6701fdfe6c6759e1443d6ef2c524352b1e4a5e18ef57cd1\",\"8fe16e88785834adb59fb1f432ee5c854798c23502670d243e250fc0546275e8\",\"de1e17fabce8fea481ad2ec9367382cb53b2ab7d1114ce3a115d8a13c53796b7\",\"35fff10286394543650c1b1452dc20e535aef32bde50606937ad1da448c67e5b\",\"f9370c6d11be7a0b4ec22f5112b1addb5206540f3faa3174fc7f7202d1bd407f\",\"c5c4d62b918386fa99ff3609b08148683fb727f136e0f6bd7e1d44577963acf9\",\"a44aa449616da83b2b0cd578c519deab82ab3d353dfbdd141eed80b19a638aa9\",\"6572f3319954ce471c61363f1dc802769a4d740cb9ea7ebd69607e46c117d0a9\",\"4b0afa439cbec722b9cbe55033f281f4e0141aed8089979b4bfceb03bed52e4b\",\"60cf54098619bbd79cac1ea4420a32138d078990354be383a56f75960f2942d3\",\"4a93cdd609ca49c5377d23a01ac461f2d74ae3fea3ca5a58d7fb181e884cc6e3\",\"76bc7f342c38a9d455bdc1720b6bbf5488e6c605ed6b124af61990b530924ec4\",\"0653536c176951e13e86f539a310cdfa16887b4cc0e5f6321028eb25b526065c\",\"1653f77bd5429e5b4fad5a72c73d21798f60cba53f896fdae7ae54358a6c2b9d\",\"0d56a81ecc8699eb025deca60b7609e8b72f3113126baeec6fd1c8479a96b440\",\"34a7e0ba88d8638bc1f20b804aae43a2cdb2897b44b5cc01999b1432902e6da5\",\"9a28d61612868da2e849391d7ab44c0097676fc47574fe93afba49b98bba3539\",\"b677a48ae1bff3579a18182ce9757987b9cb6f0575efbf650876704b8d5bed2c\",\"6ed2639cb195cee34bfce6af100faf5502de54f9cbe3610bcd5dc4ee8506b810\",\"b7ff9039640f5018d3bce42216ff0b08c02cea748a6f5b29312a80aa87bf8a68\",\"1da927ace52da48697c559fc940536669afc61001ec8a51a15b893fc36c3b702\",\"3ad4353b0d841c7d2af9d182e74d69400409f45bd9f68bd8b2065dd515410019\",\"22edf864c12aba10f2c44f13596564f8dff644dbc07e4bbae0d5839f3905dc41\",\"81450e5c668bbd7121a5ee98ad5e8c07c22e27964a241294d4a5a77a0be54cc4\",\"2401e0d497f56f519abe4956142842252f65f6415f33ed89a614911b45fa304f\",\"f7518002b705318fb87aef89f93e67fcd011f09b998a9c29c7afe76d00bd91b8\",\"1045cad6b17b30b0430be9a27671411154fceeaa6ca9169a1be636bc062e34bd\",\"f8a01ee58bf3a3912f76f467e7ea3ca423ad527dfd234089b4c55b531b5d395e\",\"ec1f424dd94e7307256159e183ce0dc8c6aa1dfae34d8fb11410b03f8b55a659\",\"215f7eae236e6369987305ed2417402b53d1aad6f6e8275f44cd2ecdedf49021\",\"2252dbd46f28d7f4947184d683286777a29b823f7ce448aa07817b8faeb77b4a\",\"2786e5ccdfce36652e261d460853a0324c86ef680d601490d9c5b6ace75deee4\",\"497027b6c7d8723d0994c5263951a6b9a2f9b449cf3b3ec547fcef157b36cbe6\",\"3b43bf5f29dd5a9259e99be8e786f17fd5e673b188bd46741161dae1b7eadd82\",\"76b90af64374013f51e681263ef4c602f7425000ec38603de2fe6a0ad40dd6c7\",\"8297f2cf7f52bed1c897f6011976d9e1df0adbd439a081e60b41ec979ac5bd33\",\"fe65e4a8d400e6f4c76add0ea571fed71ec91dace2f1b95de2a0eff5b3772c7d\",\"79a257ebde72382db3ded455badac625196270e0f0f6cbe08e77da041c2fe0e4\",\"0780bd054792640eb107b961fece191270ba8b0f8b367c3b0292292059a4af26\",\"379f4b1bfbd05068e19a035e5baa826452c59083e0039d83ee65109258d8f3d6\",\"09fddf16ffd6e18965f3adb7b7d486f91a10058f83bc7147dae3e0cca4bf5ecb\",\"bc06c50490ead3862ec03a943422db905be3bbce3ad9220c5edaa9d394c10fd2\",\"a2bbaa963372459b452624ff566c4aaf4bd6f41214da7da7768d2186f69f9595\",\"cb963152f1c23c8af3290049f1dd6afcd773c1bf0958e67bb6653ffeb1fd6141\",\"e1a256497878491dd2300f21504e742866c9d3b6bf0fd23fc10ad2f61cd76170\",\"48ef37bc1b74d965bd9c85f64e5b869c3445f8b82c1d7e3b0ea750663d3e661a\",\"2c21e4e2d45038d479412ed068ba3a9efc422bf6b82505712627d2aa4f64bc5e\",\"21743e19f76f514503e22d7e69423287a9fd3356767121eb5583f15e99ea7984\",\"d6b5bcdf8d17982f5b17a0d3010c841057c81bbfe1660171a6cd7627777deb77\",\"444ba762c695ddbd533a8604a0412b0a4425360c8e2519ab75f32488cfeac870\",\"69a58b0593dd6487f6a15891dd6b977bf561e7fa5b2389ee4ce93747855566ef\",\"5dd2727927ff63a3dbe018e8150323369937d33128f794cd33fdf2b250499edb\",\"988d3e5d42f87a0046a67afd8ce3440317b9ec481ed8f3d86843cd619a1703ca\",\"cf1b171cfd349cf0fcd93766f12b8b04a046089bd0a67ec188fdc82e32741b4d\",\"0ffefa1097c3444b5abaa80507cd50628799422207fe6dc9d349282a85166b8d\",\"96ee8bf67db244d91cc10585de41039dface1496cf38e19d0ef3fe57b059a0ad\",\"0f3479783d785e3e31c6e351cecebcd065e3d10b95e52821614e619f9a0aa499\",\"5ff7c10d32f27a4e71c74ee8fc043d22ac2b79f21a7a4159b4e702a313a42b06\",\"858c01e6567fdf60c4c2fad2e783ad624e34b8c39f1461b0a55fb073b65685af\",\"c0e16541429414292a0ea0cff6ae3dee78f458ec037e67c30754ec04f1b52e76\",\"21fea05fd21b56c4dcd05dfb0ea9de614c80b2778562c874f9a76923ea02f65f\",\"2f8e162a506670a9d96b1332b1684abaaf1d0e149ac7791cf1829065a1e86e3e\",\"6529b22b6f6467ce6108ea114bb356e08bf2409759e468caeacaf6a0e19a0796\",\"c4c6004d3071a4367e3eb9e7011130b676e5a1aab8cc07a298dfc46e1d71f3a3\",\"ee62c0c4742a589d7823643b3e6516db9ea4e855529d7e000af1fd25be2a6973\",\"519f467afcfd7a96b058ff52a02162a7f5a53b79a441f65afa6ca59c2efe5d2b\",\"56a1707513d4dffe16d545ccb63589a1a80b303990a4e95f83cc6def2db93963\",\"0f3942051a2158003d7abfb1b2686f590c3cd48acfb0e0a8d8d9ea796bb12b43\",\"7d5968d01e20bc09ed627a2ecbd73233d37c7a5a023ee43bcff30e30d438d095\",\"da8fe8f076f9d4fa5191c5d7d5ecf1be4fcbd4b5c9c5299babf447be1e8a45fa\",\"96d15843cc7746272c410e5c02dfa9431a6f0396d350f7181d5d6e09cc25e4ab\",\"e4915f3dd182069ab9901d428ec66d5d40a200151533e264f70ae8cd8749aca3\",\"20f674040215270b9abdbd06e5c18ec833b0d2b8b93cb976cdebb0d39f7f13dd\",\"b05109b3e623754de35014240606305c463785e357b3e545577747dcd92d4403\",\"97731a7955edea14dead56571f4490f19311aba0c415117bb4271d73d626346c\",\"30bd13cd0c6af7952e1db6ce2a4b54abdd8a480acff0c79bd672e18595ef584a\",\"8897929bb2569d21c5e5414fb957c0585712841c9128c1fc40c1600fd43b1a47\",\"fe858f83d0a428f2373ad446e5b0a954c9073acd6bf2e525f75f290fae10c0ce\",\"7c20263a0590f82498a59b993ffea90a72d289eab20ed32136251a6bb08f5850\",\"3e2638a63d42619d59ac5e6c208a7788a7c6246baa9fac2d8da3da96661c0aea\",\"e7b4d6e937cf9505a78b71b2052828571b880a39692d05e91271b36c472f5e28\",\"9b8c0b768206f2ecaee6a3d8442fe97e1b2d22eca14fbc0e56c31331194259e5\",\"c3411029dfb16cb14e323fbea5258847276b96bae1676b22c4f0f038e39139d9\",\"7aba5969834d33292d7af43c81d661bed8b3b35b922e3eac0405983960b2d214\",\"40fcf34869bf6998893458994361ea7eb3f994428e5d06f827b46c83d99aa56c\",\"1b8ec7b09382f7eb71d79f38706bd0790c5f4477c875c06a4411689db56464ba\",\"4194869ee45ea1cda9435fb50e13d79bcbdee6c97d70852d3133787c74e21a06\",\"8362a01950c54e2ee77eefe1d387990871a5200e41bc2860498776c1bb73d658\",\"8ee520d9cf76c6a31d555e0215ab8dffc8b9c252bfaf242e6a83fc989d5327c1\",\"a3d20b567d64af161794799d9db8f845aaccf5c29187057afc48500111716398\",\"0db5a420b86dd263ff0f4fe0a077d29a1fcf1e508e023c7dad1007cba33ebb57\",\"4e2a8079fe15c5c298fcbb1eca687936719008db92c8fa73765b6e1a3ce21ae8\",\"5f063106afdbb9cdaf2735d90f01d092ea23b267c56e97819818dbbc75b0bb02\",\"cd54b65e9c8963b10ee511e0c4b591abce2debe980312e293b3de4bb272d6f2c\",\"b81684044ca70a32403920a8f4bc5a26d29e10704cc18dee18371dbdd9275b7d\",\"e46a44a8b226dbc64c152efc745e8d29bd3e5b03a59a4d37de80206f82983e38\",\"89ee71e0061c00aa520dc5fc1f3a138d3807d9d13b454b9d6f763df30ff3ad46\",\"f0cd7b9f52bdbe94f73152a1e27fadd0485c67e1d8d3ac4f024d45e2d16ebba4\",\"4c8f9a49ed2d0dadb9e294582bbf2634e2bf6ea66f08f4606f7cd12aa134d19d\",\"84c0c76a3bd59527d0f013f73fcbfa06d9a191f3557f3083a07890fa157d044f\",\"91aa0ea5991bf04423d2551848dc4ca322b279e4efb6e874fb90a81d2eebc540\",\"d8c6480999aa1fd4fae28a941591d7c01878f6d8b5818a861e57127bb88a5d5e\",\"7df767231359ed5304d2e08c0dbb978c99dd85335ed2424a3eb2d3ad4612ce0c\",\"310bcf43aaffb0451dbde8b7191e7194f2b5293cd1ff355a44a02795a5ee34c1\",\"5c6c808dcc64ae11e7734795ea3238b4b2a9843887b7f7540e7937b9bafa2616\",\"09006993e625abbd5227e68021f875a5aba3bd46b112b0383062ab47ad180ab4\",\"64ee252b7fef6c14d6f90b23550b726735da51d123f539aa468d0e34d9ceca7b\",\"df6b4ce48c6a7bea19c7d6223041038e13d90f6b7438afa7de12dfebebc52079\",\"9900d67d8bf57839ec2b6f5c00e051aac74963c6b597ca94cec4f44c6858deba\",\"0df759697d45fee38361c12cdc6d90e62ac291405f3c20e3c41dcd8044cace7a\",\"a8a37fb62165a55ab617f55445f3bed36ffab855dcd534614a999407a7e8d0ae\",\"dc72aac00593d22f8aa3ecaf5b51fb6079d87e75a220a073d8df17e42cd4403b\",\"1b72d2b9fbc247259ea7e73ef4108be13fff911704e0a0cab41ea7d26078cae1\",\"131b5cebdce1e8e412a58bd8c4d5151bbdcf776d83165b7e6d099f88d5e262b4\",\"ba6563236caf78ad6273cfbeec38a928d18bac18d72feeec380fac9578829731\",\"b6208d6b6c1c023b3261c12b11bb25e58919e3f5aa7e2602deb7da2a278cb846\",\"91c7e709b8b60f80e870f2d45d4e616679faacb7617c100209f7716b5fa0b1c0\",\"2d51884429ed6dba64e7ec8a09bcdeaeee82b24cb3e80a3909b31078d33107ec\",\"aacc5a781b8df7a25bc345940afbd42b28965237163b44f84f332073b5074ea7\",\"60915b894e6a99fd2b08cb80542f6f1174e32815899fddf7f46ac985692fa5f7\",\"22f32c7825b2960d9a81465546404f691fa5de20a0e7de94fe6ae5b5e9143b73\",\"3e086dc0e9e3e078b9ff763396594b3df3d7782fdcc4280d977fa70ee58f137b\",\"8f704f2c653e17b8bbddada83768e19a69c5549bc701196146a99d16018b8d0f\",\"6b7f78eb9e0a94043b6519532e8565fe996cd4afcb972188c27b963326febdb6\",\"c8fe258b5597e7b65d58f06506c185603ca470866a95975a390080b4ac44d9b2\",\"d2eab2d0684a5acba11f0c334553c6347bd9f35e2e218e9e47f3f2fdb054a508\",\"1d7bdfd9b38785b5bdcc25be099452409dd5096bb9bec19d7142d69cba4c8e6d\",\"507913d00b55c6380cd30f68f570e47100995422b30158315f65f2a0a33a1247\",\"7a2c4153eff0508678232b68a8cd3df26d7f39daf2ce9c2a5685d5c21a0a63f3\",\"224f6c5526c923297045029f36a7738cecdae9315e368bc77b4274f12fd499ae\",\"cef59704e58eaef6079df2a0ff50685ef4dac87e653d7e5be145f397aac21ae4\",\"9c9fd63fe3daccd3fbe510f59e35be81402d2ec5669c3842d3ae0eb44edeac62\",\"2e242df552d038a7dd39f50677379f390feb673a6b5e3767414242ba75a6f9b7\",\"afc8b767be630c69b78e4eea933e7f0f9c8679a3c01b4b13671f9c809dfed697\",\"7fe349dc78af52aaef728689a8a29d02a93f982322635b37e6205597c8b4d44c\",\"5ca32977b4808b4e549374544f0d259f94e6116efdb1c0a56b3b1c0db353e63e\",\"d57ab76119dcd195e080edba32b8469a9d7bff6eea8fc5d09a7d24a0947041d6\",\"9167550ab0253bd995942fe111b5fda55c51f38e593daf190d943ef5a758431f\",\"c57ccc4b801aabf58197cb23e99553297e908430fd1cfead2da1a9aeb5aa115f\",\"c8bd3249869fb71db35b101e4187f11574df5d5f9efd2e03f7ffefa7be5b4dc0\",\"135aae3896466a7662749e6594e1e39f4c8cb005c31a8a5ac1d8ff67ee0a333f\",\"5a8788e88ccf76c9936efa458460d792e62068d7d74b8c6d4f1ccbc98f6b62ab\",\"ee7a75dffeb3d0cf95e9f5a22d218a50fd8da203a6bee5e3ef014d2b635071fb\",\"4bdd289b4eb8146ac0b524494e94376863113fb17d4a171b4dc1e6a7e93fb4c9\",\"2460a49891f9f60ed90c10e2ff9691fef87abcf9d0b2460ac4e6a2d071bbeec0\",\"643be7499ea0dc18e48ee9c4652bfe186ea98e292864076fe7adf8bd0580a088\",\"a39e0e06ab934e55d931d347f430dd3074e1945ec30c0f8adae87e576ad17d89\",\"0e34a5d273ae3cc97c8f2591e7ab978aed12d0ee406f85cd5e54f4d944ba5081\",\"b3867475d24788ac06ca4289f81ab55ea55908fc43f58237d8b276708a6abb05\",\"2801fde9eaa27d95237e0edecdd4a69623a37db6165e9bb8ca71b86d1d1f3052\",\"9a36cbaec3a6df3e1080acc1d89de935015029f3f0aa971fff0e086244b5f8ff\",\"4da9224bf280905e1c7fe4b331e12cd0114725173c295f60005b2460a0459fde\",\"442b84b93e44f90c27bb635079e75e983a5b09cb7575f10e62e1411c279c52de\",\"8357b5bd70db674754f991a63937a0f85fc6925bc28198cd0d23c3a54ab24a5e\",\"10481c27333a137f23927afb72eb1609e9aa11ebac3e40f2488498807213c60d\",\"fb14b04e25e460e37f1ecc4593e2588b313b52b85fd6917042dc277c732bd173\",\"138d79f3bacbcffcdb2120dbf3b9b9f0394b06eb46586cd3078b967b940d3413\",\"16577460f96ed15f7bfbf5fe833c0022fe5865f1d3a7ae745551178beb097055\",\"1cf2dd55688a35a48ba4f1f82d54f888b35e76ec56cb979f43812410f977e263\",\"f8889c1c16876c45f40ec1cbd3b771b29a35d837f02313a6b1cc2d58be7e8f1b\",\"54ca16f9e9475ff2661d8a4aff953bf1933210e70c577ba9396e1923be299f05\",\"d27a1fac45ed85d8af57e665b20b52161b48a33895b5aad9e7ab03faa7f31a7e\",\"cd91f41df1524d0c04e276493d05f9bf58d9ff5f3b9f7cbd56cfd1d15f01a6fa\",\"e7202ada9e0797258b9fe42e28349b36aa165690e9f6dfab5d288f7bfa212222\",\"1788f7318d7e5c66160c359683487069815383982906eb2c39a4d47bd528bb39\",\"2e4ded044208f3da80f76a250e79a5dfa1b578ccd86ec75aae9e0cdae902786e\",\"8f171b1a3415a977b22ab7bf741f0d42f0d296cbc93f5149205baee812c605fa\",\"7b8dc3eb08a98a6b0ec7a63f4bc55d7730e31fe018d69c54efee9293881f96c8\",\"f11bdbc99a4ddeba19b8fe1414c08edff9af7f8b541ccc234d5d2554fdfb1412\",\"7e76b7579ff1ad35af9ba821e7cce1e18929f83a1b8ed631333e6ebc87e5e601\",\"12d85e9f10548722f829490635558e1f8e8970c21f14f9bc3cbc966fbd5bcd46\",\"42082bad3742a9fb31e26aa74db1f80edf7852fdd284e43936182bff18512ca2\",\"0b4c85aa2ba646f87032aae1005c689b54654b6575bddf6142f91cfb46aceb2b\",\"f66fdb81fccb389fd3956eb7365f4069f5055ee346d75f39e011c2186a7c07c0\",\"b907af754002ac0535a0abec7bf1d26a07f47edbbdd4be2781f0279efda0841d\",\"4fe3e25c889166b36500bce7af2579ca5c9ef77bddd23df037157b0dd37a8061\",\"94b167bee68c39cd4b8aadf60d7569ca09e7f4024819da656c1d1434d9714357\",\"4d86f1b5c771adb25dc0315a2ea4572138dfb47bae9ed7b9c0dfb0328d7c46e3\",\"8901003082795fb8da6ed248cf0abe28c4106fc86feb6c016e4ee8a3d5de7eb7\",\"8a776bac1a424eb3be8977a2a52f71c4e17f738bd389501d8e42a3720ea88550\",\"b8350a28f6319de55183b81a820eb1147083711d2763fa455de39b6fe7ff6ba4\",\"1a0222bc146eb2f944e9e177a0c690682a984c1e18a9b2ad5b518bf208508d31\",\"0296b9b6ce66a28c682f71b93f7d2355144747e01d22f7a0b9661c0a2e44e927\",\"9b17ac056130ca76794d67dc5a4e3d180f53d82efb3da2c33a05a519de3229bb\",\"b4a112095c2f7e220a478ca4576a5a795301fe90d1b26cf362934f746da5b8a4\",\"b8dab4d9c04c7373ed190e24152141a6884dbf9b7d2fda2dabdaa6c557497e85\",\"c702fdd8a1d0ce154c9e285fbeebe48c24e3b4ba63e48999b4f763a9ca641173\",\"f7750cfb3a62a02800be761e2959b2dfb71e0444eec03d7b1ad99f07b1fd952d\",\"aa01dd6482aafdb0b6f851e0294308bc25aaef9f8a93665c9d2268ae3f3b17eb\",\"862df10ce74fbea700bbffb102cd8fda1566574375b1a5c0c93198bff768efdb\",\"8d336b1cce6479affde8ab5cc52c9be44f9cae4102486a7ca69aecff01400967\",\"2d870ddf8837035e97f284aea22f1443a624ee85e2950a550a5a2ed065e400a6\",\"6cb4c351be86de3c16c67a89b6c0ff51b0679eea3d5187e4858403e352542eb9\",\"d617599809761a66ddda4fe4419b3c9475b38fd0ae55e85f08cf5e71ca8d12d0\",\"9295cfc5573e3e519d1216f77b5297c0897392f6073a4dc034494e20ce823a1b\",\"b95b8ffbc4cd75bfb04fcd0472a7497cc6c3ad75ddc15e0d3c7b1a63b568d65a\",\"0a137fd88ac2847cfcab398e5afbefe1fca3157d65afc8a77877c9986d6e8df0\",\"73f98177ad76cfec4b08861e970390a01489c765802030ba180799fc58edc7fb\",\"b1103387c37f188c12d1cb9633ce2f49311815fc274f1320124df0f3920f837a\",\"04cd87719b842cc7522657cc64fbc359712937e158db3b568838ca78cac1946a\",\"fd9a3b769dbcb7dc0f2e40e40cbba7d53847c894ef321a902061555fd836c0d0\",\"3ac7660ea973c55b7be97c8937396b01906111b90e6e1b10e23061e8deee56a3\",\"77fcc59c1cd73ded325ea656a773563d215a6a14fa0d34ab17eb1af0f9716124\",\"1203aad6681f53548501d1cf510afe7930565a8c7cf9c60c900459ebdf403578\",\"a86eaae53a1c225d9fe27138b48319897c16ad62afd2ae52abb244a80eae2122\",\"dbe89fca316de2f72dd87b0aaf4274333c3a919f028260742ccfaf8e6236749c\",\"e7a1e190929c786990759fe05ee92b5553f57754a17caf10f45fb12fa8fbae51\",\"469b933a197334c25577f84452e5dadcf770cf29eed432cda3efa4af44c077f0\",\"3303f8afa1c8fa05e9d330e5a7a8fd9954d75c99040a8c0dd4415ecb7755a501\",\"d431c8c8943ce84197a0723427a091634786fd1ab8b9e9e03986a02d7948a608\",\"969515b0fdfcee341dd3ff52a20d628fcb5e7c10ea44de32c5f8b999c06fa97f\",\"0db0c4a79c4404be3ec74f3e481a06fbb64e3355ba962e35636272371c8a82a6\",\"03f1bc45acf8236cfd362da3ea357eb8e5df69f4566feabad4fe10c7b0d7d16e\",\"d366eedcfe06ed7a2ea2d3a9e7fc6228dd90e754d20b0e9cd9e436fe8a682eda\",\"40f993591c54f3c2dd2bf070e7f6c6f0dfd5b557071671dd0b65215c89f9dbb7\",\"1619c9a8aa2f43189714bbc80ca99765e62a4f84b5c11a55a0ea1f1be67dd638\",\"102dffa06f7c486c8a98903d00070b61590dc83867eae1e0a00b53c54bd16262\",\"997f03a32a0980179e643a8595324061a2df6f987342977fc888f550bd23778e\",\"c896326aebe9d736dd56ca1b8bcee5f4a18f0fe27abc411456fcef4e2bd37af9\",\"9d0e0f1db77ed79b2fe792dcc5453962d2a190f408bbeda094c42cf9a88d8536\",\"9e70082a2e33a483d97802ffe76256c20f21cd6eb03bfaece6ce460a8bd671b8\",\"32dae00fdd18307d2cb97d86eb5a905b4d59e702da12453db029fb913a6de4b7\",\"4b1e8d2047a20e7acf58d24dcc79d14cbb9638c1d96b1aa15c3e02bb3abf3c89\",\"a7312d1d3e460e2aadc51caf14858000a2d70efbea4c807a70bb73c724fc5df5\",\"8746f57951748985960fc33ac3b947496ec3cf200c6e8a545bca253fe67814b3\",\"0aa800607ef224fc32fc65ec81e69e190de1664e0410a2ff0e717e6dfa7de68c\",\"a30c525a6e06379dea76df44f6524adc9fcae9f74d1e76c242e8649e9073ec28\",\"e8227c7ebe6099fb93a16dafcbeb214898aab9d518cc9ba32a1f63aec0a6e39f\",\"38455338c396ab6fb6d403b13cc1f324b8b250e0cad5c7ed4ce2217b46140c9d\",\"b69132d1dec5f3f5ccd5e589c78415fd2ced80b510be8ac2692fda25b67407d0\",\"91d6d31dc335123f0d5c7bcf8540f7ed10404d5b7e3c208b970334acfecffc24\",\"704dac359e5fa173df01609a3fa29771ba572ecbe847ac7e4e2ff841f0e9785c\",\"135d57d824ae4996f6b5487fd49b7700880ea91bdb8db3125fd2c098c43db5ad\",\"2980a2426ba7975542ddd69fb7ab4977a249b4953519dc7ddd40ac0a4aa0eebb\",\"71e62eef8734a8ac0156296526e52ce2999bb9db7da6a88c22a104a462a683e8\",\"9aab477c0116dc24b8a69f6778668a3aac9ac4e0a6cf6d7542887953c80e5b04\",\"da0f7c1c33429e16c23f03e2c5e6fbfe9fc5777cbe3c5105887704ffd1a7da36\",\"d7f686dced0f0850de7404d55e60abd50e46a5e1b300d84f8c084361fa53dbec\",\"0c903e3eba00fabea6bfe6fc3fba01b19520fa06855f28313ee8e603ed48b501\",\"db3ebe2c39d0357afe6a011cc48c8cd53125c6954da8d7efa090fd43f986f059\",\"8e13049c46fe218700e3eec7324e0d13e97596a0484bb0d44084f12786ed132d\",\"bf747bf610c9d52d9886c1c4fd0cbccd349298e0532987cb39d441eb240f544c\",\"6a710d0119331bc26bb9df4c03a51a1a0fa74af2c93ce888b884b53e519de62b\",\"a8256d30e6654c820b2b93ee121658a8844a0a3fff9c78938264cfa301c28833\",\"ff12b6bc56e944b211d7955591bcfc182119b4313a1de61fb12f42c04cb7d233\",\"a85a3d2b8ad34b6820f1e05f39fceed8953c549c256efbcf6ad4666aa6764ff1\",\"d6719c6d40a86a1c60b86dfcd9f7621ef74d797d7a97ce7aade4a55a1bcdf8ec\",\"eab984062f2395a3220736b3922685611ef56c583da2dfc75bbfd26e2bce32c5\",\"68a65b0ceb56b271d65711079857b76840fbd51edd9394dff44d47cac3c2fa79\",\"32d924626e2641af62fa0d69184230b4bb2bb3286a3f7e91ed4ef2c5022fe493\",\"2f1582313b418262b4b99c24fbfd5d398b0958e255fbaa543b29e7262905d51b\",\"ef3494e9d4bebc1dedd37cf6408325088e8a3bf078f8fabfd2475c8e9af8d1b7\",\"d3d5d586664478e0bef5952bbf2720101937099fc3885538477e064a48c44aa0\",\"5b3d59590048c1ef8754175595a38032eedf5d7500ce896092ee8f1112c07e3a\",\"908fca5b6cd4fa8ffc016e3a0a487572fd04ffa6ffecb8f236d76bb25fd6729e\",\"5b161a5a35dc737e5020c2dd263e0f5c853b67bf9dcac0273fd64b8d8539479c\",\"3e9a57a54f8cb333f4db6b8f225b0f8043002030fa7a266014f9466d1f1bbd15\",\"8af5b72b95c275623c856a0d7355ba10a88c41db7523a1323953461586f77a3f\",\"c4c9eada75efa012126bed7ce4a414afa309d468462819bee4f7937fe963e759\",\"45413e7cee4f8f9dbcd2a4a4511c09f6c21c0ed2c25fbb699fd72f99a21936d0\",\"2ab44a0e11eaab44510909807f9ba863b7577c4263b39019abbfbb960bed7a47\",\"908a7d6c65b2671a0e839ae620466827dfb0b70500bd0b054bd4cf5821a4691b\",\"4355e604d1c60dc659874148e37363a236bb9ceb99154275755e9fda2e5b5438\",\"87d291878995a73898a3e12d6d66057943ec5d1e72cf3bf8fc285685a7254868\",\"57cfeccc14c40aa3e3a7d01b3e7430698e8a31289314e0184058e866862b0a4a\",\"61924b9b0bd58af9056a49bf0d62d28b9caec6546c84f1b1c2f1681a6635566d\",\"940156400abb56fc9c0305cf286341b4db7e3e5377cc2ebc0ff950b8a9735b60\",\"c045ccaec7c7ce43a3fbd9c92d35ae00ab66a57581e228287f7040032e2869e3\",\"38c6b0b32acd1140c87b9389688f8f39e58f1b196c1ab7cea05954d1fdf8809f\",\"8781dbb39bf037250f38a06126e3c16a746875770a2ea58cdbd3fc76778a89f5\",\"a78e2584cc73694219d92712cc5cf5a25b648ee79aaac75eb820ded7e65e1378\",\"a6bf191b91d5270b3df4f59c50f78200cae5f74da6c8df2ed502b98ae416bdb8\",\"9970a175076453c909401181a243007cc481b5ab12e7af64eb4a899ac6b3d438\",\"ff705cf5d8b8ed180d8b731d4d33d2b18cb7c03e8cae74292bc9212729181bd1\",\"74f171bc3d4347c804d204ababdce85bce754a1f87e0103dea619c72397d2486\",\"9e8adb16b6f55165f2e3836010d60ea2ab4b1d85a4d93b6d8ea2ac231a752105\",\"45943c4df01a9832677dbd7eda0fc2d0bee1bef4626cd8bbab79ab81be9e86bc\",\"9c136f5c41618e5761fff3992058f74231a9ec7cb56d1ec2cf110874516ebc11\",\"0c13d1ee77323db19ea8bef25a4f929733a941ea3907c9af8fe3504923988586\",\"73abc200e90831e6b77902201f76417e0d13ef552a08c4868d482f75946056f9\",\"7b2d5dd8ecfcadbb20d2b4b171010f353e597d11606780fd193c8c6009276557\",\"165794a43c2a94a760de3571ff1994fd527c0581414f6f33325062d845f8f55b\",\"33f18d46f533bc8ac19694ca1368ff90f585ab773090024a8765fbf570411286\",\"e699645a7b3f22fc014e9b8180398a94a91afe52c1fc67416001216368ca4e54\",\"35e987501d1a3d5c0f25e1ecb39a24a4a686d484f94f2af68660ef4be07fcf50\",\"a32132815b3ac7d98d05b2f62da870b5a721fbb5a244fecf92e0c0b3c678fb55\",\"4fe53914d0dc33eb98a44ed472b220a59bbe213b0e58d7d2d0ddb54161202e53\",\"da0f2d35ec763ebbb751d8698e36d95f2c12f88a582652d52faa159e65909432\",\"628c7b0e9b8441f1d1224732751517db074ddd576b15eacecb7d3e012298d7a4\",\"b13f7ad872aba44b283c25c174553a4863295cf8ff0426c636e7f0f4d01ed665\",\"710de590985a239f8b9d065e1e5dbb96d3d170f33956cf7946d69f763e0c488c\",\"c0e670454534f30412e9c79c0ad542e7e95d6914333e96399d1b27a957fa8baf\",\"096ac2181bcb45fbaf43289ce4624a9e39b75a15564b827c142f590254f20ac1\",\"8db7f207775c668a1892471eb903a8f38e4b74faf374afb672517473a7b0d665\",\"b4d0c5efa732ef046bde2898c924bbfe0787d02e6d15ece8ccafd4847099e2ba\",\"c67dd1e5bd846417ff33b53b3dd4e7221e9836fa6ec5e3aa8b127a7566fb58b3\",\"e971ce4e543d199c5df9f418f0c7ba72ec833aa3944afc1b708d202879889e11\",\"bcc56b972afd39b569971e94139ff0d5c3134c558bafcb9c17639f649e37bce0\",\"0e898fd404b7176c8806e8eaaffcd3775dc92a3dd0611bef6485c39080e21f6c\",\"97db03826336fdaba2a053e6d74e2cdb5fa15db8124aa90518c64b1c7f33d5fb\",\"37ea4eb443cf555c3d1c219edb5a9f3eb3de0b5827780851ee123f9c060b4b85\",\"1b82605cccd12dd677392e18c4580350c1321a72e5556fe12ec7b125cee12ee0\",\"b7b48e9add80d83ba6f3914ee047c4f4fb8a5926725321a94a872424182a6c7a\",\"905ca1baf8164602b5d6622bc27dd492c35d908eeaecbd76bab185f0adafde16\",\"d430ce856159a11f0cc8e18a799923c7292cddd0c24240dc0bb8745c987142df\",\"8281da2eb1a2ad215cc647d8dc99719419dbea8944f5edfc01ec17d6e1786713\",\"598301f708cc65d5014e48f84cba79329d25a1ee635966e271b95f40ab6aca2d\",\"9c6b4559809ebe96069cb89e8b1a1abf872e1e000e3b8bf4d5f2aa6095289d80\",\"b0cc688a19f546c215e0e90ad175ef26f68d97d11602b62fc1c3b9e1af619835\",\"fa7540e8ce07b87e4163bfac472d9762b0e28f8875db04d1ae249ffa242e645c\",\"70f38f393294e53c7af75b8f5fa3f4cd44524e2b1e65e35f0921625df7c18bd7\",\"2e848682ca1f2b0108ceea65d76538d129dfce3cd652e1da4c579347e41a74e8\",\"fa33ff8e4feccfaafdb0c5a15b479b969160d529a7e37ed3f71ea2749ce46538\",\"85855e24ce6038b3d82337295473bf0960fc3c77775e5b71ad826340b12688a9\",\"f126884dd976c1bae1eacdc750722535cb1c2e8d1c733d0b4a069a86695585d2\",\"6deda8b60e7fd31fd025d24aa1fe59bd52ea51dbf811b943a0d946fdf74907e8\",\"cefaece2c45810bad1a93483a2fd6f093fe2fb275cc77fa5370c93b584c90c7d\",\"e1295db58e9167274d76add76f71264ab3bfae95671bc5b7a2dc1276d57627b2\",\"8287b182f6e17f2c0798c2c8cdb646af8300134f1ed421e9adf24c35686d98e6\",\"aeb3eafa7f62ba58e519a112f834356a01d91e7a74a21de3515254156f8d7f38\",\"aef4d0be9ed737d0d91157ec7f30ead107478209bc5be4dcbd7fa6ce53e09423\",\"7701e153ff660305f5c150c8c98a3e6cb54c4fc785542d3241b898a0032f70e8\",\"97ba4fcaeefe6a0e4287be2cb7b4729334e12aa384f16f9b937f585e176c506f\",\"5bd2cd9031e13d00472ab4d6552ba48760f2b324ab347e39d3611d1b2df9bd83\",\"13eaae9a996a67e2b485c26f56c758bdd2166486a982bdd487e9b9e8bde90b1e\",\"7712372a2a2d768b549ca96fab288983df779f4c91f7b6fbf03c389123434e3d\",\"74ae68f1533f86bfb65b34ab3a604be53d3892a995ecb223bf3019858621dbd5\",\"2d082b7246a5d6c706e9574655e68522e412c63930d17e9485b0c45dd518e541\",\"2aa175582b5c072b65c2d6701805964cbd396cea64aacc4f4e0d44f8b35cab03\",\"5ae42304407e3f47cd3af3d1dc0ab6004a5bd150258f5be7fe3c0e8460e4b23e\",\"749436cb745e393e8332b1e24df12a1a59d33c2cf16bcfb3ae34112810617ffb\",\"decc983dfc5e73e4df6f16af69ca2f2d761ed52fc92cd4a8e0d529b602dcf3f1\",\"a96e73b5ad492a86f06979325bbeb106a49040bada479a96197df74c9780355e\",\"e4d48572a11afbf5ad594c57cec5e06e8fc5c374a39f75b200e19b3b85b27475\",\"1bfa5856741088c0654a215cc128b267cce1b82e0495ee61a4c58672a6269fb9\",\"54aaaa0c030acb781b40cffe6141e407842819b3311511d372d26b60bd77e881\",\"9599919b553d51cfb06bfbf3caa8e8c15ddb082006648bf32d88d32e62201090\",\"2f4e5674951308576292e0f87515f79dbcb500a51a9afbedbb4803bab0bb03f9\",\"8cf0b8d94c1782a7bba31c219a0b559fbc521faaf20e68cb147001ce2e1adbdd\",\"006d35c683f46f79d74a112e7d4d8c0e22a5c7cf3d51808b4c3d2ee3d2d204d7\",\"2335824ecff59056f45f93daa878b9e4786f7f8db6359acca7cd4fb6fb2628d7\",\"e4f96730c1188b02172a8a826223ae45be727aca6b65ca6567122978b0c44e5e\",\"988a03db5ccca2726ba77a854069fda85498c3767b83b3d64bc1a4df62aa7103\",\"2ae38d84ba03167b8248b1a17b1ab73f7a6d2c8bb81fbef2c3f4ca71a6c79677\",\"63c0f0fc07203d086cfa477c3611209487a6bd8ece48f06e24e451a66ce3913c\",\"f734fcccd4a2e746c95083905f652342f4ee47448a6344ac209627f9f8d916b9\",\"43069a3c6c8d6605c85b5ee2f5d01114e5d5f62158ef8634e91448f06f0fb317\",\"bd344e7677f2a71bfb10807aaf12f1a78c52017d8b0e4e061e9ac5e9a5dc42e1\",\"09920246e9419fd349730a1c1efef3579bcaca92b2171545dad03b1d69e10ba8\",\"1f8035fd666b1fa52eab7cb44de0a330be2a6203f7c67efe97ae04b360af99ca\",\"95b4b3775de1ed10a0cd1042aef73f418b110f193bb4aea6e11cc4c9cb64022d\",\"cb2511939f715fb00bec89e21fd0b3a4984149850fd380d9ab169ed05a12a76c\",\"35e0ed0821f81f63573290e1a9d2eae1c4d5f56778a20614121b5991c0aafeec\",\"356a1af2161dd3899764e8b38a28d2c3b397581b5d959e177557bf0827e4e58f\",\"1110b564c8e4042a287bb578b51c3a11a28b614cb36f445fdb6da0e5bab9ceaa\",\"6df422db239413ceb4a141034a8fe8cb8e387264f43459c0f5ed81fb88549a01\",\"ae0099bbf97e5b9f967806adb97f3e92d6a9ecfe81d4ceb71bdec332fd54a55a\",\"72396513bb0f023cab55a672ddc55ef5ef05317643f8187a916df0357505937f\",\"3d7c2a31f45df05a64be17fb40ce782852b4733f977955e99ffa144d4ee5c9e9\",\"9b3d247a59c86783f7fa00e0cf5983d8edd607b47d3c25c3cdd4f99d12ff49ca\",\"b87a7132b7e6d4c549c9e5b6341f55d4dfa5237af94b0e65bc45e8c51764b604\",\"2df83ca2663902a3ffa28ba83902d275352e5b1f9a96eb16a712c26a8775e3aa\",\"3dd9bf20537b37f3fdbb434447c3a88d461f9b700e2b34ae4d7318a0fe7d541d\",\"7e9e29fe13414bf20e49f9f208f6d3d692f63427b103de38f009f36280f7c121\",\"38a133b1fc27046e87a660b11d9dc474b766acb4dfa4c6c66f91828d030aa4ef\",\"d841c62503d27d862e11befd834f977543d2cfa79fdfe927a50dc0a94f90ca92\",\"5b25d0a5bbeccbd329fb90e5a72e1ae04d46d00adc61d680d8368d6039c760eb\",\"bd35d066a3c0edb4a6e2e993505b85da38b8db8ea78186b719a66f9ee25438ff\",\"e39c598b2597593530373b9e14bea24048c9af4836384d3b1cf7a8ac27bb795a\",\"1d8858c44788c8e7664c166246999a55b3d450bd3c685ac4c1d5d92bddeaed0b\",\"c3fcae83206be3e91c6e76c18549c758345e573b0be7856b1e897b66ec562374\",\"213d4dcf60bf2b365699a69401fb2d6e6d055fa8920875fa6f26abb82db10bcc\",\"854d9da81e1772004694ee6fbfc9cc15c85e4047e707ecda4c3cfc5ba61d1bf9\",\"5be80b4c5d3d2731d1305b646f9d5f1de8d3b5effb45c8f71efcf2d68ea49d07\",\"a9834426e2d8b67deab735f28bb35481049c411297f7d73f2733a796ccc6bfaf\",\"4c659cc8b42718c2a09406348058d095449db511892f8544dab7be49bbdbb44d\",\"0f870b819be49d6e8577d6f99b5e8aced32d85c157f73fcd19d70e01b218fdf0\",\"724fc1338380cf396d043c3d18e59d73b8d4a778a59f40fa59f9a956c7b855c5\",\"bd6e043c6a0e9a3926b8564b01766c8598dc56edf35cb094859c2c22074afad6\",\"48cc0d18cd871aea8efac647000d995c9051c8fdbcfdfa8c9f3df14ed54d4ecf\",\"88edb89916a8a19731d2f0b319eec98bcb7b2c677f26d7f09fc37739cef03b55\",\"3505707e07826c8ec6192e4c8f1505c0b83ed26426c16a08e3d5f46dbf3193c8\",\"61b05860788dbf1fb12da278d86f4a48584632d0492dd85e69e3cdcfb9eed4c8\",\"b48d918ea3c9f649b62f0a3df8a03944f98c76cc40975d648c3bba71a1260a8c\",\"ff01442fbae9ddb6b140d85c45bee812cb716e1cc427ca58c2e70413bcd8d17e\",\"5ce4d78055c2768f70c559e7b7b0e1beefba0da5ce6f4f52cf0c3a65e78926d2\",\"44b91833d54bc469469997b68857f7b35dcc04ebb57e5cfa42ffea762b2f4319\",\"2cec208bea564d381069731127d11435ab0e19f3bea11b4b3e6534f6eb1d9bcf\",\"0ef214864db23b5c7682cde3e2b87ff6d0d2298de2421d3015614af3625f1925\",\"25c504f312de96419d486ae5897490b8e695da8bc71d6016b948f5a19929fef4\",\"f89a03cebbb451e2caeab3d5a3f3f45349871088f50b2571bce02c8ab61a5125\",\"9fe2dc843f4c7a6de65fad7599d159d3340e3d6a1a2b30a34d17c53ebdd5582d\",\"44dda704b7334ac04025bc3e89a226b1b8a8066603cf453ccf2673fd04b57390\",\"f18436dce5111fb48157b37b7f42b0836379f1a7e5c0a4db2f032f2a0d78a622\",\"aaef6b64d33dc071110372152bd740e2900f602764aa1d778e41dd866a1b789d\",\"75199db79df387990ee703b91b0aadaedb602d49dda498a56d069591eaa8ef20\",\"912959b7fe2a228bbe8bc0af5e3b944db887bdb2d37e5833794ed7b22380a0a7\",\"0cfc2d8607aa6b2755b41d8da871d602fd44ac9f8403f1f890270bde2910236f\",\"c0a04c29ddd7abb49c456af8233cfa0c1513fec0a734135045da49941ea84ccd\",\"9be755327436a56c65b0b812e544b05556ea55b1578cf8a42bbea5c4d37c0096\",\"450f2a9d683c3f0486df5f9dde7e182bbcc3629b475cddf910aac5ae9d567376\",\"4ddf9234971fca7b36d531ece80175e507d07e22f80eca79a96b799ac71a68a8\",\"f4506276e2bf5867f9bb5853a6a41668acec797643e623b3b2d85731be0c049f\",\"d1fdc84464fb48e2ce3db48b3709857c7d0cbec0e2505c22fdad4728ef899007\",\"0f25b76284810676c45d4195ef88ed3ae2f3c5b640d4fb0b97f38d50ecee9160\",\"9763092beaac1dd0d5b08708ea65d6b7b51436acdf6eea725aea1543e612cae3\",\"eacc50b4c99b127726a2313bbfa73613818e7261b2873dd636d9a739b7fedbfb\",\"cf210fd9a6504f33560dca527e2e1b41ceefc66fe1ecbdc144653764bf7de388\",\"507203c48f8921bdb965c0fb034693f0b2dcdc99c830b3f99574ed18994ae801\",\"e7e9f682cf751f0697325e755849e90f098e5bc35101b6298b719219702c8e55\",\"8579124d5b8bd30bfa65938cf50b008f36679d68fabc8a91035dc9df23f30306\",\"63d98191ad6089ac572c73d5ca89e907d4f852af0043adb16fe8f798ae1c5542\",\"80638a5c8c18a21731121b7988b80ae654d672c24a4ad907249765c494815bc6\",\"bc744bcf2a5496893080e6f05aa8c8503de08ef0f445415d9094a8c81e214a75\",\"c336f037ea955a4821fd0e38e1341b9111e8e48a1bd53f2bbbaedaff23b16d45\",\"a1ec38fcc6ba98ec67f5d841ac252ce62db71c1857f8b61edd82e6fe6cd616d2\",\"dbbcb9fa7f7cbfbba8876a7b253bdfc01c77e79c00ef99741c3d1422e3793524\",\"eb7c86c8cd683cd893d0f970ace2159a32a281412ffe59a560fba8d7fedc2147\",\"625189744cce7876ea9c8de50f5e1331ebed10fde3b4f8bc9a302186d53cfdb3\",\"491132bdfb44891fe793d8513225a99c0d91d9b9e948f9cd70622daa3ed7125c\",\"b7545cdf1dfadef63f11c01170c3aaa18b261ba7b4287a592d1c4a6af6279b39\",\"f30bfa42b5b8b07d7e9edb77303a1113219921e7e8cb01acd1a21b33053bc04d\",\"6a912a3f7c8a4a6616d8b3cb83ceabe814f778e54778841ba827f836473e3b14\",\"d832afa8e164313226f36d4ecdc942f47b49cccac445104ded88eccc4f69640d\",\"d9f6fddda6b47cd4cdc6d0ae3dec15d7e09146a0d563519903e8c03f0010cbba\",\"441b34b7722b4240153a3a4026d010e34ba9d5ae9b7f208b506d7cc9228c7dcc\",\"f7943dec4cb714875775abce94176df2e57440683ada72c7caeccae6aecfc10d\",\"d15cddd7de9a9b54f223d3e660ee30440b1e6184bc9a6ca470fbb049a15b68e1\",\"3f83d56ac7b5a28ed6fc5cae7b00538ee79037401aecece6c88db1e637d26d25\",\"f582b8e6a157d4ad2d08a34e6b0077dba1e5cb7fbb3b0be201bde618e261f32d\",\"18e84bcecc6ea650a7a2b6f226175d95d66c2703e982098f5a579386f09eae8a\",\"df19f7bfe98c4fb03644b7988f93cc46aa01935f0e559308a11be174376683e9\",\"9da669a36e800d7525f11f2fc0fb851d4e03119e5ee61f093356292c92eaeea9\",\"7a75203e611887568d7fbf3855dc1b970d3bde20ba957178aff63eb01ffa389a\",\"ab41e20fe34d3209b56d0379859b2dffefc30dbdf5cceb0ce137913e5c0717dc\",\"fc7a25778e0a605132401fd22985800525c511f4125cb82c7fc5ea177222eafb\",\"5370fe1165bfddb20caeafea30563dba380e8ecee77166b46657830942a570e7\",\"b0605f4fffd16233bc3e5462a33e6e248eceb92f45022a0a31f9de3776c950c0\",\"5538b6255ca3ef4033cf124a0be926e1b3cecdc29eb419b62d4da3157dd22c31\",\"8c1ba6e032d03055fbb96c00c3248cbf5cf28f0b5fb6342d9d76e22a08933c49\",\"6965dd0279e87ab90860af2c6f3442f624049eec77604dd0e8fe302646fa07a6\",\"25ec40c355728b0a05ab16279b1135e35d536b6e6ace06723ba9468f8f32b9e9\",\"30b766c46e72511e1f50b3938c6744122e4314f5103e412c9b4e7a143a6f0e4e\",\"832596ccefc834eacd3ee18d1a499338e829d19c45d2436cdb4138303e932231\",\"b031e264bff49bbfa0852a90c75b3bef999d52a3951998ae0d8c0cbd834f3543\",\"feb4873b81c247f893a9057feb8fa96fa65402fe2f3dc76ff21c2787e06ff3d4\",\"a171550c9c228644f3f8180591a32de61541baa392a84a859c933a40bf48a66f\",\"406e1e5c0586bd0908341bc630572d47b8a244cbff897321ec6242333e105b5c\",\"ffa3d37e97dbe82db8b16c49c5355945e0de00a90433d90522ffdc2a301afbca\",\"0dc485bbdbd987cd56051fbe82762e5c1c5782c11031766f4229434d37be0a65\",\"3a2233a26f8fb3966c7fad4f3624542fc5d43e380e4a108261088de3b2dcb330\",\"e8febb902c6a4be7a7cea6803be797d614263fae66b3819b23154681032f0188\",\"0c5c658cbf9d09308971f59100021be5f373f91f974a1a8f74937fb2d35c8c6b\",\"590db3e9d537a26b4ad575f4c3cc3b4fb6622b8bccdfe80ec948c8c0e3a2967d\",\"ef74412134354729873d767cd43d74de45a260df8727c9122a0697ef9fb8498b\",\"2565de91028e5a9681ed10ea24e5f5fd80ceddaf0dfcf63c22757cde7f601ff0\",\"0965ac605b1ab9c306e5dcf168d831de31b6af957aa753cd7e1f880c7ed45427\",\"ae465d21f8611b2a7b23921c18370411b1c5a9718ec5b5bc0599b71e0ea272c1\",\"8952c35a140dd5eaf117d7fc05c26b730cd33757e18f2a095543a5dcf47195f0\",\"1c6ab03a713d0c7b5c77f8bc90bef07a1c2ce574c2b7df949b1066e464c6f281\",\"7883819c8ced1a91c78e767e9e01be8e8eb03ea1a17dc7e846f76960940a7c8b\",\"87c470b1e7be0c6b8c0177dc6c46e2ebbd2dc2214cf032c16c22dd64196531c1\",\"1ec01a40ca745d617ff5b5f650b1b18efec321ab99e3b30c667f91f305ca9e6e\",\"49a7a71bb60a1c76c5ded0e66dbedab5f43d312033b4a41b417d48aa29774b11\",\"fbed99c0e1d808f5c0319205647d91acb46885847e46eb6b2dded70b5dcf8f94\",\"d9b0f72d11ea833aeb245480c2f5b8f997b39369ee6bce13f84c67d061b48344\",\"f773830e80c72094b4ad3191838163bc59d335b1e2f45b569317ec6e91b01aeb\",\"b5b6a47d65804c8e8417064a6af272beaec039241557dadd9055ddc7ee861bf0\",\"dfc5277427bdca4a6a0d75c13bee87bde53706dc5f7294e8b4e8247a3ca71c08\",\"cddca46c5f23e520dde1c1866c2bfc62c52b54eed67b7c21bc0f470ee9cfec0c\",\"bd965053d69996e7c87b59866e5e3f33ae1e8ee927d3f6dd8bf309e9243962c5\",\"b4c181bae7b4af9de9fa95058af462deb36fb935244dbb60fa582762f7ea77c8\",\"0e6d36482744194a23ba40d2ec0dd34995e747e6583e145ffae7c5da6f8d5658\",\"d85c24f921f7daa09ede2103973d7b610d3f4cf57ac6ab69df2d2330a437c6fd\",\"a2621e68162882ac6c7e34bb85d7c975fe49bc32bba26b3c4617dc39d55e359f\",\"023a52020521722dfec8d4afca23f923e7b881cb24a55f7c3794001b09c88a58\",\"5f8dbbc5c1e4c2b42a7ae360a026a69f4aa2d99b4d77f43d94eb150696aaa2cb\",\"1e4a29598c2ba1768da96b4a2392bc4e3cfb55b0e26aa1fc114f5d49c1fe8cc2\",\"bd21d03e4ddde53a18b3b74a186f806851412fef6c7c71f6a9277aeadac34eb7\",\"5921fda005fd3c0806cc30764071b166be34f2489da49015b2a9f9604c0258c9\",\"c073405f70cdd84a33583a855e2b15f8b7fa3f772b014d508c02dee0816a5299\",\"69af1680a48d92606d5afaaf161dd8ed665a480a53eff311bbd76deed2fbf585\",\"e40aa59d3868dc0bbfb39e17e8d18767beeb385b20d0fb4b85cd3c2874a4e013\",\"84e92ea90c8a67216f0ee2ec2508b7b0ee00f81557f27c2ef399a220c94c2237\",\"a30cef48688a3b2d35ba1799754f1d6f6392664204e8f0ce3d8db83edac44962\",\"525b6de2e2c35993578409b92915efb707703515279aeb11c2d9dc86171f1dce\",\"3c72d634d97e096af986dde38804b27301cd8f023a5a7fdb149b8868ea3241e2\",\"627dff67c837060e2286828defb2e2a869c7677c078d54896eb8f3be39bc1468\",\"82c587cd332762919f3e37802c8a669a9abba37439175c06ae6f35bd5b834fc4\",\"12efecbe48c49737099a753c39ee7dbb8ac8eebd84a50dc05333f8ea4b94b803\",\"91ffce40236f2d4bd4eea22a762a0c986a5b044f6217b2a67c3ca0d1bdc145a9\",\"adc571bad2587064e4c8c1572ebff5c9f500b97d14398dd183d8c2eaead3fc7d\",\"70d6a1ff18d7f021591a0a23ecbb6cb558f88b540c0862189a2c8527522e3458\",\"9414b07e8ee76eaa62b70aafce50fb576acc079d2fa5b43543626e1703dae06f\",\"c7ae9efb42b71a86f970937f05df35ac000ffecabe7ff70b2b5cfdc4fbaab5d1\",\"4d5645a08405e8775f47fc9eba42d04ffe4fd275c8e1565efc644b4151c0a7cc\",\"e4505cdc0a77028aef04249e9bfe2253dc35ad093180464d322eae2de2695d00\",\"6735042f81e39d0a3b02f8d5be60c19a9573e5d108b53a6aaaf0a2927843b73d\",\"f7d8e6dea7b8c4ffec59d6376a777b728bcaa5bf307cf56f8ed3d5e8aedfe514\",\"261772622842864e5de0a287b3c13aeeb9f0fda1efe97c7b916cacd3e4ae0245\",\"e1f92e05f99a79b7dcb509b9d1ebf0a0b30eb19e0a41b72c43d0e3ad213d0066\",\"19166e761dee24803d863ff1061901d8fa13e7a0d75074d89a67b089b5320cb9\",\"74b7e07e7d133bced85e811e1a6399f77b8b41a401f7991a533c9445a74f0d2d\",\"dd11b312bfaae8f293b49eeb63d7f852af7bd4a86763c307eff1991d4ef11cbb\",\"17eb55cf662963a8e2c8ab3aed58697d7639f5a71203d7da2bddbb89846c3c59\",\"e58a4716814a3bb355502a6cfe2c43f932ce624b1577c4a92837d5e14cfb0826\",\"befa1a5c764dc3944da4640b2cbb0a4d43b2bc6dbfda96fb0d6fb018cbae6dfc\",\"adaac11b0e542c6fa5a97bc4ae0f627ebc8ef9f58047806081606c34895a53c2\",\"eb6839f9d1ea3ea80ac5b7bf730b6a842c547eeb2dc570969534f1260b4e695f\",\"809ae49f6322ab534bd80fef64446d432440a2f6b88db533813776c5d514e748\",\"5dfd7ca3d9aca01d06073d40ec9715bad269a4e44c79a7aa198cd67b13e79eed\",\"6c378024af4c594ff5e802ee241432c2f4868cf85fcfa876c1d11aee77a7f2bc\",\"bea872d5c414d2048bcb29e02b5935cf9c3aefb2cfd5a8b0b2e2a3f323b126ec\",\"f84b39dda069a8665285dc7e329515cf3a5efc6f91a945fbc42fe87fef8d53d8\",\"bc5ed4562d37fb3f3c669f520009d3a4080f0eb1e13edca6a70405369f68deca\",\"c7314152eae11c52e85fdc1fb79678f4681cf552c6120a3cee79bed1c6973457\",\"94dedb902525f5ac9419ebc3aa7be0433b4c7a013cbb1969be1569482d154c9c\",\"1eebf080b7b19aa5708f71784b62caaa837f784262d22be8cf577f263fe5f4b2\",\"4e57e402970707da15daf615a5459361dffdc8bc485b6edcf7162667979349b3\",\"671caa41d3eb18d485945e0722ff81158cc9dc6116de03fe9799a484bcdc7df3\",\"8eafbf059599db7f944a08aa7b3239fa6af24032cac14a7d007988452daac194\",\"5cfc811b1aeed888d2bef49dee7bb9fa9d6abd1fcd910c5a2f24853001740854\",\"ae556e8501effd94b9138b5dff4957927298b79c581a1b5cacd6fa24c033d64a\",\"c98ea217143a5625ab2285142f0c7cf3715abce91a32be56a0590b437bde5ec0\",\"cb7ca1323df4ec9c9fc782a01da1e763709730704db8951f4b86017570073c3c\",\"f9857dd635e79d701be0313f704b61c328e35cd238901f8eda20f08f6744e4d0\",\"9a3542e4a61fe64a30189826203d1ce16afa35afa3cecddd122df4d8c6d917b6\",\"5a8b58b88dde8971110cb3a49b7d32d341277404f21b8a420022c530c4700f97\",\"d1d19960d090d36b93f2bbf64813f05495306d5bc5e594de1080988e76b3fe2f\",\"1a06779c0007e5c944cf8c5082441266ab1e9318d67b3de775f6edd1245d1a48\",\"fec01bf7630730988659a39fa7b5ddbb450e87723e952fb2d6b0670cbd0e3e09\",\"16d8466cea42b0c4442581a0682e8f4aa5d22dab5fe88ddd72365fd2f35b5b11\",\"b6e2918eb00b11aa0d20b469bd46e21e21b570b5438b2c9eab4d7055a85aa0a6\",\"d644ae9cb9fcff2175cc982462bfe78bf9e4db6d211aa1d94d6e7663e207fb4f\",\"8e164fe7db6c847c63dd6bd3b2a6a6c2cddec9def7ea393e5a6013c1c14204e2\",\"1c328d5cd428af5e1d925815418d6e5c70f27943e3ac199392a64defbe1fa757\",\"db76a3a233d5f82fa02a43c50dedf040327a4739f01bf77f549e7e8c86260012\",\"e48876b778ae6b503aaaa5be6cccd4923a89179b9901614a0a1e7832ea3fe6e9\",\"ec944ad75ed5e5c29665cc57f237d6bdb835f894ebdc7a0221acceee03d77760\",\"ee9e003003e22f8340ec7078258402914b22e033b1ba113fd2f8ee28cfea5ad8\",\"051004c734163c8893e81f9f49d82c92927d364baadf0ac3a1dae1bdb0cb7710\",\"31f76725d07ff6bff35ae81d9d12ca0be32e3f288cd46e6989d011e88023ca88\",\"8cf85dcf7bd08450976a43edee3ab2ad8845da7eef704eaa27e1c7a015502d81\",\"5dcde18b66ed74bfcf716b3241f73b44ebdadf962d070cb84cf65f61d9a9ceda\",\"8d6d5946c63259cc69f19b67222dfafc75ea9a626834fa8e9aa2d791406d1c3b\",\"b64a9bff4eb671609d4a8671b50be777d256063e4fb66c4e51e022453edaaed2\",\"87f0716c75cf1759cce1f45e074c0de658cb12e76db57deca88b6ac1322ce9e8\",\"e8843599b7087d88ba0ead0a7fdde661c56b8cfec40bae32a69f09570b3d6e76\",\"3debef4c8219f6197afb188a423772dc7c4c082d9e6c1b6441364b2eed38267a\",\"2bb18aa97b733620e819cc6ce94176a04a75d79c8e5d1d86bab75c097ead7f57\",\"2cbc77e0c42846a336403e85396be17c59b89b5b8bc8771be47cb9f705e90e0f\",\"4618fdc8e5ea12687a5d6a42d491773daa715b717c2012c5e3f4d0a2cb944d01\",\"9ef9fcf9cffcebe6f9b838a5208a62c5df9112f0bd5e654963b9db33bb5876ff\",\"e3a05ee8351a48684a8f4edd4c25d3e00e9dad630b970b6c69c8b43264bfed00\",\"976bb7f1be3c3f23dcdb01c13b71af7dc1f12025d58e32b03e470e716f53e6f0\",\"4c2dec380df00549f5890cd18a8e86fbba7424ced1bd24d495c204ddb2c4d945\",\"06c144893ce9e42f9fe4c9d68c71c3a67f87fbb441a7955efd29374f43ca95f2\",\"ed1cf6207a2c18ef47985ccef39e53c1b918ac2577927c694e88aead544850e5\",\"72273eee9e96164ce10da88cb05e905641fb296e680d6313e8605408feb7efff\",\"4a41d48355c195c191c01fd0ddc7059246bc651afc6bfb1a560fb23bc7c1d3a8\",\"2222a1c4d7558cf4e7bd2dd5415637a3c7360e87015bc44d0e8ed0a8b47d45ac\",\"00180f8cd6634549e21c72ff3a234baf0af7ecb9d4c6adae67f6e32c27666490\",\"265b8fabd53e074c8d29f6831360b20471252027754c60572e1c704471c72e68\",\"152bc22f962395392a9f1e1d54d3d4393e57c42e29dd5fbdde95a9c5a44b12fb\",\"8742b5934ffcbb908ee2d055273a41b91d765a8b569c637af9dac6fb18f90a29\",\"dd07435086c3d1d23215f9614232682af37dc0b5e22fef0304c7b2ccc3aca9d3\",\"93a34a9a7352d27d1ba64c36f1d3b6e7afe84a6134239f874f671b7c238c25a4\",\"6aa1ebb8091c071ac9247944e6809c0ca2e5e0ab71136b1bf20cab62c2823ee6\",\"5fa03095c6c4dc0be170badc8dc4c6f6c4ae18ba92f32580c4b057ad0d0337ee\",\"6b0e4a0f4b2443400ffd873e200fb599c23bfa3b0b20ab12bc9e1cf9cfd78b63\",\"50df5241e9ecf906e2c1fbdc0c29c1a9d4105885c6f2ab765ab04b434763ecd9\",\"7884fca41300d3e40d7192bd4fbfb861a8e5f85dcb7bcab4435b5b3a78145c23\",\"b13c351b717fd7984476973b809b0ea783767dd0f88df76804a70b9f196db1f2\",\"32f5718d6ae181d969010d18f304e411476752d0bcb3b78b902af820eac58e80\",\"98ef3a3e01708569d0672a63717be8c527c8949d7ed863f68e37ade3d1a42efb\",\"b595747bfb15404a05f75d6633eb0f746d92f326ad883a8ccb9b419432d91a2a\",\"812daab5c35d88cd5204cea481376ae76376db0fd1a1dd78176af67455a40f9d\",\"d2f45edf6e1a11d6265155f098eb597c23e899d4b5ce61bfc4842a6a7bddefe4\",\"f2f11e04c11483fd4766018c1f3c615ab87d6d279a3197bcc83b85321878a1bb\",\"be95a61c43b8d7e394f89405dd05da74845c2253a38e30e05b58a72ffb7f78fb\",\"a60b27baf2dedc23ffacba29ebce5696d4dc230cb71e297aa2d1645df9ef168a\",\"dc18b586ff7702f809c8dbe3c6999bef38aa7de416dfc66e76f8509a8bf5a100\",\"ac48d0686522501cb7a29d7e5cc3b925669f9cf248eb7992994c639895ebf2da\",\"6abed8928b7bd33af314ba0e19618b0f376083b3a49527803aa1cf376c0b8473\",\"c75320f754716f60528c095941509ed74306c1c6aee494b4cf5c9588c9f0d64b\",\"3b9339bbd47747d8eded70844ccea45e7666939f6ba695a11b0d86aaa56c4f1b\",\"a0676ad0c475711af0fa57d5c120cc59c076c0775c368c77dfd7847dd1745e2a\",\"fc806358d00a3c959a10d5b6d0c3ab28a8775f7867eab4c1f706bb40d5ce8b5b\",\"b6a0238adeb3818ac222b5394a7ba8b148346d26c4c34c84268b6ecd9c8d5f07\",\"8058d203210a8457945a8b431b669743d82529b6679aae1580b2da09de032b0e\",\"21e2b927247e5f81f75538deb084c7e13e50a2e54910e649dc4348215aa611d8\",\"09d1387af0dfd0c073d0032f202bb5bd031b9945b2f394509b2b00b91ad6ee83\",\"da2ad2e887746b4d1ab87bb4f1fa5027768c26673b8ea9ec808d7909150c8094\",\"acf02780d5524d7bd1b063eb4c0bec21ee44421468859547b70d7a1cf8b6d84a\",\"6f745b9c06d86b8e39c8e7b1cc936c987cf4261b634c654f5c8bab5192b93b23\",\"65708dd4f5c7daa007ca754ac2c0d82cc417baf541bb346f43c273c403cf12cc\",\"c5e0d08235972211692b54e971484be07cebc8c9f7cf7f7e89e279cbb3463af7\",\"4cf4301d237261f702ae5d093dcfc0a692f45b8a6f5cbe95802d462be22bddbc\",\"e86947c7966c0819f18213b487092f6755068e3de0d0a4ef6ec8fabad654ed0d\",\"9048eff8d211e14e285d8e23cc75a0e356e4c7d42f44a09680fb71856902b10e\",\"d60d011ae14866f31478de11bd971cbff6badd1903b156769ac2174d83daecec\",\"29c7e0e074365c098fc79bb04bfd746d62acf2c99b3a99360ac850343545e117\",\"e7aaa92daf31e3de849a5bf5a531f7be7579a5da1729a4e3cb329ee95f6278b9\",\"4429c132cb2f2064c1b0621751554e619ba8cf9d02c2993643b59a8e642034a8\",\"74f94d564e6f906911ffb0873241a1b1f77ce561a5a906461ed2231a00c88751\",\"8e59822719fe6f54c6fec0586835564bbc23a20fa8c0edfa2daf42593e10a58f\",\"1c8c422c4a693b2e28238281a3acb7917ce25feb9a25c8121211d86776b64544\",\"1276eaf41afd29c42233f802833d29452e17fde69086b7c03c9997db64ebf083\",\"3027cf0a873a718507476866a7924a3f1c2c75f13a3117a68fe8cb7a59a6c925\",\"ceb9a743db4cad968214a4c0aaef85eee2b1d8ae73abfb218614a4b89a431fe8\",\"a2119797361bac5cd4b37bf5ee0d444e0b923cd570f01373c7c4989b7f1310c1\",\"6dd3293c39066d4699eddbd615121aebb028823b658a83009fe4abd09fa9d1b4\",\"3fdac9dfb6709076741dbb338c9d5c5a2ca5a1cdc489d81eaa57df33f5c7807f\",\"d636e09bc4a9560965f6864f7637291ff00d4364e30afb2a58ca680d906e76c4\",\"61a5a4b0347ee0ffae77750c59ca5cf9485e45265c66172d6f80a3f6e2c2b538\",\"6a454c54e8ba552307ba195f2081c900a3940b79290c126b858dc34df435a0fa\",\"69c920d03f02001f599b223999c8f863bcb57cb66c6ff7c880e23e1b417466cb\",\"583efb266a2d9e779d7f8d224b979e9b9ca473c64469b5a3d5de80ae68dc4d8e\",\"57aedc43faee31257c841922306803bda49bcb0ef0e5b4b114944df8df48b513\",\"c9957e4f6234258a78e1cd74fe0317560aa8d8a72087165befb57dc958103489\",\"bb78db671c1f526e32d4687cf021bbcc821dd1d6bd8b4ca962bd66ddaa2ce0e8\",\"33a80225ba02a811657e0508da7538c8bd7f60109b8dc26601249b3a459c0857\",\"17e2a0bf4e48156072697b8693173359726b8155fc84c86bf50d82d18ff1a68a\",\"70b086d6ad39f51cbae209eb55e825782fb2a1486f85243afdebf8bc36cfaf71\",\"84e10c36b9e776accc253e8d7ed172ccce6c16c1d5e8ec5c44cc57e9c25622c9\",\"d84fe35841b33c5f247d0c1959ec6624eea05076465783e522f0151c23127b0a\",\"c0c15216dddf99ada217646923985afd55170b42872d4438fbd001a5b2a03e6e\",\"4345d1faf324e1dadbabe74a1e6e113d7b3401549dfa09e60b4fa72e2d836a7f\",\"47f75b6f080ec07ecda000459d9ef7f59937bd248951b04573618b0331168ce5\",\"0868de278947d9a123391eca5fc0ca8de7f0033cf56c4dc776f88710eb084db5\",\"e81f4bbe4f6c12a6a9c121e748ebb90f2e944c63e93bd34af432047abfdfc2c3\",\"de126a5238b64c5ebf4c45687cf07426d47ed9c5d58a871350c815b2dfaa4850\",\"0fb7c472956d715ff4ecaefc392bac19b66df352946adadd1f1c93cbadd98ad3\",\"25d78ec34cc1be5776f52c4d675ab89cc5d157551494aab6adbeb1443e93c564\",\"8d5de8358b2d2588bc81b2304f4c39f1ebd0c45582c3c9bb6496ccedb9009924\",\"7228f4f4c30c6669af908aed88fa84564fdb1c053d7d9f525b563e1c8e633de0\",\"1d8949369859d8e877df9f9c9964482ddbb13f94cc01ac1a74c5072b124f76ef\",\"49e685ff9b948464adf2638b05efb0061e3392dbad13b878c6ca7f0acaa133f6\",\"638b4ac7b593b33282099c28243bb3d77cc67949263d3df2d1222150914e0cd5\",\"da9bd9bbfc7264526ec06cc1b67019ef3fe5048517a6c7564a5344fa27de6628\",\"34a6ebfcd859f114ffe558824ac583679aee47e46708300131ac8c29e162e5f0\",\"c140ccfb54b372ca07ed1f1e5cec98fc5f63a258e90b6e4247a787a0bb3d28df\",\"f0f3b058f579661b74c83f841d8ba6436c4b14a39ed17348b7469e5a220c8cec\",\"219bafdc46178047622649a486b304110f940963003538875b96b8343f921881\",\"9f8bffa6e0338d7959ac54e6f94fbee2420360c45610558c0856a5f243e6841b\",\"6dfe4f0825f205f477bb48870bed184d21e9a3cf433f4673524f9d0de447b04a\",\"d78b08219f1cc08fd0982010f4dc60d3d164181214515bce97732647806e8e1c\",\"46fe95bdd0acd2a595a328638688b25bcbe28ad11f767a2beeb1eba7da5380a4\",\"2d82fe599eb3b412422a9ed9b18ab22a46851d761aa198134a6c348d436d5afc\",\"e1e252c0e9e65e347fedfb8ac5bd7249d50da3aea5462d5f0d4e95b610480351\",\"88cb44de0ce49936dd723cb98f87a953f4e6a3f22576b6bb9897f9bafa1a83d6\",\"7c0f568e5bcc5da149cb5d110038501a12e9099c33b27407bd81c3a2fa9e7572\",\"5ea1ae394071a2b9b0c43eafb08784489e722aa21bd1a0c46021c3e51e849f20\",\"65a0790507e9ce5ecef2a64b3896e9a67ff20ef4adac987bf0ae9937a3d71d3a\",\"f73f4f4d82bca951d28220228d091ef048c670350acb1e64c334dd1c7d8c33bf\",\"4b9a73acc46191d166c5130b93eb8bae63ef89bdf123e21849ff3e8086f83340\",\"7649446f4591f5bad92d1a85afe129dfe09d82f303ade5bd3fbcf423dcdc1cbb\",\"820cbcb4b9c7f0b437fbc1e31842e785c47090c1317e2f13d93eb66480c39737\",\"c828c8c4be18522e18be6d8db4ec9a4365d292c9f2ffda3ff8b6f26e4396b81f\",\"03fa48a36b65ef008ed945f16f32a2be9f17ccee18952b7ab9f59647b084862e\",\"f3c94a2943fd931b98356646df8292f89e9152a8061035daa3d9a16abe0b88f6\",\"b24717473febb713204c7f444daba5f6d35f5a06cacebe63685c94c5ef72377b\",\"7270711ec3e9f29935614c0ce23a42ca1619d4442b2506591ddd1f66bddd5c62\",\"312de9758de3adc43ec9873313ada2e0344de39ae67b79ce0224a3ec2a0a9136\",\"2f3930abb3e917ff4f030e0de2f889b5eb03bc93cd7c11b830a741edbbdd474f\",\"5a8e6e3ec7fb9167430226596a2275dff26f993f320535cf9e4c97bb9f97e8bd\",\"c1a75d3a1fe1b8cc46a804d71b4a611ffad3bb89c1d7b086223e568190110c3a\",\"1ad1c514754e9461f36b2bcd07e187b40d0b9751334ed1799325612524d7bf97\",\"36f72c32e0a7fc91016b6de69e4a0e82db519b9f1daa6b7f49ea0ce3ea343738\",\"60e7c9a1a424bfa93fdb62d5a872a19b21382d773aa4dad0c7013b59238bd5fc\",\"8e63176281e6ebc90b9138c6f8f14db2a9b959ef6e537d6ac279bf925cf5a3b3\",\"6302e3d8ac1cec06ae0042e7bfade992764da3fa537b4dff8a138373cede0115\",\"53d46d2e5bbaf55226c608c1013aea4159f30616c1e23bcda10018cf7e03f5e7\",\"379d072e300595225e85a473cb9a7c65a42eca33c9c6e78d93ac9ad2217e28db\",\"a61e0de1cb8dd06738b337b8a5cde601abeff708050e76779d0f36dfcdf61f4d\",\"111a5afd120641db3b717c182606b8933624f44b44954ada107bf97d4326155c\",\"e7b4ead772be2724c2fbc34b4e79a710bc16cd6e48c565e7b7b4537486422929\",\"dede709fdafa30944a87cec4f7123eaaa980f1f51c233abdcb317fc2bc48000e\",\"212fdb6332b44c444166f04491f0bb2c88aaf0ffbaa98a8a5abd22688c351be8\",\"16f591f216d7406d0c693b064f79244db9b54610ccdf5bf20bb28e5a82f9c452\",\"e694b1ee5ec0e06fb664c126e360375c9d45416877faff67a9fbc6173f996d8b\",\"04353e185bb3427f6451de8b7f2436d9d89d565d8683683f8eea9c5c00c80077\",\"f1bc7943b6ed907714dbbbe8b4f97bbc11bba8b4c1535326e8d59b711f449137\",\"5c09ab25ccb54aa0c2451dcecb924d3d59fdbceb82f3efdc0f16aff0b4946227\",\"ed221bab6500de27604fe4e57bf805dba6ee0138d82f0db33501fac5c67ff83c\",\"9b3b77ebe9e57bf38483ff8638e9c1b263e987d53f4a9b6c9f0460000642ec88\",\"0dc436180fcc7cebd4e12e959fd60ad8bec420a10b7f47aee5e994e37e5a5613\",\"d72e1fff20c4de8f9b537d5126c23cbf290f4192b7e59e67806124c4643c75b4\",\"0783b021986a6ac01be3b3f3ac5d83017a3cb6234d20941bc7f34f965863a0bf\",\"de40fd581be23a7e9f15763450d640030bd5582a0fec6a901e1c4c68abe775e3\",\"7ee7cd966fd5ad406aca9a7ce8e78a2fcdaedf9c37eac760fea6837fac77435d\",\"5f91d205bee43eed9999355b9e9c998e44573ee1c30f9c1e1198a689f42b0a1c\",\"d1ef7cde3a13b289c60fedbd30076bce5713875852a2bb0412f80c44dc4d75be\",\"0a4fb7ad65384764215ea842d18780bf01a2a59ae9081c05c37e733825da4d40\",\"24b5375b938eaf384d779fe4602ca58d2971ea64a4035e40b79d9af7320fa472\",\"56fec08f9c0a7ffd2bb8f6d31771eaf25e43efb4850898532abba1d4ff5fcc0f\",\"0d1f55ecc19d2cce45e7ebe54859c5827f92b3d1e83ae1b5758bb40034b0c69d\",\"3a69d3d27d34612c4ac4ea48e073674d989f3a65c848ac4a5e9940437a555e04\",\"687febd1f7d4b36760b0a1a9584a2bd14820a200fb5a9a95965aa8d2148904a4\",\"b683f5cf9e2b4931326b3f99de4bd8aed22ce66ed1981faf96695eba7f0ddfa5\",\"1cd0143b7f6b85383086b86b24958a11c6d7ecf3283658585566be66462ea900\",\"016ca7674125e28dfa1794c5dea2caa07077824e1e5bfeb14c85e5d941e01f39\",\"9224a52fb909e1a98f40f538bd9d6fd43de651d4c2097f061e0819d134b09f97\",\"2b637fd697fdc1a7392f8da979ad7ef74353ec676e18771a1602964f21da6c50\",\"bbc243017f8af3786e38db80303dd80e2e8652498f09b15930c09fa2663bf955\",\"8d71fabb3886a8d2d407035569159347e2992b12c38b046391a296317c4eb575\",\"6b9f39134e510fbc81891c8e96ac110fa795e2e747133d1bb1f528f106915483\",\"d3627042d9dafc8280b33d59b46241f1666a0c4034087d6664ea9081bae3f57b\",\"f81e69887c243384deaa5f42663a93ae76724ef04a7e635690cb030fd82a8e5e\",\"681f3942f3367e403730bb9de371e2a2fcc67df55baec581a976b106c44d13f4\",\"d6aa2120cbb8a9b27ea8b4b0e2591fb48cd73d6a2822d875f7a226c0fdfb577c\",\"b4a95d8b4acd532934283ddacd822adf408890a9f012dd5ebcc62575f4eed364\",\"5f4795dcb064feb56fbb94c79349c9710bfd8a6b754c423aa94ed064844abed9\",\"d89bf39320a682da23d46d6b2177b6421fa45515e90cdfe31def8fe38d1690b3\",\"bf4fcc5305bcd02bdb90c11b3938fc2e3ef02543d890ff748b02ed172ae5736d\",\"12a3d3868c118511598c8e30b683095e11fc1be0317855f09283de5e31fbac62\",\"f081cab67e55e49a35bb1f7917a4d81cd6a65715bd0a6487393e4399010aa859\",\"dfde3b39fea700eb45868e505a59ec460bd152d9adaa84d5227d17ba1f634416\",\"f8267266d11cce9168f6a6090b4f1222a48a7ec485c7d546d53fb350bcd702ef\",\"1e51ccb48be1ba355232d5fd6ef35919e7f36a82084881a83c4ca92fa233bd99\",\"4f2fab07885b54dc2983067f75130ed7c54e50deb05e9b567a99ac0ebc31f2a5\",\"a18549f06d00eec8ea04d712084c1316891800a86ee5c9a473a456fc2dfbd34d\",\"a78a4e0dfb341ce2283970f112444701a8e1ea9020f66386fcd4f112ef9a72cb\",\"eba71f2e82067187d594b0e40e3a7d39039136a571116403ade753c6f617a52d\",\"65c92b0f5153e994ac2fd43ca6dd3041fe44568e7237d9ff7aa1ec96d24e0bfb\",\"25a566e93e57194fb24d26e14ef4b863a68baedf4607ff5069dffccf54c3b290\",\"44586e14c98597f3a9a206be2c359ac21d9bb0bddf57b63141f08c4d6328f075\",\"5edcbf2e5402c2a6fd24fd2a034fcf81d5e2dba701cd46a0b064c13b75f4a2e2\",\"83a47a327dfaec08b9eedcb671d784b49b44a9c61212a3facae58f5cef79133c\",\"a70e380ef49b1702b1af26dc066de0c8639a8d6d07c9e12c0eeb887e72d96ce5\",\"7aa7ea6bfec0cc269e96777c2e164e8b2f6cd07adcfde43517a2380acab5edcf\",\"a16b6939a38b5a1e81640a3dc22f55175f9ec144c09b5051939fe8915f096e99\",\"a72e72a6ec27866abb4fe3fafcf86f60e08e8fd251a5d2c5a1677a86cc743d31\",\"01d0918cc937c93fb5150c1b7e7c4c8da26ac1cdd5b6ae5a08bf56cb4490f2f8\",\"3ba9532271bf30fc81a489161afe015ec4b295cfa2771543de3c4548da7e3100\",\"ca143e8a2e4829e43641a0a34527fcfdc99b5612355c6ff556ac4a5bb8ededf5\",\"fda3cbb8493c514642255671bc8abcae226d7152cdb36f516d53573816248dc2\",\"e3b3d252e542ce34d69b76f73083db1983304811e8bea285710cad9a33a9652e\",\"37d5af1849ba275c04fa36b7864350d6ec93508f73d93795061ac479fa40894b\",\"dced7b11ec15b9a0162ffdea56d6af0e03dedfcb93549b6d6742c751e5131e81\",\"00ebfb759f75cb9ae7fdd8ababfe5d056fd05d6155331039a116fca025839141\",\"7be4e102eb90e5e9fe7f3b34f7b9fc376cf833906468b8f65a1f36ec75a74b80\",\"caedc5997df02459eb43fe8fc005e3a704919a8cfc83b9d5736b2a4adba595d6\",\"085131d405bc13a5efb631356bdb461becc9f851bb8d52491cf90efacf6426ce\",\"1c8632eeb78f0cd9de36b43987f9ec8103141eb2504d4a5e71c8733b0b5afc96\",\"915cb77b9eaf773e24786ca86cddbef9bbcd2d827655f9eb5d85618a03d89092\",\"b2f6d9bd1fbfe8e780c01e3e974b79e01c4479d0e7a9158ff35b299debc178e7\",\"076944809048e6961c0c7ca6e7b03341230b3c82834469b843845c51ce30c943\",\"a8a1df4d8c642e469bce87021c8ec692ad95acfb56d1a346683528b3f763bbe3\",\"67321cefe09c78f2986e4212dcc834f4bbd7d83f570fa211ecb2eff68cb107f1\",\"af404aa8f08a68e63573ba851125b5806fb487366c8ee45476b9fd3d280bf8c8\",\"6aa08a9954167654df1640cdf83fa496d90e812b543af02b4e3cb42a4e68f7fa\",\"289f95772ab94355568effd3f513a308ec75976004cc4c20b02ac6d20c321b0d\",\"e528ab795adf63484b0a2581c873144c504b37f66a4d1558058ac629fe2a903e\",\"7f41b81fa62569e51acdc99f54febbe80a56819c1c0493ce4d2af03834f442b2\",\"88e7a0d5ef12ef82bd7ff70447633d0a169b0ad61943738fa8aa5305293b6ae1\",\"a4f1bac50756c044432767044bb0acca43420ed1448d4268f775c87d1c1d4356\",\"022193656268bd395cfb6b458c65e29245241e4f4daed19b81e79f8dec0a08c9\",\"f3377c01873d188da2bf7dd9530d4f86a8ef3a37fb49ee2467817703f8f1a4d6\",\"bdf2a5e4098da4d1951bfb2d55e97ee75b68f5a4ef3625c4f99825714b6412ee\",\"e759cc575ccfa79ac61d92110ccb8d3da0388a7b330da37d98fa02483766185f\",\"ba3f242dff147bbf0d732409aee9a9f320e173e730c7fb617bcbfc6fdaa109d1\",\"51865cd22551ada0a8ae245ce0f6d17a2fbd52fecf7e85d901ef838ed5cb4a70\",\"52f63c0d054606c3d1eff06f5e263275e8a51eede47361e06331b023155d0928\",\"44c21266098107ad6c8a2593949bb3848a8c75a83c8a8a4ca1f9aca9979a2c8d\",\"601f101c1e0d49db6946be7bab241ed91fe43ec2dd5a6e637bc4b57f5bbe98bc\",\"3d7b51ad432808c47f5aa41b5b28151abfbe3a51b6c603bfb78688b0a32e3ac5\",\"848f61f8f22f60faa3a8611791b5151cdab15120c0862eb54f00c782e11c9301\",\"b110e374018dfef43a43373c39eb7fe8fc0babdfbc048f20ee7483b18bfc43dd\",\"8d2e2fb7bf074c50c1ab0723cd7c429125b90cb597363fe7c455fab280bc39a4\",\"992923fac76fb07459e90e8e2acb2c422f700f312385f7f8fb4a99165bea26ad\",\"73c75db49acfc8893d7dd6aac44f0bccc3a7e4c96e5c07921d324eebb79ec15c\",\"97acbafbcc5cdcf848c85d1e09f8439fd2e4c1f489272eb985eb374f2dfe9e9c\",\"b47b5ba0b037f6ed14141eb54e487e93bcd5fa4707c3180c169e8bbb532d8e23\",\"7db6a551762e1cd2d90a591e97d8808ec5eaeb3656a1319869825a4c54167bc3\",\"52d8aab25f50b447b3b254ab224a01f89d5c248f45ce5801566a01cd5d0c438f\",\"353a18165f4069e439fc514590558ee33380ce9bb776ada6a2e1242199a656dc\",\"f4a18c6455f46d1bf502735969d1213909fc7a555451920d5567825e6fe720b8\",\"e9254d30110f70e79b878a384d6a238a1f263cd74e847c6f89d1d27889d3aa92\",\"72a7f556889c8988e252c9054ad21897fe525db8efcca7c7ddd38e8506488f3d\",\"f1d7ec2c68b5cb821b8588f208c45db55f8ec6092692e2be674328bea6c5ad1b\",\"48d690b6a3adb3ae74b25bdc8bbe0aa04cc9fef11c1af26a96b9897760d37cd1\",\"04d35b41bff658bb2fe2b90ecc925c8c5b5e6b9d8bc63be4b68aa372e1784613\",\"54f946072c0b3bd052192c10fa9592518bcb965015ebd60f55c291ec3bfbad5f\",\"cfde5ba5668249465d2e05fe7197fef4f2d0bc4814e4cf0630292c4c67ce570c\",\"5acb16903f5a6e33544c874038bc2532d55a48c103688fc289a8d456b1ac3ec0\",\"6255cab83ac3cb51d0aa18aa83b41db2e6317ea3703e59f213f4621057afbe39\",\"1358e6e4e054764448174e90d73308bbe32e6bbfe90cb7dcb8d5d1da786edc6b\",\"ba61030a2d3a6588339271de944a38d0a04f7261acdcfcfac458125c16cf4d6c\",\"925ea45f7265e29b56b2ade67a3df9b5765ee9b38d36a3ff1ea13267284b7c3b\",\"701d5cd7a95c71fec46bac00980395c169a239b05ccb3f4be7101ac800666d02\",\"c087f8f3bc52dfaa04cc4fd016abcf5637461ccfda61ed64c19ffdafbd6a2894\",\"50ef255f4d5a92e637a3513ff04c52b9b1f02a0a58bd1e6446f67d0bc6b5e019\",\"8bcfaf2ed4df56bbbb6a6c2716ff8fbb1b5591e23d3ea789332141c694ca4b5d\",\"8ab935000838defa900c5a555cb6c2505e180a37289452790bfbd4b4c0ef585d\",\"bc4da24d784260d83f3fad1088f975db95e1e0425ae229f1543b5cf3397c6a46\",\"9877b361d93fc7c5d8833b1911b6a356d08a327cb99720726429f879c31b776c\",\"52ebffd7fb1288385c1f495111fb2cd8023ff514a6d284bbd9b1b6dc5ee02b73\",\"124e82d3e632fbd827bd7539b3253a418ad4a393600c19ab61e5864beace59eb\",\"1fe00105380b0d1d7822b8d1cc7b8b555e83e37e51c427d356f968867cec51be\",\"cfa4a72da471cff2f1e5f42582c2dc6ca6712f3c88876628a949976fb7159f2f\",\"1f6f1b4b0bc73e4f43b476f85123b70a37868054316a77792b17f56530923075\",\"331c8df816ecea5d251e090756abcb954f40078c169334019f50f87f6f6451af\",\"e1618d5d294863df937a4940da6a02494e0279bc540f9518ccf576321f6a4dfb\",\"118d21cc13a3e5dac728d7fed81e7abf6cf58a501ee1579bc34de00d214331ee\",\"124727380b9ae7a2e047d087503339e8cc2afdfd913a3adffab9dcde63c846c2\",\"f63bebf2c824bca37e1ed950e520b993bc416c93aea1fbda86a4490da54e25c5\",\"3fe18771d5e5b6277fb2994a1683851280c75242da615ea7281f9017398deb95\",\"56f5a6229e1ca1c43c7f7f2d661b42e26f586c06c7669d1a715f31c256e160ce\",\"9e817907e42a2c44160af92b414096874fdfc6c9879c6669226a7cdf31e71990\",\"753018bfd40abaa4e0f3f60028031509f52ad94a6d64e41842f9e649ca6a03ce\",\"4803aaf66df2a62b208edd17c746fa37f2b160b3509bd421a249910fd25ac84c\",\"c64038242a8855d1ec16dbeee67562070e6d7fdf999741204c03d0cc96eda047\",\"ce74cf31c81cf9ce3a749ed77e31b87000212cf9a74d61c8c7a483b5f906e5c1\",\"be5960e0841166cd9c814b08daf47a6479d34bf8d0570013f3e5f20d7cdfc189\",\"c078ce090b424bf2cecfda8d10e70700649db455255633be5810bb9833867143\",\"7c40e97f3ee0821eed04197fb708f7f7cbb5c9c6a42e41cda28bd419c98784ad\",\"39248bcc9aff93284af16ec3397dfb4f8acf1aa461f1c9e85de68fb154679aa6\",\"8e5196b9f3d72ec60114dac32e3b52973f616972e17f151d5e6027d6f4552dca\",\"3e12f93b0c01c5a6552127abfa6da85439eb1b8c7d965428d3b97f31670b759e\",\"d1c400e2810808824085b3fbe3cbea6c93b928a9516482e452196f5aae97bb25\",\"2954ae35b816246447a90beb1f2aae469fe33a70e7e59bc9ea7d595890fd0b62\",\"3ed4ff515af95529ac80bdcebb66a1dfc28d69c4fbd3d1773127638c479aade9\",\"69c21d1d551c981955c91d3bfd39ca855bf2e038b924b4e6ffc2e81ee7ac5510\",\"6633f3c1e4ac4185400a6c6b755413a255dba57193e62f0865d78af30e4b2d4d\",\"d3f68206308f54e31a0556bbbe5fbec36354e985f5bc4359bd541e102c234e02\",\"04d7e3144cc4d45b71d895287def3870f9355f3aadd94624f08bba3c91957f06\",\"8cf10c07f283115df4adab833c5746709c543f234b20c96459527e19e9b0c937\",\"b3f9aade5e3a18cae1ce07817908d6ca7c40f995971b1047811bc888b1e051f7\",\"1f3010e1ba64329db1a6275b41a8121bf77f43b37fa5f382cdf5e122640fdada\",\"072d6806d5b598ce11825aee0124badf70fe4d129f54bd7168c3fb2cd68cf77c\",\"6c1648f55391937b68b80e0f07a937855ad6bc1777e23e017f9df5eee1ca8d14\",\"0a8462d7c32a97fde8bcd987e3100e7bee8d11f4a6dc5cd392511d14b4a734eb\",\"07c247c1dc22f93f806137e13c1f8acd86f32418ff2810270b020cf1b6662051\",\"8c431a9c9dbae9b2c0624dacfb97b2c24e591c8547bb381d094b8a6590f83e23\",\"b64ad8f4c7241358949a4d6bed386a0d6e23e5567eac047469d7ad879223284b\",\"cdaae21fe7438a858e3bc7266dfeb8af05518c8a48b4710cccfc7d4282a3d90f\",\"03059b88bb43454ce1ed35c8d3b1675dfaf8665c9cfb47979a20b297cf7745a6\",\"5c54c68db9dea3f9b52c77f3562a416bbbb0b12115d9e048651281acffa4185b\",\"0a58c81737f4cc62dbaf8011c079f193879024714b8c6037eb33e7d4553f8594\",\"578569104b8cf50e99976de596a25a7826cb8f7fb2a4b844d94fca75a9c433af\",\"83aeec86c0edaf54efd8cd7566621d6dd21adbc241d5a174c164b25047408871\",\"0ede1a940009733ff50a4aefebee92c7cffb102fda2daba10bf1698969e422c1\",\"010d46bf559d40d64739c1438eb4e02f292358a2b92a7aa1e32857de295e5b4e\",\"1f181cd9960e8dedeb84f684ac7efbe8c2ba9f94de65960ccf432c7e10970e4d\",\"359662df582996044f3d44921b8c443e5cd6e9251481b96f4795ced72f142845\",\"10d632c61716139ac79e2d5fe23ecbba586b57bc533a8a40deae1b9a8753d235\",\"a247602fbc5df4e6fcb96f3bd2fe40f70c0c51cf0e580440fc81f741c601d394\",\"893b027440760c31e0b695662be8d57787a2a306d04239ca4145b9d442c51182\",\"406cf615161bbcc61a4bfdd666133cd488bf8b26be8212e7906af7afd1553656\",\"33b0cdca91e59285f932a35b3b15fde9503f9be3b87916dec61703c8fb36422b\",\"7a89c8d458831f91d660c6daadfd893de23efc95e9bbd20092c61c70d5ea7cd8\",\"3a8644cb84334378c192376a4f95361883f7e3a11219135122ba31c44dead8ca\",\"4f99c927f9d3b1fa75da5d0966a2f92013ae05ff7c1bcf20d7a15925e762c8ec\",\"21e05377288d73b8ded78db2e3235d13dd1c0f1908af7d73c6efb5c42d512708\",\"cd5acc5c133574d9d5e7e31ffff0c80d5ad77f78c8ba905329dc817a80081af8\",\"3d5bc572b3468aa6b1ebe5cdd5dfb144e2277b324c29f685bc640c4dca2d9383\",\"673185413b5ddaf0765f631b53a48f74651c1e6fcd09f02c869bb99862e171ee\",\"2f3c82f3418bc4c321d23793b76fd814aa1507f08b9b76843d15fc89ba0ee574\",\"3e82f4dec70d21f98b67d43443d47215a9246220686c3f089cb6085323d340a9\",\"0133a73e6fa930e1ca3251f3faca6a2e66e36b6b0f62d32da6af283529343576\",\"5ac13c610eea4d0a5ffdd4966d4488edbefca65d401aa2e291d70ba901aa508e\",\"1b637e1ab21c465b70936d4ffeff49f449435219808fc6987cc682f0ebf032c4\",\"89cb393a48faa3c9fa6315cec2dc7476ca415b61795a6e671db4259329241a25\",\"82a83e89a098b35f4d667c92746f543743087730a622265caf3aefaaf77962b3\",\"404474624067c2f486cdb2525121f72d0309055fa5206458059cacbecd0ec0d5\",\"6a7938515b357fddcb435bd7f3785b830db0752d43793af7eea88ac6699f4ab9\",\"85aca2d40f176316041fb1028fcffb851c639e4e8a7535b133cdad205390e092\",\"e553565d8d431bf616b2b0f65cf4df1c21293c3a1578b51113fdf34f78652658\",\"d5e43210c22707088d9442179afdc1c452d668d44fd0512408bb02c1c6001768\",\"04cce2f01c36e49c8eae14d2f362f6582a0dbf9ff90e7ff8450f863f13b9061b\",\"810b1666c94c7b87a5b38683f1ffe310796bc4737da218498cbf3d15cd630cce\",\"1247f585882c40d36dbcb97ec7b3de68400585adb37b191ed5714d492bca9bb7\",\"21257dd6c72044908e2ad6fd9423f4514f344c6dc8ba3087b4c72e5ae40fa4f6\",\"b264a1a11ad2b79a74e9498800a08029f96c709d8821753e8208445e9f4a47e3\",\"122fbc03db7bdd4a8c2908ad210c56562c46388c565caebde66ae1a9feac78ea\",\"30300e262958ee9ba9b1830cc7481364e9cc05973209ae523bbfcef098e55b69\",\"e9e846c28b3e6a196beed7319f105299b7dd3b1c7d6124a2d4c83929d09552b7\",\"12e4f6c68afb927005344ceb6153f5776e7c8b29c30c217b7f500c16a005a400\",\"51cd2b65d9d33facd072cbac93d965ec2a1b442061573dff544d303195b9c3dc\",\"e117e07c668978d33a7a36383505e66deb88279c865ad1dc6ce9c68ff3d63e06\",\"a5c1b5d4ef682a8b65f9fbcaa6b1ebe862517a7468ef3f02846aba785b6a7d17\",\"02e69378d7e8c5d2077876168fe73549a0dba1eeaf3a45bec5ed25e84b8162f2\",\"834077f8fc97453893eaf3920ff0a726d4a7ef05f4811ab47ad6dfaf0df24893\",\"358ba9974e18fecdb3f17167bdc65e9cc1366acf033310bccf0bddcb93a44c77\",\"0fcc86fd39e14aa9c3240168743adcc8132762e5d223f7adf33df3e23902b614\",\"147b10fb32f2118b0c42a5ad1fa504cbf0b65c40a3be7183b51767bec3f79c1b\",\"743f2d35d27247687e5c761500012ae465183d8264f8831dae4fb122e6a8824c\",\"f242d7cbaa68075a632a3cf0e3e59c89df6054475e17988182957b1226895e5b\",\"fdf6524509b96f9fdcd538fe8fb98bee7f1629b3426e5a4f5a44a1509d548401\",\"3f2012ea7546c62c30b99f3c681d53aa38f5f88325fb515292e5bb03749f1f0c\",\"6459327af1acc8fcda27c88afe30fd20cb2a9644ddeb83855148601c685c5f6a\",\"3c456c5c823298df2652fa895fc33a7d84311f0a71fec328b58a1340d447d716\",\"95a837b45037d86fa0bfef9488493ebcd629012957f048c472c7c9ff201752ca\",\"1aefe6eb40e1ae264efa1dc47b232fca455f7fd10c00fa50f58ae988b5648163\",\"47d89d3b2b7fd9deabdeea7bdab556e6c7ec11366cc3d050ffa65ef22dc4f736\",\"c6261fc6eac757a22b676c2d614571af6e98226a6d7355683182c662d446010e\",\"e67e00053638c58bcd2b30b37ceb26fc4e628322a4a66f573d3f00fbcc085d7e\",\"7d65071788201901b5f9958b4987c94fe2229cbec54bf7a6ca653743bcfa0cb2\",\"6c37c39399e326e53af11c972b5943b775b6912a8b146452381d5a0402ad18f1\",\"9d19c486541ea6b77dbb06ac11b5e4b0de16bb6784ae3eab4d269e4d51ba9de5\",\"de00ec1907a7f74f07194c214a283ffd31d6fe0aad065c9b45c06bb5ad8c7a1c\",\"efff85558d0ba2c644a4d65ad2c965b94f6c7b820abf75fc881c000cfadf3f40\",\"993910673b233f1f45a8dda9895f4644135e7054ef6b31aad86cd267228c2dc3\",\"76a6d5065f3d349027acc8685572d4dfa74b8211392935dac4a5e51be6ce06f6\",\"96be1c331f47eb25e557dd06d551ff0b5870fb267e4db3fa89fdbf9597cc21ae\",\"d73a6d5ee1e2f0d52e2bef33630c352e15bf73313dade7e0b70baac842f37c9c\",\"efc58db6dbd51c64f6a65513734d0cc278ffa04062acbac18c999fb0ad68c65b\",\"79a8a8628aa62b790108972455b38b26d7bc42dca5925274fd5fa3790e81e6ac\",\"ea3aee4b6cc53837d0bc9eb22be83e3b219d4ddd51feebc322cb04a445eeffa3\",\"8e97c759cca7b26eb11e12b8bbb667b50b7dd756f984ae62a1a2b468673d6d37\",\"267525de67737a38f0634658635042dc4910dbc31837b939b7e4e1c6aac4752d\",\"7e128da53c1c96388d8cc41b8273c1fede242a8b00b4c24c548785b7b3072b47\",\"f662cae0812de051df9d2aace0b389c1009ab12e531aaab48d70470e28964062\",\"d5d07d552f8f79286c5f8c69c0b27ff1c52e4c15d3580f5909387c0602e9a3e8\",\"890e2cf3483d5be8e6df02d3ca85700a7ada9ca89fea057fb4edbc6dae306148\",\"b7ac3375e92fe0d7bb7b25088a7c8958cefdbeb5e10937f7d684f0be42f7bdd1\",\"daaa2902778bd36471f74cb0e83b60e9b3679b7e653c41c5e1f0940f79c6d3e7\",\"9f7d639aab1f38d35a8b508d2cb2883cc023f0f17f6ae2003498f40929463c7f\",\"27b6f70a47a5093dfddeb51781a74bfe92e4bcd2ab864b69d2450c280aa3cfd0\",\"5c9702e5be89322da2113152376696d1328f3018278b966c1e283d91a656c2e1\",\"a767df799bb1a0ca286d9316da7c38d49e516f319b6b4c4b04032e6ea18b54aa\",\"a4068b9a2875c48af0a37e5663165a638ba936c2f46930068739783f615fe758\",\"9975e20907547f96b9691e530afbbfd4157c7536d44e9465936d45cb0b4749ac\",\"b5390258257feb8d643440b78635cd2b96f25e9205d572184a64c3ee1ef61222\",\"c833c5181420a62822abc554d98214083479bc3517e00f4590c7afffe6eafbfe\",\"b0ac11749387b489b372e2d9451484ef7b2ede5863ae5133b521a24cf939d89f\",\"835bf2c49ee9f942ed226e26646c8b3489027787689339608dbd44c8966454b5\",\"83ba5ed6964413e2b4abec6bfba8c174f10f87837522cda6a9f4dca97fe69da1\",\"95c9a4669b5ec76e7e3167e6c1d84bdfd78aabfeda24c7f5623b781f1d9682cb\",\"3df35af08858a79223c942f6dc8dd64e147b790f5a73eee8cff186d6e7087785\",\"eda6983953955d5e3a0c4c287c5bb4f92f959680e4d88e9c8331a2c5e68d7a23\",\"37e01273647383242ca55c164f821168baa30234eb6b924a27005cdbc6922fe4\",\"0acd079996aabad46584dd4c8e414120af3ca2d367b12fd6d60e9e240cd986c3\",\"00972e6f058b58c8902e9d40d50739fa32a17e8913eb06ca7e36edc818e640cb\",\"a675ce493b9da0d44b9f74efd24d6184489b177f914eb6e15518ee1526bab859\",\"5e9a6d1553ac5c26797484e5068fc45826daa0c1198c5e9048e5f28f46d7c761\",\"7b97d1516e324d7f4ea64508a31d7648d2a9983dc38c97bb064d17f824be7fe1\",\"370c57d70b6ec88d4d21c26deb9eb2ef19a46d602bf16e962d3ab40e97a3f922\",\"bc15b750587587436965baf5385213321457bbc43b19f53ee4e4d845a39540f5\",\"66124058ffc740789138034dfca52bdc5f44f6f59c4eb9de66535e4a9d115352\",\"25d49cf1153c2fa09ecdd07ca679acfaa85f97142fdba0fc24950dea7f743116\",\"f01d878a4f92e69244a65ee01cb6d2cfccd20be64829121d36b6cdf7cdb5179d\",\"99c5707824b833eecd386d5d259b07d32cd9ee6715a9c11565b71741c2468227\",\"9da361f1ad360cdd566a13bd88f7467f3c20ce3484af274387ead05e93cf5162\",\"a355d9bdb24fc6abfd22c2aed6f38a7814eb346ffbd2f8e7350cd77ccb88186b\",\"2e50a18baf0c7db6d860cb28665f136f2ea3e6a0f0e5501df18e428c47c652e3\",\"cea59bbf52e3831bc545b2e13fa730faa45443a231d3f8a412be92cdd00dd590\",\"798f716abe1094ff5fe15738e7fedfb3cdaebe4397a699640ff11738833d17e1\",\"11d16b2536d1bdbde3515ce39dea56a4f9f82c97a73e57680c7798799a249992\",\"2c99797693a53f2e7aa85342b5f7db7081d868ce7167da5784f2d72e709bc76e\",\"ea6b1bb8a15b24ee88d9ab75f33bc34c3b563de5a56b1f654773795d33a0581e\",\"18a36aaf2af168d0701191b34ba2f973a8f0ea297494d096ddad767101952b20\",\"c654fe27892daab8c77b236a9c8fadb97b083042f6632699250323549f32fce7\",\"024d7310d5b219d5c462ecaf9a8d7194e86f7b9c00fcba111eab92ef46d1f8fe\",\"38bf7150972d0d7e6c06b28fbfca5ae16f4e149b1d36df7515614b4cb5df9a49\",\"a3bcf55ba562565de15d79be685f24cdd2219f217f70d659754614664101f4fd\",\"44b8349950156ee2b63fbce801a00015f3f1d065d7551a0ebe4ef0d141915dbd\",\"6a94f467487fc01acb60b531af43c0fdd4b98e2df431fe7bc1a37db7c0da4207\",\"0fd6537bb177f8e770a94aeecca8862b3b929928971f60ef674ea66c0449b4d8\",\"bf4f0187cb92baeb96f46080a0ac2dc043145183c28e998498d07ebf858b0bd2\",\"decc00c8060a967d226e457d82f2de2a786c95ef81c069f213c522a9d09c86db\",\"1099b36b5d19e02cd3520cbb5616045eeba56f763d77ef670025d913ed761bbb\",\"bb00616fc7faeddf8813fa0b8a3df562642e2d45d36b917c3cd1f7c1d92c8d7e\",\"d325bc23a06351936a17ac181bb7876f57306bf0e2a58cb5a12a57b8b2a65c91\",\"94569191736ff5a60eb81835a0e4bdaf144e83fff241665097d938b489faaa1f\",\"05aa5a9dd383fdec1bff6935b9f16125986d144bb633378808eeed5d64aa33cd\",\"6cf7d17d48601bf96736507a23b4108b597adeca3ddbc96bb866e5f3518b401e\",\"e509f8b3511276ea2689123c87f6d93ae6e13e04ea17a104b56709d1fe75b37f\",\"94319a53a1117c67888eba985a524e1e2d4e3d5eaeb6bcd590ca9876ffd48808\",\"c9ae53650f1f06f724e617e750565064b1f299c7cfdd82306ae71c76fa75a014\",\"fba04708da864dc086b0b841410ee045c10310b91719b077218c1652602c8f80\",\"e902a9e5564ef5cc4ae0c7b525bc32734b6230dfdda0d8ad5ba0c4cd83b32f5f\",\"60d3209a5caec1ee49a924222aad8871d18ef0f5e512a24807c4509fb4324472\",\"b543a7e3230192ddfc294aefd8ee948dc86336585efada141c15ae4ab81bc2a4\",\"f3d8fc59eae71bfa86cfebcdb96c50722bc9ca48eb82e49e26f7beb425c0ca70\",\"9858aa874363b24bf6bf508ef19e8d81c1d0a27c0d260daba7ab2fa74d0b7d8c\",\"736f7a0f558290337ce44ed48a25ff64c816d81a47b43f25457a24e9eaa8bf06\",\"d3ff41466ebd626d4aaf372356618f743377929ca54d26342bbc6ab51ddae333\",\"a1348cb4a56a7ac047e0e3d7c45a558144d4eeabb5c425efbdf1490d09589014\",\"f84bb00bd12c04856d72230d1599cafab7410c280185069699f0f8dd1cee2efb\",\"6a95e69b89d637d0a4768391d6e63b75dadc04a843402bddcac205efbf6aeaa9\",\"430895d1924d53f005407a9ab1bc262a3e8dfc0067511cbb87acad99e3f24024\",\"4f0ef8292cfc27040f6f26bd9aec5583a2bf5d79fbfc964e9e37868f5deca26b\",\"a7cade038a95106d7cfabf42d6d6abee3f0d696c17d73139d34f553a7b5164af\",\"9e991180795dc07c06e32d89a15057823e22022ec62b7183199d58e8941f001a\",\"501b1f41e89d210fe438681575515d9a913f30575cb10362090abff80d763893\",\"22cfb8ba24a095f31deb48a2cd128ba2c3f65161eeedfd31e8e1eb7e7786b8c7\",\"d83a4061b60ffc792cd6b433ed22604f004d75ed21f27dde38b3f8bbf7b574eb\",\"70a66ffc07d5ceb8606f1e8eb83fafbdef7d09d4e0fd78b2537e1003d518198a\",\"d8cb1f560b14630ecc7540b626c8efe0ff95228be06fe597acc31fdf2a622f89\",\"8ab24bcdec136661ac7e43e2bdfa97bc2ae8eba04b505871dc5b78c329c5a5f1\",\"726c454d7e1b9f290a627636719a7b8220143ec5294582e4a69a3dd2ba43aaed\",\"0377d4f9f2c7dc61f2bd012feaf62efda12012bfb3aab79cea980cf766c6e674\",\"6f06e727b41936cefc745e6f7d18d97da64b310981e843489e0fb158c200816b\",\"f5dcd117b63858164dd622ca213804e343b9daee6e1b4b07b9f54b6d7c1112b1\",\"938163950d327a70cfeee5d4fa45dc5b4f645fe75c72a63ec6301379d7ec2e32\",\"42465704d910bb624f634a743bd0e1d0ff31f77199dcddd74a82e988f0691e3d\",\"b560762486183ffc33aafc1819e27375bb1310a00f41b22301ff5d252181cb37\",\"d24d962109f105dd53bbcb63cd8d9231f154c534ff3882a5c48968a9ebda1e69\",\"12a8e1a36e94af9faf58e9c17db42561b145f9c705485c85aaae0a82139db987\",\"7f5e9d1feba9672fc92687d8994209fd74744b271b6737221b64bc919a25b325\",\"9c470fa636c83406bb79796c6ca3bc46258ea89efd3ab243cf4c0724d8497b4b\",\"75f293693112d2880b23df8fb32b7e061fca765048756f497e37384cfe197b41\",\"19193a50af2dcc2e25f2e097760a3220c5af3c16222d4327745da08c2b26ef5c\",\"22a948fb8937d8517c344711c5900f09882db42f8d7e328399291e820e1c6308\",\"1c3175737c0d93ea1ba4f474f2adcb6d4fd3393e15e35c1c29486b6eaca36151\",\"9e079a293c12388e2df22f5dfde5d2da74c74cfcf182f85fc180ba70b86128c5\",\"8f4c49e7344f15c315685a71414b7b5e4622f2f26d0044757e009fcb3f541e62\",\"ed46b4ce6a0f5e98a4b8b95e20c2d611e77b11e9a336dcb30f3adaa04110c04a\",\"93817ee1ca421241b26c0aed2cd1a4d74d909c67c427d3f868ea7c9d41ee32e0\",\"86930737707b3ae551183681acdf7ca9983f7ffe70b20a30188837ad94a8227c\",\"351a1fae45ca7b82cfa2f6ecbea5ee1227737109bc084856b75349ca3350610f\",\"26bd30ba838c22d1580f4272c03c56078c292f72560517dba00b7d57deb9c587\",\"f05b5efddef850cc2489a7610866ba4e0d5dbc03edcb203589cd818660630932\",\"c5089355e35f97d84062cd595b89e69ab14effde55f3ec99f227681fafce8fd9\",\"a4a021530589545095c9e0b4d391b2c85d716ed0db0d52799356ee9f168f1748\",\"c16fbb5e5227b9140a2cfab35efcc71aa83eb1467d221b935fb5b85e29c49b2c\",\"14f717f2abcdd43fa9b5777d2bd66b2ab5cd228d4c2990b785da74f0382ded37\",\"47f4f95a80aee54866efab0ca58c5c5686faccd655dac2b0520bb4a626b521d9\",\"de18ca197c2068b78061533ac0df3a0014b66b38b935e25acf19e3ce5f7d96b0\",\"760d5ee1d549fbed17712127447400f1eebaf35caffe3dbaebe2a5c28f32e293\",\"68b1beae00fb937f2628751490ce66bc3b7295e3168e7324bd77ba50fc35c792\",\"9c81f66352e978a9c206b0ae6419713c6cc5b6eed45023e4e8bb34bb4fc52683\",\"ba07e49c20e8fe2fbaa3ab52b0ce037e7fd2964d8baf6d9ab05ba3e5554bebab\",\"0210132c15863358430486b58ef6b7d903dd1feddf337c006e597b1a2da8852d\",\"67fab048b089e0131f3cf01b14b0f1d277da57c0550907c7a28ccdde52f9c1d8\",\"fd49dac7c8cfc549d5bb1d4f9974e4ad22751d8edca9b3699d0c00988e936417\",\"a8724376982b81f294ab2655a5d2bec716a173dfc1fed6e30869f24984a08d18\",\"3ddee7df864c60ac0b8b444759e32f6d6f0fd5f8a91b0c669f011b078cf54514\",\"f35aa11f80bbd66c83c964ff8707e389191adf0e2e51aa17ab0752e15ccbae75\",\"3a5b14ab10bbdf564a050660e938c7699fe122836d1a921a285666fe765ebca7\",\"208fb17d9c874c4dd84746e8dbe138ca0fb01a9293aa3435ea1d492340a7caef\",\"ac0bd43db7d0f29e49afe4f0b73802d17eedef5028b2e6ea6891bb325cd974e0\",\"5cc7e2dd29d20d337fa9df88f20cc8fd3e45f943f91828afb53281339b41e6cd\",\"92baef7c984775661fcfde99718a275a6c16cd3a526955d7dc184b38c90289b5\",\"51a7d5837c94510dd87ba352bdedfb72e9ccd1d0b115026d0d775ca8d1b45551\",\"d57cce758e379b1318ace0a49f880260a06dcb7a014646a159318117f672d816\",\"675e39f4976adc23e859c20aaf1b4102f5443017ef0acfff8ff4ad57e0023aad\",\"3cde780521951d67c5932fee5d3bd2305d74a961b6a7d322e7406c9531e46841\",\"1ec03d2161df06a24747742064da080069340339ee284009fcea3eec45ff0c29\",\"3379f1a2689e8022fd7f44dab0c848ec5ee872c9816e48c36d345cd9b3b32038\",\"33e84e3ea9da92516f65d1fd083e2945c1c4e4d30c37b6fecfea4d61e212175a\",\"8aa7b9d9d7bd2aa50d5d7845414cca1e9ee3cd3a4a0a7c69c8d94573c07d49ca\",\"b1690ef7de6fa017f9d619db9abcf73aa5deeadea3863a96db11adc079915d34\",\"49969ae28015243279ee3239b99a1ffedf51ba9dda38ebe7dc21adb42f8bba4b\",\"20c76294d16ad2b513ea954b338a2322fb0bf13b7377590e6864fc33150557a6\",\"6310f7ea2f076973b9a42fbcad8d6ac6db9bb857126d980d48572ba4f50443ee\",\"e9ccb0bd8f0df48510d8d325e9783428a7001086ed5df4bae9f2f99bad802680\",\"8a5683084935e4886f8b38f86a5b3c8e15b87cf31e87d66ec109980e315a206f\",\"2d42992f13fbca4784f1a097e0474cb6462290594ede76a07f490052de66e17e\",\"682e363f1c4934ce87f807c53d80531cb6dc94a03a12056d6f119ab2b19cd1ef\",\"ef9d7617a564247efeb1ee426377ba9fa39926002aacd57ff138f9f3596b9da4\",\"526ff3e80193aa35750c4e511c88437400168b04feaeb360eb4e2c88020df08f\",\"cfd12d5f3596a980ba0678d95fbe9faffc7885b1b0ffe843826157b5116468f5\",\"d3803dd1feb4c23d7b8a0846b0c360e935954366a73d4c049b23faf3578d727e\",\"b73f488f962de84ad52c195af30c32449247eb9ed6b4fc99bf3894bb5562eb01\",\"bb86c54c77bfc78c4edfee1963e8a2347032452760f03083b712dca2868a1c8e\",\"cbffb00a1ea9adb2997862bfbdb388e26a2d0b566482a010f245441ea8a2080b\",\"1da43e8a159f01cc5c73dcbb85e0ef7f530796b427e089b2c0dd5711223d9759\",\"38313866135e04e04df48b5783dcc9af033910ad0f0b7064f10480be4652e034\",\"1db2c87f9150d4cc7210a488173926c707cad9760af5e128dee5547dc2b45079\",\"d5c1c160e9f2dfe66627ef260f762cd96bf55c734c6d9b2947439333eba86a0a\",\"af4a8c17fe1c1844680b885c4fccc280132c7d3063000e182208b0beaac76994\",\"f0f2b4c799073c147d511ae60e37b62284407f219e03841f701469944dbec6c7\",\"6cbb4b32245938f7c64fb81a4b0dd71fe9bbbfce522e167e69f16b825b39c5a1\",\"a040fe57fdcac9f77b6e17416375e32d402648ede337359380a7cd824a98e099\",\"894ca59f62dc6d2ff26e3b422e98845fed8b2a76c400df48e7fea624e16dab04\",\"920da78eed55451dbebad8191466b2dfd5a4d846fee2981de1b33d1b5f407431\",\"55cbdd9874ce04904450b0bc2007aa5d977f78756f88d92c698e732e9ac91612\",\"53ca5bb55f157df858e750c4fa4da5495fea3adf31491725c9d39801197a39e4\",\"f128c14c95931953f2d451cc7038de91fee7f0f4aefdab5b984c693df17ec083\",\"beca14161c51af8764161242c005662c7389baaee9b846264554623f19039b8f\",\"b75f93b54dedb664e32e5bf653a1935122eda208677e28eb906c68cdcffd43c8\",\"c571b92f846e685551bab24098b3295061a9a950b26488430a9078b341d6ca31\",\"0f959791c4be52e5a18d6a220b14e9b2294e67173eee831346a9f99b46ae9e5f\",\"6cc0e8b3f26687ce87fb7fb6e8b1063dc00e3cdf749107ef28a464b2884fec6a\",\"47c16f812c6581ff2c7da2a931d8d74c12ab4f40ceaf9502fdf27ae98febe4d2\",\"9272918fd8dbd253088a0567f7ab0930477373192107e2299a0ec3a39045f903\",\"fe5b779d0a286b79e681c5c7bd5c6becb1cb5f88378a39946bf52b0b83e0d962\",\"a6762e285c368e94d4a74e3fb72712e989b1a67a816fe97553bd1d49c81bf1ad\",\"e039b76f878408b44624221ad42b495f3aac4678fead37fdbc0da8af66ffc5dc\",\"f48732728961389097840acfb3f9da8de4d7a4e8d76b85c050a3c7c3cd8c0591\",\"affa899b33a35cf9feaa4ace4573906d21a150ad28bfc1fe98f0981a8a707981\",\"004dc87210b60c49727616f16054619eef54787d2e9ee37ee0ee479b8b50cde1\",\"52210a31e7899b2a6c1e26732f3dcc25edf49a64bd662b62d415749cc8955d77\",\"6202f97c48a6751a3abe28968d2405eef2f34ca43cd107da37d0106b16dd5757\",\"9dbe0e25f16bde5c69a4c0e86b50eabde333aaf0153d295daed3541910c1649e\",\"82c5552ee7ae33035ce2d0786a0a05c0a9b211a51d5daf919b0f9d46a8be12a8\",\"4a1355d8fe334afa7688c7293df76473517ffcc8fc65cfd06e16f1a4cb9f7cbe\",\"2d0a3029685bfa18b29457acaa6ff7a9ede421a2f3e848ae4e74b09cfa21c10d\",\"c36e33e5e10c9f3b1121a1c874016d937bb0cac77b2e29b9c388613ad6ce5484\",\"2e6a44f8a9d678820f1706d567eae711e8d4a276c05f34bac87c37b8fb20dc29\",\"2abcc5c16a1a7ca98469d6038815a08e22070d44ca90d6f0a57f8d12664dc575\",\"a378e1b9107b542b3f9d6fb98f0bb94cb2b99848f80837a7b6acef6256d25edc\",\"d794f5072e18a0a93093462bfac394e87fe91096d191fc20a7086ad3357a67d3\",\"f3a0588f013dc3d3d2b376d98c91797081be53a701576d5ab02f1b61a394aa5c\",\"d4a69f3885b8813231c6fba7fa2d2bec9e2704097b1aee176b7802e62566e756\",\"320f797e1868652c84e20b418c6e3bfe229684c9226c92065b2889b446468f69\",\"88363dd640bc210baf20b0eb46aff869e3e6524598c3b675999436a1f3017f85\",\"fbb6c7c306d0fd4e4f3ad9c535ec8b30a3b9906ef56b018f87d0add779c6af6f\",\"c7976db22ede9c9f3f92b252673254ed6001a503560c9b2e14a7139339351966\",\"97476aec6bf6f6745333cc12ca34ad0021126fff215ec0a1e831c3a4b2e61c79\",\"d7c81e90eb3ac496e717226dc08a0ed703af434c96116d2ff00bd001a62f174b\",\"6313bad696872bac6c0a5b1c02935788032526ee9955be558093efea9e75b28f\",\"6866e5980f1c37112715ba4aee1656f575a7298c5760fe857d83674b2e95dbd3\",\"4c3c00f4fbb7fc9537e8034dc0da53313aa784b79bcf7d92d84e38686b233e95\",\"e534b9e74e7f83a915ed516eef03d5b6159632a2c79d3ad68fbf3c5867fe5e9a\",\"322beee2dcf47caa30da11fc967719eec02cde3a79eb137caaad30983466ad53\",\"261891917acbbb4ff3e4fa581d76a6fc444f469b55447ed2ffd5e66e086cac68\",\"d1cc2f623b736bd168a91df7f4ce5f79203ed0c4ae1660968d7688f2c7fa598d\",\"d0e760a8353d626e25353ec07ae62a9c2a58395cd90775614abd11230f56db63\",\"8f4e7096da7c8d7bc861284f3c075ce249ec7bf841010c89f4b62ea8adc10838\",\"360587d525b00460a4adcf69d759a8f439f5b4a48fb609a6bfefc45467fd21f9\",\"c3c792eb51cfa455edcb8181a82b8c2cc94dbd0c0bad35f35f8b9b9c4360ebaa\",\"4b9339108ef757f455b23aa62b0828f17edda30be932d32aaf75d3ffeaa15acd\",\"ab88ee55073e591ac12e1b6263ebb70d42edb5600ed5ab2d5ccc2fdcbb927247\",\"5e82b11018cbb648a63c0927424f96007892e97e4f66805b2a8423f306ac6ee0\",\"7e18682cf9b96d11a98d88aa2ca08ad6b0e79d59931de8e47a5abdcdc452c359\",\"b90c7aada04a71d558a3d499f6dc2005f1a85476178e361f9dc7f9b47219c390\",\"7ae0b65ec9ff80a1f92f0f2508c9ce6bbb12748ac394adaeeaea6fbca7b9a13e\",\"919a50c70bf73b851eb38fedaca144848955adce7ebfe3f112cfd1b055f46139\",\"f0f2b9316e095e301ecdf3daa8ff9197677475aee1bf96bda5392dae22de7d6a\",\"61932a25163e9301f424eb9cfc553c1dca1a628ff8616332c2a18930e14e23dc\",\"3f20eba9687d112d379775e743a662c38ace406fba68d62a6216eb483857fddd\",\"37e5d3850402b92e92d27c06e0931354169aad9bd89e32abc262a1ed2aef684e\",\"3b50a6cfe8aaf0b22194e731be2c2c607226d0a673cecbd4bb4897fce671175b\",\"090656fb5f9cd1aa3bd2392bab9c2820d3457f37bf9bdc73a807b6d53e57fef6\",\"65df14eac629ec25b12b06c05554b4998b03a263d5f6c0acc8daa9ed306c50ab\",\"2f79c0e716d325dea503b8b103c2f784fd492f9728edf86b12b6e3ba0128e4e0\",\"f8798d798aaa5cc49538511ea35d94312af63e27a308500539199533502dede2\",\"579b493b0f938e62d201e408cd1642e501450b0182735e5e9b58fc754224aa5e\",\"de7abdbce30795221839d1193557e39544dfaa394a48f1de46bbed06aaabe170\",\"c8160d5a73c65563ec91bb69a80f6b210beeb134dd2cbb6513194ba25307eef6\",\"f03ae17040fe9aaf416f304bfddc6ae28610e2cab16844a0f9dc6c3fdd8282c7\",\"9237716ade79000d2b65d4b08b35f62ecc0e7bd06344c522c70db33601dd1357\",\"f18cd7d7729487cf019a1dc5c2e883e013feb528f286a9ee0b6280f0588edd29\",\"8f1c314336a80614c8198568405de5b153a74a3490fdfa6f65763b4faca13807\",\"52eeac65af6bbdce85482143ee6bb7239189b2278ce1923f591b1c1b3c28bd02\",\"5ae45d733b2918ba23f03145d1792cd2c86dc524141b6bf066735b5abe162ba1\",\"9c74bebdd5f163d8a3c2ca411d7c97f495a5f6e2d473969adf44f5c82db4e592\",\"a439596089f5cf7f00c691438317223faaffae88f5bae909fd37a6b807c4b477\",\"b676cafb964def1508f2d6a4ee5e2e8ce371afa2079b9ab0457559642d22d455\",\"981a7b08e7609a133ae00a81eadca4cc62ec434703a0150b6c48fc1e2740aca7\",\"0f0e133bca2811ed64c1f300a22c291ca310534bfab4557b17da1e1ecd6b85c6\",\"4cfa4eedcc8aad5963b418bc657f04615c14fb28f12e2ec552aef46297366782\",\"1cc63d66fe2e55913aad2586d69d2678d55fba6a670d02b60cac0f9aab1a1fc2\",\"8fa3dd0a4ae1fcafbbd162b436d0e6c2a5915b16134ef961a414608b5fa1dc72\",\"c665dece8c89f3bbbc081a4c0434b46e7772335831a5385a1fcffc060141ce0d\",\"5d0c1fed9ed8a52367145e3efba6bfe3fde27372837f9ff1e922d8b3255b73eb\",\"0190df54feb2eae8128b442e8b7f5ff02941a7f2a97cb684a70fd74ea6446e92\",\"6f068eb45a43d6de3d66d8307ec71b1657504cec925c3d0c7215c8c9197880f2\",\"a0ad203e7e6d78ae46636139ccc53fdf6a578be9e465b07b3bd5cbb78bc689a3\",\"8d2f1299b0df99532b9d6ecb546013946e7147a12c0813bbd523d8e033b20a8a\",\"6f2c375dc49ceae31eb488ca846ecd4fe3d8363e018b7fe8679349ce51156870\",\"b266618a3d4235610aee209a93ff96e2d01eb1a35436321d30f1e558033fc14c\",\"98e862ecdc63a801309c45569da717bbe52e048aec028c3bf92d94ec017151d2\",\"1c453363e9e7f2bae846d914e289a0ba8658a27388e68ce5fdd69ca378282a20\",\"ee8623a721cc6c0a1a96ce3508dcc0cc9e6aed9e6d33e3078253737727a787de\",\"2ff79fb9d5884fefdc8a9d9cb5bb938393f14daa54add8f749bf25bbba173857\",\"970425d43c7cdc3c8468e0bf3bd1026755808c6931186b48c598ef3f5691ec63\",\"3002f4b58a5d5293a5f6be58fea51f5d3cbcdf53f8c0ec09d9c3ac8ed68d6150\",\"a5dbc80efff5cd31ad21d2ec01db8d66ee4a6fd54588cfbd6dc2327c10a5fd49\",\"aab37b7484893b465d1290d523c0d952c719a059bcd16b8d22b0fd3ff8f7d945\",\"df745b054456681bc73e264023baad427119a676d0cc3821bf07341d8b60e654\",\"541d84e4a41c6be514d0fd9a7a46a82428a8609c4d237ba97521ba1a8506ea2b\",\"9b58e35c54de943c41a70d98af9ed766bc85333d8244bec35c9105147ed1b20b\",\"f6b0c9b7cd3c697de03416a6394ec1d3d8a81d116e6eb0ee4079d6d71a30244a\",\"3a157b3a296caa95c16ca2bd4b1eab72f00b538295d8d8abc67d637cdaaff116\",\"bf5035464c97869c2b6cb1093659fe30eb34526222ce232566fa83cfc47a3ec8\",\"e622592a46c1adb60a93b812f0238a30f78e9841d39a6d729cbfe5dae359638b\",\"25e3656d61abb4d8d6b4fa2f08a8a56d5f053e34918ce314ca24185fea433061\",\"0a45d56d0da2b1425ed365d804bb32eadaec6d62f0e963cee5b6a1de198a714b\",\"56b360b7c017b107a86db1bda250d232cefe82cd42edb77309234455f8e74af8\",\"7943773e6e25943688c7c692980b90d7a46dfd34884eaaab9d4b774430c8d201\",\"2bed36660f2e6a31b44345ec2070af04bebd6b4020432a8c01ea1b3f11448c70\",\"d0e1ab2c1bb0c39654130c6f14ebf0fbc801b2b88026610a9f432069a070b5b3\",\"a486e24be67437601f23b5a37dab412f9bafe8cd24e7eae98e705b2c6cd14891\",\"a53444c61cf032d621fe90aa2a471709051d6efb84e9d3dd9a031b92b2c409f4\",\"ad9b70673f82338f26c947e39d85e507b3dfd31cce02f7d15238b874a1fdf23a\",\"f8011580ffaa72e9fb49e93b4a7bd97dc3edce3495fe91739560d5ec4e3487da\",\"a971662efc44dbb1c065cadbc7f5160ac3ff1dd2c82154ad84db133c04fe5936\",\"9b10e13e3dc8f8ba7cb32f4ffbc44636bf409e0f5cf5fc74e72553e3706e85ae\",\"fec45a491e30624c8565f33ab995b28c44245161d5402defc1f6d6b2b0ca0b37\",\"4e37a8848b032f61b311b5ecf5ab944de5d4ed5ce7863648d4fe6524d482872b\",\"c2e3f7214ddf9f8fd28311795f0690fe12666614718c9f10a406a32375b68109\",\"c94588956f0ba00682092eaf35a95fd41b503dacfd9632be3f15ffab1b456405\",\"c1e23247495814c2ff67ea410f7e2b924784cee82865fb42cf5300526978b7cf\",\"0ec60e3eb78cd8316c8f4661f292349261054a9343e91b0dae7ec7a69c39601a\",\"0b7898bc4f76598d4b330e4cdb87d23c4254d95eb78244d910571d1e73c2451b\",\"e966d6ab53ae067a31ab58f9fa19ce87eb629e2e236146db523b3bd2c0c74088\",\"be5e3250d3bf21fc689ccc3f81aeed92dbdf2f8cf172d1f44f69cbbdb35bb2fb\",\"cd5db2647226284082461eb357acd469be4d94f7946f48674ad0356b25f375ef\",\"4e6392ce77a41965b89141565fc97bc470a87f11c1fd86d7d5e0e725dbfc5447\",\"435f0b83fb16cf2cf1e2f9f70debb8f738da1fa0f8ac2acfc34a55a2c6ba11f7\",\"eee712c8909b148a99728f57e06478eb5044e9dcdb6b00f8d333b545756e398c\",\"56f48af972487b5197475aabe2572a1e085ba800dcf4cc0d1aa3924d2ce8560b\",\"469f52d229d5bae95fb5dbdfeda6931c8becf52058a588290e93bbd057a596fb\",\"6f252b0cb16248fb2f0f1de370001c94622757cccda512127721b7b604fd5714\",\"e22b792130fbd607016b447c988c666a27672f493a3ed54c0580abd1dc697658\",\"a9a7091dad1e2c5caeaf41b59a7f9ccec17888183a055b38ec3b5f63c6f76e9f\",\"3dd01e4e0e7b2bf25dcb99429c30be6d7c04173c3c28dad3c072b89469dd257a\",\"a1e66a1b177cf7a14a03fbda51f33da9d0bb2b20a8ba6092125ee6145435020c\",\"cfd6bc15f983d0cae6212f735f99ec23e7549dbd5ae8b1da98edb405db7ee727\",\"06662af572653ec16ee09478067fc6990722dee5fa506a9febef47d66e915798\",\"5255d4037ea2d03a46218933e03c15c21c932b0da57630eb7f06a4f2df0af1d9\",\"0cf27f77fbf3e8ce63fc9f569ab1bbffe28d27cd146ab5ba992149a6be196788\",\"24b48e5892a0252a00a887b59ced2cabed561f158066a8fadb784ab8e0836331\",\"680589b80080065debe38d6ea7c3f7d65243540e2901360c417f94839eb8a2ba\",\"f81287d44ec92a80ac3e4661287524ae80dea8a2f1e1574b3d19b39b15d9f620\",\"88c9b3d5beaf340d6495625cc4ffcf759ddfa5f68818fdbca0d43501eb635404\",\"df8bc103d61411e30ee7f9eef0e6f860dcc079c205a600ba98763364659e1275\",\"20999c3de6c6bf2c4903dcbe53efc76e04ca97f7e567626c470bca8968413c39\",\"a9a27ba2fc65145b25a74a8944b879bbdc9e94c8eecc7f243c940c7d2ec7a1d9\",\"e5c3bddb304d23241b4cb5e448265aa69f715ca90307635af9353868bc9f921f\",\"c2be83e927a1667ac81b74b6a2ba3ed77d0e0001e9e7c7f0451d495d9b2e3259\",\"78fb96dd1f4f4560a61bfd6bdc80ca00d6935a7fd9d48110a4a21e59095b4e4a\",\"aefa5ab1acf596607c7f875509d49108f16298509d7d053dc1022b5320046fc2\",\"417b4d3a29c386ca38069eeb8d55582a4e43c11e8497778e8fea3dffd4d4fc8c\",\"4fca33387759f1e49f6380180dba58d974cfb910567e4a09f51a83f05de3cfcd\",\"ccba7877a0fa0800d70dc7d618a8e7e2adfa3ad11d1fd6a195bf4557f8c464f7\",\"b7ba246c27197184b7306158010a6bb0816cd44343914d80a75ac3ef4aedc3ed\",\"7e39855350479eecd07dbd2f848bc3db581d87f8e8825c0fde817823ba1fdf2b\",\"9622f52bd83943b78b1d796ddc1ef0a51329c29bebc910b8979ec355665c9471\",\"e5c2d65e1ab79e8052bf9608ec237fb0b517d6076bfc031773a4494cce27b542\",\"d331b32a824e1011074b9893442680f9c9910a817c84f06bdb72addcf5e055dc\",\"485a398673db6f1beab2f92a6cabfae220a82652af938fb176d13df6654bb807\",\"752a370b9d4938f8de58cb5d11e7f6c5ff3bac9d968ff9670effbe1c6b86ed7c\",\"97a99c2e165d7ea816636eaf1eae0fe06aaa2d6232bc62825c66b59caad3c1d4\",\"bd7c2526b2bf23a3281e0752384b39eef33ff0640d0c94bcf2469cda18edc798\",\"25786cca750ead541b8b228efceb44f9736e7b02502d14b142490992caeacef1\",\"fe9baa6a1de371828fb8be489a37e61174fc5d5653648860f1b4b6e860ca199c\",\"d065b79a04691abc67fdc1067b6026cb91a14d5f8031417564d2f571e4a6b144\",\"ce087ea447661302d353b830ebcf44a8a94ced7b56f91ebb28e6fbe2b4b04ee4\",\"eec684729940142cc2cd5ebea70619ea2bdface2d52e0557e666a8007a1a89ba\",\"ea7b84a56282105288effdc6ef29eb81691282b5e12d083bfaa94a6d580fe900\",\"2d7cbed5571a5d5e6199a90709db391e09f444c1110e311ae6a50938e68b10ab\",\"f839b88f8897452b1290809cf525f81f3f8c4961471da9260b70780e0fe3590c\",\"82dbcb32effe792da6f305d2bc552c92aa68d2b29b4e242e72b90cdd0609d955\",\"4db5c14f6aabbabc435d066d13753d62f5ec9a465170292da333cfce8c0640b2\",\"a85e6234475623a80c41443873ea9d6b304f3d5313c91d0f3bd699b1bea90f26\",\"159004392940c8ff7210ad444f35c7092314d44b6ca551a9218961f66e339187\",\"f096f6f2537d625a7924bc5ac755289611241afde89766887e97cc7c240c89f5\",\"0d5e133eb36e7241d879cf34e899652a5a5f5c6c015e88881012fd07ab5663ba\",\"9350466dcd79dc223e514d1ecd490e9d157f7a5f291e718cd1d65e5dc4ac96b1\",\"0cb476032ec4514ce53d3d0eca28fa14a180e071145787ecd3cd730a000a488f\",\"73836c14336ed3db3db78b5fa29230dc96725bde7c08d25efb0583b3c68470d1\",\"8c581493c42663c92b3b4aea2f84742af08d202859182f8c040499af3489f4c7\",\"6370e2767863ecf13cbff54b21c2da9475338c66a6645eaa80c9ab8b87c21010\",\"554616b09222bee5fe6591580f37fed5a6d691ed45133a04e7eaf0564becebeb\",\"87db3eb08c3dfe269d9471827c0f624da2c5315685aee125fa62bf5422cf74aa\",\"d68beab0e9f2a91eed2b728c82f7b3bad4c39446c21fe0bd3b0510c47761c81d\",\"605ad1836756c02f89b4b0bdf538139aee5ade3bcfc4d8c35829fb4f63991138\",\"c660d0c31e4a54bf4c4a5362797c1367b64385fe9ef933c806149d85281731cf\",\"b470848e3d70458a20673cc87a376b26001a73b41042deab343dd77c485a5712\",\"de23752cb66e117b0d1b3d347fa9773a17781730dbb48079493304ca3e6c279e\",\"65d69a760bea13783a8bf1b769b4356e37d6f713a0671956f374b90b55b19751\",\"00da2cab18ef0adb3f5bd1b589d7dc1fea8fbec709ba33ead48a5ad7cd9ec43d\",\"6f2a1cc8a4ee968f6fcb1ac32b8dae87769a962b21b48b9e15412230ab472b2d\",\"11a6144559b69687abddc775cf41cd3d1cbdf7a587cac23d3422c30d41bc610f\",\"181b5ff9085589288b26d767cd8ade4b1d4a3436abad664cc7a588d8116b6882\",\"b50b27a22ca333826eb48fc3e21d9a99fcec231c9f200d12ba182dc22a6ff947\",\"54e09f3ec2aa088d4ca98666899d2e227ac7e0d8f1a88f5a0475970df79628d3\",\"dd240796bf653afe631eca7e5587498f80db000630955cb7785f0cdcfa446828\",\"1c935868a589367a5e28ce216e88ab373a9845a943e0b5cde3baa5fbd02a2eac\",\"25be415ce338f474e3b4392878d85702bc29955aff6534eac8da53d7e732adc5\",\"a78b3fb9740344ae304b6675796c76e1e8edcfb595f402cbe7ad00b1e160ceab\",\"4b6940243c18dfe178c85018e3a10d93503825ce85647147ee5c31be7534dbf7\",\"052566b05a299858059db659519242770f984498a94825b9523cd354d56b59ab\",\"c409ece9329ded7cdd1025d1874325af673a2c2323bb0d0f1dee8d9ac6644caa\",\"edccaa22e6122eb142c71cccb190aa30bcac284f2442c6e6bf472498e7101eb9\",\"61cbea1c0cf12368865a572be40e1658e348481309e19fc2d56f91bff3e9780e\",\"09e364917e956f9e9c1e4412345dfb66b510c82320cab9018d4e0eb38d272d91\",\"46ee4930343268526f4bcd99faf6e35722392d5d1ae3cc8720a7cbacda86855d\",\"7709741666a7fc9eb3a93584d007e8daef64394bcddf35cdea9b30a5409f0f46\",\"1513ba4141d1bd8123c58254b03297c66496d2987be67a437fd338b590a8fa71\",\"9eb93487125c6fa986053942bd394bad18629fbe458c035087e3833fd050fe6d\",\"e4cd1e6151c6ebf839093d415cd2625fdc0b1eaf79d87d52f1ea50582e883d15\",\"155c9b3f540ed2809b5c10589d08121296412ff5acb7eaf873eb966e1fcc6a38\",\"cee5c36af4e9d8ec416d29ab76c62ad0fc982e70ef8fb6498e5f61fc099e25ac\",\"82af8feee36767ab826c7e6a55a251e95276877b3ad0ccf81dacf48ee76bf4a9\",\"f485f9ff408f21e3e62eeceff0f3e605d299ce47efad9bd6ae2d1006b4ed6990\",\"5fffacf28ca63c6f383cc6e30e1c88a17ddac029dd95c4b32e9b91377910cc96\",\"74c4249dcc23f1a37288331f7b26a5915364c6ccaa7e67786d322d454c46aa5f\",\"d4ba7f26809a72127cd7d6457fda84bd42d0b1eeda36cf3bb55ad39d1358f085\",\"9c12b78f5b62504f201713e8cbed8d8312bb32a22c4a3ba2c0e647244bc8db4b\",\"3213d12262b1e401a176efa737a63dcfd7b9c752597a1033058f1136c93a1521\",\"4126453b8a0c8c9b849c244d504afb6de5c6b04aeb65e7fe150c997bd44cf8dc\",\"b2a685b3d19a07ede751dc11b347fd0b43c8ae8002bd3ba69f327ccc4ea1f454\",\"9f442fb8cd4f0fc8d50c06b2034404a98b97c3cbded3c914cc54eb51d237aec8\",\"c19ebaf48a602d979dbe2e448000f9894e30e994a23e83fd1152b6b6359d1596\",\"6b12c60d38912cb4a16efa7eceed9333b4822b38c50c6d2f07bf38ed2672ac5a\",\"d88c275c622aedc0c214cef624a2da258d56a222a4a446d6031f1171d2d71f01\",\"708c4e81c04e9778ad1b8272d1e877cc0f20f78ae9caa6565a5c7bb082e518c9\",\"750edc6318b59eb73bca7cc2fbaa67c7288be409eaebf34fad8e747b98cca4d5\",\"881d92bd48bc43e2f91bd3e21c10fe2432b91ccd4a31acefb89414f7481217fc\",\"d36a1ed5f543b70be173f665a2859b83bee5d1960376e5fea5a41701535b86f8\",\"b39cf432afaa98ea2067bc8a2befa4229cb0f022623cd31f0bcf2a96c9ec9818\",\"a9e9cd56d6688def056ea20f98e6155e35bf3b8bf2b4f41a1cb682e2dc7eabfb\",\"e75c006c0c755c90790f84baffdba5e265fe7e0588645bfb44d12f81fb7930c2\",\"0966b21749965b86bb0ce41422bcdd4718fc7c5873c981eb4672d179ff99576b\",\"ee0b1004789abb561bb1217fdb7f351cde266595f5a633214fa19d3f75702999\",\"612f733d8f64629778e66f798c6f43d1c5a855647a1894ddb5a1f4585e1e821d\",\"fa584307566400b8efb682524144cfd76dd14a95fb2cff5e3cb178ba47d7f940\",\"3e94292c111cc8a70e9ef9643c32e70fb21c20508abe64b17b8f1fd7e8a8093b\",\"739af7ff6f341ae994c5e40375061bcc46b20cee37a2553264671caf412d325a\",\"b5e9f29f2607327d44dcc709546aa8d11755220b5a440d404a54c58312ca69f4\",\"c196e661d797a4ebd1670726b5b6a86d24b4aefb2e0119406a5a0b1351be0c2c\",\"1facb059605b897099320187b57fd4424c55b7ace6d4451dd6a5c8625cd563ca\",\"893fd815f1c4d9775875c743c79351d596ee460199f929faed81aa9a7d7c3c28\",\"3ba526934c433da848df78b033ba105e4e2e7208a9fc7c7a39f50286c5a38c0b\",\"be847249664d6ea76c70ff344393e1a6452b8032827a578794b9db901abcd4c9\",\"0888b73fc2232047d6b569021a099f27b945e5b653a5e9447bf7d8667e708b58\",\"906ec6f63605f18098e17d325be24434e24ca42be1f8802de44506dcc9298098\",\"10b37fca14d0cab597803046f7576b9fdc0bc8352cf61ed986682253d597a898\",\"fadcdee3447a4fdff20dd19d64943762670c0eb4b7361bb36be357583ce76c72\",\"2709e54a998749c25dc104a8f3ee706ff7bcab2ab54dba0547a17bf307c793dd\",\"de2fd1dfda0f17d5858deb6e82221129f7cffe3ea8bdd5fcafa7f021b28db1c7\",\"9defa160a07d59057b7bd8641233824fefee5ddfc7af3d4ec4861f096c376c37\",\"2cb98d2fd32db6077351d1c0065ead559129e6c37f7f9e34228b94773b8ed68b\",\"a1e22280ecf7dc0c95afb0bf9f6e2bbc765e8f457fed046417272e1921b2a59c\",\"8e3c4e8155fcbb35038f18663f4406d0df507ebffd85e9c59011853605929b9a\",\"8b831da754d77c27f4666fa3eb551b72475ce0a1b27c923e09c8d175dcf13a29\",\"a9d3ead246a4c8496439c32cb9df2c1f80b84ce8825db9ec84b6a3793fb5563b\",\"fb2e357d2f252a23d0227e0e6296ab332d9e8e8e5bb7c959947e88adf669ab4c\",\"38724dcbf8cccc8830042671cd7f41b5c0ea3407cb65b509f1bc202c4f1008be\",\"a0556bc749e3dd7eefa79f111bd5efa8c89ea4ea88580122535a8ea2b4980403\",\"58037f8db9d7918966d5f4433183da8127217c4bae82baa50530ad0287d057d9\",\"fe288992bcd1666adbe7f87efbd910615c75fd9df3447cb68e4ae07aad122005\",\"cd7de602df6651feada431f623ddc7a302e74ac77d01eeb326e9e38908992c65\",\"0ea9ee9bdd13ed199cd9b3e60063b09fadae23360217b26335d63b0258656966\",\"f3b5af658503a07a7b6374796189c00370cc48c7bea4d79b6bdec484399458f6\",\"9465d28461cd1e583c4b282e86b3b6560faba76e20691047917fe5de50c6611b\",\"b510ee3550cb312c9d18f3213c39eb76f5d03fea3c765d64d5a9184c3d4060ed\",\"20872083119181877897857e844ec06736fd0e937ee4b23642facde77e81d3de\",\"7f2022a4ca0df42e31dfe02fdee7ef4c776a00d3a6e644cce62dcddd08fc82d3\",\"0265cfe8d921d6e875a0fbd36cdc19c979f349e861270305428c08ba8b069eaa\",\"ec9490a7b60be04e6ca3010190ddded78c2be1ab8c9ae770b8eeb0c5e868f7e4\",\"577bfd147f193b2b036915e1714b5f12836579fe0b7618cb1f00240e5e5263b0\",\"ab3b062fdb5efd693c7613727248d14ef660c79bde7e411ce240c1b7af29f7ba\",\"4039b46120c6415c375b26052fc43a5328dc6fbaf55b2fe2a72b676cff5680f9\",\"f9f09b64b0704c1f9913077a57497aa065c115f0e1b2be29f8a266b213972a4e\",\"c436855247cb7c651b9fff378604ebc3aa7968185fa995935654ef921abea331\",\"8329ff3a7f345665f841a2bc150e3c218cf794b41b8757850f286577caab6ecc\",\"be3afba78f70a9698f296a95b7394b94037344c159676271f42f59fd47e4bade\",\"cf6b2bd3a3546ad5df70c1b979ff0095f826d6b5b551d73317f7f1b88839de95\",\"f800f7d8111e0636054b3dd485e24712284293486d50042c4794898b93c21923\",\"c1cc196776c680b009b3fb96d1dcce4bca95934a1ecabf59d3084a618760bc19\",\"2177aeeccf9d748fc95c423f7bd0f08ea340cc8b78d2e7b3812f0a7cdeaa107c\",\"067a142a086b58ac81a7de300ee7326332ae19ef65137d3bbb2762bf406ea3b7\",\"cf51e7d1dc3ca71e5ce7fd0ca5fea77a95c40b454feed67cf4b6fe6ae45570aa\",\"b5d7caabd93588f280e656b6c6cefb289120ab33bc9ab75def9d890195f29154\",\"36091854d57c0c987438995f47810bd94dd8d3a764a13e5c45c0ee6b15ad97ef\",\"ec8ace09b847a43374fee84b0bfaef143cca1a1b1dfa9b4130c3b97069730643\",\"4458017e8c11a07c03b51497d392c34a2bd42d82baf769a5b77634c3a94f46e9\",\"97e73c3fb7a9bfe03242b8e2cfa0149f6baadda0992f78c1784158f9d637c4f7\",\"0b239ad9ffdaed91763bf3b4db0352eeb938649b12711d5327406b7f8927826b\",\"737d885e6fc11e00cd9585dc4907d1735b6fb16da60fc735df11fb48dc7d8090\",\"93bf8fbf5135bbd38ac11128152e6229a6a026d2c2eb3505e3af0bae783e790a\",\"5bd60dc227a6d1cd67fcc32a449e87298cfe7b874aba3ed352bc9563dbda4089\",\"a55fe78026e402b563b4a10beaeee2d81767ab700bbf50db3b62bd9dd807ffcd\",\"962325786b0541c900b416725ced1ed6b6770ac093eeda2dabab4e91ee9f3b64\",\"97935761b96f585957a29da1df7f6d7446d3199a19fc818e9b2d97cdb13db0ef\",\"427ce4208c200ff98b62432b4c0f49e8cc2989730122d2443def7620bce0b457\",\"51bbc75e70f55adb8e3b1759267e3aae9450bc192a813fdd71bf475de5edf959\",\"a18edd0f63d6b83046977fef16722480ee989710b47d1b149b60aaebf1339720\",\"c1920d7394a0982f8fa1a98f076bb0d484d9f2a284b2742047c82185ef41a6ba\",\"b9f5504f8b5b381097632942806368bf00c2b29111ea2a226e2fd36d303e8642\",\"4455ce9129dcce46af1c25aaf9d5532b6efe379d1478dcaa68d3336b769f6900\",\"572df974e11378e8423c67adc03a9f566359d17f33a333d27a7b9ab62f2d9709\",\"66f04d910b8147e53a24f8a685c89ec3e28f61c8852d776246ea0b10e64e796d\",\"8d0e5a7e2315467e346bf3d18730a92326ae5f618491942bae170636ab5c72cb\",\"3ab9dd31d371aef7f49fbe5bca0515a79b20f682b4c47c66ebfba81434f82fe3\",\"c562bb60bba4687d5db1109d82cdd32330b0b3671d19bfccb791117eba89b357\",\"348dc8aa0d315de996b5805abaca8e74a8e1c7a48eac35488e7aee090ce64c17\",\"6b3b5da33f387bd7af5b69308a635f28d6123fbc4394cfce70ade44e4b1637dd\",\"49a4752206829f0498db620089791195fbbf45f656938210491933798e20f4c5\",\"8f30cb09b9bd8d49fec90762943ecc50e800f9d34d2c2192fd54519fd7d42a29\",\"bd44a718cbb29cc79a16b604f55e4a0c382c223b1e40fb780a847ae10420256d\",\"8a5292dc0ad68f216cc26306580670f57dfe45b547241ed141327b011f0abaa1\",\"08c46a079a91c81160bb8e00cf07bdb7444078ea8afe6dd44a28759c8c8a7731\",\"4074b71eb6c26b903827bdbeaaa274d74044967e8eb391fbd60367711f5c116e\",\"6f8ee308760334c993a6a8e434c8453a3722551322a8196a1847d3bbe46159ec\",\"7f9c74e0c6b7a89e9cad9d83fc29495543584e9990b2321aa123dd219cdfeccb\",\"a65747d520a152ad150b69d71acee0913fe55a3aa1cca4179b7e3394e9b38771\",\"fb66910ffdadbede24e6df96a278ddad54410ad3e2148d0ffa6aef8517fac921\",\"10db9ace94610ca3003eba7a0cded8f318a30b5d84b7ca13af5b324d83bd8c73\",\"605c0a8b3f6a63e67e19673936fe7d082648826710dba61d4ef7d7cac33cf1d2\",\"244542b455b17372a7df8b1a27ce8bdf0180b0a147290ac260d6b737a17f48dc\",\"c84273719295a2d48a70c04aaf5880637e94d6120c0568d51039a7dae38bf8d6\",\"cf78fba03726a6f00efc1bd95cbd48ea16dda27a521ad028020528de83273e49\",\"e6a03e5487166508584833fdf7297d125d73d86615494c4c0bed7beb57dc2bce\",\"a6a727f4742cf9f4b7b2a93582588b2662fc94be0e8895bb564e33ffd6e91bf2\",\"6ed7db67f601bd2500859617be8a2e1949992671018dce61e647cbbab41c7c27\",\"f506190a01b446d0a9a587889ed1ff706ec8af14c2543b44e14e556ffb31bd51\",\"dcc7c284963720b7831b634377e38391a04fc1d9bc7879f2c20bf225acb8080a\",\"9f6536642c862fe916177c35034568d2b230ed9e00161054af8bdff1fbbaa775\",\"a88e51d0150ace07efc61e6e212ace2a676b16b45e77f47cf5625a87546c644c\",\"e5b350c20e4b55a2e180c8a18fc0078237b2759bb6eb0a2c1a779c445491054c\",\"093a1310b6bd6b6919a382efa368afabd7106aff7a5f8769efbfcc4cc2c8f3b0\",\"e8050ebaafc24d075488ee1c6b8d52fcfebdd567712357e445339495615fecba\",\"58c0f98bf55c33320e7ab82aa586be0f81e2b46356a266b30d4c233976c070cd\",\"f920234b9067055d4a4ad452bc807a29f1620f1997d7090802a99104685e5b40\",\"72f03658e9467fe9a164904029b6fe547b34a5454931eeb111780f0b23c24a62\",\"305fd8e3e2e8006d8735b77196ccfd97d64732c9f199f8022941dd5644c6c848\",\"3e5ab57c5c877d4f7aaa0658a805ab0bdc39d65ec5d99bdf75ac17d7cb65ccb6\",\"57e823724ae68661c029a1cfeb54ff4f412d0d68614fb1f53924966c6a177adc\",\"52c646ed2a562f5574f561cdf1078045900e15eff6fcfb16af49e5ac36103545\",\"ee77666a1ea13a54fc4c33a751a1c9776b5ea5c21f67a01138ab782ddae6f97d\",\"425d862cd7c36a9ddae85ed0d15fd74d20aa2067e3c03471291cb97868e1f764\",\"20410078dc2ef26142824d63fa4b118638186662aa9fbf9d0c1d42a697f23af9\",\"3cd4ba67b90a49981f1b2dc50ce7cc7cd352eed4f9503332b2503cc14500f341\",\"687fe626a0a72e663db6d02d60ad6e60b746ce3f1c579fbf98148c80279c2aed\",\"82e7ab1395b488ba92a04eb4c78c39f0a48ccb92831218aeb157d5f9cb99b987\",\"549ed1a89978ca96cae8a68eeb916e112ff70032c455650fb9e1e2e919f6e4f4\",\"cc3c104586db8c4dadd8e93ed21b307278505842ce671bcd04fa0221a77fb144\",\"876baa5fe2804649fbe7326c436bdf2e4a4745a4b730f0905f741af0fe07e379\",\"b7d0bad6435623281e2e2f153519dc63f9a1e44ffeb9f44d499674a7e0274296\",\"04dc7a7d44ab80bd2501f0dd8c359ab2d309ac87c469069d1f69cdbe7d8e7490\",\"f5bafc09230f0f466e14069ee5a7c24759bce03623bee43721c266f152270842\",\"7ce6bf0a323b87660191c3875350e7109657f7c734050308293978e12f621231\",\"8fdb163368b535511530ed9f6f3639c6517c82d3e0c054ac67680c59f93f4c85\",\"7a9859b6ad5e0b1853dcbfca0c67382979ae7ef6fc2617c392d266cc1bad6244\",\"e44d778353adfb0024c29fc102d99aea49ad6aed718ae05638e011b3d1f01179\",\"99d71b8cc820c9882d28b3e55c277362c91edc354bd36a03a25201f91d0cbf19\",\"ecb75b29d0344f1170e7229c5bef37ac8bfd254d1979056bd487518d4b7bf60f\",\"b0d198ebfa43eb2138485104a78aab763daf79be3b216023d79a58953be5325e\",\"9e954213dd344dc0ef9f72b680601890963e3bd204c7920d11eb3304ff53187e\",\"82e83c9c3de085a1bf68cc48dfe9617c3dd46e7db6e6e78083906e586d3707b8\",\"ae8f5bedd0d9de0449518a8553e5157898105f90620613893212a35f4d7e4d68\",\"16ddea06aacb6d139db8e271b19695f064acd7f967998138cd7720bbe0a08290\",\"0022f22ca33fa1dbdf1c2c816520dac0c02df73c69e57fe07ed1235ed4f55501\",\"9e44f7ead2bbef38dd6d998cdfe51377ab0e943e6772b8adffffa2234b8d594e\",\"b7f05a590fdf42482bebf66012cec134373b55ee482b7793255cf04dbbf045c4\",\"7d4ae0440f2d63d7d836f1f576f32ff9e9017cd7561600f1b9e7833baa739391\",\"02edd54d7f5a5fa512bf2c6745dda34a9fd9d809f86f252b8af7e053cefec4f4\",\"1c6af341f1f9182b75174d729697a93186c622efc7c6209e390bcbd40d40a08a\",\"c33fe319e7be9961e591cd0a69a97f941e3348c206d3f8fc3fb37734c5cbce13\",\"f22a886e423983277c5f455bbbe7c4b077efe967aa3ed22d23799b896f42a54a\",\"ac9fb68234e7df6dad3ec5cd74dbf9e1b476e4b5ea299e4871bf7834988b111c\",\"25c8f178f3014879fcc694bd41bb1a6bad024c8ad16d632a0246f694ca250238\",\"209e34cefaee4dadca4ade901e9addcc5a40997bebdeabcd5eb2fb14c008fa1e\",\"6b12e24d42d77353e57116771a7c0f0e6b1a448d372aa9897d5cb8d9b2aa80eb\",\"485903cf65b733dad9cac6d42d5188e2b688bc7fa70f1f90b5f19747c3d34ef0\",\"2902805c8d55f56752f30264762b43998106ca922fbe3e63338603a64d93dd06\",\"751b17277ce8f4c907ef652cc7bb8e3bf40cabdbf9996c75f12463e9089c4353\",\"c83713244f684b4956b4f57e668fd103d85d7280d75a3649f8851d009d9d9a9a\",\"0f5dc28b289ab623472ecd84cd20157dc97c3f0e24a62f93b538e88e0c39f2be\",\"6d3901bc980b252ec7fbef279253079eaca35d5d7ef1dd83ddf7d5648060cf0b\",\"4baa3f5aabb43399e8eaf287410ffd0afbf0ac60d8b16066da245c05b3e5f053\",\"c7ef78768b182fe4dfd9b10a298cd0c0bda99fae57e29c73b49a32e3f2b1db5a\",\"85abbc41e331ef6809850d8b76ddf19f1e38fc0f5fa8fc3555d5471b442335e5\",\"78e44995951f780b97fcf430786b538e399824aedef7613598ae15acc58cb650\",\"85fcc5e815051534013df942f2a9e62116dc3268dc666d51e4a4a8ace30fbd33\",\"ec42608ddf01da65e79b3f88d5b8caba54a88c64f815b720273e9cbf5739eebe\",\"2c811cb46d291d5a4c85b54de8cdac74b037f8eb63630ee65324370240f03134\",\"f21c46f25cd60018d980024d6038a31ab909fad01b1dfe83b98042b811c3c205\",\"706225274a2c4488b8363b3845be4b4a072b69a380f5f8ea7e8ac5993ede3f9b\",\"2a64fe57d37184967289a6064233ba5c6dd6d561c399c3e90da896b30eb97b06\",\"9305024a0ec52ac1ceba03abc335cd348b8dbe9b870808c35e6b7812ad1bf22a\",\"291953df1892a364d7d4c4dffeab5b13075ef5bdbbb4805b856ed3cdab47a003\",\"21afdb2a7293c13b89489e458e2c865d9cd0f900fdcc47f8e3c48cceafb7a327\",\"9bd934e3df7a31143a2b8c0f8fad07e9ef5f3a80ba72b7eddaa10df5c37d1c42\",\"7b3bf03836716a7f8e4fcdd5edc55ab33d385d2550894607d0a794af8dd39275\",\"293f7fe145986c31302cfdf2999021091dce4a2e06f5d1a4d14c61a4e082ce66\",\"fb6278ab2dc81d0a843ae1874db5ac5ecfed7daa4781446ef4350474df9d9fb6\",\"7098cff3ee59ca062b1396a430a6c093fd7394eed3badf4428594b719b1ee942\",\"4005565b45880271d1bd5845472de832e4da9c2170e4d4fdbe9f1af838b4ec5f\",\"572809a8caf24560d01a6e738e09b76628b781a73d3262c23e6c8699923785ff\",\"b5f8c5ae9eb16ac7f9c3d0d2b8fcb8516b32a717e240fbed77da4eb86e397bf9\",\"e1b2d18ada7370bffee6373c055e8466c5de29064a861161b4f5b156aa3302ca\",\"c63beb54edf50b0018cfbd2ac5e09107ad2d7368eda9e108adb53d2f8144b37f\",\"f0d774a2c82ca44d6263bc2b68b0935e529b12d853f1092a17e431cd2b5ec41b\",\"e74d26cf90e3c674197920b2cef0474d93e1cdf2d4b078bd1f83dd90dfc8458a\",\"871f6166830e2ddfc0e8872a258ea57de8c0bae430b78184c91ebcce644c0862\",\"2b871eba695d23ddeb4cf88bd3276bba1a380b742d66b4fed4678538847b0bef\",\"f2db73a9bfb80f23c7a88f2f671bff59492e43eeebf7b607131c37c71f1e0317\",\"ff75904e3ac73e3c813cb00c5bdd0d7a0887ef6885ff6c319f6fecfb34e4a307\",\"74d8b88326b4f83284e1368ffe6f6f68a1114a6c2a2ac7fddc1d62f35a905480\",\"95e5ec7baeb1ab8b19d059a9ea71b569399fa2bdbb4f3b62764cf419722491d0\",\"d0d55c7de0c20f19fdca9ada0512aea0a67e2f8484ac80add11ed0dd1ad8a798\",\"6c9a0e45fc845d7222b5a996d611430045f2237d7ba28b0981e6c42263ca48f2\",\"8ba3b6e5cff61ed859cb7d851347f165a9ab0ca12524d2fb9fe04303db81a244\",\"bbb4cec1cb707aad2546c081422d7b06c9a9bea892814cd82f4e8700995bb28f\",\"befb4fb8748a10457012e0f5a5caa46f6b4b68337ca3e1c82460b09e359756e8\",\"521f5964118186c9c1bc14fabf883252df3404b515d73f25731bdca017c3913c\",\"07edca0fc3dac5c9636279799930e8f9d917c46230677c7b07d6da578daf2f3e\",\"187f555642c96452ad7f0546b173740a7908b230cebebcad1b9b0bada7059a1a\",\"1fcc11e5445babc4dcdb8e0e0f8205955e146a71d0f166a67b88e7d0563a5be0\",\"fa50e5af5b3dd41bde98cb80770aa0eca5efba953e822aba19532b75a6de96eb\",\"6585690bb2926e021d3211b4bc62af6da136d83582e363db3234f495a0008004\",\"557a3ede1da136fde097655fe3de0cfaeefb4b76463fd550cacea7393d935265\",\"031cb589454d55a19c0dfe5d2652a5fb34e8cbec3f9e2181a08826a60dfd6165\",\"317fb56699bcc64f09f029ad7a4dd6a90d00931884b91ec3360d4b292e1f1d76\",\"7e2639b7f50b8d9497c5473f62e4eac20baff72ed88babb01f538aaf6bd4d2ca\",\"fe8aef68edd1416fa8f18e6427f1efe4d7cdc41a3fdd21c00af45da948b80f3b\",\"fdc988be3a9fb0fe8b49b7a3ee97c96bcac61da5762f2ab5a5b0b74f1125d580\",\"afe644fefe5efd68fedad78f4d5d3445c3c0560abbd2f85eaae66fba538d9f6a\",\"d4bd29b2ff601875009f74b622dcc1c3876be894dc17c0e5404a422ff4d01358\",\"3654f804c76d8fa128a8aa49bfef3ccdccdbeb0e414f12f6b8a617a82afa16b7\",\"5ce5aede02ea7ecfca165850d871ce3cd1247b8242aa9d98b12e5e46b7b1390c\",\"f99e50552d221d324a1fed78b46dff02be70308d9764e20ba9eca286c953f61c\",\"1d6132f3a5bb559fbfdeb4419124d1eb152b169680ce3a69a5e21ea1c7449fc5\",\"ce769fd55cc801f712d039e51b9f1b0fb9668603ab5ae29521f5adb00100b05d\",\"863eb001825b18cd267f41e10eb9cf5dc097cfe83e60fe17d1a062d722d38943\",\"466735ed0b9bccabd38196becc60ccccc020dd5bba483bdffa685f6a768b8721\",\"5bebd8c644a5fa2697bdefba5d2ecd4da30258750b854b254c3ec0380a688cd2\",\"a76b8f8f44f74290d202091fc815b5d964944f3318ca252b2699addd54219841\",\"8ab138516c06b6e8c11061fdcb3e963bacdaf4e295fd4bb702cb041bb5325236\",\"915058db622738bc1bef1098841b3b94580d9c2f2b118630606d72d02708c3fa\",\"a69aa193b13e9095e17291067a2f77dbe29a8896defffbb82af3d7aca2c78215\",\"f5764270f883116d1739e125c126a6484c9e4acec7a6815d4033baa1830c2c91\",\"7065d9caee42e1d206719f877a5eadbf13aec0f868512be6a3ab85bde66ae597\",\"77afb76d8362879be0737efc1266a66ab6fc551fe3a21448cf0491c1a8c974ad\",\"aaf243207153ba358c25082b4f3b18fea02fd0de98eafa8f9006b841f6232ac0\",\"c348a5e905e3b76d78f11b54d5417857a192c58a066760f1933fe1571e69cfc8\",\"627b3f8351845d1c810e207983393a6def3b0d80d7ea7164db2ae524acbf0c92\",\"e17775a7c6b6dea5cc293b5641f6dbda2cdd34693bb3c34d8c374d62c62c7ccc\",\"aee379593fc92927366f92d253e32a134a18ec4141443701a350748c41566e85\",\"d639c0407083dbe4db8b5ddaf9ce5e1a80ce59dbb1ad27dd53fab7214a0ee430\",\"5d5cb7ad65bd695506cd32f3517299839c5c7c7f5345d8b66f76a8e9f3b9c7a8\",\"5f80892ec6d11dbdc75a9c2f658a765baacc35d3595ed83d0f71ab4d550555da\",\"888c2feab76d9772eb3510cbb368983d1958ef764080862b7cd097f10f7781eb\",\"2128c953585e10396f1240926545b669ed9ca3bcd52a1288bb8965348c407271\",\"98038321f08f8cd84ecaf8a928bb68cbaef24b2d3ed66d4cf996ee2307c4c6f8\",\"fa3c5c74e97338a9793c30109ec2058992cf28235ec4934e2f9c2a049b0265dd\",\"fc3b6104abf26190104e7fb0049c3adef3659c29ec0667d5eb7a07002ab2a65a\",\"5db1f5e10599f39038d84b42ef41b421041e94279e46e3428c38959429a5883e\",\"bd1de16556e8eff33e2717b900b3d23b83986e9127cb1f92135e053568e2e6f9\",\"57b4981e412334b19bbf60b22667c7c3914817e871bfc2b6d18c03eeb3db610c\",\"0f93802487d3ec13fbd93a3d4f8701f8f841d16e722d6b935a7cbfdcba73d630\",\"41781b10deed2e7bade7d2300f6cb3fadf2bae4fb5d8b27f6fe01e2e650ac1b1\",\"320a54169837302602afed2836c5b5258a788872b3970cc11a9399fbd244cf20\",\"73e47319f2b48bc01add84e550acab7536618f1dcfaf5330fb187caa1ff67aa6\",\"15e84ea3cd8c7e161a53412ef9a687d7c018b555e243c97e52b48a2c89fb3208\",\"1ce3140465e9015023a4f1299e1695bac2661abacdeefaf18f8fe7c3ec641ecb\",\"62aebcd7bd2fc15ff7d5a182a2812970ae1a423939edb5adda1b5326be244707\",\"ea3aac58b74dea11b14a9317c8c373b550644608dec4dae4dc434ca2f77765d1\",\"66207f5e9157711e4d0e340bef3cdeffdc0d9e4391f0721619bb278e8b153818\",\"e95aa6e673a5cf921c6f28b69946df4a4995550ab0da6a511ea7c4b4208419e1\",\"c0c4da8c5e9d97cd182686973bb99f1b29be33ab6b30b020a3fbd892fb09a7a2\",\"2885ed77e3d2521497197981ec481dafa67439cde796e0d639d974134b498d38\",\"86b098d607cd581ebe16399459a2502966d6a1ec44542c0282254f9fbd6cdde4\",\"842191061318505b4af85be714d98767fc168c138943d21b35cb3b75cb615bc5\",\"2e20cc216c2028e20b6c87a084d6e31ecc602fca6614eb3408cd0fc8032b62a0\",\"3eea96774138000e59ad702f3bf55d88f981bbbce7814296cff4c7f600797d9a\",\"bf9792f45e29faa2a3b2692aec69bb38e4f00f99f181c8a4d75e70759967f2de\",\"8a08d227193b86edb38e56801dd91f56693370548aaab836663d6b56f4912a72\",\"8fd54662618f999635a0a4ffbb6915cda9de7ccc6f3fb28dd5badbf35d4109bc\",\"259b10717686ddbde614c0bb54448e821554fd8c96e53b94159a03d704961006\",\"f14f68b929387536ac53308078b7558357bb16d944cfccb479e427a95457e24f\",\"449b035b9a0d90d1434c8f4824b56759b8c4e325aa499450cdea5b0d9f8bc459\",\"2b6e7f0744c770f5513639b32d2253ae342df4fd9c7e856080bc83f0d306dd07\",\"bfe556f2c3e1aee262a7f46acc3e3e4e6ad03443651c80054d94df85925853b4\",\"26a3af0ca9949f29e601cd2d98930a37430bda0efdd24a8664048d1ee14971b1\",\"1f4f2e474ecda00b19c2c9a4aa4d8e31ee4e451fdeeb485e03bc60c81e2e0d3a\",\"254ac9a6677e65be74db4192e3afe8f4031fd721ad77caacd92643de469e50fc\",\"be0f28ec76b5c8f4739ed4169bd4a1146292afd1c6132efc0c197507d87dfe1d\",\"33c6580b6be915d6dff5493b965049bca29053c79341acda68ad5412d4b9292d\",\"8956f19749813389572e630752a23b3aae045adf5368a3d69d1898fab8cf5bbc\",\"7f03707c1a07d651916505321ebaa27cbe9588b4f2e69c4388a1b1e78c9aa6e2\",\"5d1c50a1369d9d463cec016422a18b2b9430ac3104a156075368ec88f9b750b3\",\"3d8c5107b6696bc169ba79eeb3c17bccacaadb5bae05563f330cb6b0545eb70d\",\"216e1db529cef9f3ab45160c3c02c2e1b82f2fe4bbc0dd25e3b80fd9bd4194c6\",\"c0e06428cf23c15d51785ea948bb943e7c01fada0c08e4337fa678a974772dcf\",\"a320abe31f2217b0fef67330cd7c9d54873179104f96eb71c44e93e4f7393d40\",\"97673317da3389cb8ccdd30ecff1fbbd6fbcf031af6afa583b80ae67c8abe036\",\"726ce517d01642088de80614b004ffdcaf184fb9a44fa079755298c8e7b7b868\",\"b17894d895bd09c9a3d75e0cee6802fea960956545f28f216f6295b116212f9c\",\"8f5b0bbbbf1cc08c51d34f661f5c9cdb649bee5941d167ed72fbc3805c5ec95c\",\"c4005873529249cf5bf59dea0f00f124f28c3e26ffe81b09b1f904d32ea893d2\",\"02cb1eb59dced6ea690ff10f4d4ea0830d174da8166f37f5c8d24380743b0ed3\",\"f12a2c2573f9c8f1bc52eed8cb786bab6e73f0fa203ae61bd1f1c3f9630697f9\",\"597f67d3054a22ac9fd20198f7731942b7c841dec5a5248cf35c797da131d549\",\"4c3309585667a4c526d181f5d864d18d593bfb53bab8cb737c76f98792032952\",\"11db04fc216f3ada1980fe4d5db2bbb70b7edee03ace37e895369c0a2368945d\",\"3414687373ecae152ba5d82783931163b51096a1f82e1691cb9466095006bcd9\",\"7ab1e96a90d33aafd8ff4f0ac9da9da9a175fbfcec8c62ef39959c51d5d6fbe3\",\"f0a033c039c2d4f1ba7a3e655ef88e8c14b009fd0d57a3585f16837dba753eaa\",\"46b000697fd682fd81531b83677bd23eb926dfb9eb281557a78ff05c45a7ef94\",\"3ea21652ca784fc85ea0e5596cd2bd15ab9b108243f5ef790c787d84f00db7e8\",\"f3a2f520113db9b52daaafdd78ff1f0eac5c369bf97bbf5038c2e4606cad99ba\",\"031b66b5ffb7717ce407e4d2dacfe1053ad7261f08ea7e7e22be1e4fd86e471a\",\"5937bf236856fc8524421eba8786d11386ef029fd92806d4ff77205fb4e9927e\",\"685846919bab69d782ee14976645fdd4d35ebe1fa2d8d02673655bf99091f09e\",\"64debf16a5da428bfb83320e15cfe9b4898a5c6d8daebb5d4e9c6f520ef2810c\",\"25950f85d7443665cd0f93abecffc7f79735ecc645550b6d883319303d3031be\",\"59cb8688a6580fbe1db8c6428d4cbaf8819556000c307671ce3cdfead8d54c5e\",\"b670c1e24322ba8fa37303eea661ddedd52ee2be6714c6162c0841533d2e16be\",\"78cd08d267b43597f462dabcc4bd3d7a8bbfee9b02b63ff2cb9952f18e682b72\",\"6479809b39e58cfebb2089c3860a5594ba657a319142edd0ac9af6f19ab6492b\",\"d4a361f6c87fab4a3e2e3c1bd4ea93ce2e4694fcb1cee20e09bcf1bffe7bd488\",\"bf6f546e7b58889df7d4b06ca3d5bc8b7fa2186b569ba9b26a33aa5a0bea50db\",\"19bd089f9aae101d67f3b0ca0440da11692f1c00156ce7c8b8a7655d676a4f7b\",\"b5154006f09a6f65770aaa26e57397803f774d75b8bb0ea85405f235c4ac9def\",\"657d65f7b7eed8266e787460fc7583da21fdfc8604ffb843341d6d5ca6646e8d\",\"fb1325ec22ecd20ea319bf53b3d09512b1776dc94b073c1770fc52b6369aedd6\",\"f0e3effb16ff563f262b8c52c9952a013c652c4017de08763fbb62a19be23896\",\"528de5f78c2423605057fe4e161dd8de660499f675ddd0f33692b7a74e38ae41\",\"d81c9713e0197ecd59ff26a3393697ca09aeb62e6ce2f0528d0bb2040316402b\",\"74747b721ea46739fcd1db53200feca93fbfd211b2012d41dd470953317a9ea6\",\"90b5aa5cd1a463893ef371b70be7defd4cd6a61dabc3e09be27c332f633e9f4a\",\"0d2c92784cbdaa23a37eb81feb2bb4037eeb438f342f70ca9bbd25ce0f363c14\",\"5f36a248c006bc32e0a02985c0941c26e4c8af662567ab6aaafed1661059167e\",\"4a3cb1af8901e27f88526c5ab61aa56f36d802279a41ad2da2a11c7d7957c039\",\"4d5677a8a19aff391324df7b3edb143e6f4f7d950a7a3627d38aced36aa84c12\",\"ff11e2294618b3e7b84c1da4080e641f1a6289fc128cb24a46f97fa7bf6b3887\",\"29c4e17a6919c7701b4f11055eb37f081eaf2638a40e8a48e7063befcf6c2366\",\"bd06c139846d1dee9945cd49401cc0b496aa9692c4431fc3d6e6ebb0d8e3cdb5\",\"c8a73527787f3dab4dc9adf0a668d245ff713f18132a675e6a6f41c27eae5586\",\"4c176ed98fbeda31989add16c9e11711bda0e7b8c021a07faf0b250be1512268\",\"1064f193744b6da719c97c6c64a165d053c2e3c162c1aca7b7c63ec41d3c29fe\",\"bb8eb2485338435f45a455f5abe71ad289ca10dc8976ffe98172abee4891e48e\",\"4a0d9ab9638dc8efd0801f2a51fbf46c05a156ea05b7ecaaa67d848e4496a04a\",\"31a3b2f89edee145b07f1ee126a3a6150c30ca4ba5590ab2a7dd785e156d3095\",\"52175e7049b0b70e97ab27a45abe58f6ef03ffe4c55d9e0205009658b73bafa3\",\"55a0b7eedff3032b40d2815123d33b1e1c6d958647b21fd3040b9b3ff6f660fa\",\"271c022ffa205ab0110c9c4564663cf8ed213b2800343fa83eaf1d9dcfc25712\",\"44ac954cf2ddf0e731784ee23d631735b4dc2db4dff6809363c32a33c89c502d\",\"899644173b58d09f41fb6376c1e6305461d8cf949cfb7673784f0f1077be42f1\",\"7bfa66a7d1a8e69c288dc574cdb7d552e342c6d296694ead9a11dc5d4bbf27d2\",\"e940dcd27e2e175221fb81b31f1d0b6a8fa7877ef3f795b117cea9dc7292424c\",\"93ba106f569244d88d06251fe95eee58b5dd8fe51a699c8d7deffc7c08a009f7\",\"ca746c4ab780b2d419fda4e5eed34228ba613137b87d8941132a83169c10d019\",\"fb9afd365a69639eb27b89daad49f2489c1a9071342f8d87908ee2d6b6041550\",\"055a493911a907bff433c5c406d31f3a65c8c6c18f0a415dfe4ad57df7a3c3ba\",\"3d9d812525e5dd2369351140314e64e98ae6e23cacd67af10da0818d93e3cb22\",\"1e129a910cf477f9d0905050f4687343f3f3cbda059b0459dd89f90057992ce9\",\"421df71e8d191cf246cad92eb1c1035f3675b8d47a6849894405796859dfd3ad\",\"4f8d62b73b169f166c18fdd8e4e3378e831b5a6f86cf42ec504a5ca32aa88772\",\"bdc212bf9f85f11152f8d1cf1be7b2dbce048950b18554ae97c849b25639aabc\",\"cdf6ddb6a84243d3c470cac04be1bb90cfa2616e87ad700ebac890a7541b146d\",\"3365af22415e53febc274e4ff85d538d74fb59bce0a948a8344c1861d98860e7\",\"19374bd4cc1837e72f7f56ff9ccd52467daf46beabbf8bf7357965d7c180f830\",\"550a90387cc05ade6df35a30b4b4b4f8f3be3053ad63718c99c82a8ef4a4971b\",\"55943dad344fc927d72fd1cd1f1b409d94a19725ad88410f7bb87a02c7d2e3fa\",\"4f5ced37e30e1549f57804dbe1f76fe466f019560ba93d1f7ed257f6cd0d53a3\",\"c63b4e2a17034e9254dee4d6ce39c271a3322b72994db23da56f329bce1c902f\",\"6042bf2a22db4511614c12d9e641755b211c16d270e53e8793ee2c5a830673ea\",\"3ba8cb49e0e808f5527f856a651975eaa1047402804643cbddd2be7f125a8d46\",\"f919c04c03329c6726abd0ad7d6700a5d568209bb9fde1f3eadffd2bce653e0c\",\"99fed90e3224b60b9057aca66f83701cd320af723ab10f3017da9f03f6030f46\",\"747d25e291aea7160179c8f29995d96de17c900524fa5f72a6db56e3a9bb1061\",\"7a28f95a50ef1678b67b932baa3c505b776129d7f1f893ccb75068babb37d537\",\"6650fd207f912e5bb46d1967f3244ccde8ced0722e5aab43255e0d919cb3ac0a\",\"50b73492f43d7f20e041e972b4f41ae5b809a281faedc4b466ff4dd1e80b7282\",\"efed74ca0c4bb1e8c89d150b80acf7c8a50bd098556fc696d0168779c320ebea\",\"237d443a01989bca5aad24fab9ec46d650a91e0c09779c2f952200a6ef5c1778\",\"fc88057885bc33c0df095637f792e276c07e3f348fedc27171b0f661f424f8e0\",\"ff6e0bd9d5a339be5c36491db91fa454059372e0875e6e72efeb2650825a13e6\",\"a292671bc8ffa6c0ae77647feecee6be7531db302486499ed0fe746889f219d3\",\"e447a1a02f2604908ca2d77c989705d818babf9334c17098d44c940be1bb9432\",\"d16f2ff32fbc37a927a2e84d901ad9719e9eb7698637833e77a44a77addf0354\",\"6ef12fe7e747addb3e89fbfb971480830a8a24aa09f5dbe67e6be367152bacbd\",\"e980b23727096f31553b8703b0f75048b993d26e2ce3650efb04388fc028fe7b\",\"5b7d7719766b8bd3cf88aaffb07c28c684a9bf2a70e60eea78247c4fca2a2e44\",\"e08db0df20c0a3bf5268467a587142684abd3f5e3a16e1c118ee1aa3a2e9a0e6\",\"aae39afca4482617fc6224bd6142841fb8b5c368f24582e2c5fa3bfb8d79c36d\",\"ac7bdfb3842922c27cc0a7e34fe2fdd47d1312cc4f3686c2f26b741ad101b4b2\",\"a527889f1f4a2401e29387d9b29b8c6a99d5f03826acf1d017f888550bbe8083\",\"9d1aecf775ed9791792c38e0f717619a891aa10cf4ff8837e811f242da386f38\",\"053cd5aa08cac69b22d9ada71eebd565d5a2d34d211eb331f31ccff9238841eb\",\"0b104d01fc09521ed02abe2a613177be050cbae19714dbdc47bf877a5837cb58\",\"a37df63c73710b33c276474ebab602c798103e970a1e20865f2127cb2a915e74\",\"bf0cf67fc9e9a003748aed492a17c57b10a99c493c65e13e9e6322f4f4b19686\",\"501be1b4111d209ff7dfb998cfe22cbbf5d73333ecd89115195a5b86a0900fd5\",\"025fa4a1b8c7b09ccef13b4d169e08da9ed6fd4620597bf1ff358e9ed3e685a6\",\"6af499f66c50dcd51c074c8f8d9bb267b984833526ee56b460f4526d79d34046\",\"2df68d0980d048074ce3aa181696eb45249741a0695c39778d60ae046454b6e8\",\"290f1a5b08c6b061d746b0d5a3b3913e8c8986bba9d694699beb9426b7f4102b\",\"6e1ecaafc8c8b8c669d257cc147d818dd9bb461e048532b06c51c967f3ffe25e\",\"f1ecdcf57f4efb3085b41f617e2e664bc4bfde8ed3df26466ab717b468b5e4a0\",\"4c56bc259e78b801eb9b0a6e51f803c16ef9cfdfcba1477f191e9f182f8cb1fb\",\"2a2af89dde6e85360a20ebe422db63ec6d0b3b90c411b4a8bc92968807fb9d4b\",\"211673ed47fb96ecebd32719ccad3df6f46e73dc9826c97e3ee774ada51d84df\",\"1a2423f5f05875f840ec8a327ff2e1e9bfb998a144739a1851c967e85e7252a5\",\"5bc306e3d9da499598699d625211dfc0667263c54012924b2e2512addd79ba81\",\"8c00e44922af2705f4d8b09c6ddc6a71de4f7b5807f8baab648d9b21cda7db65\",\"1e109bf78759f63fd1234ec4dc98f776aad280a1d8779df5cc8e74a0cd45bbd2\",\"c92bd88ca45e87bf71f9ae2feac7a9117fa28ae8de2db03949deb4fd23580d51\",\"3bf9e58a86ae4dd422c784d63b85afdaa4a3d19f40b00e99574714d6b5b59581\",\"d01d55c41269a8aa6e7bc3159dac80dab82d24401155fb7f286ef72956b27e5d\",\"9655bce718528327f5026a03e94a782101ccda19ba9dd5ec6243c1ecb41a989d\",\"d942fe5fd81388b732d84c51eb41ec89dcb1aee1c19684ec2bca40874a6e0aea\",\"bd58796831bb302c893d807107e4840c3490de63710fc5ddf7f373ec0a0667ec\",\"94e398653d3a3f09e09a43523292452f4eabb230c654fefb31fb40e0504b9fdb\",\"e7f8e987b353af1ab41b65fc1bb8deb6f2b10680446b1092f627c99cffdb52d8\",\"c3ca239d1ef7d2c4e8e57a0bdc63cd8f09219c8e153a5e636cbb633e6cb88391\",\"49db1336e03775a9ee7b6b01dd3142d7a5c3b7ca1ea4bc2ca3a6e037623e4ce9\",\"9153663fc2a72a03e9cf175397945a04aaccaf91db207713acd4d544920c3f38\",\"aae136f98d2a20753b111c4337dd8e02e96650f88291a51cc05f4b6f54e48926\",\"8243afa9a5337725ef322a5a90d51cc3288d01f5e0594fa865f8c8442a8da390\",\"2b031c3bb91634851ca1a412a2086487a6ddb8881235f2441aa8b0e22050b0e0\",\"c9d395d7439c3533f1cd2746dbf79af75756eb8497a617aadb33ff393ca7ca46\",\"c567e4e0226903cf15e06dc6aefd6a837c1ce0ce5ef99354107a4866883a86b4\",\"b6b12d1c48a7ae7021e46058e403735a46d0c04aa683170de1a153b17b600fbf\",\"6922690e9d4e971cad7f79430b326b8168ec9474ef7bf4288289a8ef5d7d8277\",\"74144321dec17a5f3893cc0571d86da3db4036e3e31716c396d84fbf492a2c35\",\"f693884d20612991ad959baf2446d929b6592ec459888418c55abe3d33fe817d\",\"ae87331fd0cf12890e703531047d95ca04f74bcfc0129aa522b7836ec4c0fa51\",\"06cefb67880d4c34434756cae4b96c963791b5efd8203044f6523c7fd754588e\",\"a090e7a08765f18092f0183d0a1bb6d2329f34efd23d869f62b65821d0e72e4b\",\"eaa22ae4a7fc338ca2f81878fdb3fd9f050e13ffbb63ea3e786a82a820bc7436\",\"70af932e0bdc45f622d580770fff4f8bd2afd49d9f1eb3940c0e29e4994f44f2\",\"ff9ce8cd072d3990a74586d09dad47145f576729dd52ef02c0fa9a981fa8aea5\",\"899173ad8d9a33bfc6a090c6eadcd71dceb296b7c7b6f21756b3c40b65b04044\",\"4bb4d2e04c37f1e9f5fbe76f28a05e1b2372c224593af53f9c1de60e73bda380\",\"6f5f314b0cda0d3f800bd0b0b30361349896a5bd158eb223dce4ad754e5fb395\",\"31b4f58977e37420919158a9c6a85b207c6e534d8cfb92d19452e8d3daba8fa0\",\"51f97cabfd2464f5cb994b68cc177eefddf608a197ed3283c4bc2ec82bd67929\",\"3b919137a3d8dc0ea835ee8ea2dc1dc751e271edc85d51b9d6b4cbffc6cbd5c8\",\"6c8086810c45fd5abe67cdc10bd09bdb4eafe2748e87b78c0b449429983607f9\",\"0068a6d75412bb4da8b7d99e7fcac5b5160a10eecafa2bad3ad98e0631fc61a5\",\"bd3c86b5340dbcd6610b300e52a1db4e2bd11d12e92d165567296c821a2078b1\",\"dec9863b9b55f94235d158980227403a17b5d738cfdc0d537374f62c835c608d\",\"810e1ed6e4621ff43474776c3f8e36d738212c13d495a89fd9bf9c3160d097f6\",\"28c5dee3c2955709bed2ef740037ee8a2c4eb994ae275228ae10dab6cac8a7de\",\"230fe67bdae85a7efcd3ea9e35814fdda47bb0dbc13b8fab48702a79730acb07\",\"0015aeddd0d5263f86f36c36f9b95cb976367e86740d2723ec26b4b315710712\",\"a6991009a7904515dd3d9394e07c36e61a2a4ae3223a087fe289ff8ac17d70fe\",\"f355933791973ddcee26d0e871caaf5f9a372aea3989868acd2eb8fff4b6497f\",\"b0dda4a375e6dcf31acaa7d4029fab6b6aab05236680d8612a2686c058b31fe6\",\"0635f5d3224bbc7f9e5236fcc5513cfaf37fc3771fd731244fa0fd7b8c0a7cb8\",\"77665b4aebc69c21b14f74ef69a15296ad509898ad38ae5296298ef1b198c77f\",\"f7cefce1b461d0157b264a4afd4ef3c110431393cd2c1b3b501701a46412de83\",\"aba684b2f6996f64d65471b09b40ffd7515627fef33d0f5891aac5a814b6f0d6\",\"90603ae988c72ae2710a836b0cb6cad8d037dc032ddbc7da0353e76f7fb1edff\",\"e0f96739cdfbfde918d79760f645077cd257d41b947376c1a2989cc68e1adedd\",\"7d7be7ec5fe76550240d0b711b8c2522ccd7598f9073d48fb61933a9974bc644\",\"a22fb675f8dc46acc8575b05bce6fae637f4ce2abbeb67b9a6a765b429046ec9\",\"e9f63275ea80af6e745b81454219cb6cc3cf7df3325496b3ef475661d082f869\",\"fd41ada2e5cd8af8a0190f4e8717ad88c11f131d22a60a598ee78ad55e3048c1\",\"ed771fd8bcf4bab1889bd6a05498cc4bb539d1262f9249f60f4ec96dfa27d3a9\",\"35e2d16636f2fc7b20c6cb2cb29f34e2d82e2ae68aeedae09f671ee03f0f4e59\",\"ad481c152c7edc2700332a8e6009bcd96e98ca0571ccb02e3f261939455b0aae\",\"3daa624a59d0f0847db3388cb1df5740b0ac235df323fa36b2c2f66482bb59a5\",\"339a509136590d4409e81ce86f656e55ea2b586ab01b1c236a92073423299468\",\"c5b3721d0fc671e1b86dac22f3aaea7e5f16ee568f43f1cdeb9b0366aa5226fc\",\"55cbbd6d9e9349eb4e311284faa0bd41f05f638f616fab5e0e9b48d376874faf\",\"4ddabdeff7813fb323d99dd1f67c12c74f046285fbb7992ddc77fd91d54f5d64\",\"ad04830dc71ea6bc6efb6a4bcf9d0b316a8d0054867b43341b0df59753ec206a\",\"2312b53f8ead0f9d006be12e4d55dfe20bb92fb01b713e926d82e9a4643fdb09\",\"ca1b805d176f9d963854bdaf66020e3153606f6b379e0df9b85538277d26242f\",\"d9acb2554409410b7f8c8a7f7b25d0c5c5d2a86d42ea20d57f5534907f10c1d4\",\"33346a0c7746e578245ace0e397bff896bc656d79661e008446f59a666ed83d2\",\"1e0adce71c9ef97c7959de20844f66d2768bdd95b52beea929abd4e74c1f60ab\",\"d383cc992494f8c4994f7ba9eac8c96f8dad6be4de659a5e69cc306a52a18055\",\"f44cd4ddfb6322c3e111855bd9b73aa0af9f349c0a2976783b21e5d92ea58c89\",\"ea164efdd02f4683eac079d1e47f618625ac04cc2854329999986558305aae4a\",\"b4a87d63b5102ab1dca08c701dd47b8885b39eef1f3f37e423c5e2967fc2302b\",\"8751ae74f5b0a7336750c460e2bf59d7737ac2bbd21c8815894c0c5e850f073a\",\"d9298818ade32bc42c29e52da904918e78d827641c11bff33a78087426d4a700\",\"90ac05bbb5c082d0605aad21c183cd16bc4f29eee3f7968bbee34180370c40ae\",\"3656f9353a0702d037db7a4cfb15501468c1d68ba2c8697e4f0bb4f3bb768ccd\",\"afc32ce6dea00ce438c713158b7f70d881ed5717bf377e75925f8f31d7a927bb\",\"7e3970ee1aed3b095331e8c5d0cc13bb342613377ba1f6aed4d30f0976d2fe6d\",\"fceca5bef21c7bd95ad5b7de1f1bfa059031b89f51f560b7fe4b6ecfc9630a2a\",\"3a1ee9af28ddbfe9c44ac5b11e02ea11d6dada14f0325b16a423316ef5fd5e3b\",\"8ef2e5469e5748bc8a0c51994d51c16cf5b72dc8158c1f7c6706e8ec50dc3337\",\"cf20fe76bc9999aff0d1a52d72f1335c57e56359fc2d328b1af7f6846e493864\",\"35b34128ff87926e19cdb445dbfdcc572b51e5b319c0b8551f026dc764d3fc12\",\"94460d0e9cf43a0f6ab2cbbbb919c5b28c9c6b7a8908d2fcaf998867978cd2b3\",\"f5f4626f8d0d13e9f0a120aba4be53fa5a9a7d45ffaa2597a6ce1a1743985f09\",\"f258ab071d6181b4b9ed97f0620e095724a360a4de66fd85380ce22a5d306bc5\",\"441784d1390117b81d9e909d37098d94ed26a4f4599692a89ae199685ff34278\",\"f35238a85d63716388bbcac61dd4362c9aef002e8fab34a8bee744ae486d291b\",\"6be9e0334bd231103ad460b187d9f993f0bb8fc3b8be107ee6c97f708dd4fa4c\",\"fbe54075f697890a2230bf5070a2c18a36779830f52c77a717a41788b72872f5\",\"b75799f358e94d2b6dd8d15fe1f88bdaf4ea971cbc68b0d86f718bb9c2a0b77a\",\"6d0c69fc3e0e459b0d5d7ea99a7404577d2c7d4efbfc67f76ff2159a505ca762\",\"7137a780c579ed71e7f8d856b84ffbdfa52042c79475200d8d76aba237f7c642\",\"9785e2711fe07c12b0cead4b401839165e3644ebfaaac600b91ddd6da2d4525c\",\"3b0c234a17cea5e604bd7e56a775abec8a4169ab524e10903c88adb23e5f747d\",\"3447ce6ca3d7f69cb1aabde45292f80b9915b3bc8f6e80e37069b2f8ca928686\",\"f415498ae8d0af4067ae00d5514b517edeec55a312e2e357cec4e9f3bd0c63e2\",\"b19ba045189688001b75abe165f4bedead96696f8dd86ce56015d5d5cac24fc9\",\"4b93614d192f93421394963abf9618c9b70bfcafe46dc2c9995e5c655aa42c12\",\"df29da0f93c72163329c78401f0ce989c34e81232b5fde4e803e7d4cee0e29a2\",\"a5079add6e3ddfd71186ef45020b5fa2936d6f83ccb52e2a896716a75a5dcf17\",\"7db8d8b034848f6d6f27f9ac27250ba4ff1a8ca6197428b95cfba0bd6f85cb94\",\"281016846c43656cf0f8ef2f539eaeb7fcd8c84da25425ac156cd1fe8b3c8c03\",\"8a6217c592e0e1e69f736029f2ed4a839a9701bb9c0d1a9eab639ad5496b1e00\",\"89ead7aa0316efa7711cd53257d8344101bb71e25568fadceee1943037aa5773\",\"d4eb05ef139ad7432d6dc15f0b457e75bcc5bef86f10e3a51a7b5950b2c4672d\",\"dbcfe361c5f90a321da96f80a04655f004d3dfc007dcdca16bbd5a456066df13\",\"b19c9c62bbfeba937a055efd549b424cfe86ccf0ba695bec816972f951d60d94\",\"bcd01d3f0b9b78281ba55c2ea1d82e3aa2ecb4707e7bed6854d872a29bd2a8c9\",\"7320a257631ab772fba3fd924890889bc3a7572bdb9a32cf295117005dc0f173\",\"8b6c8b5affe7f82c64fb08dcb92effefca133cca0b92edb853796719bfd59d88\",\"4b81860b298d4537bac5fb30ba1ae0862d9e6adcf09673dcd4013ae4972ea8a6\",\"0c40b903537e4aaaf73c148694c161c2a254fc155b707cfe0f0b160e484df2bb\",\"869bb7f6333311a6e7ba32380dea3b7698340907383922022a9f0e8b94ab76cb\",\"b234a50ce31c2efe70cae343e85200a5b77e7ee49d0bfc1ab55cf66010920805\",\"e0633048703da3ff73ec0521f7376864f2e9d26305d41c78c5fafdc30bc2652d\",\"556017ea38458b31e78ffb9d81e3c8e9ba7624eef2fa877a6db425ef2e1f535a\",\"452548723b22327d20aa832a14c843f0918564c96ce76e7df46d3ab974a44878\",\"62653ae70ddd980c225543d49a0bc58bd6d9c9cc6f090bf9f725b3c812657cc5\",\"62b5a2b4eb421ee3741ca7576be63f0818ae60dda45dd4c1116d5489e47d8680\",\"34bc1caceb8f52d80a71187f1f84a5350b24302e0594d06d4826556b942617fa\",\"84f21e61221a0a221dbb878a8c1183f35fe4564b07bf4eec1b6b9f73d4aaeecc\",\"5dd33604049b3a3dd0d2ea01a51731217d6aa6145d03a958241968e6d087ccbe\",\"5d76f027b43170d55bd143d983c326e284dd35c168e7f2943ceb5f40c6f1d160\",\"7e7f2de29de53d5f6f82a6952dfef0ddb9b71b7a8fb2b72538626c1b8aff8066\",\"23a57c6070a3e40d09a9783997d89071390d956e43e7f922d53c9b974d12c670\",\"2f76715bef5a7bd9e13c200938cf8c7114d3097e55b473e525ff686a8ade4ae0\",\"4d5cc16e99428e791e70581918fdd9c3eda9b6ed56980ecb1940c61e5ed9e197\",\"5f886ab4d1b72920ba1a827c4d39a339e3e0eb9205ba82d416a60ce5bd4213b0\",\"80380daa1113fb805eaa6dff4823af5190709f20a8084a0330df508a11a33bad\",\"fd3fcaf4ddffd5bbadf8b0820cfee904e00dac328d0fb43667300dd84efe95ce\",\"b5ed0c50e95a91d18e008436a7ec1f61d034d8e4338d48aadd603ad014400f55\",\"1b273efc980dc9686740d82b44b1b92c23f63320f210a0505aa449c3cbc8b17a\",\"1f9253507992dbdba7553ddcdd861ae364ebd3d03eba68a55908fe95a5f8806b\",\"3ad16fcdbc1dd84cdd688dd5e4de46a3c2601dd1f2cd5a8a76655dd1c33879ae\",\"7b0cf664861ebb0288a844f5cb774efc8b569dfc4347b2a5e9b8bb781956c86e\",\"c074e4f71d4049700131ae27acf261ae310451cc55da366ea4de28d9790de254\",\"93ed33b8f29907c04bfd02f731e09ca12855aa13c7077068c53289972016d019\",\"70f3d0a4bcc83cd4337a7645b16e6a0b2e8481cf5cd9d17548c349812790adec\",\"00ddf26a7bf1f2109db134b506572c657c25a7de3f6cfc9844c9132fc02f38e2\",\"36ec7ffd6a9d0e9998dd68335a923774e90cf4f9765b0b660ed24898426e647d\",\"1ff7395718579dc16644f237dc2ff049dc7ebe8d735d8d7a8b23fa86dd7f9140\",\"77219e8a92e9625ac2df23fb223b10651673db990fc186063746b8535cda63c5\",\"d4f4d377ff28bffced39c46188290a619d44c1a8ac2e21c634394be419fecc0d\",\"e7926d92061ec2cf2db9e36f4c8ff4ab5c4f2a4923ced53d319a0ad6164926ac\",\"90a7dce85d8924732fa912c7560b129a04ee058d91c0dfb65310a74f625b55a7\",\"5f237137ea2d0430dbdac138d47e5e6529ea2f3f8f604d05e09258bff1c3eb7f\",\"00e6893b901bb694ef4e9017c9687d06143e6a3ef067fbd2a10f4e51c4ec30a6\",\"44380730340062b993be5ea4506dcbf88f4ef45845719bb4b997e5c0463b7c8a\",\"4706ed1dcf0ba812010d1f810bb9e7bf8b77a427387bf3fa702030468fbfd9a4\",\"78486c45589ce75c5a23741b39b9fb41523f45cd0b4bcd12d4e6aa0df2798336\",\"8df42964a6ae2e1ed51fed32116321fc31e48fa22c1cb15dd7c96f8986d2053d\",\"d18c191c4d5e4bbace1338760fff3324d298d79a629cf704f7a3d4ec00e1913d\",\"eebe5778ac76cc92c44cbea8f3d21b3e4f3e26c3b07e5590c1139fc5a52081d4\",\"0ec9162e4e4e79e2aec1386c47ab405807ef12078b994952727ff9dd29b713de\",\"e909b5d0e160a46e840976d4c851fe5a4dc44ca539dace118f25dbe337903e41\",\"ebe18e933cf9d4c20a87451aeecbeed3bc294b95a5b2e9d14d8924e84c09184a\",\"dedce30bacaf2ee9fdb855b022d1c2274c1f0dcc640857c8345b678ce0bcf504\",\"ff6360508fcc99c2aacb55e8258b544704539177af928550ef1f40d58b6bf960\",\"f6ce060da78eb95aaa12067ad2d59be52aa192a77a88f160fc8c4770a4d80df0\",\"3bfe3be3b586a9dfa5e586189c865da608feff5dc0d7d2b310d8fb1316c5ab01\",\"2f30e5974cabeb6ad5937f2083dc7b2701fe53db1e253c3cd04d6d0397098b9e\",\"0a4b94935e1d2a498d0ecbc1904996fca9be3635c5cdbeb5e2415692dffca5d8\",\"f4ced452b8d2f04e52080281660f2eec90d65c2d772c5252d32c3cdcfbefc8a2\",\"dd5c54f8fe529943cfafd306b71cf5b62e8223e0ffdf7b3411414d8557f06fae\",\"18e97dfb44f196e20b9314497a9484bd7a940860ffdf9785e988b174dde0f543\",\"4403c39f1b82cd5ac257d041b01168b0e9d50c8470dd7a287e95fd79bb439342\",\"9e16fb2b1eed21027121d58de498a5ea27b7191fe2b1129c5132eea3261a6d11\",\"54ee1988c311755a10f8fb3003599ffa99f4c19ac35fb84d0ecf606a877aa1c4\",\"1bea34d16ba650764a17cc0408d10282701e10b0f4cd8354adba28e9abdccdcf\",\"1b253fa094747ff03106c575641537ed8399e7371d3fdd5a8d951c7964aed251\",\"5dec8c24c2916b1624b801b7af8b6d01fe4877fef1151f7c24383303b605076c\",\"57cbf4cbf31e138fa30f897d6b000e33cb1f5caecbcaaead47db0ae0b09dc1c7\",\"3aadd9f469e1851b7f76dc9a7a9a9829243c53505b0e61f7c9625047cce4f9a5\",\"7ad41c50f0324c599c4bbd5243ed353a17148baf004052a12d89287e4f966f77\",\"e8872092f9165322d4360de4e8d9c777ce64760c6f95900dad5257f9a3840fd8\",\"f55ad96258820ffffb74b509aa33579eeb93bd93872d1472ab23532df18d16b1\",\"1b67ffaba9a74dd7b53728b8606b3f2f5dad434f5f128c831dc7012123b24227\",\"48ce28f83bef1193e79fb760f503a6008f4731eef96fbb6a08ce8ab292c9a7b6\",\"47fe79c9ea19e4ab5a53cd16917b9b78e1001c6bbab7f9e2682328e6f07acba5\",\"1fbf348350c52ca77438e79259d21180a6052be6f3b4f0b80673cced77dae9c2\",\"a387cf1571d4e3888c8a2dbf5402d0c21f13d9c934e2efa8a825895975037018\",\"b9b1bf5a8ae801b53651d05187a2a9ceea656f69016417ddf2a27dde95b247b8\",\"67a51455e056ebc29e13b7687cbad8d52e2238567e6e74411df5ec4d208ccdef\",\"dfb62d926ccfc3fffb6311d521dda2e7c220d978b34e07a0d371608ce6e03518\",\"5a95d87b9351665cbad0eb5a0e298ea1098f2e609e3a7254ff7ba2a951069f74\",\"7a3a2cdfe1f8690353960307c5b7533af22589700ff1895f0935f8f286676de0\",\"eddb8cec02ffced86cda280e5eeb9304ae0eb5b1b150f52977448e6aac2ccebf\",\"2a084210fa28185319057fcd82bd71d415652611f6d7eca60f811a32d500fca1\",\"ab94050fb60b6a1280bc56541763f1a1ec8319bd43278165506692885b7f5d95\",\"0f73c71dcc7643624979983f8e28ecd9c5813cbf99ca1880295322fccecc5de5\",\"72dbd50665bcdebe733015f311cc1f2e833583afa52c51c261915a477dce9aa0\",\"1e4267ba98841cc32cd44c37a9873e17abd1d9d0c44752a6f07ba1d8afedfd00\",\"5da0232f981f573ec9ef0e531d8d628cd86889335e24841850566bb0dcec74ad\",\"5829c9951e129f957c6a518c2f45923fc82ab20cf4972c70c9423f7aa66a7bc8\",\"44625bef01ba0ee760fc313c458e480716e814c5e7e59413b81a86f3eb821df5\",\"5339e00c9e9f1cda95ce2cf284b534715cd5184cb09a02b65f9890881155caa9\",\"8af1765a4a4ab5f7ffcc8a2206c6edd3bfdf37a1429f8b1fbaff1f486c39bdc9\",\"f3fd65babc81cf9c1c7690a8de71b1b3097a273ac8eaa0f4421f653ab4b40e48\",\"b4b06c0dabf572a84d279282199233c8e96d3a6bc5ed5f54bf2aea0ff0a0926c\",\"b2ccf5be36db8d312a62643c040b192fa60e016cd3d2643188415c496f7d5b1e\",\"93fc06fdb76378748255d217b8e24c5027abbaca6c599a73467efc9316ec8db4\",\"74c725a2dbae47772489b0a9b5007cbbd890d7e36eaa2410f2abcabc3a29dc80\",\"01cf22a78a24736e1ede24ed5d60718b95bdf95d469135fd2429efdf47f7494d\",\"e73db334f9cd898d0e97f4a51aaadbc18bc1b12516b94e81eb115f5549d0671e\",\"3519ae7a96aa766da4b2b9ecd00c3ee5f8268d83b16f2bc6cfc33c70626105a7\",\"c92225faf6d74689764f64527b34c0bf11da36d99cd324fb0e89aeaa6c05750d\",\"525d298844132f3fe6fadbf0b67223be3595510b2825f75d1dc3168e302a8711\",\"d42020c44576c4dd32b8adee5eb166b2b71ca3339baab7dc1518cbff282cb3f8\",\"cdcf97fef2be7839ee986cff6d6e3fa988273db912c05b62381fa2d29671ccaf\",\"a30bc681889a3484937eae1705a8eb4e2cea80608634c95bb7367cec68400f35\",\"ed5aa6c10b7c251515510cfe176474ae503700b25e0831c5d13c6944923b41af\",\"3bd454f98d27af4d1af8e6130932b6356eae86840cb33265ec4b92bf0edfb0ad\",\"4c0c0627aa0bc21510a7465007fba8c115cb76396f6ee643782fed89cb715731\",\"d94659f93822d21b689780ab931b9470915726fcffaa6d7e179132f6cdbc3ad4\",\"49fd3765f04b4041f58ff98596d8d5fe4085d9ff20d3188e030379c2e6a6c7c2\",\"f2c933e582c8a786b848971278313e16d3ed5968880e371d33b8a0f10ad67333\",\"a5d8598a01eac4d2c28d0a05d6d68db8d0a853c06ccc6ecb7085e8b74b719186\",\"a05cbd074a6246e9069c58d15ce63bdb1aa67f4d5cf28162a77526bb3d0b2571\",\"10febf49a230905c1bbf8553c14cdc8a67e686845378b277c1e0a99793aff143\",\"0d48a0604d5c422739a39ac0c0e09a58d24dc5adc59b85560a278c09f67a6c37\",\"f3c9b658f25f5e65bbf554e620a14358cca4ee572062e20214c8505a9aab45f4\",\"8676a69b38afe2105ecf2458cb9a5927ef547426f8506f534d6c41c7f991562f\",\"9bea26155a2e06b46547c3c0864a52421fd379e58889c1c4a63f6d31b8ecff59\",\"32c527937f824bcf1de54832be99436494dd079a6fb8333b2ae5271029951164\",\"0541c8d6a51dc1809c1e2e6e189244c9e9eb8b474c65bf1b84c972b043643df7\",\"e84726dc3a21d0ee543cf916b4b7c74d829e07a6c62a2c384a7de59c492f482d\",\"b8c7e1ecb0e7a71f7eacfa8054639ba7a5ba1df2417a9b0864ccb3dd2675a31f\",\"80c8b458d44e0d48a174f3c7912717390182ae9b649b5028c40786df54993fbc\",\"0930485b767d10d447730b924062e538db19a6c852cbc4ee74a66d499f1607e3\",\"71e99ac4693f1eeb8545d634b447cc0639d8d31e4d9061675363da133d3351d8\",\"4767ef2a48aa55b0478e0c7b638c92554382a8e8a2c3eda2bfa51517ec65f83a\",\"a3a95a3b542f730d3091fb9cf14356312d867af62f026182690e1c15c4d57d88\",\"ad80c438add4aca2604aa85a1e3dc8bae0f66192cb883a5de49efc19667aca01\",\"0f2b7c2f9719078112e83a44e642502332df92eab31a16752974e8ec118ad9dd\",\"d41eed01ab92da750b8f22782ee1efc9fa9272293c1a3a29beadff83f89547bb\",\"4a7db3a9b9571ece3b4ac9972a4455a5300408111fc542a54bc124df1d240594\",\"15545164547208d4712db2aa5227bd7238ec37f2221c3681c1cdefc2cc88c988\",\"59384094801b1fb016a87f02f74d6f52e33b8b4016cd7672b78bd1564199f32a\",\"64d8e229363ec76ba062ed7810771651d5e8c122ac8a12fac600fd6a35e17a17\",\"ce07359dfd4343dceaaf61af014037989426b00879b5fc0c9c68ebd76e2f77e0\",\"1bda6db5211ec0ffaa067627b426d5e8b53c65dc7d5263bdf93841cf19c3032c\",\"33f0b73d16353662e7079ed00baac9e1e7af1124031b15945068264894d7201a\",\"92a9bc935dedd274701db29e9f64565e85ebd0e520912d7518bd237ac6d14c05\",\"8529039a0b817ebe4ca620cb8142652ee61ab7446064fb40f52646cfbaa1be91\",\"30dea800e953a63b424c8c4986402421c5faad618d5d1d6bbf23299c86e5a222\",\"a00a77653ae5fdc02c2502fe41055e1ffefaf02d2ed3ad84915bbbd8670e3b6a\",\"7b9fc05792aca9c50bf5b298cfd6c5347738df14e82db41672660e530c7f5681\",\"7d45b24b7c02c592e818545f26c885ace7fff9d8e15ad741a1c3c1759f46e49c\",\"ef8776a4fa76faa7457834f86bcf5bfd6df6f0332009a429a90213edeeb84f3b\",\"eb0f010f02fa81fa305c98e95248930c8766d9e39ef189c8ac8c1ec69c2b08ca\",\"660d9cdcf31102a705afe5cc5ccf72d21bb11132c68339454e5320a5eb22cb65\",\"7e98438558538a2fdca06cf7ee3f08b9ec2b72382b0f8cc73859337a0525c8dd\",\"aeb1d13aaf8715f11e0b7b6e2472b85fd17df40967bc89f6331a4a8fa43ccf0d\",\"0a29295a0e8860a64637fad583e5209b4407e45ee4ba73232636119bd9940b81\",\"8fa964cecaf6f7ddcf3bf6dd672d7cfac426334a32b60a593dce5246bdf5f358\",\"cd428f66787bd54bb71b7800e406ea69d24ccf6c9919ae0e1d11b6fd0ed29be1\",\"32bfc4ccf0225e11eeb6680047d59455f5224961a4c389af164a1174bf665cd3\",\"9d29acecdab121608c3d6ecdf4ae99b22e9647cc41b29573b957f17b4538b410\",\"e0c45ae801a914f1126d952dfbec09b3eed20d57f2e943b7648e006965c31515\",\"869d42d5f0c54ac22550fe4776ead22bc9a95a7ae8dc785ff0660a8d60bcf0a5\",\"87ead910a2735a060a5cc2214f2ab8ba48b2407c27ebdcd1512f70a8ed16bfe1\",\"7ddd609b4d4d0db2258041b55fd4bd9b5eeb8a3b2323e3dfb0ec940a48fd63cf\",\"a682933160ecd588fcaab7e908ff402c48b8f3e08168ab20e47534f5cbf8638c\",\"3708e6cdfa1777f0ee30101cb75688fcfc9e58cdea6f5062a3ea4a72e69e2222\",\"5582aab02133ef5b67d5766125dab1d935e0a2215010e9f23bc07f4ea87aabae\",\"f3cb401b85b31db35b245bc4fdd665055c04a3f00cf10143c282c8098285c4b9\",\"eab8709df9cef47d06c677d1afddb022d1c928d9e27c56ebc03ee9e3a9aa93db\",\"a41576c3f4ae5d85fcc7936f065dd6937941c474c6b862a6ce78992ace35a0bb\",\"c41e48e8fa2684e6b57781187234cb8612a1844f43426efec95e5122507761e5\",\"6466ba2a4b9e0627733f3318410887b12b66797c85c1e9c24304e0ba80313207\",\"820c3657e9821515687a9f8fdeb203179430fafb04e8c988a71184580f4469a0\",\"d5524949ef7345a8ec91c6edae49de2f41533851b63334cda6178f66b634321f\",\"dbda1e4c48a650b39e2eed0bfedfd39e94b44b02f29d51b09d6a74ca59890d8d\",\"fe867cba598ed2bd2f399e36ca991677cf9a7fbc9877f07e23b8c425e1b0b1d1\",\"3e3136a51dd6a3b4a88254c30faa31a99194dafd1b7e98d160a62760b2fd5c79\",\"b08f8bc2ec3a887e4752b660579c72e8d8fcf4f7ec25ec0af0e9707df6165438\",\"dfe694c69016844c06b042d3e6cb0e05712ffd9e5dbdd4657497059fb47cc213\",\"1e9727f07276330f6bfe8e374bdf1ab9c2872feb34b26641b1cc2b141d00b1be\",\"b99363912fdd84a5092a3faeb4b7add9dbb91717b7d64da3262613aed10bba98\",\"17aeadebe68fc85f80e09cc883d56dfbf10606428998d6f5cdab02935c90a7e1\",\"e1a78e4994769300421496cba20b552246567bd82073e1816c1ffb4e485b11dd\",\"feb5352781ea58e286d598696dc3987eb7cafb0a5e4997a76a25ae89cdc750da\",\"9e26b3ecb08cc053f4e68d3675498934a02b60ff14ba7d5a1dc85fdf5e74360b\",\"f58c7f58e62b3529a835a2e754b9b9def798331af40292586e1c7d9b5ddd802c\",\"47b68a561024718bd8724cb2c3ab9a18410b0e49253974ab2380eb7de7dda17f\",\"cad2531988dd01acbe523626b25965670229f92458a08531ce118d3cd71e8f9e\",\"cb458791f8f4bc233b36e270646d4cbbae13d9acee8563e7e7679ff8dfc81c51\",\"cd6952181b948f8b40db83cba5549f71d79e817a3c93f4057b10808b628fde42\",\"b7161dc1c969fea9c735cd3684b2bfc6e62e80730271dde11193630911d0462b\",\"a8ebf91702489f202b65c470c4de44beaf4338fd2e3dce59b5a7c0eee8f000d9\",\"c8dd80d7aade29c3b48a2c97c44fb31c69413bac8b67d7e872af8e2d85086f25\",\"9fb3cdd6c900f03122505255d07f42e4400d65ba9e47d940e09b8b91debf0f9a\",\"7029697e3696a00886483236c22b3f665d572eabe6e4ff6d2bb5cdc6632e2480\",\"a912b9b5bd6e7785c14008e1b1d78d2b31045cca01a583ddb72dc5b5676daef4\",\"89273b25a4b8ff4328d0d838f1bd994c6bef6ae07c5b8b69de7211ab9d3328fe\",\"d414bcae0fcbc2219ca0131cd9304b2ac4e59dea4b7cbb28a6225ae9e25d7907\",\"4da8611d7b171eeb81b6e466c00e41e91f7a4df7dd85c11248c1f4c9ed9cdc29\",\"57edcd0e53e5bbe71b125d828f4a09dba683b8b77a1b67fb68718ed0387a2843\",\"e3ce1aceda23845823b6a7fb53e74c35f4209c4b37c5948e0caa60dbe339ca8e\",\"6a03dc4f9e94269b92f3d85b12d31ffbb09854dc7994df5ebb54dfcc630d070b\",\"c8be795ba9968dd155f4c6a045febd658b16017b07cdb396d53e8146dd741644\",\"eca2a3fd63b277dd67a2151ff3b5a535d9df656acbfdb0a26b52b4bed8349b41\",\"6024ff76bbe946efdd3b3cb6f274172db7389727678263cbd8c4c2cec37d0607\",\"fe39ca723077bec4febc1e6c0be034e2a30adc1c7bbb324bcf0cffd186831277\",\"2bd537e9cc1a40ff8e64c3942dac2f24c540aa8a50d3f55d970b3eaad2aa6dac\",\"d70099192dc58be6a0b12fb09261792057f0b816026ef1713d5d2d45f6bad3a4\",\"7311b0ea50a5e3bd06508cac6d79c0c89fefc0ab06a5ac934fdf0e249f1060d0\",\"6a1e2f0aeb16bea7f4ed84ef8c31f7f007a41163eb20cfd9fc048282a5c0d09a\",\"32f68d94d977b7ef8e6efb436591c1a32943204492a4d3ab764d33e7fbdd506b\",\"6ef574c3cb2c7dbf4ec5bb68e3bd25cec7f3c8a48bd355bfcd18a9a6a199de70\",\"e08062ca0e6a02c87c7a78be0028e21ca852e3dfaf36b7fd1e1af6db76080b9a\",\"de722c8ecdb319ed1cc3ae9c6f696cfe959f22138cb7ff7f534a2655e4d606fb\",\"1ec71b7eb469fb689be72dfcb92c014f8a39a343af0d90cebc7ccd066a85472f\",\"9fb9b101d8bd6fd9c3f6cb3c80866bcc234471efb70fdff461458a8b6c33bf75\",\"5b0305697989f66374894b62ac870a04a2085fa554785dc8faaf961edd0c24de\",\"69f88f1c4f252ec8896b0abccf8b4b97eb2461ba6748e76344e8ab8bfc2e155b\",\"515b23f4a54449a5100df98e7c40b1d1f7545b21b599320ddefb9c9520da62c7\",\"ac0220c35c8b8d6ca9eaefaf178425f702924300498f6f00690ddb46951eb1a3\",\"b34f415b30282353c6cbeeaee6661452760a15ed5d3af5f4e7c2cd0d7b866d8f\",\"666efa7a942a3406b7d0970d25678dce81b17ff83f6a88f2aa589580eb985e56\",\"b6cbf804e68705a153b5a382da532cedaaa790030e07d8104403a8e110b128d7\",\"137784c947d71e8ecfb77490419de762bcc3c6e464edad137b4144c23260c719\",\"6857ae5fa20a954f03cb451a4136ba0e418f8b755cae5be35fae1ef12e911cb3\",\"2bc485b949a639dcb8a6c5d479a146cff87d8e482599d2b1effcf7421df1dd3e\",\"275441099e30ba6a0f75ab2a870772b3cf797083683234b5e4453232ea2e8a48\",\"7fa85681b601ed787a325a1820c38a30d99d0420cf64333643727680b35a6042\",\"16e8113578a01b507b99895a445ee02f5701004fd59959de32f18747401a789a\",\"7f1d914eb82940f3301f217e593b636533e2df2843b917a52831a920a2c4ff73\",\"e63b3a1d02ecb98be9de7e30f3abf258333d88913a96b340d9173c097019ed8a\",\"20c8edf1583d17374f8aa86ab20a82625f062ebae09cba16bc7ac0c452a332ab\",\"ce9ab28a88962c27ccd72e8f11675962531cd785487215131953f08b1e3322b5\",\"d22db577dcbac4237d38274bce07c9b9d2a3f8e0a735f4c426891a9d0d5c9437\",\"ce868d3cb2e930f1c2239cb8c0749d9583427fe1a647ca7dd69deb226631a7c7\",\"8517ef52df474307c10d2526b96dfc2cde284fdcfefb8016bb95f3678f890cd8\",\"33027a93c39ac7dc7ef9bff286efa001860c4cb3f3750ee0e30ac8caae991e10\",\"887252e1cac8b75d01d34ce65c04cd095011a6a2c2bea0d10de8b4d0a359d0a7\",\"f9b315d7f28ebd35343202001d1f0bcaba7cf00db7bdd2d88878a7351b551165\",\"6ddd16465fc1716cc6514c9614ba933ad01c7b138e4ffa604f05b1f4419f02fd\",\"af90eb408b78948d4644dd76957a008efa20f666617311ee7377d8715fcd7cc9\",\"8491fc36bf00591b47636bf5c01e60a6c0644ff2f10af51986b578dc651d55d9\",\"5c2aabccda0746d962d584fdbd056bda5153cffb87fcd6b2c9b32cc24925655d\",\"73e1e708f29c3c9ce2cb2c3473c574fd02588338fa26083376175324a09c1175\",\"06b9af21f6a0596b504801c1e5a0aef4c2a482b068774503bf682a835780b240\",\"bc58c322e14104be2365b4a4107aaead945296c0988e4fad17adbc870ac8f9d2\",\"d4d00f9f1501a87751d079d7203e0ff2d684897db80c87a7fca623a61c6c0da3\",\"35cbfb5d4d66dddd10b7c357de0a05a73e199c1a174e1c02ab42ce8f3d88495e\",\"442acfbf10e28571d65956eaf9f340f596f9dc3c6e7e1a6ccf7f083e590de209\",\"109d914fb3a913b8e00ea295bb9f2ed1e1b5514eb466025c109e4744327db181\",\"18017073e8dbb58b9ec585040c2f0e26577e03ec1985e614270e78d15eaceefe\",\"08ab2298f80fd4b3c29ca4dbad56cb8a5d60ac869953c0308f768550f9ce8887\",\"f708576cece2ebf41ca3b368c63f6fe81ef0d14147c9b0f13578ee93ba651bd3\",\"1a7ed2ddbe5ab1f11670b3d5f75f690e4d09e238651593bd3e9944726f71e51f\",\"9f689e734d6a6418a1d7f9738f8f99f246124d1d649c72e992aba529abc8e46e\",\"b01b79ee96300213d164755150ee4378e44bb8c6b907eff1297c6193097d59dc\",\"05f43e8796b1daa3be1eed3cb6d3d6335a487f44a5df67f4a89be667b651b95d\",\"96589ac82d0b7103a1c7a9986a753cc74640415fceeb27be0827c58319f34a23\",\"418d74293123c43c803a78b0878f43f569fa29eb13266c9be4d48613532d21fe\",\"eb914c85c01c36a3a482c2b078ac5f3ce09ec37b70c77c869969e0bb595d6671\",\"1514729bf88b61b50943dc5bcab2d51ec473d1d5d012882102db8710c859c76d\",\"d047161e033815d26aae4e3324c9e4b736ca091c95bf585143c439b31ae9346f\",\"ee79de045984256fae4ffb4b6694a850cadf56924c8b5c3a4082cbfa7991397b\",\"9af416674b1af1ce341794c09444a8ccb14909d45974a31404555983e42781c2\",\"129f7cb856cec277c9cd33dae13cb01418f782c1a68556489b1b83464cb256ca\",\"9a061053011f5c3f0dd002634ffd1699d322319c71cffd8c9aafebabad74cc66\",\"12676afb27fd721d1c658790c1f9d7726ab67056d9f7f25d6ba0b3b341de0960\",\"a415164f4f02e851b8341f0cfdc584412aa8e647d9d5485e33b4848a07af0e60\",\"cf4dcb80c39b21d99a67f9a24cc2d312298c4b8e479214b59130320370065ca1\",\"70a08640a67e9a824271dd91b5071330c96d13ecab9510eaceb30fbbec614b87\",\"dbe4beb5ddf3e3eb9f93178db649960cb61fce9400e95068f15bb1468cd7d842\",\"4857f4bfe9e4cbc7f706a878b05d5146627086c091774c58c992d85bbee9e56c\",\"ca92e12e201132f6aeb8c0b7af46d973be887c60a241e5afb70cdff95a851f94\",\"b259e7419f068f21ce2701cbdc574e9623906dda96ed4c38b5e718ffa1d45f36\",\"b307e3c5ef19527053a25a76a65bee2ab91a5ec04db45d648a4315c83066af41\",\"4babe6da0defb0f18a78961c58991f679396471b081fc33be6a444462ddef86f\",\"dfa0424b4a263005fc04542de0dfa4c13a2962dd7217c702ed4ce2c22df20013\",\"8ed3db9b7ffd096380ee6ffc5ef492d9112c1ce75631b5b445c44fc2f1fd67bb\",\"5c9428b68b2c7d5ce2e5d248e6287e8f248bfdba1ea6c878d49526174366362e\",\"2e222bb974066f181163e6e4df51ca2caed1398bd94cd802897a34b2721aefab\",\"0f873febc8230be48577d4e01eff480a6314a59659bab95bd88b786ae2b18781\",\"df951ba0dcc10e2c93d2b6abb6e712d9b1a4b1eb6bc906dc18f085cf976239ca\",\"b2fd064134aee46b05036b6f40bb34bbfc3f7682448b8d4c3dbd8b682b73ab82\",\"c3c1db5cbb2cf73497cf46091ec80729a9fd4599c283c3d557a219583687d00b\",\"9362b95ddb6cd2a39f9f93dfaf4c2ab2eec5c19c90016c6a94663b70cba5d3ef\",\"d2b20a956922b5ce772ef2576318010d5fb1ef5a572a213a7e45d3b53232e3f9\",\"a3e8f54ec01de93fe55bc4adb3b41d9941d13ba538483873cb31fca1519925d2\",\"b70968a834aeede4ef0a9404f771e3e436cf43568683e43303b52e8ef622f6f2\",\"382c63b26b4de115fc39aa6d9037888a7c12564e8f00768704637363f97c1423\",\"83508b80162118edef27da1c50b7572eff64696dc7bd41c38e0e57b2b5e3a9ab\",\"7a9b764c9883d5c9fdfc71da1a2de49ae38bc098aa5fe23f6e50b5934594def7\",\"dbed06239fd314e8cf318eba2bde2aa748deb05282a4dc45796d9cdb21ce581c\",\"457ad13e58d921f84e60d20a816ad181c7c90359960f98c8cae91fb44bf52b5e\",\"5d25815482893aeea538b5338115baaab1f6cd2917136d4b93236dce47558e06\",\"22af2ab931d2b08b01b03d732a6c222fd2bf620e93e6ccd9f890bae9259271d4\",\"3d2c7da85f48ea39a818a5097920d8162bbc4b4d6fce1bbc28883b7601903fcf\",\"51504a31435cc4f29665508949797b0f96f51a71fa37afce68c4b69863692624\",\"4100c799907d6e20d3db19b9a4fefea089966858b50007beb350cb090555134b\",\"ec5b5766149d3b2fcba292f45eb88015a19b30cc60a6da0b7727e1587c375254\",\"8d270ab1377f6bafefe47838ab7405ba24801f456bdfcac95161b2deb3b33b39\",\"ea6802c1326d1bbb1b1789ea861bb5b2366c35b135f298c1115daa2238373720\",\"9437927f41836f885856f980b35e613fc6fa69d2f3543346081875e091442930\",\"2809c63215d3aa3c788a664010e3bac1c18c05daed8ca97633910faaeb8603fa\",\"4a4ad3e002ce86ac8a75898ae6de230c5ecb881f7e2b04e6778b232970fe6bd8\",\"f00aac29efc86b2b8a95e1f03618349d714a6b69e2b1c1eab49ea7b708fc6510\",\"014236f2ad24e4905dbea498262b276f28975b6ef9c2b4b0824c730321809c2d\",\"fb0640c42ccf05c109fd72e44650de2f523ccb306886e65bd49b9aaf187d2401\",\"186d18dbf1b89b3243e6dc688f03047a5fffb88eeed51f6e3109e66d5a6c27ea\",\"3e07300df75f102c05bcc5cf54770009c072702f38d7db5db9fd0b970690ebe9\",\"347eea5a4f7fd944f483eb704d27f169e45095dcf14767a927222040b1560f3d\",\"7ec6560908d4967c5078df6436e7cae3d7744866380b10e34f47fd9aa7ad331a\",\"4d7746d3529c2c4c7d71676bce276b11b7449cc5f41f695a9b2cd1cbb6bb3028\",\"16220ce4f8a477fd7ec6f61163de72f4907e439e32eeb3bf118ae1a40f510860\",\"6fb8cdad2b750463862423525e4160625ee25c251717f4d85062eea1173eabaa\",\"1d43e216b80fe9d19d5f0576e763114aba98ba4328a24299acf62af3bfc2e363\",\"5a453a490293dde316ed2de3d3a50ff91bf7238018ed9793499b079a52d98e73\",\"4405f63dbf5aa025d4defdd9c01ae0270783b5d52c66437e7925ac5bab24cb82\",\"be41f983d5b9cd54534e7ba88886188ceddd3b033544fabf5bffa5b4ddc5cc54\",\"11959e6c05019a6762c5b2a22d549dc0407f24d6d800f9bc026c31842fb7bb9a\",\"c86f666892ee8258420f46e8924d0b918e648ea01b0c489169f9446411cc6b46\",\"3116de3d4c3c1031066fb6d54f597b839a63238cddbddd3db34b4153d3392248\",\"2aca0c2a94129a89a4524e4b642f227a55733918b6512677d94e04daf9459aa1\",\"5ab8ee1e22d321dbb46875caa96ea4610db69f559f55f744f2ad5af49634a52a\",\"dd3aa7be1fa19e27c2d4235192d32c18112db25d41eb7d6f4316df5da248115a\",\"885d5dbd564d12cebc6bdbe163407f3c38544003d4bc6109acf2db4caf6f88dd\"]}", + "headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "yeEzxlYrYiAUNHKmb3vXDzElib2opAdAO_eLwqPuipnwFn41apyDzT4e3hvbZcOBr4RXNZdDw0Wf4G9BUWlrAw" + }, + "read_headers": { + "Codeaf-Identity": "F4BvPyD8vGkQ1ueSxfJ9ff_qTxJSr9-lJ9dCqGjth4M", + "Codeaf-Cert": "eyJWIjoxLCJkZXZpY2UiOiJlOWQyNDI4ZjVmYjdhNTc2NzM1NDliZWViMjZmNjQ0NTkwOTU0N2ZlZTA3YzgyODM0NzYwYjM1MTgxNWIxNTcwIiwiY3JlYXRlZCI6MTAwMCwic2lnIjoiNWI4ZmI5NTc3MTcxNDAzOGYzMjUzOGI2ZWJmYThiODkwYTYwYzg4NDRmMDBmZTg3MTQ5YTdjZjZlOTNlNjE2Y2JmNzI3MDEyZTczZDI1YTlhMGUzZjRiMDVkOWJiOTcwOWQwMjY4YTU3MzZhMTZjN2Y0NWI1NjA0ZWE3OGE2MDAifQ", + "Codeaf-Time": "1000000", + "Codeaf-Sig": "HncEhmglc5KEGe8G8CIRosuXkJiAX4U9ADWNgb9RkJawYa40Zx498xTpf5UonqRvqkcx74GSMP6V1OwUZOKZAQ" + }, + "body_sha256": "b240b845c9e2c5d9df5c51ac4c0887f6de7f36492c49cc0b84fb5efedfe57c25" + } + ] +} From e953ea0bac37a78cbdf329a406742a02363e0fe0 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 14:06:44 -0400 Subject: [PATCH 072/103] Commit older requests through native and personal leaders. The native ingress waits for durable staged completion and resolves large forwarded views from the exact applied result. Archived clients retain their signed bytes, safe refusals and complete saved metadata after recovery. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/raftdirectory/apply.go | 6 +- internal/raftdirectory/forward.go | 109 +++++++++-- internal/raftdirectory/native.go | 57 +++++- internal/raftdirectory/v1.go | 14 +- internal/raftdirectory/v1_stage_test.go | 171 ++++++++++++++++++ internal/relayserve/archived_client_test.go | 143 +++++++++++++++ .../relayserve/compatibility_refusal_test.go | 23 +++ .../testdata/preconditional/archive.json | 10 + .../testdata/preconditional/bridge.go.txt | 49 +++++ .../testdata/preconditional/client.go.txt | 85 +++++++++ .../testdata/preconditional/http.go.txt | 167 +++++++++++++++++ .../testdata/preconditional/main.go.txt | 99 ++++++++++ .../testdata/preconditional/records.go.txt | 157 ++++++++++++++++ .../testdata/preconditional/wire.go.txt | 92 ++++++++++ internal/relayserve/v1_large_test.go | 96 ++++++++++ internal/syncsetup/legacy_large_test.go | 105 +++++++++++ internal/syncsetup/member_forward.go | 3 +- 17 files changed, 1347 insertions(+), 39 deletions(-) create mode 100644 internal/raftdirectory/v1_stage_test.go create mode 100644 internal/relayserve/archived_client_test.go create mode 100644 internal/relayserve/compatibility_refusal_test.go create mode 100644 internal/relayserve/testdata/preconditional/archive.json create mode 100644 internal/relayserve/testdata/preconditional/bridge.go.txt create mode 100644 internal/relayserve/testdata/preconditional/client.go.txt create mode 100644 internal/relayserve/testdata/preconditional/http.go.txt create mode 100644 internal/relayserve/testdata/preconditional/main.go.txt create mode 100644 internal/relayserve/testdata/preconditional/records.go.txt create mode 100644 internal/relayserve/testdata/preconditional/wire.go.txt create mode 100644 internal/relayserve/v1_large_test.go create mode 100644 internal/syncsetup/legacy_large_test.go diff --git a/internal/raftdirectory/apply.go b/internal/raftdirectory/apply.go index ef9bac71f4..84573e3d5f 100644 --- a/internal/raftdirectory/apply.go +++ b/internal/raftdirectory/apply.go @@ -69,7 +69,9 @@ func Apply(ctx context.Context, store *raftstore.SQLite, b raftcore.ApplyBatch, if old != nil { size -= clusterstate.RecordBytes(w.RecordKey, old) } - size += clusterstate.RecordBytes(w.RecordKey, w.Data) + if !w.Delete { + size += clusterstate.RecordBytes(w.RecordKey, w.Data) + } overlay[w.RecordKey] = w.Data } update.LogicalTime = e.Now @@ -77,7 +79,7 @@ func Apply(ctx context.Context, store *raftstore.SQLite, b raftcore.ApplyBatch, results = append(results, e.Result) } for k, v := range overlay { - update.Writes = append(update.Writes, raftstore.Write{Key: raftstore.Key{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key}, Value: v}) + update.Writes = append(update.Writes, raftstore.Write{Key: raftstore.Key{Namespace: raftstore.Namespace(k.Namespace), Key: k.Key}, Value: v, Delete: v == nil}) } slices.SortFunc(update.Writes, func(a, b raftstore.Write) int { if v := cmp.Compare(a.Namespace, b.Namespace); v != 0 { diff --git a/internal/raftdirectory/forward.go b/internal/raftdirectory/forward.go index 7695920b1b..6172a9b76f 100644 --- a/internal/raftdirectory/forward.go +++ b/internal/raftdirectory/forward.go @@ -8,6 +8,7 @@ import ( "net/http" "github.com/Agent-Field/codeaf/internal/clusterctl" + "github.com/Agent-Field/codeaf/internal/clusterstate" "github.com/Agent-Field/codeaf/internal/directory" "github.com/Agent-Field/codeaf/internal/raftcore" "github.com/Agent-Field/codeaf/internal/raftstore" @@ -70,44 +71,116 @@ func (g *Native) leaderID() uint64 { } return g.status[i].Leader } -func (g *Native) forward(ctx context.Context, node uint64, work clusterctl.Forward) (clusterctl.Forward, error) { - var empty clusterctl.Forward + +type forwardCall struct { + id string + link *forwardLink + result chan clusterctl.Forward +} + +func (g *Native) beginForward(node uint64, work clusterctl.Forward) (forwardCall, error) { if work.ID == "" { g.serial++ work.ID = fmt.Sprintf("forward-%d", g.serial) } - empty.ID = work.ID - link := g.forwarders[node] - if link == nil || len(link.pending) >= rafttransport.MaxInflight { - return empty, directory.ErrUnreachable + call := forwardCall{id: work.ID, link: g.forwarders[node]} + if call.link == nil || len(call.link.pending) >= rafttransport.MaxInflight { + return call, directory.ErrUnreachable } - work.Scope = g.peers.view.Scope work.Generation = g.peers.view.Generation work.From = g.manifest.Config.Anchors[0] work.To = node work.Sign(func(b []byte) []byte { return ed25519.Sign(g.peers.keys[0], b) }) - result := make(chan clusterctl.Forward, 1) - link.pending[work.ID] = result - defer delete(link.pending, work.ID) + call.result = make(chan clusterctl.Forward, 1) + call.link.pending[work.ID] = call.result select { - case link.send <- work: + case call.link.send <- work: default: - return empty, directory.ErrUnreachable + delete(call.link.pending, call.id) + return call, directory.ErrUnreachable } + return call, nil +} +func (g *Native) waitForward(ctx context.Context, call forwardCall) (clusterctl.Forward, error) { + reply := clusterctl.Forward{ID: call.id} g.mu.Unlock() select { - case empty = <-result: + case reply = <-call.result: case <-ctx.Done(): } g.mu.Lock() - if ctx.Err() != nil { - return empty, directory.ErrUnreachable + delete(call.link.pending, call.id) + if ctx.Err() != nil || reply.ID != call.id || reply.Error != "" { + return reply, directory.ErrUnreachable + } + return reply, nil +} +func (g *Native) forward(ctx context.Context, node uint64, work clusterctl.Forward) (clusterctl.Forward, error) { + call, err := g.beginForward(node, work) + if err != nil { + return clusterctl.Forward{ID: call.id}, err } - if empty.ID != work.ID || empty.Error != "" { - return empty, directory.ErrUnreachable + return g.waitForward(ctx, call) +} + +// Body pages have no directory effect. Pipeline only the existing voter +// reservation, then wait for every durable reply before submitting the final +// mutation. No single proposal or transport envelope grows with the body. +func (g *Native) forwardV1Pages(ctx context.Context, commands []clusterstate.Command) (clusterstate.Result, error) { + calls := []forwardCall{} + defer func() { + for _, call := range calls { + delete(call.link.pending, call.id) + } + }() + if g.peers != nil { + g.peers.sleeping = false + g.peers.now = max(g.peers.now, commands[0].Now) + } + node := g.leaderID() + for _, c := range commands { + raw, err := clusterstate.EncodeCommand(c) + if err != nil { + return clusterstate.Result{}, err + } + call, err := g.beginForward(node, clusterctl.Forward{Action: "submit", Command: raw}) + if err != nil { + return clusterstate.Result{}, err + } + calls = append(calls, call) + } + var result clusterstate.Result + var applied uint64 + for _, call := range calls { + reply, err := g.waitForward(ctx, call) + if err != nil || reply.Result == nil { + return result, directory.ErrUnreachable + } + result = *reply.Result + if result.Code != clusterstate.OK { + return result, nil + } + applied = max(applied, reply.Applied) + } + if _, _, err := g.appliedReplica(ctx, applied); err != nil { + return result, err + } + if g.peers != nil { + if err := g.installPeers(ctx); err != nil { + return result, err + } + } + if g.Changed != nil { + index, app, err := g.appliedReplica(ctx, applied) + if err != nil { + return result, err + } + if err = g.Changed(g.stores[index], app); err != nil { + return result, err + } } - return empty, nil + return result, nil } // ForwardHTTP authorizes every frame against CURRENT committed enrollment. diff --git a/internal/raftdirectory/native.go b/internal/raftdirectory/native.go index 94554e9bc4..1e046b446e 100644 --- a/internal/raftdirectory/native.go +++ b/internal/raftdirectory/native.go @@ -17,6 +17,7 @@ import ( "github.com/Agent-Field/codeaf/internal/raftcore" "github.com/Agent-Field/codeaf/internal/raftsnapshot" "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/rafttransport" "go.etcd.io/raft/v3" pb "go.etcd.io/raft/v3/raftpb" "google.golang.org/protobuf/proto" @@ -39,6 +40,8 @@ type Native struct { forwarders map[uint64]*forwardLink pendingCommand []byte pendingAdmission uint64 + pendingAfter uint64 + pendingForward bool outcome *appliedCommand peers *nativePeers manifest clusterstate.LegacyManifest @@ -267,7 +270,7 @@ func (g *Native) drive(ctx context.Context, i int, input raftcore.Input) error { if entry.GetType() != pb.EntryNormal || len(entry.GetData()) == 0 { continue } - if len(g.pendingCommand) > 0 && bytes.Equal(entry.GetData(), g.pendingCommand) && entry.GetIndex() == g.pendingAdmission && g.pendingAdmission != 0 && g.outcome == nil { + if len(g.pendingCommand) > 0 && bytes.Equal(entry.GetData(), g.pendingCommand) && entry.GetIndex() > g.pendingAfter && (g.pendingForward || entry.GetIndex() == g.pendingAdmission && g.pendingAdmission != 0 && g.outcome == nil) { g.outcome = &appliedCommand{result: results[resultIndex], index: entry.GetIndex()} } resultIndex++ @@ -395,8 +398,27 @@ func (g *Native) submit(ctx context.Context, c clusterstate.Command) (clustersta if err != nil { return clusterstate.Result{}, err } + g.pendingCommand = raw + g.pendingAdmission = 0 + g.pendingForward = false + g.pendingAfter = 0 + for _, status := range g.status { + g.pendingAfter = max(g.pendingAfter, status.Commit) + } + g.outcome = nil + defer func() { + g.pendingCommand = nil + g.pendingAdmission = 0 + g.pendingAfter = 0 + g.pendingForward = false + g.outcome = nil + }() + for j := range g.results { + g.results[j] = nil + } i := g.leader() if i < 0 { + g.pendingForward = true node := g.leaderID() reply, e := g.forward(ctx, node, clusterctl.Forward{Action: "submit", Command: raw}) if e != nil { @@ -432,15 +454,14 @@ func (g *Native) submit(ctx context.Context, c clusterstate.Command) (clustersta return clusterstate.Result{}, e } } + if reply.Result.ViewIndex != 0 { + if reply.Result.ViewIndex != reply.Applied || g.outcome == nil || g.outcome.index != reply.Applied || g.outcome.result.Code != reply.Result.Code || g.outcome.result.Now != reply.Result.Now || g.outcome.result.Version != reply.Result.Version { + return clusterstate.Result{}, directory.ErrUnreachable + } + return g.outcome.result, nil + } return *reply.Result, nil } - g.pendingCommand = raw - g.pendingAdmission = 0 - g.outcome = nil - defer func() { g.pendingCommand = nil; g.pendingAdmission = 0; g.outcome = nil }() - for j := range g.results { - g.results[j] = nil - } if err = g.drive(ctx, i, raftcore.Propose{Command: raw}); err == nil { err = g.drain(ctx) } @@ -591,7 +612,25 @@ func (g *Native) CommitV1(ctx context.Context, c clusterstate.Command) (clusters defer cancel() g.mu.Lock() defer g.mu.Unlock() - return g.submit(ctx, c) + parts, err := clusterstate.V1Commands(c) + if err != nil { + return clusterstate.Result{}, err + } + var result clusterstate.Result + for offset := 0; offset < len(parts); { + end := offset + 1 + if len(parts) > 1 && parts[offset].Change.StageV1 != nil && !parts[offset].Change.StageV1.Commit && g.leader() < 0 && g.forwarders[g.leaderID()] != nil { + end = min(len(parts)-1, offset+rafttransport.MaxInflight/2) + result, err = g.forwardV1Pages(ctx, parts[offset:end]) + } else { + result, err = g.submit(ctx, parts[offset]) + } + if err != nil || result.Code != clusterstate.OK { + return result, err + } + offset = end + } + return result, nil } func (g *Native) ReadV1(ctx context.Context, c clusterstate.Command, committed bool, inspect func(context.Context, *raftstore.SQLite, raftstore.Application) error) error { g.opMu.Lock() diff --git a/internal/raftdirectory/v1.go b/internal/raftdirectory/v1.go index 9f94f844c4..323040533b 100644 --- a/internal/raftdirectory/v1.go +++ b/internal/raftdirectory/v1.go @@ -42,7 +42,10 @@ func (d *V1Directory) For(actor string) directory.Client { func (d *V1Directory) ForRequest(actor string, r *http.Request, raw []byte) directory.Client { p, err := reqsign.Capture(r, raw) b := &v1Backend{owner: d, actor: actor, err: err} - if err == nil { + if err == nil && len(raw) > clusterstate.MaxV1BodyBytes { + b.err = directory.ErrTooBig + } + if err == nil && b.err == nil { now := time.Now() if d.Now != nil { now = d.Now() @@ -51,14 +54,7 @@ func (d *V1Directory) ForRequest(actor string, r *http.Request, raw []byte) dire if b.err != nil { b.err = directory.ErrBadRequest } - if b.err == nil { - encoded, e := clusterstate.EncodeCommand(b.command) - if e != nil { - b.err = directory.ErrBadRequest - } else if len(encoded) > clusterstate.MaxCommandBytes { - b.err = directory.ErrTooBig - } - } + if b.err == nil && b.command.Actor != actor { b.err = directory.ErrUnauthorized } diff --git a/internal/raftdirectory/v1_stage_test.go b/internal/raftdirectory/v1_stage_test.go new file mode 100644 index 0000000000..2c30e47460 --- /dev/null +++ b/internal/raftdirectory/v1_stage_test.go @@ -0,0 +1,171 @@ +package raftdirectory_test + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/reqsign" +) + +// R25/R3/R27 require restart and abandoned staging to preserve the old directory +// atomically, with one bounded staging slot even across different requests. +func TestR25V1StagingCrashAndAbandonment(t *testing.T) { + raw, err := os.ReadFile("testdata/v1-large-requests.json") + must(t, err) + var f v1Fixtures + must(t, json.Unmarshal(raw, &f)) + one := f.Cases[1] + proof, err := reqsign.Capture(frozenRequest(t, one, "http://localhost"), []byte(one.Body)) + must(t, err) + c, err := cs.V1Command(cluster, f.Now, proof) + must(t, err) + parts, err := cs.V1Commands(c) + must(t, err) + if len(parts) < 3 { + t.Fatal("large request was not paged") + } + clock := func() time.Time { return time.UnixMilli(f.Now) } + path := t.TempDir() + binary, err := os.Executable() + must(t, err) + child := exec.Command(binary, "-test.run=^TestR25V1StagingCrashHelper$", "-test.timeout=15m") + child.Env = append(os.Environ(), "CODEAF_V1_STAGE_CUT="+path) + output, err := child.CombinedOutput() + if exit, ok := err.(*exec.ExitError); !ok || exit.ExitCode() != 86 { + t.Fatalf("mid-staging process cut: %v\n%s", err, output) + } + source, err := directory.OpenSQLite(filepath.Join(path, "source.db"), clock) + must(t, err) + defer source.Close() + manifest, err := source.InventoryLegacy(ctx) + must(t, err) + must(t, os.MkdirAll(filepath.Join(path, "members"), 0700)) + host, err := raftdirectory.OpenNative(ctx, filepath.Join(path, "members"), source, manifest) + must(t, err) + defer func() { + if host != nil { + must(t, host.Close()) + } + }() + inspect := func(wantCell bool) { + t.Helper() + must(t, host.Inspect(ctx, func(owned context.Context, s *raftstore.SQLite, app raftstore.Application) error { + rows, _, err := s.Read(owned, app.Applied, []raftstore.Key{{Namespace: raftstore.State, Key: cs.CellKey(one.Name).Key}}) + if err != nil { + return err + } + b := rows[raftstore.Key{Namespace: raftstore.State, Key: cs.CellKey(one.Name).Key}] + if (b != nil) != wantCell { + t.Fatal("staging changed directory visibility") + } + if wantCell { + var cell cs.Cell + var init cs.CellInit + must(t, cs.Decode(b, &cell)) + must(t, json.Unmarshal([]byte(one.Body), &init)) + if cell.Title != init.Title || cell.Head != init.Head || cell.Lease.Fence != 1 || !reflect.DeepEqual(cell.Frames, init.Frames) || !reflect.DeepEqual(cell.Keys, init.Keys) { + t.Fatal("committed data changed") + } + } + n, size := 0, 0 + after := "" + for { + staged, _, err := s.ScanPage(owned, app.Applied, raftstore.Outbox, after) + if err != nil { + return err + } + if len(staged) == 0 { + break + } + for _, row := range staged { + after = row.Key.Key + if strings.HasPrefix(row.Key.Key, "v1-stage/") { + n++ + size += len(row.Value) + } + } + } + if n > cs.V1StagePages+1 || size > cs.MaxV1BodyBytes+cs.MaxCommandBytes { + t.Fatalf("unbounded staging: %d rows, %d bytes", n, size) + } + if wantCell && n != 0 { + t.Fatal("committed staging was retained") + } + if !wantCell && app.VisibleVersion != manifest.Version { + t.Fatal("partial staging changed visible version") + } + return nil + })) + } + cut := len(parts) / 2 + inspect(false) + // Repeated abandoned prefixes must reuse the same fixed slot after restart. + for attempt := 0; attempt < 3; attempt++ { + for _, part := range parts[:cut] { + r, err := host.CommitV1(ctx, part) + must(t, err) + if r.Code != cs.OK { + t.Fatal(r.Code) + } + } + inspect(false) + } + for _, part := range parts[cut:] { + r, err := host.CommitV1(ctx, part) + must(t, err) + if r.Code != cs.OK { + t.Fatal(r.Code) + } + } + inspect(true) +} + +// R25/R3 terminate the process immediately after durable intermediate pages. +func TestR25V1StagingCrashHelper(t *testing.T) { + path := os.Getenv("CODEAF_V1_STAGE_CUT") + if path == "" { + t.Skip("process-cut helper") + } + raw, err := os.ReadFile("testdata/v1-large-requests.json") + must(t, err) + var f v1Fixtures + must(t, json.Unmarshal(raw, &f)) + one := f.Cases[1] + proof, err := reqsign.Capture(frozenRequest(t, one, "http://localhost"), []byte(one.Body)) + must(t, err) + c, err := cs.V1Command(cluster, f.Now, proof) + must(t, err) + parts, err := cs.V1Commands(c) + must(t, err) + clock := func() time.Time { return time.UnixMilli(f.Now) } + source, err := directory.OpenSQLite(filepath.Join(path, "source.db"), clock) + must(t, err) + root := key("root").Public().(ed25519.PublicKey) + cfg := cs.LegacyConfig{Cluster: cluster, Generation: 1, Identity: cs.IdentityID(root), SigningKey: root, EffectsKey: key("effects").Public().(ed25519.PublicKey), Anchors: []uint64{1, 2, 3}, Now: f.Now} + must(t, source.FreezeLegacy(ctx, cfg)) + manifest, err := source.InventoryLegacy(ctx) + must(t, err) + must(t, os.MkdirAll(filepath.Join(path, "members"), 0700)) + host, err := raftdirectory.OpenNative(ctx, filepath.Join(path, "members"), source, manifest) + must(t, err) + for _, part := range parts[:len(parts)/2] { + r, err := host.CommitV1(ctx, part) + must(t, err) + if r.Code != cs.OK { + t.Fatal(r.Code) + } + } + os.Exit(86) +} diff --git a/internal/relayserve/archived_client_test.go b/internal/relayserve/archived_client_test.go new file mode 100644 index 0000000000..9fe6788f31 --- /dev/null +++ b/internal/relayserve/archived_client_test.go @@ -0,0 +1,143 @@ +package relayserve_test + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/relayserve" +) + +// R25/R29 execute an archived pre-conditional HTTP client unchanged, with only +// its unrelated types supplied by a bridge. Its blind force must be refused, +// ordinary save/release/revoke must work, and large archived bytes must match. +func TestR25ArchivedPreconditionalClient(t *testing.T) { + data := filepath.Join("testdata", "preconditional") + raw, err := os.ReadFile(filepath.Join(data, "archive.json")) + if err != nil { + t.Fatal(err) + } + var manifest struct { + SHA map[string]string `json:"sha256"` + } + if json.Unmarshal(raw, &manifest) != nil { + t.Fatal("archive manifest") + } + dir := t.TempDir() + if err = os.Mkdir(filepath.Join(dir, "directory"), 0700); err != nil { + t.Fatal(err) + } + for name, want := range manifest.SHA { + b, err := os.ReadFile(filepath.Join(data, name+".txt")) + if err != nil { + t.Fatal(err) + } + hash := sha256.Sum256(b) + if hex.EncodeToString(hash[:]) != want { + t.Fatal("archived client changed") + } + if err = os.WriteFile(filepath.Join(dir, "directory", name), b, 0600); err != nil { + t.Fatal(err) + } + } + for _, one := range []struct{ src, dst string }{{"bridge.go.txt", "directory/bridge.go"}, {"main.go.txt", "main.go"}} { + b, err := os.ReadFile(filepath.Join(data, one.src)) + if err != nil { + t.Fatal(err) + } + if err = os.WriteFile(filepath.Join(dir, one.dst), b, 0600); err != nil { + t.Fatal(err) + } + } + root, err := filepath.Abs("../..") + if err != nil { + t.Fatal(err) + } + mod := "module github.com/Agent-Field/codeaf/internal/archiveprobe\n\ngo 1.26.5\n\nrequire github.com/Agent-Field/codeaf v0.0.0\n\nreplace github.com/Agent-Field/codeaf => " + root + "\n" + if err = os.WriteFile(filepath.Join(dir, "go.mod"), []byte(mod), 0600); err != nil { + t.Fatal(err) + } + binary := filepath.Join(dir, "client") + build := exec.Command("go", "build", "-mod=mod", "-p=1", "-o", binary, ".") + build.Dir = dir + build.Env = append(os.Environ(), "GOPROXY=off", "GOSUMDB=off", "GOWORK=off") + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("archived client build: %v\n%s", err, out) + } + largePath, err := filepath.Abs("../raftdirectory/testdata/v1-large-requests.json") + if err != nil { + t.Fatal(err) + } + raw, err = os.ReadFile(largePath) + if err != nil { + t.Fatal(err) + } + var f struct { + Now int64 + Cases []struct { + Name, Body string + Headers map[string]string + } + } + if json.Unmarshal(raw, &f) != nil { + t.Fatal("large archive") + } + var mu sync.Mutex + matched := map[string]bool{} + blind := 0 + svc := relayserve.New(relayserve.Config{Store: t.TempDir(), Now: func() time.Time { return time.UnixMilli(f.Now) }}) + defer svc.Close() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + return + } + r.Body = io.NopCloser(bytes.NewReader(body)) + mu.Lock() + defer mu.Unlock() + if strings.HasSuffix(r.URL.Path, "/acquire") { + blind++ + if string(body) != `{"force":true}` { + t.Errorf("archived blind body changed: %s", body) + } + } + for _, c := range f.Cases { + if r.Method == "POST" && r.URL.Path == "/v1/dir/cells/"+c.Name { + if string(body) != c.Body { + t.Errorf("archived client did not emit frozen %s bytes", c.Name) + } + for k, v := range c.Headers { + if r.Header.Get(k) != v { + t.Errorf("archived %s proof changed: %s", c.Name, k) + } + } + matched[c.Name] = true + } + } + svc.Handler.ServeHTTP(w, r) + })) + defer srv.Close() + smallPath, err := filepath.Abs("../raftdirectory/testdata/v1-requests.json") + if err != nil { + t.Fatal(err) + } + run := exec.Command(binary, srv.URL, smallPath, largePath) + if out, err := run.CombinedOutput(); err != nil { + t.Fatalf("archived client: %v\n%s", err, out) + } + if len(matched) != len(f.Cases) || blind != 2 { + t.Fatal("archived contract calls did not all run") + } +} diff --git a/internal/relayserve/compatibility_refusal_test.go b/internal/relayserve/compatibility_refusal_test.go new file mode 100644 index 0000000000..d1483cabc8 --- /dev/null +++ b/internal/relayserve/compatibility_refusal_test.go @@ -0,0 +1,23 @@ +package relayserve_test + +import ( + "testing" + + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" +) + +// R25 pins compatibility sentences separately from wire codes and presentation. +func TestR25CompatibilityRefusalSentences(t *testing.T) { + for _, one := range []struct{ got, want string }{ + {cs.EnrollmentMessage(cs.UpdateRequired), "this codeaf build must be updated before it can join this cluster; your saved chats are still here"}, + {cs.LeadershipUnavailable, "this cluster cannot coordinate right now; reconnect enough of its voting computers and try again; your saved chats are still here"}, + {directory.ErrConditionalTake.Error(), "the sync service must be updated before chats can move; codeaf's operators update the hosted service, or run this build of codeaf relay for your own service"}, + {clusterctl.CommittedRefusal{Code: cs.Conflict, Sequence: 1}.Error(), "the leadership request could not complete; refresh cluster status and try again"}, + } { + if one.got != one.want { + t.Fatalf("compatibility refusal changed: %q", one.got) + } + } +} diff --git a/internal/relayserve/testdata/preconditional/archive.json b/internal/relayserve/testdata/preconditional/archive.json new file mode 100644 index 0000000000..553772404f --- /dev/null +++ b/internal/relayserve/testdata/preconditional/archive.json @@ -0,0 +1,10 @@ +{ + "commit": "5a65274e364e4f83bbf2bdcae9f103f4210450d5", + "path": "internal/directory", + "sha256": { + "http.go": "cfc2527428b29ad080a22e5239bb2ca0cfe30ecbe4be04ab3f0f5b39175a5a8d", + "client.go": "663253ed2352bc4c2e1372dfb837132800694550db50d1cdbfcdd64471ba4478", + "records.go": "a253b30a5a6aa423f434e2f0260ab9d0a04838369a79d53f6829706991ff47df", + "wire.go": "eecbaa325d4808374a19cee0b1178ea0b28e20636a1520f9ed93bf55145948e7" + } +} diff --git a/internal/relayserve/testdata/preconditional/bridge.go.txt b/internal/relayserve/testdata/preconditional/bridge.go.txt new file mode 100644 index 0000000000..a96193317f --- /dev/null +++ b/internal/relayserve/testdata/preconditional/bridge.go.txt @@ -0,0 +1,49 @@ +package directory + +import ( + "context" + current "github.com/Agent-Field/codeaf/internal/directory" + "net/http" + "strconv" +) + +type Rotation = current.Rotation +type RotationReq = current.RotationReq +type RotationView = current.RotationView +type PresenceView = current.PresenceView +type Approval struct { + Device Device `json:"device"` + Cert string `json:"cert"` + Grant string `json:"grant"` +} +type vaultSwap struct { + Old string `json:"old"` + New string `json:"new"` +} +type fenceBody struct { + Fence uint64 `json:"fence"` +} + +const MaxBody = 1 << 20 +const VersionHeader = current.VersionHeader + +var ( + ErrStillPending = current.ErrStillPending + ErrRotated = current.ErrRotated + ErrRotationStep = current.ErrRotationStep + ErrBadGrace = current.ErrBadGrace + ErrTooManyWatchers = current.ErrTooManyWatchers + ErrRequestGone = current.ErrRequestGone + ErrAlreadyDecided = current.ErrAlreadyDecided + ErrTooBig = current.ErrTooBig + ErrFull = current.ErrFull + ErrBadRequest = current.ErrBadRequest +) + +func (h *HTTP) Presence(ctx context.Context) (v PresenceView, err error) { + hdr, err := h.doHeader(ctx, http.MethodGet, dirBase+"/presence", nil, &v) + if err == nil { + v.Version, _ = strconv.ParseUint(hdr.Get(VersionHeader), 10, 64) + } + return v, err +} diff --git a/internal/relayserve/testdata/preconditional/client.go.txt b/internal/relayserve/testdata/preconditional/client.go.txt new file mode 100644 index 0000000000..5c482e094c --- /dev/null +++ b/internal/relayserve/testdata/preconditional/client.go.txt @@ -0,0 +1,85 @@ +package directory + +import ( + "context" + "errors" + "time" + + "github.com/Agent-Field/codeaf/internal/wireauth" +) + +// Client is the whole directory as one device sees it. The device is a fact of +// the Client itself (the relay reads it from the request signature), so no +// method takes one. Every write is a compare-and-swap decided by rules.go. +type Client interface { + List(ctx context.Context) (Listing, error) // everything a home list needs, one request + Cell(ctx context.Context, id string) (CellView, error) + + PutDevice(ctx context.Context, id string, d Device) error // upsert this device's record; never touches Revoked + Revoke(ctx context.Context, id string) error // stop another device of this identity; ErrNotFound, ErrSelfRevoke + SetVault(ctx context.Context, old, new string) error // CAS identity.vault; ErrCAS + + Create(ctx context.Context, id string, in CellInit) (CellView, error) // ErrExists + Acquire(ctx context.Context, id string, opts AcquireOpts) (CellView, error) // ErrLeaseHeld unless opts.Force + Heartbeat(ctx context.Context, id string, b Beat) (CellView, error) // ErrFenceStale + Publish(ctx context.Context, id string, p Publish) (CellView, error) // ErrFenceStale, ErrHeadMoved + Release(ctx context.Context, id string, fence uint64) error // ErrFenceStale + Archive(ctx context.Context, id string) error // idempotent + + // Presence says which devices hold a watch socket now, and when the + // others were last seen. Live changes arrive on the watch socket instead. + Presence(ctx context.Context) (PresenceView, error) + + // ApproveRequest lets a device that asked to join (see Requests) into this + // identity: it writes the device's record, keeps the grant for it and + // tells the other devices. The first decision wins; repeating it is a + // success. ErrRequestGone, ErrAlreadyDecided, ErrBadRequest, ErrTooBig. + ApproveRequest(ctx context.Context, code string, a Approval) error + // DenyRequest turns a request down. ErrRequestGone, ErrAlreadyDecided. + DenyRequest(ctx context.Context, code string) error + + // Rotate makes one move of the identity's replacement (freeze, thaw, + // retire); Rotation reads where it stands. A replaced identity answers every + // write with ErrRotated and every read as before. + Rotate(ctx context.Context, req RotationReq) (RotationView, error) + Rotation(ctx context.Context) (RotationView, error) +} + +// AcquireOpts says how a device takes a lease. Its JSON is the acquire +// request body, and an empty body means the zero value. +type AcquireOpts struct { + // Force takes the lease from a live holder. It is for a person who chose to + // continue a chat where it runs now: waiting out the holder's lease would + // make them sit through the time a dead holder's row takes to go off. + Force bool `json:"force,omitempty"` +} + +// Directory is one identity's shared directory; For gives each device its own +// Client. The relay handler calls For(deviceFromSignature) once per request. +type Directory interface { + For(device string) Client +} + +// The lease timings every device and the directory agree on. A publish renews +// the lease and a heartbeat is sent only while nothing was published, so the +// relay sees one request per HeartbeatEvery at most and the TTL still covers +// two missed beats. +const ( + LeaseTTL = 90 * time.Second + HeartbeatEvery = 30 * time.Second +) + +// The errors a Client returns. Callers compare with errors.Is. +var ( + ErrNotFound = errors.New("directory: not found") + ErrExists = errors.New("directory: exists") + ErrLeaseHeld = errors.New("directory: lease held by another device") + ErrFenceStale = errors.New("directory: fence superseded") // L7 + ErrHeadMoved = errors.New("directory: head moved") + ErrCAS = errors.New("directory: compare-and-swap failed") + ErrRevoked = wireauth.ErrRevoked // every verb of a revoked device; the wires share this one sentinel + ErrSelfRevoke = errors.New("directory: a device cannot revoke itself") + ErrUnauthorized = errors.New("directory: unauthorized") + ErrUnreachable = errors.New("directory: unreachable") // transport; callers degrade + ErrTooOld = errors.New("directory: this relay is too old for that") +) diff --git a/internal/relayserve/testdata/preconditional/http.go.txt b/internal/relayserve/testdata/preconditional/http.go.txt new file mode 100644 index 0000000000..cb25b7f643 --- /dev/null +++ b/internal/relayserve/testdata/preconditional/http.go.txt @@ -0,0 +1,167 @@ +package directory + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + + "github.com/Agent-Field/codeaf/internal/wireauth" +) + +// HTTP is a Client that speaks the directory wire. Which device it acts as is +// decided by the Sign func it was built with, never by anything it sends. +type HTTP struct { + base string + sign wireauth.Sign + hc *http.Client +} + +// NewHTTP returns the Client for the device that sign stamps requests as. +func NewHTTP(base string, sign wireauth.Sign, hc *http.Client) *HTTP { + return &HTTP{base: base, sign: sign, hc: hc} +} + +var _ Client = (*HTTP)(nil) + +func cellPath(id, verb string) string { + p := dirBase + "/cells/" + url.PathEscape(id) + if verb != "" { + p += "/" + verb + } + return p +} + +// List reads the directory. The listing carries the version the relay read it +// at (0 when the relay does not say), which a feed compares with the versions +// it is told of. +func (h *HTTP) List(ctx context.Context) (l Listing, err error) { + hdr, err := h.doHeader(ctx, http.MethodGet, dirBase+"/list", nil, &l) + if err == nil { + l.Version, _ = strconv.ParseUint(hdr.Get(VersionHeader), 10, 64) + } + return l, err +} + +func (h *HTTP) Cell(ctx context.Context, id string) (v CellView, err error) { + return v, h.do(ctx, http.MethodGet, cellPath(id, ""), nil, &v) +} + +func (h *HTTP) PutDevice(ctx context.Context, id string, d Device) error { + return h.do(ctx, http.MethodPut, dirBase+"/devices/"+url.PathEscape(id), d, nil) +} + +func (h *HTTP) Revoke(ctx context.Context, id string) error { + return h.do(ctx, http.MethodPost, dirBase+"/devices/"+url.PathEscape(id)+"/revoke", nil, nil) +} + +func (h *HTTP) SetVault(ctx context.Context, old, next string) error { + return h.do(ctx, http.MethodPost, dirBase+"/vault", vaultSwap{Old: old, New: next}, nil) +} + +func (h *HTTP) Create(ctx context.Context, id string, in CellInit) (v CellView, err error) { + return v, h.do(ctx, http.MethodPost, cellPath(id, ""), in, &v) +} + +func (h *HTTP) Acquire(ctx context.Context, id string, o AcquireOpts) (v CellView, err error) { + return v, h.do(ctx, http.MethodPost, cellPath(id, "acquire"), o, &v) +} + +func (h *HTTP) Heartbeat(ctx context.Context, id string, b Beat) (v CellView, err error) { + return v, h.do(ctx, http.MethodPost, cellPath(id, "heartbeat"), b, &v) +} + +func (h *HTTP) Publish(ctx context.Context, id string, p Publish) (v CellView, err error) { + return v, h.do(ctx, http.MethodPost, cellPath(id, "publish"), p, &v) +} + +func (h *HTTP) Release(ctx context.Context, id string, fence uint64) error { + return h.do(ctx, http.MethodPost, cellPath(id, "release"), fenceBody{Fence: fence}, nil) +} + +func (h *HTTP) Archive(ctx context.Context, id string) error { + return h.do(ctx, http.MethodPost, cellPath(id, "archive"), nil, nil) +} + +func (h *HTTP) ApproveRequest(ctx context.Context, code string, a Approval) error { + return h.do(ctx, http.MethodPost, requestPath(code, "approve"), a, nil) +} + +func (h *HTTP) DenyRequest(ctx context.Context, code string) error { + return h.do(ctx, http.MethodPost, requestPath(code, "deny"), nil, nil) +} + +func requestPath(code, verb string) string { + return dirBase + "/requests/" + url.PathEscape(code) + "/" + verb +} + +func (h *HTTP) Rotate(ctx context.Context, req RotationReq) (v RotationView, err error) { + return v, h.do(ctx, http.MethodPost, rotationPath, req, &v) +} + +func (h *HTTP) Rotation(ctx context.Context) (v RotationView, err error) { + return v, h.do(ctx, http.MethodGet, rotationPath, nil, &v) +} + +// do sends one signed request and decodes the answer into out (when non-nil). +func (h *HTTP) do(ctx context.Context, method, path string, in, out any) error { + _, err := h.doHeader(ctx, method, path, in, out) + return err +} + +// doHeader is do that also hands back the response headers, for the reads +// that carry a fact in a header as well as in the body. +func (h *HTTP) doHeader(ctx context.Context, method, path string, in, out any) (http.Header, error) { + body, err := encode(in) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, method, wireauth.Endpoint(h.base, path), bytes.NewReader(body)) + if err != nil { + return nil, err + } + h.sign(req, body) + resp, err := h.hc.Do(req) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrUnreachable, err) + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + return resp.Header, refusal(resp) + } + if out == nil { + return resp.Header, nil + } + if resp.StatusCode == http.StatusNoContent { + return resp.Header, ErrStillPending // a long poll that ended with nothing new + } + return resp.Header, json.NewDecoder(resp.Body).Decode(out) +} + +// encode marshals a body; a request without one sends no bytes, so the signed +// body is exactly what the server reads. +func encode(in any) ([]byte, error) { + if in == nil { + return nil, nil + } + return json.Marshal(in) +} + +// refusal turns an error response back into the sentinel its code names. +func refusal(resp *http.Response) error { + var b errBody + raw, _ := io.ReadAll(io.LimitReader(resp.Body, MaxBody)) + if json.Unmarshal(raw, &b) == nil { + if err := errorOf(b.Err, resp.StatusCode); err != nil { + return wireauth.Wait(err, resp.Header) + } + } + if resp.StatusCode == http.StatusNotFound { + return ErrTooOld + } + return fmt.Errorf("directory: %s", resp.Status) +} diff --git a/internal/relayserve/testdata/preconditional/main.go.txt b/internal/relayserve/testdata/preconditional/main.go.txt new file mode 100644 index 0000000000..bcca2b2d8f --- /dev/null +++ b/internal/relayserve/testdata/preconditional/main.go.txt @@ -0,0 +1,99 @@ +package main + +import ( + "context" + "crypto/ed25519" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + old "github.com/Agent-Field/codeaf/internal/archiveprobe/directory" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/reqsign" + "net/http" + "os" + "time" +) + +func key(label string) ed25519.PrivateKey { + seed := sha256.Sum256([]byte(label)) + return ed25519.NewKeyFromSeed(seed[:]) +} + +type signer struct { + label string + cert identity.Cert +} + +func (s signer) IdentityKey() ed25519.PublicKey { return key("root").Public().(ed25519.PublicKey) } +func (s signer) Cert() identity.Cert { return s.cert } +func (s signer) Sign(b []byte) []byte { return ed25519.Sign(key(s.label), b) } +func check(err error) { + if err != nil { + panic(err) + } +} +func main() { + raw, err := os.ReadFile(os.Args[2]) + check(err) + var f struct { + Now int64 + Cases []struct { + Name, Method, URI, Body string + Headers map[string]string + } + } + check(json.Unmarshal(raw, &f)) + clock := func() time.Time { return time.UnixMilli(f.Now) } + client := func(label, which string) *old.HTTP { + for _, c := range f.Cases { + if c.Name == which { + raw, err := base64.RawURLEncoding.DecodeString(c.Headers["Codeaf-Cert"]) + check(err) + var cert identity.Cert + check(json.Unmarshal(raw, &cert)) + return old.NewHTTP(os.Args[1], reqsign.SignFor(signer{label, cert}, clock), http.DefaultClient) + } + } + panic("missing archived certificate") + } + a, b := client("p17-a", "put-a"), client("p17-b", "put-b") + ctx := context.Background() + for _, item := range []struct { + c *old.HTTP + label string + }{{a, "p17-a"}, {b, "p17-b"}} { + check(item.c.PutDevice(ctx, cs.DeviceIDOfKey(key(item.label).Public().(ed25519.PublicKey)), old.Device{V: 1, Name: "c2VhbGVk"})) + } + _, err = a.Create(ctx, "archived-client", old.CellInit{Head: "saved", Class: "work", Keys: map[string]map[string]string{}}) + check(err) + _, err = b.Acquire(ctx, "archived-client", old.AcquireOpts{Force: true}) + if !errors.Is(err, old.ErrLeaseHeld) { + panic(fmt.Sprintf("blind force: %v", err)) + } + check(a.Release(ctx, "archived-client", 1)) + taken, err := b.Acquire(ctx, "archived-client", old.AcquireOpts{Force: true}) + check(err) + if taken.Cell.Lease.Fence != 2 { + panic("wrong fence") + } + _, err = b.Publish(ctx, "archived-client", old.Publish{Fence: 2, OldHead: "saved", Head: "published", Class: "work", Frames: []string{"frame"}}) + check(err) + check(b.Release(ctx, "archived-client", 2)) + check(a.Revoke(ctx, cs.DeviceIDOfKey(key("p17-b").Public().(ed25519.PublicKey)))) + _, err = b.List(ctx) + if !errors.Is(err, old.ErrRevoked) { + panic(fmt.Sprintf("revocation: %v", err)) + } + large, err := os.ReadFile(os.Args[3]) + check(err) + check(json.Unmarshal(large, &f)) + for _, one := range f.Cases { + var init old.CellInit + check(json.Unmarshal([]byte(one.Body), &init)) + _, err = a.Create(ctx, one.Name, init) + check(err) + } +} diff --git a/internal/relayserve/testdata/preconditional/records.go.txt b/internal/relayserve/testdata/preconditional/records.go.txt new file mode 100644 index 0000000000..03def23f46 --- /dev/null +++ b/internal/relayserve/testdata/preconditional/records.go.txt @@ -0,0 +1,157 @@ +// Package directory is the small shared table that says which device holds +// which chat and where its last durable turn is. Every write is a +// compare-and-swap decided by the pure rules in rules.go; the fake and every +// real implementation share those rules and one conformance suite. +// +// # Link pairing: the client API +// +// A new device joins an identity by asking, and an already-paired device +// answers. The wire is docs/ux-pairing-contract.md; this is how Go calls it. +// +// The new device has no identity yet, so it uses Requests (open, unsigned). +// Over HTTP that is NewLinkHTTP(base, hc); in process it is Links.From(peer). +// +// asked, _ := reqs.CreateRequest(ctx, directory.NewRequest{ +// Pubkey: pub, X25519: box, NameSealed: sealed, Platform: "linux"}) +// // show asked.Code, asked.Check and the link https://codeaf.agentfield.ai/p/# +// for { +// r, err := reqs.GetRequest(ctx, asked.Code, 25*time.Second) +// if errors.Is(err, directory.ErrStillPending) { continue } +// // r.State is RequestApproved (r.Grant holds the sealed grant) or RequestDenied +// } +// +// An already-paired device is a Client (signed). It reads the request with +// GetRequest on the same open Requests, compares Request.Check with the one +// the new device shows, and then answers once: +// +// err := client.ApproveRequest(ctx, code, directory.Approval{Device: d, Cert: cert, Grant: grant}) +// err = client.DenyRequest(ctx, code) +// +// The first decision wins. A repeat of the same decision succeeds; the other +// decision is ErrAlreadyDecided. ErrRequestGone covers an unknown, expired or +// deleted code. Rate limits answer ErrRateLimited (wireauth.After says when to +// retry); the numbers are LinkLimits, read with LinkHTTP.Limits. +package directory + +// IdentityRec is the per-identity root record. +type IdentityRec struct { + V uint16 `json:"V"` + Identity string `json:"identity"` // id_…, derived from the signing key + Vault string `json:"vault,omitempty"` // rid of the current vault object + // Rotation is set once the identity has been replaced by a rotation: it + // takes no writes from then on. Nil means live. + Rotation *Rotation `json:"rotation,omitempty"` +} + +// Caps says what a device can do. +type Caps struct { + OS string `json:"os"` // runtime.GOOS + Arch string `json:"arch"` // runtime.GOARCH + Sandbox *string `json:"sandbox"` // "seatbelt" | "landlock" | null + Container *string `json:"container"` // null in Stage 1 + GPU *string `json:"gpu"` // null in Stage 1 + Cow string `json:"cow"` // "apfs" | "btrfs" | "xfs-reflink" | "overlayfs" | "none" +} + +// Device is keyed by device id (dev_…). +type Device struct { + V uint16 `json:"V"` + Name string `json:"name"` // b64, sealed under the metadata key + AddedBy string `json:"added_by"` // identity id that signed the device cert + Revoked bool `json:"revoked"` // set only by Revoke; a fresh pairing makes a new device id + Caps Caps `json:"caps"` + + // Added by link pairing. Old records decode with zero values, and a client + // value for Created or LastSeen is ignored: the directory sets both. + Platform string `json:"platform,omitempty"` // darwin|linux|windows|ios|android|other + Created int64 `json:"created,omitempty"` // directory ms when the device joined + LastSeen int64 `json:"last_seen,omitempty"` // directory ms of the last watch socket close or hello +} + +// visibleAt hides LastSeen: a device coming and going is not a change a person +// sees in the list, so it moves no directory version. +func (d Device) visibleAt(int64) any { + d.LastSeen = 0 + return d +} + +// Lease says who may write a cell. Only the directory clock sets Expires. +type Lease struct { + Device string `json:"device"` // device id of the last holder, kept after release + Fence uint64 `json:"fence"` // +1 on every acquire, never decreases + Expires int64 `json:"expires"` // directory ms; held iff Expires > now; release sets 0 + Pending uint32 `json:"pending"` // turns sealed on the holder and not yet durable +} + +// Cell is keyed by cell ULID. It exists only once it has a durable head. +type Cell struct { + V uint16 `json:"V"` + Head string `json:"head"` // last durable turn id (hex64) + DurableAt int64 `json:"durable_at"` // directory ms + Class string `json:"class"` + Size uint64 `json:"size"` + ParentCell string `json:"parent_cell,omitempty"` + Title string `json:"title,omitempty"` // b64, sealed + Keys map[string]map[string]string `json:"keys"` // cell_key_id -> identity -> wrapped + Lease Lease `json:"lease"` + OrphanTurns uint32 `json:"orphan_turns,omitempty"` // turns in a death branch + Archived bool `json:"archived,omitempty"` // a merged or discarded branch + Frames []string `json:"frames,omitempty"` // hex64 ids of the frames holding the head's closure; a hint, never a condition +} + +// Listing is everything a home list needs, in one read. +type Listing struct { + Now int64 `json:"now"` // directory ms when read + Identity IdentityRec `json:"identity"` + Devices map[string]Device `json:"devices"` + Cells map[string]Cell `json:"cells"` + // Version is the directory version the records were read at, which is + // sent as the VersionHeader and not in the body. + Version uint64 `json:"-"` +} + +// withoutFrames drops every cell's frames from a list answer: the list is the +// home screen's hot read, and frame plans are for takers, who read the cell. +func withoutFrames(l Listing) Listing { + for id, c := range l.Cells { + c.Frames = nil + l.Cells[id] = c + } + return l +} + +// CellView is one cell with the directory time it was read at. +type CellView struct { + Now int64 `json:"now"` + Cell Cell `json:"cell"` +} + +// CellInit creates a cell. The calling device holds the new lease at fence 1. +type CellInit struct { + Head string `json:"head"` + Class string `json:"class"` + Title string `json:"title,omitempty"` + ParentCell string `json:"parent_cell,omitempty"` + Size uint64 `json:"size"` + Keys map[string]map[string]string `json:"keys"` + OrphanTurns uint32 `json:"orphan_turns,omitempty"` + Frames []string `json:"frames,omitempty"` +} + +// Beat renews a lease the caller holds. +type Beat struct { + Fence uint64 `json:"fence"` + Pending uint32 `json:"pending"` +} + +// Publish moves a cell's durable head under a lease the caller holds. +type Publish struct { + Fence uint64 `json:"fence"` + OldHead string `json:"old_head"` + Head string `json:"head"` + Size uint64 `json:"size"` + Class string `json:"class"` + Title string `json:"title,omitempty"` // empty keeps the current title + Pending uint32 `json:"pending"` // turns still not durable after this one + Frames []string `json:"frames,omitempty"` // empty or absent replaces the record's plan with none +} diff --git a/internal/relayserve/testdata/preconditional/wire.go.txt b/internal/relayserve/testdata/preconditional/wire.go.txt new file mode 100644 index 0000000000..0064066006 --- /dev/null +++ b/internal/relayserve/testdata/preconditional/wire.go.txt @@ -0,0 +1,92 @@ +package directory + +import ( + "errors" + "net/http" + + "github.com/Agent-Field/codeaf/internal/wireauth" +) + +// wireError is one row of the wire's error table: the sentinel a Client +// returns, the code the body names it by, and the HTTP status that carries it. +type wireError struct { + err error + code string + status int +} + +// wireErrors is the single place a sentinel and its code meet, so the handler +// and the client cannot drift apart. Order matters only for errors.Is chains. +var wireErrors = []wireError{ + {ErrNotFound, "not_found", http.StatusNotFound}, + {ErrExists, "exists", http.StatusConflict}, + {ErrLeaseHeld, "lease_held", http.StatusConflict}, + {ErrFenceStale, "fence_stale", http.StatusConflict}, + {ErrHeadMoved, "head_moved", http.StatusConflict}, + {ErrCAS, "cas", http.StatusConflict}, + {ErrUnauthorized, "unauthorized", http.StatusUnauthorized}, + {ErrRevoked, "revoked", http.StatusUnauthorized}, + {ErrSelfRevoke, "self_revoke", http.StatusBadRequest}, + {ErrRotated, "rotated", http.StatusGone}, + {wireauth.ErrGone, "gone", http.StatusGone}, + {ErrRotationStep, "rotation_step", http.StatusConflict}, + {ErrBadGrace, "bad_grace", http.StatusBadRequest}, + {wireauth.ErrRateLimited, "rate_limited", http.StatusTooManyRequests}, + {wireauth.ErrTooManyIdentities, "too_many_identities", http.StatusTooManyRequests}, + {wireauth.ErrSkew, "skew", http.StatusUnauthorized}, + {ErrTooManyWatchers, "too_many_watchers", http.StatusTooManyRequests}, + {errUpgradeRequired, "upgrade_required", http.StatusUpgradeRequired}, + {errBadRequest, "bad_request", http.StatusBadRequest}, + {errTooLarge, "too_large", http.StatusRequestEntityTooLarge}, + // The link-pairing rows (docs/ux-pairing-contract.md). "gone" is 404 here + // and 410 for a deleted identity, so a row is matched by code and status. + {ErrRequestGone, "gone", http.StatusNotFound}, + {ErrAlreadyDecided, "already_decided", http.StatusConflict}, + {ErrTooBig, "too_big", http.StatusRequestEntityTooLarge}, + {ErrFull, "full", http.StatusServiceUnavailable}, +} + +// Codes that have no sentinel a Client returns: the wire's own refusals. +var ( + errBadRequest = ErrBadRequest + errTooLarge = errors.New("directory: request too large") + // errUpgradeRequired answers a watch request that is not a WebSocket upgrade. + errUpgradeRequired = errors.New("directory: upgrade required") + errInternal = errors.New("directory: internal error") +) + +// classify finds the table row for err; any error the table does not know is +// a server fault. +func classify(err error) wireError { + for _, w := range wireErrors { + if errors.Is(err, w.err) { + return w + } + } + return wireError{errInternal, "internal", http.StatusInternalServerError} +} + +// errorOf turns a code and the status that carried it back into its sentinel, +// or nil when the pair is unknown. +func errorOf(code string, status int) error { + for _, w := range wireErrors { + if w.code == code && w.status == status { + return w.err + } + } + return nil +} + +// errBody is the JSON every refusal carries. +type errBody struct { + Err string `json:"err"` +} + +const dirBase = "/v1/dir" + +// WatchPath is the signed WebSocket route that says the directory changed. +const WatchPath = dirBase + "/watch" + +// rotationPath is the one signed verb for replacing an identity. It is the +// identity's, not the directory's, so it sits beside the directory's base. +const rotationPath = "/v1/identity/rotation" diff --git a/internal/relayserve/v1_large_test.go b/internal/relayserve/v1_large_test.go new file mode 100644 index 0000000000..7166ae13dc --- /dev/null +++ b/internal/relayserve/v1_large_test.go @@ -0,0 +1,96 @@ +package relayserve_test + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "reflect" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/relayserve" +) + +// R25 preserves archived legal requests at both compatibility size boundaries. +func TestR25ProductionArchivedLargeRequests(t *testing.T) { + raw, err := os.ReadFile("../raftdirectory/testdata/v1-large-requests.json") + if err != nil { + t.Fatal(err) + } + var f struct { + Now int64 + Cases []struct { + Name, Method, URI, Body string + Headers map[string]string + ReadHeaders map[string]string `json:"read_headers"` + Hash string `json:"body_sha256"` + } + } + if err = json.Unmarshal(raw, &f); err != nil { + t.Fatal(err) + } + store := t.TempDir() + for pass := 0; pass < 2; pass++ { + func() { + svc := relayserve.New(relayserve.Config{Store: store, Now: func() time.Time { return time.UnixMilli(f.Now) }}) + srv := httptest.NewServer(svc.Handler) + defer srv.Close() + defer svc.Close() + for i, one := range f.Cases { + wantSize := 65537 + if i == 1 { + wantSize = directory.MaxBody - 1 + } + if len(one.Body) != wantSize { + t.Fatalf("archived body %d", len(one.Body)) + } + hash := sha256.Sum256([]byte(one.Body)) + if hex.EncodeToString(hash[:]) != one.Hash { + t.Fatal("archived request changed") + } + method, uri, body := one.Method, one.URI, one.Body + headers := one.Headers + if pass == 1 { + method, body, headers = "GET", "", one.ReadHeaders + } + + req, err := http.NewRequest(method, srv.URL+uri, strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + for k, v := range headers { + req.Header.Set(k, v) + } + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + result, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + t.Fatal(err) + } + want := 200 + if resp.StatusCode != want { + t.Fatalf("%s pass %d: status %d: %.200s", one.Name, pass, resp.StatusCode, result) + } + { + var view directory.CellView + var init directory.CellInit + if json.Unmarshal(result, &view) != nil || json.Unmarshal([]byte(one.Body), &init) != nil { + t.Fatal("response decode") + } + if view.Cell.Title != init.Title || view.Cell.Head != init.Head || view.Cell.Lease.Fence != 1 || !reflect.DeepEqual(view.Cell.Frames, init.Frames) || !reflect.DeepEqual(view.Cell.Keys, init.Keys) || view.Cell.Size != init.Size || view.Cell.ParentCell != init.ParentCell || view.Cell.OrphanTurns != init.OrphanTurns { + t.Fatal("large request lost content or changed fence") + } + } + } + }() + } +} diff --git a/internal/syncsetup/legacy_large_test.go b/internal/syncsetup/legacy_large_test.go new file mode 100644 index 0000000000..27ec2754fd --- /dev/null +++ b/internal/syncsetup/legacy_large_test.go @@ -0,0 +1,105 @@ +package syncsetup + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "os" + "reflect" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/reqsign" +) + +func legacyLargeLeaderRequests(t *testing.T, base string, hc *http.Client, p *Sync, clock func() time.Time) { + t.Helper() + t.Logf("leader identity %s", p.Identity.ID()) + raw, err := os.ReadFile("../raftdirectory/testdata/v1-large-requests.json") + mustNil(t, err) + var f struct { + Cases []struct{ Name, Method, URI, Body string } + } + mustNil(t, json.Unmarshal(raw, &f)) + for _, one := range f.Cases { + request, err := http.NewRequestWithContext(t.Context(), one.Method, base+one.URI, bytes.NewBufferString(one.Body)) + mustNil(t, err) + reqsign.Sign(request, []byte(one.Body), deviceSigner{p.Identity, p.Device}, clock()) + started := time.Now() + response, err := hc.Do(request) + mustNil(t, err) + t.Logf("large create %s completed in %s", one.Name, time.Since(started)) + body, err := io.ReadAll(response.Body) + response.Body.Close() + mustNil(t, err) + if response.StatusCode != 200 { + p.member.mu.Lock() + app, _, _ := p.member.store.ApplicationView(t.Context()) + rows, _, _ := p.member.store.Read(t.Context(), app.Applied, []raftstore.Key{{Namespace: raftstore.State, Key: cs.CellKey(one.Name).Key}, {Namespace: raftstore.Outbox, Key: cs.V1StageKey().Key}}) + t.Logf("leader response state: failure=%v status=%+v cellBytes=%d stageHeader=%s pending=%d", p.member.failure, p.member.status, len(rows[raftstore.Key{Namespace: raftstore.State, Key: cs.CellKey(one.Name).Key}]), rows[raftstore.Key{Namespace: raftstore.Outbox, Key: cs.V1StageKey().Key}], len(p.member.forwardPending)) + p.member.mu.Unlock() + + t.Fatalf("large request through personal leader: %s status %d: %.200s", one.Name, response.StatusCode, body) + } + var init directory.CellInit + var view directory.CellView + mustNil(t, json.Unmarshal([]byte(one.Body), &init)) + mustNil(t, json.Unmarshal(body, &view)) + if view.Cell.Head != init.Head || view.Cell.Title != init.Title || !reflect.DeepEqual(view.Cell.Frames, init.Frames) || !reflect.DeepEqual(view.Cell.Keys, init.Keys) || view.Cell.Lease.Fence != 1 { + t.Fatal("forwarded result lost original fields") + } + // A small resume also returns the retained large view through the same wire. + started = time.Now() + resumed, err := p.Dir.Acquire(t.Context(), one.Name, directory.AcquireOpts{}) + t.Logf("large resume %s completed in %s: %v", one.Name, time.Since(started), err) + if err != nil { + if p.member.mu.TryLock() { + t.Logf("resume member state: failure=%v status=%+v pending=%d sent=%d", p.member.failure, p.member.status, len(p.member.forwardPending), len(p.member.sent)) + for _, work := range p.member.forwardPending { + t.Logf("pending action=%s barrier=%d applied=%d commandBytes=%d", work.Action, work.Barrier, work.Applied, len(work.Command)) + } + p.member.mu.Unlock() + } else { + t.Log("resume member serialization is busy") + } + t.Fatal(err) + } + if resumed.Cell.Lease.Fence != 2 || !reflect.DeepEqual(resumed.Cell.Frames, init.Frames) { + t.Fatal("large retained result was not returned") + } + mustNil(t, p.Dir.Release(t.Context(), one.Name, 2)) + } +} + +// R25/R6 keep legacy body and reply limits usable when an external core leads. +func TestR25PersonalLeaderLargeLegacyRequests(t *testing.T) { + r := newMemberRig(t) + p := r.home(nil) + personalLeader(t, r, p) + legacyLargeLeaderRequests(t, r.server.URL, r.server.Client(), p, r.clock.Now) +} + +// R25/R6 use the same body and reply assertions through public workerd ingress. +func TestR25HostedPersonalLeaderLargeLegacyRequests(t *testing.T) { + r := newHostedLeadRig(t) + p := r.home(nil) + hostedPersonalLeader(t, p) + // R6/R25 keep routing through the authenticated personal leader while cold + // followers have forgotten their volatile leader hint. + payload, _ := json.Marshal(map[string]string{"op": "forget-follower-hints", "identity": p.Identity.ID()}) + request, err := http.NewRequestWithContext(t.Context(), http.MethodPost, r.server.URL+"/fixture", bytes.NewReader(payload)) + mustNil(t, err) + response, err := r.server.Client().Do(request) + mustNil(t, err) + var route struct{ Before, After string } + mustNil(t, json.NewDecoder(response.Body).Decode(&route)) + response.Body.Close() + if response.StatusCode != 200 || route.Before == "0" || route.After != route.Before { + t.Fatalf("cold follower lost its authenticated route: %+v", route) + } + legacyLargeLeaderRequests(t, r.server.URL, r.server.Client(), p, time.Now) +} diff --git a/internal/syncsetup/member_forward.go b/internal/syncsetup/member_forward.go index 6f6dca326a..a9e3b40648 100644 --- a/internal/syncsetup/member_forward.go +++ b/internal/syncsetup/member_forward.go @@ -270,7 +270,8 @@ func (m *personalMember) appliedForward(entries [][]byte, indices []uint64, resu } for i, raw := range entries { if bytes.Equal(raw, work.Command) && work.Barrier != 0 && indices[i] == work.Barrier { - work.Result = &results[i] + result := cs.BoundedV1Result(raw, results[i], indices[i]) + work.Result = &result work.Applied = indices[i] m.answerForward(work) delete(m.forwardPending, id) From 4fc523cca83f7b8a7bc04ea0d29f5006d8cbe47a Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 14:06:45 -0400 Subject: [PATCH 073/103] Preserve large requests across hosted member recovery. Hosted members apply the shared paged representation in their existing atomic SQL transaction. Exclusive durable-object replacement fences stale actors, while bounded temporary accounting and portable snapshots preserve retained capacity and complete metadata. Earlier automatic collection keeps repeated large views within the reactor budget. Co-Authored-By: Claude Opus 5.5 (1M context) --- relay/hosted/src/raft/abi.js | 9 ++ relay/hosted/src/raft/actor.js | 17 +++- relay/hosted/src/raft/directory.js | 48 ++++++--- relay/hosted/src/raft/hosted.js | 32 +++++- relay/hosted/src/raft/member.js | 15 +-- relay/hosted/src/raft/storage.js | 35 +++++-- relay/hosted/src/raft/v1.js | 7 +- relay/hosted/src/raft/wasi.js | 3 +- relay/hosted/test/raft/abi.test.js | 2 +- relay/hosted/test/raft/actor-unit.test.js | 34 +++++++ relay/hosted/test/raft/hosted-fixture.js | 9 +- relay/hosted/test/raft/hosted-go.test.js | 9 +- relay/hosted/test/raft/hosted-worker.js | 23 +++++ .../hosted/test/raft/v1-large-leader.test.js | 33 +++++++ relay/hosted/test/raft/v1-large.test.js | 99 +++++++++++++++++++ 15 files changed, 333 insertions(+), 42 deletions(-) create mode 100644 relay/hosted/test/raft/v1-large-leader.test.js create mode 100644 relay/hosted/test/raft/v1-large.test.js diff --git a/relay/hosted/src/raft/abi.js b/relay/hosted/src/raft/abi.js index dceeee84e4..2092ee5dea 100644 --- a/relay/hosted/src/raft/abi.js +++ b/relay/hosted/src/raft/abi.js @@ -108,6 +108,15 @@ export function snapshotNeeded(bytes) { export class HostCodec { constructor(runtime) { this.runtime = runtime; } call(w) { return response(this.runtime.call(w.finish())); } + stagingPolicy() { + if (!this.staging) { + const r=this.call(frame(46)); + const max=Number(r.u64()),keys=new Map(); + for(let n=r.u32();n>0;n--)keys.set(r.str()+'\0'+r.str(),r.u32()); + r.end();this.staging={max,keys}; + } + return this.staging; + } inspect(kind, bytes) { const r = this.call(frame(7).u32(kind).blob(bytes)); let v; diff --git a/relay/hosted/src/raft/actor.js b/relay/hosted/src/raft/actor.js index c3606f6a1e..e80750fd47 100644 --- a/relay/hosted/src/raft/actor.js +++ b/relay/hosted/src/raft/actor.js @@ -19,10 +19,17 @@ export function installRaftModule(compiled) { export function sharedRuntime() { if (!module) throw new Error('raft member: core unavailable'); return runtime ??= new Reactor(module); } export function runtimeMetrics() { return { ...sharedRuntime().metrics(), active: active.size, reservedBytes: active.size * RESERVATION }; } +// A new durable-object constructor proves its previous activation is gone. +// Retire only that object's stale handle, independently of memory pressure. +export function replaceMemberActivation(identity) { + for(const actor of active)if(actor.activation===identity)actor.replaced(); +} + // A member queue spans awaits and remains bounded even when storage stalls. // Rejected inputs are retryable; a failed persistence stops this handle. export class MemberActor { - constructor(storage, reactor, evaluate) { + constructor(storage, reactor, evaluate, activation = null) { + this.activation=activation; this.reactor = reactor; this.codec = new HostCodec(reactor); this.store = new MemberStorage(storage, this.codec); this.evaluate = evaluate; this.tail = Promise.resolve(); this.pending = 0; this.handle = 0; this.stopped = false; this.generation = 0; this.checkpointIndex = 0n; } @@ -38,6 +45,7 @@ export class MemberActor { active.add(this); try { const recovered = await this.store.open({ cluster: config.cluster, id: String(config.id), protocol: 1 }, config.ceiling ?? DEFAULT_ROWS_BYTES); + if(this.stopped)throw new Error('raft member: activation replaced; recover from durable storage'); const cache = recovered.entries.reduce((n, b) => n + b.length, 0); if (cache > CACHE_BYTES) throw new Error('raft member: log cache capacity; recover with checkpoint'); const registry = this.store.read('membership', 'registry'); @@ -172,6 +180,13 @@ export class MemberActor { begin(data) { data = data.slice(); return this.serial(() => this.store.beginSnapshot(data)); } chunk(id, offset, data, hash) { data = data.slice(); return this.serial(() => this.store.putChunk(id, offset, data, hash)); } complete(id) { return this.serial(() => this.store.completeSnapshot(id)); } + // An exclusive object replacement cannot resume its former volatile actor. + // Fence pending continuations before recovering the same durable identity. + replaced() { + this.stopped=true;this.store.stopped=true;this.generation++; + if(this.handle){const r=response(this.reactor.call(frame(2).u32(this.handle).finish()));r.end();this.handle=0;} + active.delete(this); + } // Explicit owner erasure has already durably fenced every ingress. Dropping // this handle reclaims memory; it is never a pressure eviction of a leader. disposeErased() { return this.serial(() => { diff --git a/relay/hosted/src/raft/directory.js b/relay/hosted/src/raft/directory.js index 0b67c558eb..7561ea067f 100644 --- a/relay/hosted/src/raft/directory.js +++ b/relay/hosted/src/raft/directory.js @@ -1,22 +1,46 @@ -import { frame, response } from './abi.js'; +import { frame, response, Reader } from './abi.js'; import { unb64 } from './storage.js'; // JavaScript supplies immutable dependency bytes and persists the Go write-set. // It never interprets leases, clocks, revocations, CAS or request sequences. -export function directoryEvaluator(runtime, cluster, ceiling) { - return (command, read, app) => { - let r = response(runtime.call(frame(11).blob(command).finish())); - const keys = Array.from({ length: r.u32() }, () => ({ namespace: r.str(), key: r.str() })); r.end(); - const w = (app.index === undefined ? frame(10) : frame(15)).blob(command).str(cluster).u64(app.now).u64(app.version).u64(app.size).u64(ceiling).u32(keys.length); - for (const k of keys) { const data = read(k.namespace, k.key); w.str(k.namespace).str(k.key).u32(data == null ? 0 : 1).blob(data ?? new Uint8Array()); } - if (app.index !== undefined) w.u64(app.index).blob(unb64(app.conf)).u64(app.term ?? 0); - r = response(runtime.call(w.finish())); - const now = r.u64(), version = r.u64(), result = r.blob(); - const writes = Array.from({ length: r.u32() }, () => ({ namespace: r.str(), key: r.str(), data: r.blob() })); r.end(); - return { now, version, result, writes }; +function applicationPages(runtime,initial,read){ + let r=initial;const id=r.u32(),chunks=[];let size=0; + try{ + for(;;){ + const kind=r.u32(); + if(kind===0){ + const namespace=r.str(),key=r.str();r.end(); + if(!read)throw new Error('raft application: unexpected record request'); + const data=(read.page??read)(namespace,key); + r=response(runtime.call(frame(42).u32(id).u32(data==null?0:1).blob(data??new Uint8Array()).finish())); + }else{ + if(kind!==1)throw new Error('raft application: invalid result page'); + const more=r.u32(),chunk=r.blob();r.end();chunks.push(chunk);size+=chunk.length; + if(!more)break; + r=response(runtime.call(frame(43).u32(id).finish())); + } + } + const joined=new Uint8Array(size);let offset=0; + for(const chunk of chunks){joined.set(chunk,offset);offset+=chunk.length;} + return new Reader(joined); + }finally{const dropped=response(runtime.call(frame(44).u32(id).finish()));dropped.end();} +} + +export function directoryEvaluator(runtime,cluster,ceiling){ + return (command,read,app)=>{ + const start=response(runtime.call(frame(41).blob(command).str(cluster).u64(app.now).u64(app.version).u64(app.size).u64(ceiling).u64(app.index??0).u64(app.term??0).blob(unb64(app.conf??'')).finish())); + const r=applicationPages(runtime,start,read),now=r.u64(),version=r.u64(),result=r.blob(); + const writes=Array.from({length:r.u32()},()=>({namespace:r.str(),key:r.str(),delete:r.u32()!==0,data:r.blob()}));r.end(); + return {now,version,result,writes}; }; } +// Typed projection streams large logical JSON without widening an ABI buffer. +export function projectRecord(runtime,kind,raw){ + const start=response(runtime.call(frame(45).str(kind).blob(raw).finish())); + const r=applicationPages(runtime,start),result=new TextDecoder('utf-8',{fatal:true}).decode(r.blob());r.end();return result; +} + // The consensus host implements commit and barrier, not a local-success cache. // Calls return only committed/durably applied bytes; a leader hint is routing. export class RaftDirectoryFacade { diff --git a/relay/hosted/src/raft/hosted.js b/relay/hosted/src/raft/hosted.js index d23d827f0e..ef95e196ee 100644 --- a/relay/hosted/src/raft/hosted.js +++ b/relay/hosted/src/raft/hosted.js @@ -8,6 +8,7 @@ import { sharedRuntime } from './actor.js'; import { encodeCanonical, decodeCanonical } from './codec.js'; import { b64, unb64, bytes } from './storage.js'; import { v1Command } from './v1.js'; +import { projectRecord } from './directory.js'; import { RaftEffectsFacade } from './effects.js'; import { Wire, empty, json, unwrap } from '../wire.js'; import { signedUri } from '../base.js'; @@ -62,6 +63,12 @@ export class HostedIdentity { async forward(node,work){ const token=await this.serial(()=>this.forwarding.begin(node,work)); const reply=await token.promise; + // Cold followers may temporarily lose their leader hint. An authenticated + // forwarding reply preserves routing, while every operation still requires + // its ordinary quorum barrier and exact durable result. + const hint=String(reply?.status?.Leader??'0'),term=BigInt(reply?.status?.Term??0); + const known=[...this.states.values()].reduce((max,state)=>state.term>max?state.term:max,this.routeTerm??0n); + if(hint!=='0'&&term>=known){this.routeHint=hint;this.routeTerm=term;} if(!reply||reply.error)throw Object.assign(unavailable(),{forwardID:token.id}); return reply; } @@ -279,7 +286,15 @@ export class HostedIdentity { await this.elect();const id=this.leader(); if(!id){ let reply;try{reply=await this.remote({action:'submit',command:b64(raw)});}catch(error){if(this.leader())return this.commit(raw,deadline);throw error;} - if(!reply.result)throw unavailable();await this.appliedReplica(reply.applied,deadline); + if(!reply.result)throw unavailable();const replica=await this.appliedReplica(reply.applied,deadline); + if(reply.result.view_index){ + if(BigInt(reply.result.view_index)!==BigInt(reply.applied))throw unavailable(); + const result=await this.members.get(replica.node).hostResult(raw,BigInt(reply.applied)-1n); + if(!result||BigInt(result.index)!==BigInt(reply.applied))throw unavailable(); + const value=this.project('typed_result',result.raw); + if(value.code!==reply.result.code||value.now!==reply.result.now||value.version!==reply.result.version)throw unavailable(); + await this.peers.refresh();return {raw:result.raw,value,index:BigInt(reply.applied)}; + } await this.peers.refresh();return {raw:data(reply.result),value:reply.result,index:BigInt(reply.applied)}; } let proposal; @@ -288,20 +303,29 @@ export class HostedIdentity { for(;;){ if(this.leader()!==id){await this.refreshRoute();if(this.leader()!==id)return this.commit(raw,deadline);} const version=this.progressVersion,result=await this.members.get(id)?.hostResult(raw,proposal.proposalAfter); - if(result){if(this.peers)await this.peers.refresh();return {raw:result.raw,value:decodeCanonical(dec.decode(result.raw)),index:result.index};} + if(result){if(this.peers)await this.peers.refresh();return {raw:result.raw,value:this.project('typed_result',result.raw),index:result.index};} // All hosted persistence and replies have drained. Only an external // voter can still make progress without another input to this operation. if(!this.externalVoter())throw unavailable(); await this.progress(deadline,version); } } + async commitV1(raw){ + const commands=Array.isArray(raw)?raw:[raw],width=this.peers.limits.Slots[0]; + for(let offset=0;offset{ + const result=await this.commit(command);requireOK(result.value.code);return result; + })); + } + const result=await this.commit(commands.at(-1));requireOK(result.value.code);return result; + } async commitEffect(change,now=Date.now()){ const r=response(this.runtime.call(frame(30).str(this.manifest.config.cluster).u64(now).blob(unb64(this.provisioning.effects)).blob(data(change)).finish())); const raw=r.blob();r.end();return (await this.commit(raw)).value; } async barrierRecord({namespace,key}){const b=await this.barrier();return (await this.members.get(b.node).hostView(b.index,namespace,key)).raw;} async barrierOutbox(after,limit){const b=await this.barrier();return (await this.members.get(b.node).hostView(b.index,'outbox','',after)).rows.slice(0,limit);} - projectJSON(kind,raw){const r=response(this.runtime.call(frame(31).str(kind).blob(raw).finish())),value=dec.decode(r.blob());r.end();return value;} + projectJSON(kind,raw){return projectRecord(this.runtime,kind,raw);} project(kind,raw){return JSON.parse(this.projectJSON(kind,raw));} async authorize(device,certificate,write=false){ await this.wakeMaintenance(); @@ -602,7 +626,7 @@ export class HostedIdentity { const join=/^\/v1\/dir\/requests\/([^/]+)\/(approve|deny)$/.exec(path); if(join){await this.authorize(device,cert,true);const code=decodeURIComponent(join[1]).trim().toLowerCase();const box=this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(code));await this.effects.admit(box,code,Date.now());} let raw;try{raw=v1Command(this.runtime,this.manifest.config.cluster,Date.now(),this.proof(request,body));}catch{throw new Wire('bad_request',400);} - const result=await this.commit(raw);requireOK(result.value.code); + const result=await this.commitV1(raw); const b=await this.barrier(),snapshot=await this.members.get(b.node).hostView(b.index,'state','identity'),identity=this.project('identity',snapshot.raw),version=snapshot.app.version; await this.rememberRetirement(identity); this.watchers.broadcast(version); diff --git a/relay/hosted/src/raft/member.js b/relay/hosted/src/raft/member.js index 2343042a6b..bef1b74175 100644 --- a/relay/hosted/src/raft/member.js +++ b/relay/hosted/src/raft/member.js @@ -1,9 +1,9 @@ import { clusterIngress } from './ingress.js'; import { workerCell,workerAssignment } from './worker.js'; import { DurableObject } from 'cloudflare:workers'; -import { MemberActor, sharedRuntime } from './actor.js'; +import { MemberActor, sharedRuntime,replaceMemberActivation } from './actor.js'; import { MemberInbox, INBOX_BYTES } from './inbox.js'; -import { directoryEvaluator } from './directory.js'; +import { directoryEvaluator,projectRecord } from './directory.js'; import { DEFAULT_ROWS_BYTES, MemberStorage, b64 } from './storage.js'; import { HostCodec,frame,response } from './abi.js'; import { encodeCanonical, decodeCanonical } from './codec.js'; @@ -17,7 +17,7 @@ export { installRaftModule, runtimeMetrics } from './actor.js'; // public Worker routes identity coordination through its authenticated source. // Enrollment and evaluation use the durable Go registry and shared Go policy. export class RaftMemberDO extends DurableObject { - constructor(ctx, env) { super(ctx, env); this.actor = null; this.loading = null; this.ctx = ctx; this.env = env; } + constructor(ctx, env) { super(ctx, env); replaceMemberActivation(ctx.id.toString()); this.actor = null; this.loading = null; this.ctx = ctx; this.env = env; } evaluator(config) { const evaluate=directoryEvaluator(sharedRuntime(),config.cluster,config.ceiling??DEFAULT_ROWS_BYTES); this.results=new Map();this.resultBytes=0; @@ -53,7 +53,7 @@ export class RaftMemberDO extends DurableObject { if(!await this.importReceipt(manifest))throw new Error('raft member: import provenance absent'); await this.load(); await this.actor.serial(async()=>{ - const raw=this.actor.store.read('state','identity'),record=raw&&decodeCanonical(new TextDecoder().decode(raw)); + const raw=this.actor.store.read('state','identity'),record=raw&&JSON.parse(projectRecord(sharedRuntime(),'identity',raw)); if(record?.rotation?.state!=='retired'||record.rotation.retire_at>Date.now())throw new Error('raft member: erasure not authorized'); await this.ctx.storage.put('raft-erased',manifest.config.cluster);await this.ctx.storage.sync();this.erased=true; }); @@ -174,8 +174,9 @@ export class RaftMemberDO extends DurableObject { const previous = await this.ctx.storage.get('raft-config'); if (previous && JSON.stringify(previous) !== JSON.stringify(canonical)) throw new Error('raft member: provisioned identity cannot change'); if (!previous) { await this.ctx.storage.put('raft-config', canonical); await this.ctx.storage.sync(); } - const actor = new MemberActor(this.ctx.storage, sharedRuntime(), this.evaluator(config)); + const actor = new MemberActor(this.ctx.storage, sharedRuntime(), this.evaluator(config),this.ctx.id.toString()); const result = await actor.init(canonical); + if(actor.stopped)throw new Error('raft member: activation replaced; recover from durable storage'); this.config = canonical; this.registry = actor.store.read('membership', 'registry') ? committedRegistry(sharedRuntime(), actor.store) : await this.ctx.storage.get('raft-registry'); this.actor = actor; return result; } @@ -190,8 +191,8 @@ export class RaftMemberDO extends DurableObject { } async restore() { const config = await this.ctx.storage.get('raft-config'); if (!config) throw new Error('raft member: not provisioned'); - const actor = new MemberActor(this.ctx.storage, sharedRuntime(), this.evaluator(config)); - this.recoveredEffects = await actor.init(config); this.config = config; this.registry = actor.store.read('membership', 'registry') ? committedRegistry(sharedRuntime(), actor.store) : await this.ctx.storage.get('raft-registry'); this.actor = actor; + const actor = new MemberActor(this.ctx.storage, sharedRuntime(), this.evaluator(config),this.ctx.id.toString()); + this.recoveredEffects = await actor.init(config);if(actor.stopped)throw new Error('raft member: activation replaced; recover from durable storage'); this.config = config; this.registry = actor.store.read('membership', 'registry') ? committedRegistry(sharedRuntime(), actor.store) : await this.ctx.storage.get('raft-registry'); this.actor = actor; } // Alarm effects wait in one bounded durable transport batch. The router // drains original bytes and acknowledges the batch ID; arrival is never an diff --git a/relay/hosted/src/raft/storage.js b/relay/hosted/src/raft/storage.js index 1c8e7860d2..82508597a6 100644 --- a/relay/hosted/src/raft/storage.js +++ b/relay/hosted/src/raft/storage.js @@ -134,17 +134,23 @@ export class MemberStorage { if (e.type === 0 && e.data.length) { check(evaluate, 'raft storage: application evaluator is absent'); let readBytes = 0, writeBytes = 0; - const change = evaluate(e.data, (ns, k) => { const data = this.read(ns, k); readBytes += data?.length ?? 0; check(readBytes <= ROW_BYTES, 'raft storage: evaluation read-set capacity'); return data; }, { ...app, index: e.index, term: e.term }); + const read=(ns,k)=>{const data=this.read(ns,k);readBytes+=data?.length??0;check(readBytes<=ROW_BYTES,'raft storage: evaluation read-set capacity');return data;}; + read.page=(ns,k)=>{const data=this.read(ns,k);check((data?.length??0)<=ROW_BYTES,'raft storage: evaluation record-page capacity');return data;}; + const change = evaluate(e.data, read, { ...app, index: e.index, term: e.term }); check(change && !change.then && Array.isArray(change.writes)); + const retainedBefore=app.size>this.identity.ceiling?app.size-this.stagingBytes():app.size; for (const r of change.writes) { check(namespaces.includes(r.namespace) && typeof r.key === 'string' && enc.encode(r.key).length > 0 && enc.encode(r.key).length <= ROW_BYTES && r.data instanceof Uint8Array && r.data.length <= ROW_BYTES); writeBytes += rowSize(r.namespace, r.key, r.data); check(writeBytes <= ROW_BYTES, 'raft storage: evaluation write-set capacity'); const before = this.read(r.namespace, r.key); - if (before) { xor(aggregate, this.checksum('records', r.namespace, r.key, before)); app.size -= rowSize(r.namespace, r.key, before); } else app.count++; + if (before) { xor(aggregate, this.checksum('records', r.namespace, r.key, before)); app.size -= rowSize(r.namespace, r.key, before); } + if(r.delete){check(r.data.length===0);if(before)app.count--;this.rows('DELETE FROM records WHERE namespace=? AND key=?',r.namespace,r.key);continue;} + if(!before)app.count++; xor(aggregate, this.checksum('records', r.namespace, r.key, r.data)); app.size += rowSize(r.namespace, r.key, r.data); this.rows('INSERT OR REPLACE INTO records VALUES (?,?,?,?)', r.namespace, r.key, r.data, hex(this.checksum('records', r.namespace, r.key, r.data))); } - check(app.size <= this.identity.ceiling || app.size <= this.application().size, 'raft storage: application capacity; retry after capacity is raised'); + const retained=app.size>this.identity.ceiling?app.size-this.stagingBytes():app.size; + check(retained <= this.identity.ceiling || retained <= retainedBefore, 'raft storage: application capacity; retry after capacity is raised'); if (change.version !== undefined) { check(BigInt(change.version) >= BigInt(app.version)); app.version = String(change.version); } if (change.now !== undefined) { check(BigInt(change.now) >= BigInt(app.now)); app.now = String(change.now); } } @@ -155,6 +161,21 @@ export class MemberStorage { app.applied = String(index); app.conf = b64(a.conf); app.hash = hex(aggregate); this.put('application', app); }); } + stagingCost(namespace,name,data) { + const policy=this.codec.stagingPolicy(),limit=policy.keys.get(namespace+'\0'+name); + if(limit===undefined)return 0; + check(data.length<=limit,'raft storage: temporary command record capacity'); + return rowSize(namespace,name,data); + } + stagingBytes() { + let size=0; + for(const pair of this.codec.stagingPolicy().keys.keys()){ + const [namespace,name]=pair.split('\0'),data=this.read(namespace,name); + if(data)size+=this.stagingCost(namespace,name,data); + } + check(size<=this.codec.stagingPolicy().max,'raft storage: temporary command capacity'); + return size; + } generation(id) { const r = this.one('SELECT data,hash,complete FROM snapshots WHERE id=?', id); check(r); const data = bytes(r.data); check(this.hash(data) === id && this.hash(data) === r.hash); @@ -162,7 +183,7 @@ export class MemberStorage { } async beginSnapshot(data) { const p = this.codec.snapshot(data), s = this.codec.inspect(3, p), m = JSON.parse(dec.decode(data)), id = this.hash(data); - check(m.cluster === this.identity.cluster && m.protocol === '1' && BigInt(m.bytes) <= BigInt(this.identity.ceiling), 'raft storage: snapshot capacity or identity mismatch'); + check(m.cluster === this.identity.cluster && m.protocol === '1' && BigInt(m.bytes) <= BigInt(this.identity.ceiling)+BigInt(this.codec.stagingPolicy().max), 'raft storage: snapshot capacity or identity mismatch'); await this.transaction(() => { const active = unb64(this.get('snapshot')), at = active.length ? this.codec.inspect(3, active).index : 0n; check(s.index >= at); @@ -185,7 +206,7 @@ export class MemberStorage { chunk(id, offset) { const r = this.one('SELECT data,hash FROM chunks WHERE generation=? AND offset=?', id, offset); check(r); const b = bytes(r.data); check(this.hash(b) === r.hash); return b; } walk(id, visit) { const { manifest: m } = this.generation(id); - let state = EMPTY, digest = this.codec.hash(EMPTY).digest, pending = EMPTY, count = 0, total = 0, size = 0, lastNS = '', lastKey = ''; + let state = EMPTY, digest = this.codec.hash(EMPTY).digest, pending = EMPTY, count = 0, total = 0, size = 0, staging = 0, lastNS = '', lastKey = ''; const aggregate = new Uint8Array(32); const chunks = Number(m.chunks); check(Number.isSafeInteger(chunks) && chunks >= 0); check(this.one('SELECT count(*) AS n FROM chunks WHERE generation=?', id).n === chunks); @@ -199,13 +220,13 @@ export class MemberStorage { if (pending.length < length) break; const namespace = dec.decode(pending.subarray(12, 12 + n)), name = dec.decode(pending.subarray(12 + n, 12 + n + k)); check(namespaces.includes(namespace) && (count === 0 || compare(namespace, lastNS) > 0 || (namespace === lastNS && compare(name, lastKey) > 0))); - const data = pending.slice(12 + n + k, length); xor(aggregate, this.checksum('records', namespace, name, data)); count++; size += rowSize(namespace, name, data); + const data = pending.slice(12 + n + k, length); xor(aggregate, this.checksum('records', namespace, name, data)); count++; size += rowSize(namespace, name, data); staging += this.stagingCost(namespace,name,data); visit?.({ namespace, key: name, data }); lastNS = namespace; lastKey = name; pending = pending.slice(length); } check(pending.length <= 2 * ROW_BYTES + 28); } check(pending.length === 0 && String(count) === m.records && String(total) === m.bytes && hex(digest) === m.digest && hex(aggregate) === m.records_hash); - check(size <= this.identity.ceiling, 'raft storage: snapshot exceeds retained identity capacity'); + check(staging<=this.codec.stagingPolicy().max && size-staging <= this.identity.ceiling, 'raft storage: snapshot exceeds retained identity capacity'); } retain(id) { const others = this.rows('SELECT id FROM snapshots WHERE complete=1 AND id<>?', id).toArray().map(r => this.generation(r.id)).sort((a, b) => BigInt(a.manifest.index) > BigInt(b.manifest.index) ? -1 : 1); diff --git a/relay/hosted/src/raft/v1.js b/relay/hosted/src/raft/v1.js index 6fd6c465d4..1de904aa10 100644 --- a/relay/hosted/src/raft/v1.js +++ b/relay/hosted/src/raft/v1.js @@ -1,11 +1,12 @@ import { frame, response } from './abi.js'; import { encodeCanonical } from './codec.js'; +import {unb64} from './storage.js'; // Preserve the original public proof; Go alone translates legacy bodies and // applies the compatibility rule. The surrounding ingress authenticates first, // binds the committed identity, and supplies its own current clock sample. export function v1Command(runtime, cluster, now, request) { - const proof = new TextEncoder().encode(encodeCanonical(request)); - const r = response(runtime.call(frame(29).str(cluster).u64(now).blob(proof).finish())); - const command = r.blob(); r.end(); return command; + const proof = new TextEncoder().encode(encodeCanonical({...request,body:null})); + const r = response(runtime.call(frame(40).str(cluster).u64(now).blob(proof).blob(unb64(request.body)).finish())); + const commands = Array.from({length:r.u32()},()=>r.blob()); r.end(); return commands.length===1?commands[0]:commands; } diff --git a/relay/hosted/src/raft/wasi.js b/relay/hosted/src/raft/wasi.js index 43c67c031b..112b3f0840 100644 --- a/relay/hosted/src/raft/wasi.js +++ b/relay/hosted/src/raft/wasi.js @@ -10,7 +10,8 @@ export function wasiPreview1() { return new Uint8Array(memory.buffer, p, n); }; const u32 = (p, n) => { bytes(p, 4); view().setUint32(p, n, true); }; - const env = new TextEncoder().encode('GOMEMLIMIT=48MiB\0GOGC=50\0GOMAXPROCS=1\0'); + // Dense metadata needs earlier collection of temporary trees between calls. + const env = new TextEncoder().encode('GOMEMLIMIT=48MiB\0GOGC=10\0GOMAXPROCS=1\0'); const offsets = [0]; for (let i = 0; i < env.length - 1; i++) if (env[i] === 0) offsets.push(i + 1); const implemented = { diff --git a/relay/hosted/test/raft/abi.test.js b/relay/hosted/test/raft/abi.test.js index 2d9be3bf4b..1e76ae6c51 100644 --- a/relay/hosted/test/raft/abi.test.js +++ b/relay/hosted/test/raft/abi.test.js @@ -16,7 +16,7 @@ test('R27 WASI shim has fixed environment, no I/O and rejects blocking schedulin assert.equal(wasi.imports.environ_sizes_get(0,4),0);assert.equal(v.getUint32(0,true),3); const size=v.getUint32(4,true);assert.equal(wasi.imports.environ_get(8,32),0); assert.deepEqual([...Array(3)].map((_,i)=>v.getUint32(8+i*4,true)),[32,49,57]); - assert.equal(new TextDecoder().decode(new Uint8Array(memory.buffer,32,size)),'GOMEMLIMIT=48MiB\0GOGC=50\0GOMAXPROCS=1\0'); + assert.equal(new TextDecoder().decode(new Uint8Array(memory.buffer,32,size)),'GOMEMLIMIT=48MiB\0GOGC=10\0GOMAXPROCS=1\0'); assert.throws(()=>wasi.imports.poll_oneoff(),/blocking poll/);assert.throws(()=>wasi.imports.sched_yield(),/scheduler yield/); assert.equal(wasi.imports.fd_read(),8);assert.equal(wasi.imports.sock_accept(),8);assert.equal(wasi.imports.fd_write(9),8); assert.throws(()=>wasi.imports.random_get(65536,1),/bounds/); diff --git a/relay/hosted/test/raft/actor-unit.test.js b/relay/hosted/test/raft/actor-unit.test.js index dbea105c3d..d653c47b5c 100644 --- a/relay/hosted/test/raft/actor-unit.test.js +++ b/relay/hosted/test/raft/actor-unit.test.js @@ -160,3 +160,37 @@ test('R3 R27 a fresh hosted store accepts platform name metadata and refuses unk const unknown=new Host();unknown.db.exec('CREATE TABLE evidence(body TEXT);INSERT INTO evidence VALUES(\'retained\')'); try{await assert.rejects(new MemberStorage(unknown,codec).open(identity(2)),/invalid|damaged/);assert.equal(unknown.db.prepare('SELECT body FROM evidence').get().body,'retained');}finally{unknown.db.close();} }); + +// R25/R27 keep temporary body pages separate from retained capacity through +// snapshots, while unrelated growth still refuses without deleting records. +test('R25 R27 staged legacy requests preserve retained capacity across snapshots', async () => { + const host=new Host(),source=await seeded(host);let index=1; + const apply=async(store,writes)=>{ + index++;const entries=[entry(index)]; + await store.persist({hard:hard(index),entries,snapshot:empty}); + await store.apply({index:BigInt(index),entries,conf,snapshot:empty},()=>({writes})); + }; + while(source.application().size0);await apply(source,[{namespace:'state',key:name,data:new Uint8Array(length).fill(19)}]); + } + const retained=source.application().size; + const policy=codec.stagingPolicy(),keys=[...policy.keys.keys()].map(pair=>pair.split('\0')); + const staged=keys.slice(0,2).map(([namespace,key],n)=>({namespace,key,data:n?new Uint8Array(32768).fill(31):text.encode('{"id":"staged","next":32768}')})); + await apply(source,staged); + assert(source.application().size>retained);assert.equal(source.stagingBytes(),source.application().size-retained); + const snapshot=await source.checkpoint(),info=codec.inspect(3,snapshot),manifest=JSON.parse(new TextDecoder().decode(info.data)),id=sha(info.data); + const destHost=new Host(),dest=new MemberStorage(destHost,codec);await dest.open(identity(2));await dest.beginSnapshot(info.data); + for(let n=0;n({...row,data:empty,delete:true}))]); + assert.equal(dest.stagingBytes(),0);assert.equal(dest.application().size,retained);assert.deepEqual(dest.read('state','opaque-0'),original); + const before=dest.application().applied; + await assert.rejects(apply(dest,[{namespace:'state',key:'extra',data:Uint8Array.of(1)}]),/application capacity/); + assert.equal(dest.application().applied,before);assert.deepEqual(dest.read('state','opaque-0'),original); + host.db.close();destHost.db.close(); +}); diff --git a/relay/hosted/test/raft/hosted-fixture.js b/relay/hosted/test/raft/hosted-fixture.js index b93051a74b..b1b32ca5cf 100644 --- a/relay/hosted/test/raft/hosted-fixture.js +++ b/relay/hosted/test/raft/hosted-fixture.js @@ -7,14 +7,15 @@ import { build } from 'esbuild'; import { mkdtemp, copyFile, rm, readFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { generateKeyPairSync, createHash, sign } from 'node:crypto'; +import { generateKeyPairSync, createHash, createPrivateKey, createPublicKey, sign } from 'node:crypto'; const sha=b=>createHash('sha256').update(b).digest('hex'); -export async function hostedFixture({fixture=false,base='',go=false}={}) { +export async function hostedFixture({fixture=false,base='',go=false,rootLabel,now}={}) { const prefix=prefixOf({BASE_PATH:base}); const out=process.env.CODEAF_RAFT_PROOF_DIR;assert(out,'build the native/WASI proof first'); const dir=await mkdtemp(join(tmpdir(),'p17b-')); - const root=generateKeyPairSync('ed25519'),rootKey=root.publicKey.export({format:'der',type:'spki'}).subarray(-32); + const privateKey=rootLabel&&createPrivateKey({key:Buffer.concat([Buffer.from('302e020100300506032b657004220420','hex'),createHash('sha256').update(rootLabel).digest()]),format:'der',type:'pkcs8'}); + const root=privateKey?{privateKey,publicKey:createPublicKey(privateKey)}:generateKeyPairSync('ed25519'),rootKey=root.publicKey.export({format:'der',type:'spki'}).subarray(-32); const identity='id_'+sha(rootKey).slice(0,32); const client=()=>{ const key=generateKeyPairSync('ed25519'),pub=key.publicKey.export({format:'der',type:'spki'}).subarray(-32); @@ -40,7 +41,7 @@ export async function hostedFixture({fixture=false,base='',go=false}={}) { let text=await readFile(args.path,'utf8');const needle=cut.needle;assert.equal(text.split(needle).length,2,'cut must identify exactly one boundary'); text=text.replace(needle,needle+(cut.after??"throw new Error('H2 injected crash after durable boundary');"));return {contents:text,loader:'js'}; });}}); - await build({entryPoints:[fixture?'test/raft/hosted-worker.js':'src/raft-worker.js'],bundle:true,format:'esm',target:'es2022',external:['cloudflare:*'],plugins,outfile:join(dir,'worker.js')}); + await build({entryPoints:[fixture?'test/raft/hosted-worker.js':'src/raft-worker.js'],bundle:true,format:'esm',target:'es2022',external:['cloudflare:*'],banner:now===undefined?undefined:{js:`const realNow=Date.now;Date.now=()=>{const value=realNow();if(value===0)return ${now};globalThis.fixtureClockOffset??=${now}-value;return value+globalThis.fixtureClockOffset;};`},plugins,outfile:join(dir,'worker.js')}); const saved=join(dir,'bundle-'+bundles.size+'.js'); await copyFile(join(dir,'worker.js'),saved);bundles.set(variant,saved);compiledCut=variant; }; diff --git a/relay/hosted/test/raft/hosted-go.test.js b/relay/hosted/test/raft/hosted-go.test.js index 297ebd8602..3793af00c5 100644 --- a/relay/hosted/test/raft/hosted-go.test.js +++ b/relay/hosted/test/raft/hosted-go.test.js @@ -1,9 +1,14 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; +import {mkdtemp,rm} from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; import { hostedFixture } from './hosted-fixture.js'; const root=new URL('../../../../',import.meta.url); +const temp=await mkdtemp(join(tmpdir(),'codeaf-hosted-journey-')),binary=join(temp,'client.test'); +test.after(()=>rm(temp,{recursive:true,force:true})); const footprint=new URL('../../../../scripts/raft-proof-footprint.sh',import.meta.url).pathname; async function prepare(args){ const child=spawn('sh',[footprint,...args],{cwd:root,stdio:['ignore','pipe','pipe']}); @@ -14,7 +19,7 @@ async function prepare(args){ // R27: compilation can leave a disk backlog after initial admission. Compile // before starting workerd and admit the actual runtime separately, so package // linking cannot consume the journey's ordinary network request deadlines. -await prepare(['env','GOMEMLIMIT=192MiB','go','test','-p=1','./internal/raftdirectory','-run','^$','-count=1']); +await prepare(['env','GOMEMLIMIT=192MiB','go','test','-p=1','./internal/raftdirectory','-c','-o',binary]); await prepare(['true']); // H4/R1/R2/R23/R25/R29 run ordinary Go Sync clients and the unchanged real @@ -23,7 +28,7 @@ test('H4 R1 R2 R23 R25 R29 two Go homes continue, take back, quit, switch off an const h=await hostedFixture(); try{ await h.start();const url=await h.mf.ready;assert.equal(url.hostname,'127.0.0.1'); - const child=spawn('go',['test','-p=1','./internal/raftdirectory','-run','^TestR1R2R23R25R29HostedTwoHomes$','-count=1','-timeout=120s','-v'],{cwd:new URL('../../../../',import.meta.url),env:{...process.env,CODEAF_HOSTED_PROOF_URL:url.origin,GOMEMLIMIT:'192MiB'},stdio:['ignore','pipe','pipe']}); + const child=spawn(binary,['-test.run=^TestR1R2R23R25R29HostedTwoHomes$','-test.count=1','-test.timeout=120s','-test.v'],{cwd:new URL('../../../../internal/raftdirectory/',import.meta.url),env:{...process.env,CODEAF_HOSTED_PROOF_URL:url.origin,GOMEMLIMIT:'192MiB'},stdio:['ignore','pipe','pipe']}); let output='';child.stdout.on('data',b=>{output+=b;});child.stderr.on('data',b=>{output+=b;}); const code=await new Promise((resolve,reject)=>{child.once('error',reject);child.once('exit',resolve);}); console.log(output);assert.equal(code,0,output);assert.match(output,/--- PASS: TestR1R2R23R25R29HostedTwoHomes/);assert.doesNotMatch(output,/--- SKIP:/); diff --git a/relay/hosted/test/raft/hosted-worker.js b/relay/hosted/test/raft/hosted-worker.js index e726d79719..0c0b50cf8c 100644 --- a/relay/hosted/test/raft/hosted-worker.js +++ b/relay/hosted/test/raft/hosted-worker.js @@ -16,6 +16,7 @@ import { policyOf } from '../../src/rules.js'; export class IdentityDO extends Source { constructor(ctx,env){ + Date.now();if(globalThis.fixtureClockOffset){const setAlarm=ctx.storage.setAlarm.bind(ctx.storage);ctx.storage.setAlarm=at=>setAlarm(Number(at)-globalThis.fixtureClockOffset);} super(ctx,env); const accept=ctx.acceptWebSocket.bind(ctx); ctx.acceptWebSocket=(ws,tags)=>{accept(ws,tags);const send=ws.send.bind(ws);ws.send=raw=>{if(ws.deserializeAttachment()?.clusterPeer)measurements.peerOut++;return send(raw);};}; @@ -148,12 +149,30 @@ export class IdentityDO extends Source { for(let n=21;n<=140;n++)await host.maintenance(base+n*host.policy.tick_every,host.started+offset+n*host.policy.tick_every); return {sleeping,membershipCalls:measurements.membership-membershipBefore,appliedBefore:String(before),appliedAfter:String(host.states.get(id).applied),recurringMemberCalls:measurements.turns-turns,idleMilliseconds:120*host.policy.tick_every}; });} + forgetFollowerHints(){ + const host=this.current.host,before=host.leaderID(); + for(const [node,state]of host.states)if(state.role===0)host.states.set(node,{...state,leader:0n}); + return {before,after:host.leaderID()}; + } + progressState(){ + const host=this.current.host;if(!host)return {ready:false}; + return {ready:host.ready,leader:host.leaderID(),members:[...host.members.keys()],states:[...host.states].map(([node,state])=>({node,role:state.role,term:String(state.term),commit:String(state.commit),applied:String(state.applied)})),forwarding:host.forwarding.pending.size,queue:host.queue.length,reads:host.reads.length,pumping:!!host.pumping}; + } inspect() { return {gate:this.ctx.storage.sql.exec('SELECT doc FROM legacy_gate WHERE one=1').toArray()[0]?.doc, originals:this.ctx.storage.sql.exec('SELECT kind,id,doc FROM dir ORDER BY kind,id').toArray()}; } } export class RaftMemberDO extends Member { + constructor(ctx,env){Date.now();if(globalThis.fixtureClockOffset){const setAlarm=ctx.storage.setAlarm.bind(ctx.storage);ctx.storage.setAlarm=at=>setAlarm(Number(at)-globalThis.fixtureClockOffset);}super(ctx,env);} + async staging(){await this.load();return this.actor.serial(()=>({rows:this.ctx.storage.sql.exec("SELECT count(*) AS n, coalesce(sum(length(data)),0) AS bytes FROM records WHERE namespace='outbox' AND key LIKE 'v1-stage/%'").toArray()[0],app:this.actor.store.application()}));} + async replaceActivation(){ + await this.load();const old=this.actor,before=old.store.application().applied; + const restored=new Member(this.ctx,this.env);await restored.load(); + let refused=false;try{await old.input(1);}catch{refused=true;} + for(const key of ['actor','config','registry','results','resultBytes','recoveredEffects'])this[key]=restored[key]; + return {before,after:this.actor.store.application().applied,refused,oldHandle:old.handle,active:runtimeMetrics().active}; + } alarmState(){return this.ctx.storage.getAlarm();} alarm(...args){return measured(()=>super.alarm(...args));} importReceipt(...args){return measured(()=>super.importReceipt(...args));} @@ -205,6 +224,8 @@ export default { if(op==='member-alarm')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).alarmState()); if(op==='peer-idle')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).peerIdle()); if(op==='sequence-proof')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).sequenceProof()); + if(op==='forget-follower-hints')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).forgetFollowerHints()); + if(op==='progress-state')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).progressState()); if(op==='transport-state')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).transportState()); if(op==='pause-public'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).pausePublic();return Response.json(true);} if(op==='resume-public'){await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).resumePublic();return Response.json(true);} @@ -222,6 +243,8 @@ export default { if(op==='retained-control')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).retainedControl()); if(op==='discard-member'){await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).discardStore();return Response.json(true);} if(op==='has-member')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).hasStore()); + if(op==='replace-activation')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).replaceActivation()); + if(op==='staging')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).staging()); if(op==='member')return Response.json(await env.RAFT_MEMBERS.get(env.RAFT_MEMBERS.idFromName(cluster+'/'+node)).inspect()); return new Response(null,{status:404}); } diff --git a/relay/hosted/test/raft/v1-large-leader.test.js b/relay/hosted/test/raft/v1-large-leader.test.js new file mode 100644 index 0000000000..aa3a5f3097 --- /dev/null +++ b/relay/hosted/test/raft/v1-large-leader.test.js @@ -0,0 +1,33 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {spawn} from 'node:child_process'; +import {mkdtemp,rm} from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; +import {hostedFixture} from './hosted-fixture.js'; + +const root=new URL('../../../../',import.meta.url),dir=await mkdtemp(join(tmpdir(),'codeaf-large-leader-')),binary=join(dir,'client.test'); +async function run(args,env={}){ + const child=spawn('go',args,{cwd:root,env:{...process.env,...env},stdio:['ignore','pipe','pipe']});let output=''; + child.stdout.on('data',b=>{output+=b});child.stderr.on('data',b=>{output+=b}); + const code=await new Promise((resolve,reject)=>{child.once('error',reject);child.once('exit',resolve)}); + console.log(output);assert.equal(code,0,output);return output; +} +try{ + await run(['test','-p=1','./internal/syncsetup','-c','-o',binary]); + const admission=spawn('sh',[new URL('../../../../scripts/raft-proof-footprint.sh',import.meta.url).pathname,'true'],{cwd:root,stdio:'inherit'}); + assert.equal(await new Promise((resolve,reject)=>{admission.once('error',reject);admission.once('exit',resolve)}),0); + + // R25/R6 keep the original HTTP and forwarding deadlines while a real + // personal core leads the hosted voters and returns complete large views. + test('R25 large legacy requests and replies through a personal workerd leader',{timeout:180000},async()=>{ + const h=await hostedFixture({fixture:true});try{ + await h.start();const url=await h.mf.ready; + const child=spawn(binary,['-test.run=^TestR25HostedPersonalLeaderLargeLegacyRequests$','-test.count=1','-test.timeout=15m','-test.v'],{cwd:new URL('../../../../internal/syncsetup/',import.meta.url),env:{...process.env,CODEAF_HOSTED_PROOF_URL:url.origin},stdio:['ignore','pipe','pipe']});let output=''; + child.stdout.on('data',b=>{output+=b});child.stderr.on('data',b=>{output+=b}); + const code=await new Promise((resolve,reject)=>{child.once('error',reject);child.once('exit',resolve)}); + console.log(output);if(code!==0){const metrics=await h.control({op:'metrics'});console.log(JSON.stringify({leaderFailureMetrics:{peakMemory:metrics.peakMemory,active:metrics.active},progress:await h.control({op:'progress-state',identity:/leader identity (id_[a-f0-9]+)/.exec(output)?.[1]??h.identity})}));}assert.equal(code,0,output);assert.match(output,/--- PASS: TestR25HostedPersonalLeaderLargeLegacyRequests/);assert.doesNotMatch(output,/--- SKIP:/); + }finally{await h.close();} + }); + test.after(()=>rm(dir,{recursive:true,force:true})); +}catch(error){await rm(dir,{recursive:true,force:true});throw error;} diff --git a/relay/hosted/test/raft/v1-large.test.js b/relay/hosted/test/raft/v1-large.test.js new file mode 100644 index 0000000000..86b86d11b4 --- /dev/null +++ b/relay/hosted/test/raft/v1-large.test.js @@ -0,0 +1,99 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {createHash,createPrivateKey,sign} from 'node:crypto'; +import {hostedFixture} from './hosted-fixture.js'; + +const fixture=JSON.parse(readFileSync(new URL('../../../../internal/raftdirectory/testdata/v1-large-requests.json',import.meta.url))); +const sha=b=>createHash('sha256').update(b).digest('hex'); +const send=(h,c,read=false)=>h.mf.dispatchFetch('http://local'+c.uri,{method:read?'GET':c.method,headers:read?c.read_headers:c.headers,...(read?{}:{body:c.body})}); +async function exact(h,c,read=false){ + assert.equal(Buffer.byteLength(c.body),c.name==='over-command'?65537:1048575); + assert.equal(sha(c.body),c.body_sha256); + const r=await send(h,c,read);assert.equal(r.status,200,(await r.clone().text()).slice(0,200)); + const view=await r.json(),init=JSON.parse(c.body); + assert.equal(view.cell.title,init.title);assert.equal(view.cell.head,init.head);assert.equal(view.cell.lease.fence,1);assert.deepEqual(view.cell.frames,init.frames);assert.deepEqual(view.cell.keys,init.keys);assert.equal(view.cell.parent_cell,init.parent_cell);assert.equal(view.cell.orphan_turns,init.orphan_turns);assert.equal(view.cell.size,init.size); +} + +// R25/R3 preserve complete archived bodies through the real public workerd +// route, all three durable stores and recovery after the final commit. +test('R25 archived requests above 64 KiB and just under 1 MiB commit in workerd',{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true,rootLabel:'root',now:fixture.now}); + try{ + await h.start(); + for(const c of fixture.cases)await exact(h,c); + const before=await h.control({op:'metrics'});console.log(JSON.stringify({peakCoreBytes:before.peakMemory,activeHandles:before.active,reservedBytes:before.reservedBytes})); + assert.equal(before.active,3,'cold object recreation leaked live handles'); + const cluster=JSON.parse((await h.control({op:'source'})).gate).config.cluster; + for(const node of ['1','2','3']){const state=await h.control({op:'staging',cluster,node});assert.equal(state.rows.n,0);assert.equal(state.app.version,'2');} + const metrics=await h.control({op:'metrics'});assert(metrics.peakMemory<=metrics.coreBudget,JSON.stringify(metrics)); + console.log(JSON.stringify({peakCoreBytes:metrics.peakMemory,coreBudget:metrics.coreBudget})); + for(let round=0;round<2;round++)for(const node of ['1','2','3']){const cold=await h.control({op:'replace-activation',cluster,node});assert.equal(cold.after,cold.before);assert(cold.refused);assert.equal(cold.oldHandle,0);assert.equal(cold.active,3);} + for(const c of fixture.cases)await exact(h,c,true); + await h.start();for(const c of fixture.cases)await exact(h,c,true); + }finally{await h.close();} +}); + +// R25/R3/R27 require a process cut between durable body pages to expose no +// partial cell and leave only one bounded, replaceable staging slot. +test('R25 interrupted workerd staging stays atomic and bounded after restart',{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true,rootLabel:'root',now:fixture.now}),c=fixture.cases[1]; + try{ + const cut={needle:'result=await this.commit(command);requireOK(result.value.code);',after:"if(Array.isArray(raw)&&command===raw[16])throw new Error('interrupted body staging');"}; + await h.start(cut); + for(let attempt=0;attempt<2;attempt++){ + const failed=await send(h,c);assert(failed.status>=500,`interruption returned ${failed.status}`);await failed.text(); + await h.start(); + const absent=await send(h,c,true);assert.equal(absent.status,404);await absent.text(); + const cluster=JSON.parse((await h.control({op:'source'})).gate).config.cluster; + for(const node of ['1','2','3']){const state=await h.control({op:'staging',cluster,node});assert(state.rows.n>0);assert(state.rows.n<=33);assert(state.rows.bytes<=1048576+65536);assert.equal(state.app.version,'0');} + if(attempt===0)await h.start(cut); + } + await exact(h,c); + const cluster=JSON.parse((await h.control({op:'source'})).gate).config.cluster; + for(const node of ['1','2','3']){const state=await h.control({op:'staging',cluster,node});assert.equal(state.rows.n,0);assert.equal(state.app.version,'1');} + }finally{await h.close();} +}); + + +// R25 preserves the old body limit even when legal JSON needs six-byte HTML +// escaping in the committed logical row and the returned original view. +test('R25 maximum legacy title escaping remains lossless in workerd',{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true,rootLabel:'root',now:fixture.now}); + try{ + await h.start();const c=structuredClone(fixture.cases[0]); + c.uri='/v1/dir/cells/escaped-title'; + const title='<'.repeat(1048576-Buffer.byteLength(JSON.stringify({head:'escaped',title:''}))); + c.body=JSON.stringify({head:'escaped',title});assert.equal(Buffer.byteLength(c.body),1048576); + const seed=createHash('sha256').update('p17-a').digest(),key=createPrivateKey({key:Buffer.concat([Buffer.from('302e020100300506032b657004220420','hex'),seed]),format:'der',type:'pkcs8'}); + c.headers['Codeaf-Sig']=sign(null,Buffer.from(`codeaf-req-v1\nPOST\n${c.uri}\n${c.headers['Codeaf-Time']}\n${sha(c.body)}`),key).toString('base64url'); + const r=await send(h,c);assert.equal(r.status,200,(await r.clone().text()).slice(0,200)); + const view=await r.json();assert.equal(view.cell.title,title);assert.equal(view.cell.head,'escaped');assert.equal(view.cell.lease.fence,1); + for(let fence=2;fence<=3;fence++){ + const request=structuredClone(c);request.uri+='/acquire';request.body='{}'; + request.headers['Codeaf-Sig']=sign(null,Buffer.from(`codeaf-req-v1\nPOST\n${request.uri}\n${request.headers['Codeaf-Time']}\n${sha(request.body)}`),key).toString('base64url'); + const resumed=await send(h,request);assert.equal(resumed.status,200,(await resumed.clone().text()).slice(0,200));const held=await resumed.json();assert.equal(held.cell.title,title);assert.equal(held.cell.lease.fence,fence); + } + await h.start();const headers={...c.headers};headers['Codeaf-Sig']=sign(null,Buffer.from(`codeaf-req-v1\nGET\n${c.uri}\n${headers['Codeaf-Time']}\n${sha('')}`),key).toString('base64url'); + const read=await h.mf.dispatchFetch('http://local'+c.uri,{headers});assert.equal(read.status,200,(await read.clone().text()).slice(0,200));assert.equal((await read.json()).cell.title,title); + const metrics=await h.control({op:'metrics'});assert.equal(metrics.active,3);assert(metrics.peakMemory<=metrics.coreBudget,JSON.stringify(metrics));console.log(JSON.stringify({escapedTitleCoreBytes:metrics.peakMemory,activeHandles:metrics.active})); + }finally{await h.close();} +}); + +// R25/R27 keep an ordinary small resume usable after a large frame plan was +// retained. Validation and visibility must not exhaust the reactor by decoding +// several complete copies of the same immutable metadata in one transaction. +test('R25 small legacy resumes preserve large retained frame plans in workerd',{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true,rootLabel:'root',now:fixture.now}),c=fixture.cases[1]; + try{ + await h.start();await exact(h,c); + const seed=createHash('sha256').update('p17-a').digest(),key=createPrivateKey({key:Buffer.concat([Buffer.from('302e020100300506032b657004220420','hex'),seed]),format:'der',type:'pkcs8'}); + for(let fence=2;fence<=3;fence++){ + const request=structuredClone(c);request.uri+='/acquire';request.body='{}'; + request.headers['Codeaf-Sig']=sign(null,Buffer.from(`codeaf-req-v1\nPOST\n${request.uri}\n${request.headers['Codeaf-Time']}\n${sha(request.body)}`),key).toString('base64url'); + const response=await send(h,request);assert.equal(response.status,200,(await response.clone().text()).slice(0,200)); + const view=await response.json();assert.equal(view.cell.lease.fence,fence);assert.deepEqual(view.cell.frames,JSON.parse(c.body).frames); + } + const metrics=await h.control({op:'metrics'});console.log(JSON.stringify({resumedFrameCoreBytes:metrics.peakMemory,activeHandles:metrics.active}));assert.equal(metrics.active,3);assert(metrics.peakMemory<=metrics.coreBudget); + }finally{await h.close();} +}); From 4fed31b2d626e2cdfd824d6e971380efb76ff645 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 14:06:45 -0400 Subject: [PATCH 074/103] Document complete metadata saves and retained capacity. Older builds can save metadata up to the existing request limit. An interrupted save exposes no partial metadata, and users check the latest save before retrying. Storage capacity continues to apply to retained records. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/clusterstate/README.md | 50 +++++++++++++++++++ .../continuing-a-chat-on-another-computer.md | 5 ++ internal/raftabi/README.md | 39 ++++++++++++++- relay/hosted/src/raft/README.md | 22 ++++++++ 4 files changed, 115 insertions(+), 1 deletion(-) diff --git a/internal/clusterstate/README.md b/internal/clusterstate/README.md index 013fdb7ca0..613461fda4 100644 --- a/internal/clusterstate/README.md +++ b/internal/clusterstate/README.md @@ -90,3 +90,53 @@ time-aware view for worker learners; the older view fails closed for them. A new worker incarnation retires the old peer view; removal/re-enrollment uses a fresh node ID. Worker voters and promotion are refused. Rotation/removal cannot erase bytes already downloaded by a worker. + + +## Large requests from older clients + +The legacy HTTP body limit remains 1 MiB. A translated request fitting the +64 KiB command envelope keeps its ordinary representation. Larger requests +retain their original certificate, URI, timestamp, signature and body hash in +32 KiB body-page proposals. All pages use one replaceable staging header and +32 fixed page keys per identity. Abandonment cannot allocate more keys or more +than 1 MiB of body scratch. Replacing scratch never removes directory records. + +Each committed page rechecks the original signed hash against the current root, +revocation and committed time. Pages change no directory row, visible version or +logical time. The final proposal requires every ordered page and the complete +body hash, decodes the closed legacy request again, and rechecks the original +proof and ordinary mutation rules. Its directory writes and scratch removal +commit in the same application transaction. A crash before this transaction +leaves the previous directory; recovery after it retains the complete change. + +Large logical records and retained results use a tagged deterministic gzip +representation with compact HTML escaping when smaller, preserving all fields +inside the existing physical row envelope. Ordinary canonical wire bytes retain +their established HTML escaping. Canonical command bytes never accept this representation. The +shared decoder checks the complete compressed stream and bounds expansion by +the legacy JSON escaping envelope. Snapshots retain the physical bytes and +checksums; reads and public projections decode the original typed record. + +The reactor pages immutable records and result bytes synchronously within the +member's application transaction. Each record page respects the 1 MiB evaluation +envelope and each result byte page is at most one quarter of it. Ordinary +commands retain their complete read-set cap. Staged final evaluation streams +its body and dependency pages without putting them in one ABI frame. The host +never yields or changes the applied view between these pages. The 64 KiB command, +2 MiB ABI buffer, 1 MiB physical row and application capacity caps are unchanged. + +Temporary request pages occupy one fixed slot outside retained application +capacity. Their extent is derived from the original HTTP body and command limits. +Snapshots preserve these pages and account for retained records separately; +abandoned requests cannot evict records or consume more slots. + +Intermediate JSON and compressed physical records leave opaque HTML characters +unexpanded. Hashes and public canonical bytes keep their established escaping. +Comparison and compression consume final bytes directly, avoiding simultaneous +expanded copies of a legal request inside the bounded reactor. + +Large retained cell and result records decode directly into their typed shape, +checking decimal counters and the complete canonical spelling before exposing +the value. A single-cell update reuses its validated cell and typed write for +visible-version comparison, preserving opaque metadata without repeatedly +decoding the complete frame plan. diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index 4828894bfc..3767bb4792 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -272,3 +272,8 @@ No. The worker saves its claim and an execution intent before invoking tools. A ## Why can an older computer list chats but not continue a chat that is still running? Older builds can still list, save, publish and release chats through a compatible service. A service using quorum-backed coordination refuses an older request that tries to take a live chat without naming its observed holder and ownership number. It uses the existing refusal, `another device continued this chat first`; an old build does not necessarily show an extra update hint. Quit on the computer holding the chat before continuing it elsewhere, or update codeaf on the computer trying to continue. A released or expired chat can still be taken. A conditional choice never silently changes to a newer holder. + + +## Can an older build save a chat with large encrypted metadata? + +Yes. Directory requests from older builds keep the existing 1 MiB request limit, including large encrypted metadata and frame hints. The service receives the complete signed request before making the change visible. A failed or interrupted save never exposes only part of that metadata. Reconnect and check the latest save before retrying. Encrypted names, key wraps and frame hints are preserved. The identity's storage capacity still applies. diff --git a/internal/raftabi/README.md b/internal/raftabi/README.md index e8f0f15069..8c5ab72e13 100644 --- a/internal/raftabi/README.md +++ b/internal/raftabi/README.md @@ -45,7 +45,7 @@ runtime; a new runtime discards all old completions. The JS guard stops the whole instance if linear memory exceeds 64 MiB or a call traps. This is a fail-closed guard, not a substitute for the host's preallocation -admission/cache/eviction policy. `GOMEMLIMIT=48MiB`, `GOGC=50` and +admission/cache/eviction policy. `GOMEMLIMIT=48MiB`, `GOGC=10` and `GOMAXPROCS=1` are the entire runtime environment. Empty args, unsupported fd/ socket operations, runtime-only clock/entropy and discarded failure output are the other imports. Poll/yield throw; a build that uses them cannot pass the @@ -210,3 +210,40 @@ JavaScript. Scope, router generation, connection and request token are checked before dispatching completion. Hosts use only provisioned anchor keys to route original signed commands, reads and member coordination. The reactor imports no network client, and application frames never enter the Raft peer message path. + + +## Paged legacy application and projection + +Operation 40 receives a canonical proof header with a null body, followed by +the original body as a separate binary blob, and returns a count and bounded +command blobs. This avoids base64 copies of large bodies and never constructs +an unused oversized command digest. A single command preserves the original fast path. The coordinator commits +every returned command in order and replies only after the final applied result. + +Operation 41 begins a synchronous evaluation with command bytes, cluster, +time/version/size/ceiling/index/term counters and configuration bytes. Its response +starts with a cursor ID, followed by a page. A page has tag 0 and a requested +namespace/key, or tag 1 and a more flag plus output bytes. Operation 42 supplies +the requested record's presence flag and bytes to the cursor. Operation 43 reads +the next output page. Operation 44 drops the cursor on every exit; dropping an +already stopped cursor is safe. Operation 45 begins a paged typed projection. +Each incoming record is bounded by the existing evaluation envelope; outgoing +byte pages are at most one quarter of it. No pointer or cursor crosses an await. + +Completed evaluation bytes contain time/version, the retained result blob and +writes with namespace/key, an explicit scratch-delete flag and data bytes. +The host applies these writes in its existing atomic application transaction. +Only temporary staging records are removed. Large retained result blobs use the +same lossless tagged representation as large records. A projection prefixes its +logical blob length while streaming, without duplicating the complete blob. +Typed result projection preserves decimal uint64 values before JavaScript +parses the result, using compact HTML spelling internally. Public legacy +projection retains its established field order and HTML escaping. + +Operation 46 returns the fixed temporary request keys, per-record bounds and +total extent from shared Go constants. Storage uses this accounting when checking +retained capacity and portable snapshots. + +Dense retained metadata uses an earlier automatic collection target so temporary +JSON trees do not accumulate across synchronous calls. Linear-memory and member +admission budgets continue to govern the same shared reactor. diff --git a/relay/hosted/src/raft/README.md b/relay/hosted/src/raft/README.md index 1439f65708..cdbdd76f95 100644 --- a/relay/hosted/src/raft/README.md +++ b/relay/hosted/src/raft/README.md @@ -239,3 +239,25 @@ Remote idle scheduling uses the same safe idle read, durable marker and final all-voter sleep acknowledgments as local leadership. Forwarding waits, voting delivery and retry hints retain active work. A final sleep clears maintenance only if no newer ingress has woken the coordinator. + + +## Durable object replacement and large legacy requests + +The isolate may outlive an individual durable-object activation. A replacement +constructor fences that object's previous volatile actor and releases its handle +before recovering the same durable store as a follower. Pending continuations +from the old activation cannot emit consensus effects. Other identities and +members keep their handles. This lifecycle boundary never evicts a serving +leader to satisfy memory pressure, and changes no admission or memory limit. + +Large legacy HTTP proofs use the shared Go staging representation. Body pages +commit without changing directory rows or visible version. The complete proof +and ordinary mutation rules are checked again at the final commit, whose writes +and scratch removal share the existing SQL transaction. A cold object can resume +from bounded intermediate pages without exposing partial metadata. All original +frame hints and sealed key wraps survive the existing request limit. + +Authenticated forwarding replies preserve the current-term personal leader +route when a cold follower temporarily forgets it. The hint supplies routing; +publication and reads continue through their quorum barriers and exact durable +results. A delayed older-term reply cannot replace a newer route. From 2bacbe9613b7078a8c5f277dbdc7084a180c2a07 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 14:31:52 -0400 Subject: [PATCH 075/103] Document current cluster leadership and worker limits. Joined computers can coordinate through either relay. Manual searches explain phone availability, saved leadership requests, fallback, scoped workers and retained capacity with the product's own status and refusal text. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/codeaf/cluster.go | 4 +- cmd/codeaf/cluster_test.go | 12 ++ .../manual/chat/coordination-and-workers.md | 25 ++- .../manual/chat/relay-limits-and-refusals.md | 2 +- .../chat/relay-what-it-stores-and-sees.md | 2 + internal/manual/chat_test.go | 13 ++ internal/manual/cluster_truth_test.go | 187 ++++++++++++++++++ 7 files changed, 235 insertions(+), 10 deletions(-) create mode 100644 internal/manual/cluster_truth_test.go diff --git a/cmd/codeaf/cluster.go b/cmd/codeaf/cluster.go index aa2b04fe52..cfb461946a 100644 --- a/cmd/codeaf/cluster.go +++ b/cmd/codeaf/cluster.go @@ -15,8 +15,8 @@ import ( const clusterHelp = `codeaf cluster status Show this joined computer, learners, voters and the current coordinator. codeaf cluster lead - Ask this joined computer to coordinate through a self-hosted relay. - Persistent storage is required; hosted leadership is not available yet. + Ask this joined computer to coordinate through a hosted or self-hosted relay. + Persistent storage is required; keep codeaf open while this computer leads. Three voters tolerate one unavailable member. ` diff --git a/cmd/codeaf/cluster_test.go b/cmd/codeaf/cluster_test.go index 060de1645f..3b5df588c9 100644 --- a/cmd/codeaf/cluster_test.go +++ b/cmd/codeaf/cluster_test.go @@ -51,3 +51,15 @@ func TestR6R8R30ClusterCommandReceiptAndRefusal(t *testing.T) { } } } + +// R6/R30 require command help to offer the leadership that both relays serve. +func TestR6R30ClusterHelpOffersHostedLeadership(t *testing.T) { + for _, phrase := range []string{"hosted or self-hosted relay", "Persistent storage is required", "Three voters tolerate one unavailable member"} { + if !strings.Contains(clusterHelp, phrase) { + t.Errorf("cluster help omits %q", phrase) + } + } + if strings.Contains(clusterHelp, "not available yet") { + t.Fatal("cluster help denies available leadership") + } +} diff --git a/internal/manual/chat/coordination-and-workers.md b/internal/manual/chat/coordination-and-workers.md index b9744853bc..1eec93fe47 100644 --- a/internal/manual/chat/coordination-and-workers.md +++ b/internal/manual/chat/coordination-and-workers.md @@ -5,6 +5,12 @@ A relay directory upgrade can briefly pause pairing and chat moves. Retry after the relay is ready. The upgrade refusal is `codeaf: directory upgrade is incomplete; retry shortly; saved chats are still here`. It does not delete saved chats, files, branches or the encrypted vault. The relay keeps the old directory as a backup and never makes it a second writer after the new directory takes over. A row that cannot be copied safely leaves the upgrade paused for the operator; it is never truncated to make it fit. +## Can my phone lead, be the leader or coordinate my computers? + +There is no codeaf phone app yet, so a phone cannot lead yet. A phone browser's pairing page only helps approve a computer; it does not join the phone to the cluster. A joined laptop or desktop with persistent storage and sync on can lead: run `codeaf cluster lead` on that computer and keep codeaf open. `codeaf cluster status` shows its joined node and the current coordinator. + +A phone would need a supported client, persistent storage, enrollment and a live connection before it could take this role. Those requirements are not a working phone capability today. Leadership coordinates metadata; it does not move a chat or run its tools. + ## Joined computers, learner capacity and an outdated build An open, paired home with sync on joins the hosted relay, or a self-hosted `codeaf relay --store `, as a durable learner. Its store lives under the home, and reopening reuses its node. Losing that store enrolls a fresh node and retires the old authority. Three anchors vote until a personal computer is explicitly asked to lead, so pairing alone never requires either laptop to stay awake. Joining starts while the paired home is open, or when you ask for `codeaf cluster status`. @@ -13,7 +19,7 @@ If joining has no new data to apply, the client keeps its store for the next ret The membership protocol admits at most 32 live learners and retains retired records instead of reusing their authority. A protocol mismatch says `this codeaf build must be updated before it can join this cluster; your saved chats are still here`. Its capacity refusal is `this cluster has reached its enrollment limit; remove a joined computer or ask the relay operator for capacity`. A worker setup command is not available through the production client yet. -## Which computer coordinates: cluster status and cluster lead +## What is cluster lead — what does codeaf cluster lead do, make this computer the leader `codeaf cluster status` on the hosted relay, or a self-hosted `codeaf relay` with storage, reports the committed voters and current leader after a fresh majority read. A paired computer joins with persistent storage as a learner before status shows its node. Readiness requires its own durable application through the advertised join position. Status names `this computer`, `learning computers`, `ready computers` and `voting computers` when known. `codeaf cluster lead` on an unjoined computer says `this computer must join the cluster with persistent storage before it can lead`. @@ -31,11 +37,11 @@ The hosted relay keeps serving directory requests, watches, pairing decisions an A lost reply or restarted home keeps the original signed leadership request. Retry `codeaf cluster lead` to resume it; a pending request cannot choose a different computer. A completed refusal says `the leadership request could not complete; refresh cluster status and try again` and is acknowledged before another request is saved. Only one configuration change can run at a time. A different personal coordinator cannot directly replace an active personal coordinator; let the anchors resume, then request leadership again. -## Sleeping coordinator, fallback timing and no automatic regain +## What happens if my leader goes away — my leading laptop went to sleep or the leader computer was turned off -On the hosted relay and a self-hosted relay with storage, when a personal coordinator sleeps, exits or turns sync off, the two remaining anchor voters can elect a coordinator and restore the third anchor after it catches up. The missing computer need not approve its removal. A returning computer catches up as a learner and does not automatically regain leadership: request `codeaf cluster lead` again. +On the hosted relay and a self-hosted relay with storage, when a personal coordinator sleeps, exits or turns sync off, the two remaining anchor voters can elect a coordinator and restore the third anchor after it catches up. The missing computer need not approve its removal. A returning computer catches up as a learner and does not automatically regain leadership: request `codeaf cluster lead` again. Losing a majority pauses coordination and keeps saved chats. The refusal is `this cluster cannot coordinate right now; reconnect enough of its voting computers and try again; your saved chats are still here`. -Coordination uses five-second ticks. The provisioned anchors’ ranked election windows are six–seven, eight–nine and ten–eleven ticks (30–35, 40–45 and 50–55 seconds); other computers use twelve–thirteen ticks (60–65 seconds). Healthy fallback targets about 65 seconds. Scheduling delays or a partition can take longer. Fully idle groups stop periodic ticks only after every voter has applied the same idle barrier; a new operation or a leading computer’s disconnection wakes coordination and needs a fresh majority response. +Coordination ticks every 5 seconds. The provisioned anchors’ ranked election windows are 6–7 ticks (30–35 seconds), 8–9 ticks (40–45 seconds) and 10–11 ticks (50–55 seconds); other computers use 12–13 ticks (60–65 seconds). Healthy fallback targets about 65 seconds. Scheduling delays or a partition can take longer. Fully idle groups stop periodic ticks only after every voter has applied the same idle barrier; a new operation or a leading computer’s disconnection wakes coordination and needs a fresh majority response. ## Coordinator and anchor both unavailable, lost majority @@ -58,7 +64,7 @@ A compatibility events viewer pings every 10 seconds. The platform answers witho A connection naming held chats still renews through committed changes every 30 seconds while fresh hold evidence is available. Presence alone never renews a chat. -## Can a worker access all my chats or approve pairing? +## What can a worker do — can a worker read all my chats or approve pairing? A delegated worker uses its own key and an expiring certificate naming selected chats. It receives only sealed grants for those chats, rather than the full exported identity. It cannot approve pairing, rotate the identity, read the identity vault, enroll other devices or request leadership. Richer capabilities, worker labels and execution credentials stay encrypted; only the existing platform and sandbox capability shape is readable by the relay. @@ -107,8 +113,13 @@ instead of reopening the old writer. Keep the whole store directory when restarting or backing up the relay. Chat content stays encrypted in the existing blob store. No switch selects the former directory writer. -Encoded directory commands currently have a 64 KiB limit; a larger older request -can be refused before it changes ownership. Larger imported records stay retained. +Legal directory requests from older builds up to 1 MiB keep their original HTTP +limit on both relays. Large signed metadata is staged in bounded pieces and becomes +visible atomically, after the complete original proof and current authority are +checked. Interrupted requests can recover without exposing partial changes. All +frame hints, sealed names and key wraps are preserved; over 1 MiB is refused. The +identity’s storage capacity still applies, and existing records are never removed +to make room. ## Hosted relay directory upgrade and older clients diff --git a/internal/manual/chat/relay-limits-and-refusals.md b/internal/manual/chat/relay-limits-and-refusals.md index ca68be69d3..fe9d0f4ae8 100644 --- a/internal/manual/chat/relay-limits-and-refusals.md +++ b/internal/manual/chat/relay-limits-and-refusals.md @@ -18,7 +18,7 @@ It depends on the relay. A chunk the relay already holds costs nothing again, so a computer that sends it twice is never refused for that. -**A relay you run yourself has no limit of its own on an identity.** It stops only when its disk is full. +**Coordination storage has a separate limit on both relays.** Each identity starts with an 8 MiB allowance for directory and coordination records. A larger imported identity keeps its measured allowance. New growth beyond it is refused; existing records and saved chats are retained. A relay you run yourself also needs enough disk for encrypted chunks and all three coordination copies. Older directory requests keep their 1 MiB HTTP limit; staging a large request does not bypass retained storage capacity. ## Too many new identities from one network diff --git a/internal/manual/chat/relay-what-it-stores-and-sees.md b/internal/manual/chat/relay-what-it-stores-and-sees.md index 30e817450d..dd851248c3 100644 --- a/internal/manual/chat/relay-what-it-stores-and-sees.md +++ b/internal/manual/chat/relay-what-it-stores-and-sees.md @@ -22,6 +22,8 @@ There is no account on the relay. No email, no password, no name. Your identity - for each chunk: its size, and the ids of the parts inside it; - when you connect, how often, and how much you send; - the network address each request came from. +- coordination metadata: cluster and member ids, roles, term and log position, learner progress, request sequence and digest; +- worker routing: opaque assignment id and revision, selected chat and worker ids, process number, ownership number, status and times. A relay you run yourself logs only the first 8 characters of the identity id, the kind of request, the status and two byte counts. It never logs a body. An operator of a relay you do not run can do more than that; you cannot check it. diff --git a/internal/manual/chat_test.go b/internal/manual/chat_test.go index 203529e795..8f0197cfa4 100644 --- a/internal/manual/chat_test.go +++ b/internal/manual/chat_test.go @@ -33,6 +33,19 @@ func TestTheChatManualAnswersTheQuestionsPeopleAsk(t *testing.T) { question string page string }{ + // R6/R7/R8/R15/R30 keep leadership, fallback and worker limits reachable. + {"can my phone lead", "coordination-and-workers"}, + {"can my phone be the leader", "coordination-and-workers"}, + {"can my phone coordinate my computers", "coordination-and-workers"}, + {"what is cluster lead", "coordination-and-workers"}, + {"what does codeaf cluster lead do", "coordination-and-workers"}, + {"make this computer the leader", "coordination-and-workers"}, + {"what happens if my leader goes away", "coordination-and-workers"}, + {"my leading laptop went to sleep", "coordination-and-workers"}, + {"the leader computer was turned off", "coordination-and-workers"}, + {"what can a worker do", "coordination-and-workers"}, + {"can a worker approve pairing", "coordination-and-workers"}, + {"can a worker read all my chats", "coordination-and-workers"}, // R25/R29 keep older-client refusals and failed quit saves reachable. {"why can an older computer list chats but not continue a chat that is still running", "continuing-a-chat-on-another-computer"}, {"what happens if the final save fails during quit", "continuing-a-chat-on-another-computer"}, diff --git a/internal/manual/cluster_truth_test.go b/internal/manual/cluster_truth_test.go new file mode 100644 index 0000000000..2b61837c95 --- /dev/null +++ b/internal/manual/cluster_truth_test.go @@ -0,0 +1,187 @@ +package manual + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "regexp" + "strconv" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/chatlist" + "github.com/Agent-Field/codeaf/internal/clusterctl" + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftcore" +) + +// R6/R7/R8/R15/R30 require the retrieved answers to state today's capabilities. +func TestR30ClusterQuestionsReachCompleteAnswers(t *testing.T) { + for _, probe := range []struct { + question string + facts []string + }{ + {"can my phone lead", []string{"no codeaf phone app yet", "cannot lead yet", "persistent storage", "codeaf cluster lead"}}, + {"what is cluster lead", []string{"joint change", "transfers leadership", "receipt", "saved signed request"}}, + {"what happens if my leader goes away", []string{"restore the third anchor", "does not automatically regain leadership", "saved chats", cs.LeadershipUnavailable}}, + {"what can a worker do", []string{"selected chats", "cannot approve pairing", "read the identity vault", "not available as a terminal command yet"}}, + } { + t.Run(probe.question, func(t *testing.T) { + var answer strings.Builder + for _, section := range Chat().Search(probe.question, DefaultResults) { + if section.Page == "coordination-and-workers" { + answer.WriteString(section.Body) + } + } + for _, fact := range probe.facts { + if !strings.Contains(answer.String(), fact) { + t.Errorf("retrieved answer lacks %q", fact) + } + } + }) + } +} + +// R8/R9/R25/R30 bind every quoted coordination refusal to its product definition. +func TestR30CoordinationQuotesMatchProductStrings(t *testing.T) { + messages := []string{ + cs.LeadershipUnavailable, + cs.ErrMigration.Error(), + cs.EnrollmentMessage(cs.Capacity), + cs.EnrollmentMessage(cs.UpdateRequired), + (clusterctl.CommittedRefusal{}).Error(), + chatlist.LostRace, + } + for _, source := range []struct{ path, declaration, prefix string }{ + {"../syncsetup/cluster.go", "LeadCluster", "this computer must join"}, + {"../syncsetup/member.go", "errMemberWaiting", "this computer is still joining"}, + {"../syncsetup/worker.go", "ErrWorkerIncomplete", "this work may be incomplete"}, + } { + found := false + for _, text := range clusterDeclarationStrings(t, source.path, source.declaration) { + if strings.HasPrefix(text, source.prefix) { + messages = append(messages, text) + found = true + } + } + if !found { + t.Fatalf("product declaration %s no longer defines its refusal", source.declaration) + } + } + seen := make(map[string]int) + for _, text := range messages { + seen[text] = 0 + } + quotes := regexp.MustCompile("`([^`\n]+)`") + for page, body := range flatChatPages(t) { + for _, match := range quotes.FindAllStringSubmatch(body, -1) { + text := match[1] + for _, prefix := range []string{"this cluster", "this computer must join", "this computer is still joining", "this codeaf build", "the leadership request", "codeaf: directory upgrade", "this work may be incomplete", "another device continued this chat first"} { + if strings.HasPrefix(text, prefix) { + if _, ok := seen[text]; !ok { + t.Errorf("%s quotes a sentence absent from the product: %q", page, text) + } else { + seen[text]++ + } + break + } + } + } + } + for text, count := range seen { + if count == 0 { + t.Errorf("manual omits product sentence %q", text) + } + } + labels := make(map[string]bool) + for _, format := range clusterDeclarationStrings(t, "../../cmd/codeaf/cluster.go", "clusterCommand") { + if label, _, ok := strings.Cut(format, " %"); ok && strings.HasSuffix(format, "\n") { + labels[label] = true + } + } + page, _ := Chat().Page("coordination-and-workers") + _, status, ok := strings.Cut(page, "Status names ") + if !ok { + t.Fatal("manual omits the status labels") + } + status, _, _ = strings.Cut(status, ".") + statusQuotes := quotes.FindAllStringSubmatch(status, -1) + if len(statusQuotes) == 0 { + t.Fatal("manual status sentence has no quoted labels") + } + for _, match := range statusQuotes { + if !labels[match[1]] { + t.Errorf("manual status label is absent from the command output: %q", match[1]) + } + } +} + +// Unexported client refusals stay private to the client; syntax reads avoid +// linking its entire runtime merely to check the manual's spelling. +func clusterDeclarationStrings(t *testing.T, path, name string) []string { + t.Helper() + file, err := parser.ParseFile(token.NewFileSet(), path, nil, 0) + if err != nil { + t.Fatal(err) + } + var values []string + ast.Inspect(file, func(node ast.Node) bool { + var declaration ast.Node + switch n := node.(type) { + case *ast.FuncDecl: + if n.Name.Name == name { + declaration = n + } + case *ast.ValueSpec: + for _, id := range n.Names { + if id.Name == name { + declaration = n + } + } + } + if declaration == nil { + return true + } + ast.Inspect(declaration, func(n ast.Node) bool { + if lit, ok := n.(*ast.BasicLit); ok && lit.Kind == token.STRING { + text, err := strconv.Unquote(lit.Value) + if err != nil { + t.Fatal(err) + } + values = append(values, text) + } + return true + }) + return false + }) + return values +} + +// R7/R9/R25/R27/R30 keep published timing and limits tied to their shared values. +func TestR30CoordinationTimingAndLimitsMatchProduct(t *testing.T) { + page, _ := Chat().Page("coordination-and-workers") + seconds := cs.TickEveryMillis / 1000 + facts := []string{ + fmt.Sprintf("Coordination ticks every %d seconds", seconds), + fmt.Sprintf("at most %d live learners", cs.MaxLearners), + fmt.Sprintf("up to %d MiB", cs.MaxV1BodyBytes>>20), + } + fallbackTicks := raftcore.ElectionTicks + 4*raftcore.ElectionWindowTicks - 1 + facts = append(facts, fmt.Sprintf("Healthy fallback targets about %d seconds", int64(fallbackTicks)*seconds)) + for rank := 0; rank < 4; rank++ { + low := raftcore.ElectionTicks + rank*raftcore.ElectionWindowTicks + high := low + raftcore.ElectionWindowTicks - 1 + facts = append(facts, fmt.Sprintf("%d–%d ticks (%d–%d seconds)", low, high, int64(low)*seconds, int64(high)*seconds)) + } + for _, fact := range facts { + if !strings.Contains(page, fact) { + t.Errorf("manual omits current product timing or limit %q", fact) + } + } + for _, section := range Chat().Sections() { + if section.Page == "coordination-and-workers" && len([]rune(section.Body)) > 2000 { + t.Errorf("coordination section %q exceeds 2,000 characters", section.Title) + } + } +} From 0d376959a50f34c33aa3e3077b7aa4d63b3adba8 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 14:31:52 -0400 Subject: [PATCH 076/103] Explain relay storage, recovery and compatibility. Three persistent voters coordinate directory changes while encrypted chat content keeps its ordinary storage. Relay guidance explains the one-way upgrade, personal leadership, idle traffic and atomic older requests through the original body limit. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/relay.md | 204 +++++++++++++++++++++---------- docs/ux-pairing-contract.md | 27 ++-- internal/raftdirectory/README.md | 12 +- relay/hosted/README.md | 21 ++-- 4 files changed, 175 insertions(+), 89 deletions(-) diff --git a/docs/relay.md b/docs/relay.md index e051bfad69..50707a9156 100644 --- a/docs/relay.md +++ b/docs/relay.md @@ -1,84 +1,156 @@ # Running your own relay -The relay is the service your computers sync and pair through. It stores ciphertext and -sees the short list of metadata in the manual page "What the relay stores and what it can -see". This page is for the person who runs one. +The relay is the service your computers pair and sync through. It stores encrypted +chat content and coordinates which computer holds each chat. Chat titles, computer +names, messages, files and keys stay encrypted. Routing ids, ownership, sizes, +times, platform capabilities and coordination progress remain readable metadata. -## Run +## Run and connect -``` +```sh codeaf relay --listen :8787 --store /var/lib/codeaf ``` -`relay --listen :8787 --store /var/lib/codeaf` (the separate `relay` binary) is the same program. +The separate `relay` binary accepts the same flags. | Flag | Meaning | |---|---| -| `--listen` | address to listen on, default `:8787` | -| `--store ` | where the directory and the chunks live. Empty runs the blind pipe and pairing only: no chats are kept | -| `--quiet` | no log lines | -| `--status=false` | do not answer `GET /status` | -| `--trust-proxy` | count the first `X-Forwarded-For` address as the caller's network in the pairing limits. Only behind a proxy that sets it | - -Point each computer at it with `CODEAF_SYNC_URL=https://relay.example.com`, or pass -`--relay` to `codeaf pair`. +| `--listen` | Listen address; default `:8787`. | +| `--store ` | Persistent coordination and encrypted chunks. Without it, only the blind transport and pairing mailbox run; chat sync needs storage. | +| `--quiet` | Suppress arrival, departure and request logs. | +| `--status=false` | Disable `GET /status`. | +| `--trust-proxy` | Use the first `X-Forwarded-For` address for pairing limits, only behind a proxy that sets it. | + +Set `CODEAF_SYNC_URL=https://relay.example.com` on each computer, or use +`codeaf pair --via https://relay.example.com`. Pair the computers with the same +identity and relay. `CODEAF_SYNC_URL=off` stops sync, including existing clients. + +The relay supplies no TLS or list of allowed people. Use a TLS proxy and a private +network or a proxy that checks callers: any valid signature can create an identity. +Logs contain a short identity id, route pattern, status and byte counts, never a body. +The native relay serves at the root of its host; give it a host of its own, or have +your proxy preserve the client's signed request path. + +## Three anchors, joined computers and leadership + +Each identity has three local persistent voters, called anchors. A directory change +commits on a majority before success; an authority read needs a fresh majority +response. One unavailable member is tolerated. All three anchors on this installation +share its host and disk: losing that host stops service. This command does not +configure independent anchor hosts. Encrypted content storage and the relay transport +must also remain reachable; an election does not replace them. + +An open paired laptop or desktop with sync on joins as a durable learner: it receives +coordination metadata without adding a vote. Pairing two computers does not require +either to stay awake. From a joined computer using this relay: + +```sh +codeaf cluster status +codeaf cluster lead +``` -The relay has no TLS and no allow-list. Put a TLS-terminating proxy in front of it, and keep -it on a private network or behind a proxy that checks who calls: any valid signature may make -an identity and store on it. Coordination rows have an 8 MiB default admission ceiling; -a larger imported identity keeps its measured ceiling. Reaching that ceiling refuses -growth without deleting saved records. Sealed blobs still need operator disk capacity. +Status shows this computer, the leader, voters and learners when known, after a fresh +majority read. Lead catches up this computer, replaces the third anchor's vote through +a joint membership change, finishes that change with the anchor still learning, then +transfers leadership. Success prints `computer is coordinating` followed by its +term, revision and read barrier. A pending request is not success; retrying lead reuses +the saved signed request. Keep codeaf open: without an open terminal the member runs +only for that command. Leadership does not transfer chat ownership or fetch content. -It logs the first 8 characters of an identity id, the kind of request, the status and two -byte counts. Never a body. +When that personal leader sleeps, exits or turns sync off, the two remaining anchors +can elect a leader and restore the third anchor. Healthy fallback targets about +65 seconds; delays and partitions can take longer. Returning computers remain learners +until explicitly asked to lead again. Losing the personal leader and either voting +anchor leaves no majority. The refusal is `this cluster cannot coordinate right now; reconnect enough of its voting computers and try again; your saved chats are still here`. +Saved local chats remain. There is no phone app yet, so a phone cannot lead yet. -## Back up +## Storage layout and capacity Everything is under `--store`: -- `raft//{1,2,3}.db`: three independent FULL-synchronous WAL stores for coordination; -- `raft//provision.json`: the immutable cluster ID and private protocol keys; -- `directory/.db`: the preserved frozen source and migration ownership receipt; -- `blobs//`: that identity's chunks and the pointers that find an object inside one. - -Stop the relay (SIGTERM; requests under way finish) and copy the whole folder, or copy a -filesystem snapshot. Copying the `.db` files while the relay runs can give a torn copy. -Restart using the same complete store. Never replace a surviving member with a fresh -import under its old ID. Two actual surviving anchor stores can coordinate; fewer than -two refuse service. Keep damaged or unavailable stores for diagnosis and recovery. - -A stale backup is not an ordinary restart. Do not start it beside a live cluster or -reactivate its frozen source. Disaster recovery needs a new epoch and exclusion of the -old cluster; automatic stale-backup recovery is not offered by this relay. - -The relay cannot decrypt what it keeps, so its backup is ciphertext too. It holds no key that -could recover a lost identity. Protocol keys cannot decrypt chats, but keep the store -private and back up its provisioning file together with its databases. - -## Upgrade - -Stop it, replace the `codeaf` binary, start it with the same `--store`. Back up -first. There is no migration command or feature switch. First use freezes an -existing identity, imports the same rows into three local anchors, checks a -fresh majority and publishes one durable owner. Interrupted imports resume with -the same UUID, keys and ceiling; they never reopen the legacy writer. The -original source stays as a backup until the identity is retired. - -All directory, rotation and blob authority requests use consensus. Three local voters -still share one host and disk: losing that host loses service. This setup does not -claim independent host failure tolerance. The current v1 adapter bounds encoded -commands at 64 KiB; a legal older request with a very large frame plan can still be -refused before proposal. Larger v1 requests are not supported yet. -A client newer than the relay by a pairing change sees `that relay is too old for pairing`. - -## The hosted relay - -`relay/hosted/` is the same relay as a Cloudflare Worker with Durable Objects and R2, with -per-identity limits (5 GiB, rate limits, a daily cap on new identities per network, set in `relay/hosted/src/limits.js`). Its own -README says how to deploy it to an account. Nothing in this repository deploys it. +- `raft//{1,2,3}.db`: separate FULL-synchronous SQLite WAL coordination stores. +- `raft//provision.json`: immutable cluster id and private protocol keys. +- `raft//transport-generation`: durable transport generation. +- `directory/.db`: frozen original source and migration ownership receipt. +- `blobs//`: encrypted chunks and their object pointers. + +Keep associated WAL files and the rest of the store together. Coordination records +have an 8 MiB default allowance per identity; a larger imported identity keeps its +measured allowance. Capacity refuses growth instead of deleting existing records. +Snapshots compact redundant coordination history, never saved chats, branches, vaults +or encrypted chunks. Allow disk space for three copies, snapshots and SQLite overhead, +as well as the separate encrypted content store. + +## Back up and recover + +Stop the relay with SIGTERM or ctrl+c and wait for in-flight requests to finish, then +copy the whole store. A consistent filesystem snapshot is another option. Copying +individual live database files can produce a torn backup. Restart with the same +complete store; keep its provisioning keys private even though they cannot decrypt chats. +The relay has no key that could recover a lost personal identity. + +Two actual surviving anchor stores can coordinate in the normal three-anchor topology; +fewer than two refuse. Preserve damaged stores for recovery. Never replace one with a +fresh import under its old node id or reopen the frozen source as a writer. A stale +backup is not an ordinary restart: do not start it beside a live cluster. Automatic +recovery from a stale backup into a new writable cluster is not offered. + +## Upgrade and older builds + +Back up, stop the relay, replace the binary and restart with the same store. There is +no migration command or feature switch. First use freezes the old identity directory, +imports its rows into three anchors, checks a fresh majority and publishes one owner. +An interrupted import resumes the same cluster, keys and capacity allowance; the source +is retained and a damaged import refuses requests instead of truncating records. + +This upgrade is one-way. After the cluster acknowledges changes, an older relay binary +cannot resume the frozen writer. A code rollback must preserve the migration gate and +consensus routing. Encrypted chunks stay in their existing store. + +Older clients keep pairing, listing, saving, publishing and releasing. Blindly forcing +a live chat without its observed holder and ownership number is refused with +`another device continued this chat first`. Quit on the holder or update the other +computer. Released or expired chats can still be taken. Safe conditional takes keep +their existing behavior. + +Both relays preserve the original 1 MiB HTTP body limit for legal older directory +requests, including large signed metadata and frame hints. Bounded body staging makes +the final change atomic, rechecks the complete original proof and current authority, +and recovers interrupted requests without exposing partial changes. Sealed names, key +wraps and all frame hints remain intact. Over 1 MiB is refused, and retained storage +capacity still applies. A pairing protocol newer than the relay can still say +`that relay is too old for pairing`. + +## Workers and idle traffic + +A delegated worker uses its own expiring certificate and sealed grants for selected +chats. It cannot approve pairing, rotate the identity, read its vault, enroll devices +or become leader. Availability alone starts no tools: an assignment needs a committed +conditional claim. Interrupted work keeps its records and is never automatically +replayed. Worker setup and execution are not terminal commands yet. A scoped API worker +does not receive the full coordination log; explicit learner consent exposes +identity-wide routing metadata, with content and names still encrypted. The production +client does not activate delegated learners yet. + +Caught-up idle homes with no work or held chats send zero HTTP requests, peer or +forwarding frames, or client attempts, including under personal leadership and after +fallback. All voters first apply the same idle barrier. Fresh input wakes coordination +and requires a majority response. Compatibility presence viewers still ping; only a +viewer causes presence sweeps, and presence alone never renews chat ownership. + +## Hosted relay + +The hosted relay in `relay/hosted` uses the same Go coordination core with three +separately addressed Durable Object anchors per identity and encrypted chunks in R2. +Those anchors share their provider and relay path, so they do not survive a +provider-wide outage. Joined computers can lead through the same commands. First use +upgrades an existing identity with the same one-way rule; per-identity and shared +storage and request limits still apply. Hosting account operations are separate from +running the native relay described here. codeaf's own deployment is called the fabric and is served at `https://codeaf.agentfield.ai/fabric`, -the address every computer syncs through when nobody names another. It sees only ciphertext, the -time and size of each request, and the network address of each computer; it keeps no request log. -The address has a path, and every client keeps it. `codeaf relay` (this page) always serves from -the root of its host; behind your own proxy, pass the path through unchanged. +the address every computer syncs through when nobody names another. It stores encrypted chat +content and the readable coordination metadata described at the top of this page, sees the time +and size of each request and the network address of each computer, and keeps no request log. The +address has a path, and every client keeps it. `codeaf relay` (this page) always serves from the +root of its host; behind your own proxy, pass the path through unchanged. diff --git a/docs/ux-pairing-contract.md b/docs/ux-pairing-contract.md index 790b4a443e..a847540a86 100644 --- a/docs/ux-pairing-contract.md +++ b/docs/ux-pairing-contract.md @@ -150,11 +150,12 @@ Called by an already-paired device of the identity that is approving. } ``` -Effect, in this order and in one turn of the identity object: verify `cert` is for the request's -`device`; write the Device record (as `PUT /v1/dir/devices/{id}` would, `created` set by the relay to -directory time); bump the directory version (watchers hear `{"v":N}`); emit `joined` (section 5); set -the request `approved` with `grant`. A retry of the same approve with the same `device` is `204` -(idempotent). Answer `204`. +Verify `cert` is for the request's `device`. Commit the device authorization, Device record, +sealed decision and delivery outbox together through the identity's Raft group; `created` uses +committed directory time. The directory version advances only after application. Deliver the +recorded approval and `grant` to the request object afterward, and emit `joined` (section 5). +Interrupted delivery resumes from the same recorded decision. An identical approve retries +idempotently and answers `204`; no grant becomes visible before its authorization commits. | Status | err | When | |---|---|---| @@ -181,9 +182,11 @@ Any paired device may approve or deny; the first decision wins. "dev_B": { "online": false, "last_seen": 1789999000000 } } } ``` -One entry per non-revoked device in the directory. Cost: reads socket state only, no storage write. -Answers with `Codeaf-Dir-Version` like `/v1/dir/list`. Used at app open and as the fallback for a client -whose socket is down. +One entry per non-revoked device in the committed directory, combined with advisory socket +presence. Authorization and directory rows pass a fresh majority read; sampling current committed +time can also write coordination state. Answers with `Codeaf-Dir-Version` like `/v1/dir/list`. +Compatibility clients use this on app open or when their watch is down. A joined Home follows +its local applied learner for display hints; taking or publishing still requires fresh authority. ### 3.6 Limits: `GET /v1/link/limits` (open) @@ -220,7 +223,8 @@ Rules: - `created` is set by the relay on first write of a device id and never changes afterward; a client value is ignored. - `last_seen` is set only by the relay: at watch connect (hello) and at watch close. It is not written - per ping, so an open quiet socket costs no storage write. A client value is ignored. + per ping, and a client value is ignored. An idle presence-only socket does not write ownership; + a socket naming held chats still needs separate committed renewals and hold-until receipts. - `platform` is client-supplied, same value set as section 1, written by self-registration or approve. - Rename and `caps` changes keep using `PUT /v1/dir/devices/{id}` (self only). `Revoked` is still set only by revoke. @@ -242,8 +246,9 @@ frames. An event frame has a string key `t` and no key `v`: {"t":"revoked","device":"dev_B","at":1790000700000} ``` -- `joined`: sent to every open event socket of the identity (the new device itself excluded) in the same - turn as the approve. The directory version also moves, so a client that only lists still sees it. +- `joined`: sent to every open event socket of the identity (the new device itself excluded) after + the approval's device record and decision commit. The directory version also moves, so a client + that only lists still sees it. The UI string is " is now paired", decoded from `name` with the metadata key. - `presence`: sent to the other event sockets when the first socket of a device opens (`online:true`) and when its last socket closes (`online:false`). A device is online iff it holds at least one open diff --git a/internal/raftdirectory/README.md b/internal/raftdirectory/README.md index 8b29e73548..c2c9339af0 100644 --- a/internal/raftdirectory/README.md +++ b/internal/raftdirectory/README.md @@ -147,8 +147,10 @@ adapter checks current root/device authority inside that callback. Missing request proof is refused; no in-process `For` call can bypass authentication. The host supplies watches, personal-member lifecycle, worker/v2 HTTP ingress -and the wake/outbox coordinator around this translation adapter. Encoded -commands remain bounded at 64 KiB, so large legacy HTTP bodies need a supported -representation before full legacy compatibility can be claimed. Persistent -namespace assembly must route the real HTTP doors through consensus, with one -writer and an unchanged quorum. +and the wake/outbox coordinator around this translation adapter. Individual +commands remain bounded at 64 KiB. Legal original v1 bodies up to the 1 MiB HTTP +limit use bounded staged body pages; only the final application changes directory +rows, after rechecking the full original proof and current authority. Interrupted +staging can resume without exposing partial metadata. Both production relays use +this representation and the same consensus path. Capacity refuses retained growth +without removing records or increasing the quorum. diff --git a/relay/hosted/README.md b/relay/hosted/README.md index b20838461d..c4938215f0 100644 --- a/relay/hosted/README.md +++ b/relay/hosted/README.md @@ -3,7 +3,8 @@ The relay of `internal/relayserve` as a Cloudflare Worker. The Go client holds both implementations to `internal/relayconf`; run the local matrix below before claiming conformance for a build. -The relay sees only ciphertext and keeps no account table: an identity is the hash of its key. +Chat content, names and credentials stay encrypted. The relay also reads routing, +ownership and coordination metadata and keeps no account table: an identity is the hash of its key. ## How it is built @@ -61,7 +62,7 @@ caller itself 400 `self_revoke`; twice is fine). A revoked device is `401 revoke a device record cannot set or clear its own flag. Directory watch is `GET /v1/dir/watch`, a signed WebSocket upgrade on the same object -(`src/watch.js`). It replaces the home screen's polling: the object keeps a durable version of the +(`src/watch.js`). It supplies compatibility version and presence hints: the object keeps a durable version of the directory (the replicated application version), bumps it on every change a person could see (a heartbeat that only moves a lease's expiry is not one), and sends `{"v":N}` to every socket. It accepts with the hibernation API and answers the client's `ping` with the platform's auto-response, so an open, quiet socket wakes @@ -87,11 +88,17 @@ the object to be evicted does not reset them. Link pairing (`src/link/`, `docs/ux-pairing-contract.md`) is the sibling for a new device that asks a paired device to approve it: `LinkGate` holds the per-IP decisions and allocates the 8-character code, one `LinkRequest` object per code holds the record and moves it once (pending, then approved or denied), and the -signed approve and deny routes under `/v1/dir/requests/` are served by the identity's own object, which -writes the device in one synchronous turn. The relay stores the grant and the sealed name as opaque bytes. - -The frozen Go code is not compiled into the Worker. The three pure parts (frame decode, request -verify, lease rules) are ported to JavaScript and pinned to Go by vectors that Go generates. +signed approve and deny routes under `/v1/dir/requests/` enter through the identity coordinator. +The shared Go state machine commits device authorization, metadata and the sealed decision together. +A retained outbox delivers that decision to the request object afterward and repairs interrupted +delivery. The grant and sealed name remain opaque bytes. + +The pinned modified Go Raft core and directory state machine compile into the Worker's WASI reactor. +JavaScript owns authentication, storage, transport and encrypted blobs; it does not decide lease or +membership policy. Original v1 bodies keep their 1 MiB HTTP limit. Large signed requests use bounded +body pages and an atomic final application, preserving all metadata without exposing partial changes. +Frame decoding and request verification still have JavaScript ports pinned by Go vectors; the old +JavaScript directory rules serve only source conversion and legacy fixtures. ## Run it locally From af64fdfa6160a097facbc814eaf1fb55b6e245e9 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 14:31:52 -0400 Subject: [PATCH 077/103] Describe the relay coordination change. The relay uses a majority per identity, joined computers can lead, and fallback preserves saved work. Older requests retain their original size limit while scoped workers and irreversible upgrades keep their stated boundaries. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/changes/unreleased/raft-cluster.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 docs/changes/unreleased/raft-cluster.md diff --git a/docs/changes/unreleased/raft-cluster.md b/docs/changes/unreleased/raft-cluster.md new file mode 100644 index 0000000000..fc061d556d --- /dev/null +++ b/docs/changes/unreleased/raft-cluster.md @@ -0,0 +1,23 @@ +--- +kind: changed +title: Relay coordination uses three voters and joined computers can lead +pr: 0 +surface: [chat, remote, build, docs] +invalidates: + - "One relay directory writer previously decided each identity's changes. The hosted relay and codeaf relay --store now use a three-voter Raft group per identity, with majority commitment and fresh authority reads." + - "Paired computers previously used only the directory API. Open joined computers now keep durable learner copies without increasing the voting quorum; worker churn does not add voters." + - "Joined computers had no leadership commands. codeaf cluster status now reports committed membership and the current leader, and codeaf cluster lead safely replaces one anchor vote, transfers leadership and returns an elected read receipt; saved requests retry unchanged." + - "Coordination depended on the relay's single writer. Two remaining anchors can now elect a fallback and restore the third after a personal leader disappears, targeting about 65 seconds in healthy conditions; a returning computer remains a learner and never automatically regains leadership." + - "A majority was not required by the former directory writer. Losing two of the three current voters now refuses coordination and preserves saved chats instead of creating a writable minority." + - "Caught-up open homes previously polled shared state. Idle homes now send zero HTTP requests, peer or forwarding frames, or client attempts after all voters apply the idle barrier; presence sweeps recur only with a viewer and fresh input requires a new majority response." + - "Older clients could blindly force a live chat without its observed holder. Both relays now refuse that request with another device continued this chat first; releasing the chat or updating the client permits safe continuation." + - "Legal older directory requests could be refused above the smaller coordination envelope. Both relays now retain the original 1 MiB HTTP limit, stage large signed bodies and commit complete metadata atomically, recovering interruptions without exposing partial changes or deleting retained records." + - "Workers previously needed personal-device trust for shared access. Delegated workers now have expiring selected-chat grants and conditional assignment claims; they cannot approve pairing, read the identity vault, rotate it, enroll devices or lead, and terminal worker setup is not available yet." + - "A single directory database was formerly the live source. First use now freezes and imports it into the three anchors while retaining the source, and the one-way upgrade cannot return to that writer after cluster changes are acknowledged." + - "Self-hosted identities were described as limited only by disk space. Coordination records now have an 8 MiB default allowance, with larger imports retaining their measured allowance; capacity refuses growth without deleting saved data." +--- + +Leadership moves coordination metadata while chats keep their ordinary ownership, +encryption and save rules. Three voters tolerate one unavailable member; local +anchors share one host, and hosted anchors share their provider and relay path. +Phone leadership and terminal worker provisioning are not available yet. From aeb5d27b83c7c398510354ef8e0ec918bda463ea Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 17:07:43 -0400 Subject: [PATCH 078/103] Reuse durable directory authority without repeating unchanged work. Unchanged peer acknowledgements reuse validated routing while every applied change refreshes authority. Reads authenticate original authority at a fresh majority barrier and append only changes to durable time or records. Preserve committed revocation and bound unchanged transport work without relaxing durability. Co-Authored-By: Claude Opus 5.5 (1M context) --- PERF.md | 12 +++ internal/raftdirectory/native.go | 57 +++++++++++++ internal/raftdirectory/personal.go | 8 +- internal/raftdirectory/read_time_test.go | 80 +++++++++++++++++++ internal/raftdirectory/transport.go | 45 ++++++++--- internal/raftdirectory/transport_work_test.go | 73 +++++++++++++++++ internal/raftdirectory/v1.go | 4 +- internal/syncsetup/member_client_test.go | 35 +++++++- 8 files changed, 297 insertions(+), 17 deletions(-) create mode 100644 internal/raftdirectory/read_time_test.go create mode 100644 internal/raftdirectory/transport_work_test.go diff --git a/PERF.md b/PERF.md index 5039ba4873..cf48fceda5 100644 --- a/PERF.md +++ b/PERF.md @@ -83,6 +83,18 @@ migration flag skips that check. `BenchmarkOpenSettledSessions` measures the whole public open/close operation. `TestSettledStoreTerminalStartup` separates fixture construction from the first interactive frame of the running terminal. +Native peer acknowledgements reuse the validated routing view only for the same +store and durably applied index. Every applied change, including revocation, +invalidates that view. `TestR11R27UnchangedPeerAcknowledgementWork` exercises the +real transport and caps an unchanged acknowledgement at 1,000 allocations with +no application append; the measured cost fell from 3,273 allocations to 192. + +An authenticated native directory read with already durable ingress time uses a +fresh quorum view and the normal state evaluator before omitting an identical +append. New time and changed records still commit before inspection. +`TestR4R26R27ReadCommitsOnlyChangedTime` counts applied entries for unchanged, +advanced and repeated time, and requires invalid original authority to fail. + ## What the gates check `make test-quick` checks compilation, vet, formatting, packed manuals, change diff --git a/internal/raftdirectory/native.go b/internal/raftdirectory/native.go index 1e046b446e..c86bda32f8 100644 --- a/internal/raftdirectory/native.go +++ b/internal/raftdirectory/native.go @@ -639,10 +639,34 @@ func (g *Native) ReadV1(ctx context.Context, c clusterstate.Command, committed b defer cancel() g.mu.Lock() defer g.mu.Unlock() + if ctx.Err() != nil { + return directory.ErrUnreachable + } if !committed { if c.Change.AdvanceTime == nil { return directory.ErrUnauthorized } + if c.V1 != nil { + if i := g.replica(); i >= 0 { + app, _, err := g.stores[i].ApplicationView(ctx) + if err != nil { + return err + } + if c.Now <= app.LogicalTime { + i, app, err = g.barrier(ctx) + if err != nil { + return err + } + unchanged, err := g.unchangedReadTime(ctx, i, app, c) + if err != nil { + return err + } + if unchanged { + return inspect(ctx, g.stores[i], app) + } + } + } + } r, e := g.submit(ctx, c) if e != nil { return e @@ -657,6 +681,39 @@ func (g *Native) ReadV1(ctx context.Context, c clusterstate.Command, committed b } return inspect(ctx, g.stores[i], app) } + +// A read whose time is already durable still authenticates its original proof +// against the fresh quorum view. Only an identical evaluator result can avoid +// an append; time advances and quiescence changes retain their durable writes. +func (g *Native) unchangedReadTime(ctx context.Context, i int, app raftstore.Application, c clusterstate.Command) (bool, error) { + raw, err := clusterstate.EncodeCommand(c) + if err != nil { + return false, err + } + _, size, err := g.stores[i].ApplicationView(ctx) + if err != nil { + return false, err + } + read := recordReader(ctx, g.stores[i], app) + e, err := clusterstate.Evaluate(raw, clusterstate.ApplicationMeta{Term: g.status[i].Term, Index: app.Applied, Configuration: raftcore.Configuration(app.ConfState), Cluster: g.manifest.Config.Cluster, Now: app.LogicalTime, Version: app.VisibleVersion, Bytes: size, Ceiling: g.manifest.Ceiling}, read) + if err != nil { + return false, err + } + if e.Result.Code != clusterstate.OK || e.Now != app.LogicalTime || e.Version != app.VisibleVersion { + return false, nil + } + for _, w := range e.Writes { + old, err := read(w.RecordKey) + if err != nil { + return false, err + } + if w.Delete && old != nil || !w.Delete && !bytes.Equal(old, w.Data) { + return false, nil + } + } + return true, nil +} + func (g *Native) CommitEffect(ctx context.Context, c clusterstate.Command) (clusterstate.Result, error) { g.opMu.Lock() defer g.opMu.Unlock() diff --git a/internal/raftdirectory/personal.go b/internal/raftdirectory/personal.go index adb9307325..e566b61ab2 100644 --- a/internal/raftdirectory/personal.go +++ b/internal/raftdirectory/personal.go @@ -47,7 +47,13 @@ func (g *Native) membership(ctx context.Context) (int, raftstore.Application, cl return i, app, clusterstate.Membership{}, nil, err } read := recordReader(ctx, g.stores[i], app) - h, _, err := clusterstate.MembershipView(read, raftcore.Configuration(app.ConfState), app.Applied) + // Membership planning needs the registry. Routing separately validates + // each peer before installing its authority at this applied index. + var h clusterstate.Membership + raw, err := read(clusterstate.MembershipKey()) + if err == nil && raw != nil && (clusterstate.Decode(raw, &h) != nil || h.Validate() != nil || h.Index > app.Applied) { + err = clusterstate.ErrDependency + } return i, app, h, read, err } func (g *Native) controlChange(ctx context.Context, change clusterstate.Mutation, now int64) (clusterstate.Result, error) { diff --git a/internal/raftdirectory/read_time_test.go b/internal/raftdirectory/read_time_test.go new file mode 100644 index 0000000000..8567bcd4d9 --- /dev/null +++ b/internal/raftdirectory/read_time_test.go @@ -0,0 +1,80 @@ +package raftdirectory_test + +import ( + "context" + "crypto/ed25519" + "path/filepath" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/raftdirectory" + "github.com/Agent-Field/codeaf/internal/raftstore" + "github.com/Agent-Field/codeaf/internal/reqsign" +) + +// R4/R26/R27 require fresh quorum reads without repeating an already durable +// logical time. New ingress time still becomes durable before inspection. +func TestR4R26R27ReadCommitsOnlyChangedTime(t *testing.T) { + f := frozenV1(t) + source, err := directory.OpenSQLite(filepath.Join(t.TempDir(), "source.db"), func() time.Time { return time.UnixMilli(f.Now) }) + must(t, err) + defer source.Close() + root := key("root").Public().(ed25519.PublicKey) + cfg := cs.LegacyConfig{Cluster: cluster, Generation: 1, Identity: cs.IdentityID(root), SigningKey: root, EffectsKey: key("effects").Public().(ed25519.PublicKey), Anchors: []uint64{1, 2, 3}, Now: f.Now} + must(t, source.FreezeLegacy(ctx, cfg)) + manifest, err := source.InventoryLegacy(ctx) + must(t, err) + host, err := raftdirectory.OpenNative(ctx, t.TempDir(), source, manifest) + must(t, err) + defer host.Close() + var command cs.Command + for _, one := range f.Cases { + if one.Name == "list" { + proof, err := reqsign.Capture(frozenRequest(t, one, "http://localhost"), []byte(one.Body)) + must(t, err) + command, err = cs.V1Command(cluster, f.Now, proof) + must(t, err) + break + } + } + if command.V1 == nil || command.Change.AdvanceTime == nil { + t.Fatal("no original signed read fixture") + } + var before raftstore.Application + must(t, host.Inspect(ctx, func(_ context.Context, _ *raftstore.SQLite, app raftstore.Application) error { + before = app + return nil + })) + read := func(c cs.Command) raftstore.Application { + t.Helper() + var got raftstore.Application + must(t, host.ReadV1(ctx, c, false, func(_ context.Context, _ *raftstore.SQLite, app raftstore.Application) error { + got = app + return nil + })) + return got + } + unchanged := read(command) + if unchanged.Applied != before.Applied || unchanged.LogicalTime != before.LogicalTime { + t.Fatalf("unchanged time appended: before=%+v after=%+v", before, unchanged) + } + command.Now += 1000 + advanced := read(command) + if advanced.Applied != before.Applied+1 || advanced.LogicalTime != command.Now { + t.Fatalf("new ingress time was not committed before inspection: before=%+v after=%+v", before, advanced) + } + if repeated := read(command); repeated.Applied != advanced.Applied || repeated.LogicalTime != advanced.LogicalTime { + t.Fatalf("repeated time appended: before=%+v after=%+v", advanced, repeated) + } + invalid := command + invalid.Cluster = "another-cluster" + called := false + if err := host.ReadV1(ctx, invalid, false, func(context.Context, *raftstore.SQLite, raftstore.Application) error { + called = true + return nil + }); err == nil || called { + t.Fatalf("invalid proof reached inspection: error=%v called=%v", err, called) + } +} diff --git a/internal/raftdirectory/transport.go b/internal/raftdirectory/transport.go index f05205daab..6006a18c44 100644 --- a/internal/raftdirectory/transport.go +++ b/internal/raftdirectory/transport.go @@ -11,6 +11,7 @@ import ( "github.com/Agent-Field/codeaf/internal/directory" "github.com/Agent-Field/codeaf/internal/raftcore" "github.com/Agent-Field/codeaf/internal/raftsnapshot" + "github.com/Agent-Field/codeaf/internal/raftstore" "github.com/Agent-Field/codeaf/internal/rafttransport" pb "go.etcd.io/raft/v3/raftpb" "google.golang.org/protobuf/proto" @@ -25,17 +26,19 @@ type snapshotTransfer struct { started int64 } type nativePeers struct { - incoming map[int]*incomingSnapshot - router *rafttransport.Router - view rafttransport.View - keys [3]ed25519.PrivateKey - sequence [3]uint64 - ack [3][3]uint64 - applied map[uint64]uint64 - snapshots map[uint64]*snapshotTransfer - now int64 - sleeping bool - checkpoint bool + viewStore *raftstore.SQLite + viewApplied uint64 + incoming map[int]*incomingSnapshot + router *rafttransport.Router + view rafttransport.View + keys [3]ed25519.PrivateKey + sequence [3]uint64 + ack [3][3]uint64 + applied map[uint64]uint64 + snapshots map[uint64]*snapshotTransfer + now int64 + sleeping bool + checkpoint bool } // AttachTransport restores authority from applied membership before exposing @@ -67,21 +70,33 @@ func (g *Native) installPeers(ctx context.Context) error { if p == nil { return nil } - _, app, _, read, err := g.membership(ctx) + i := g.replica() + if i < 0 { + return directory.ErrUnreachable + } + // Peer authority changes only with durable application. Acknowledgements + // and read barriers reuse the view for that same store and applied index. + if p.viewStore == g.stores[i] && p.viewApplied == g.status[i].Applied { + return nil + } + app, _, err := g.stores[i].ApplicationView(ctx) if err != nil { return err } + read := recordReader(ctx, g.stores[i], app) v, err := rafttransport.CommittedView(p.view.Scope, p.view.Generation, app.Applied, raftcore.Configuration(app.ConfState), read) if err != nil { return err } if reflect.DeepEqual(v.Members, p.view.Members) { + p.viewStore, p.viewApplied = g.stores[i], app.Applied return nil } if err = p.router.Install(v); err != nil { return err } p.view = v + p.viewStore, p.viewApplied = g.stores[i], app.Applied // Installing a changed voter class discards expendable deliveries. Start a // real read round so append progress is retried immediately, including while // a public transfer owns opMu and the periodic maintenance turn is waiting. @@ -233,7 +248,11 @@ func (g *Native) pumpPeers(request rafttransport.Request, now int64) error { return rafttransport.ErrAuth } if request.Applied != 0 { - _, app, _, _, err := g.membership(ctx) + i := g.replica() + if i < 0 { + return directory.ErrUnreachable + } + app, _, err := g.stores[i].ApplicationView(ctx) if err != nil { return err } diff --git a/internal/raftdirectory/transport_work_test.go b/internal/raftdirectory/transport_work_test.go new file mode 100644 index 0000000000..fd5380a048 --- /dev/null +++ b/internal/raftdirectory/transport_work_test.go @@ -0,0 +1,73 @@ +package raftdirectory + +import ( + "crypto/ed25519" + "path/filepath" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/rafttransport" +) + +// R11/R27 bound the work of acknowledging an unchanged committed peer view. +func TestR11R27UnchangedPeerAcknowledgementWork(t *testing.T) { + ctx := t.Context() + var keys [3]ed25519.PrivateKey + var public [][]byte + for i := range keys { + seed := make([]byte, ed25519.SeedSize) + seed[0] = byte(i + 1) + keys[i] = ed25519.NewKeyFromSeed(seed) + public = append(public, keys[i].Public().(ed25519.PublicKey)) + } + root := ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize)).Public().(ed25519.PublicKey) + effectSeed := make([]byte, ed25519.SeedSize) + effectSeed[0] = 4 + effects := ed25519.NewKeyFromSeed(effectSeed).Public().(ed25519.PublicKey) + now := time.UnixMilli(1700000000000) + source, err := directory.OpenSQLite(filepath.Join(t.TempDir(), "source.db"), func() time.Time { return now }) + if err != nil { + t.Fatal(err) + } + defer source.Close() + cfg := cs.LegacyConfig{Cluster: "peer-work", Generation: 1, Identity: cs.IdentityID(root), SigningKey: root, EffectsKey: effects, Anchors: []uint64{1, 2, 3}, AnchorKeys: public, Now: now.UnixMilli()} + if err = source.FreezeLegacy(ctx, cfg); err != nil { + t.Fatal(err) + } + manifest, err := source.InventoryLegacy(ctx) + if err != nil { + t.Fatal(err) + } + host, err := OpenNative(ctx, t.TempDir(), source, manifest) + if err != nil { + t.Fatal(err) + } + defer host.Close() + if err = host.InitializeMembership(ctx, keys[0]); err != nil { + t.Fatal(err) + } + router, err := host.AttachTransport(ctx, 1, keys) + if err != nil { + t.Fatal(err) + } + request := rafttransport.Request{Scope: host.peers.view.Scope, Node: 1, Incarnation: 1, Generation: 1, Applied: host.status[0].Applied} + if err = request.Sign(keys[0]); err != nil { + t.Fatal(err) + } + before := host.status[host.replica()].Applied + allocations := testing.AllocsPerRun(5, func() { + if _, err := router.Exchange(request, now.UnixMilli()); err != nil { + t.Fatal(err) + } + }) + if after := host.status[host.replica()].Applied; after != before { + t.Fatalf("an empty acknowledgement appended application work: %d -> %d", before, after) + } + t.Logf("unchanged peer acknowledgement: %.0f allocations", allocations) + const maxAllocations = 1000 + if allocations > maxAllocations { + t.Fatalf("unchanged peer acknowledgement allocated %.0f times, limit %d", allocations, maxAllocations) + } +} diff --git a/internal/raftdirectory/v1.go b/internal/raftdirectory/v1.go index 323040533b..c3e520928f 100644 --- a/internal/raftdirectory/v1.go +++ b/internal/raftdirectory/v1.go @@ -13,8 +13,8 @@ import ( ) // V1Host holds its actor queue through commit and through the read callback. -// Commit returns only a durable applied result. Read commits the GET command's -// ingress time, obtains a fresh current-term ReadIndex and waits for durable +// Commit returns only a durable applied result. Read ensures the GET command's +// ingress time is durable, obtains a fresh current-term ReadIndex and waits for // application before calling inspect. A POST already committed by this request // uses that time and a NEW barrier without repeating its mutation. Callbacks // receive the actor-owned context: a caller losing its response cannot cancel diff --git a/internal/syncsetup/member_client_test.go b/internal/syncsetup/member_client_test.go index cfbad9942f..a0b1157223 100644 --- a/internal/syncsetup/member_client_test.go +++ b/internal/syncsetup/member_client_test.go @@ -180,7 +180,9 @@ func newMemberRig(t *testing.T) *memberRig { s.stopMember() } r.server.Close() - r.svc.Close() + if r.svc != nil { + r.svc.Close() + } }) return r } @@ -234,6 +236,37 @@ func readyMember(t *testing.T, s *Sync) { mustNil(t, s.memberRound(t.Context())) waitMember(t, s, func(m *personalMember) bool { return m.idle && m.status.Applied >= m.record.Grant.Barrier }) } + +// R16 denies a saved peer request as soon as device revocation commits, even +// when the membership revision and transport generation have not changed. +func TestR16CommittedRevocationInvalidatesPeerView(t *testing.T) { + r := newMemberRig(t) + a := r.home(nil) + id := a.Identity + b := r.home(&id) + mustNil(t, b.Dir.PutDevice(t.Context(), b.Device.ID(), directory.Device{V: 1, Name: "c2VhbGVk"})) + readyMember(t, b) + m := b.member + m.mu.Lock() + request, err := m.request() + m.mu.Unlock() + mustNil(t, err) + b.stopMember() + client := rafttransport.Client{URL: r.server.URL, HTTP: r.server.Client()} + _, err = client.Exchange(t.Context(), request) + mustNil(t, err) + before, err := a.clusterClient().Status(t.Context()) + mustNil(t, err) + mustNil(t, a.Dir.Revoke(t.Context(), b.Device.ID())) + after, err := a.clusterClient().Status(t.Context()) + mustNil(t, err) + if after.Membership.Revision != before.Membership.Revision { + t.Fatal("revocation fixture changed the membership revision") + } + if _, err = client.Exchange(t.Context(), request); !errors.Is(err, rafttransport.ErrAuth) { + t.Fatalf("saved peer request after committed revocation = %v, want refused authority", err) + } +} func nodeOf(s *Sync) uint64 { s.memberMu.Lock() defer s.memberMu.Unlock() From c1838b3a06c8eb77104a1e58f5849a511b7634bd Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 17:07:43 -0400 Subject: [PATCH 079/103] Preserve directory isolation and exact relay lease expiry. Recovered personal leaders keep independent directory state and obtain a fresh majority receipt before serving. Lease expiry and maintenance follow one relay timeline, including the instant before expiry and expiry itself. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../syncsetup/conformance_fixture_test.go | 75 +++++++++++++++++++ internal/syncsetup/drive_test.go | 43 ++++++++--- internal/syncsetup/leadership_native_test.go | 5 +- 3 files changed, 108 insertions(+), 15 deletions(-) create mode 100644 internal/syncsetup/conformance_fixture_test.go diff --git a/internal/syncsetup/conformance_fixture_test.go b/internal/syncsetup/conformance_fixture_test.go new file mode 100644 index 0000000000..a3691991b4 --- /dev/null +++ b/internal/syncsetup/conformance_fixture_test.go @@ -0,0 +1,75 @@ +package syncsetup + +import ( + "os" + "testing" + + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/directory/directorytest" + "github.com/Agent-Field/codeaf/internal/relayserve" +) + +// A closed bootstrap retains real independent stores and enrollment. Restoring +// it leaves each directory empty while its recovered member wins leadership +// through the ordinary transfer and fresh majority-read receipt. +func personalLeaderDirectories(t *testing.T) func(*testing.T) (*memberRig, *Sync) { + t.Helper() + r := newMemberRig(t) + p := r.home(nil) + personalLeader(t, r, p) + p.stopMember() + mustNil(t, r.svc.Close()) + r.svc = nil + storeImage, homeImage := t.TempDir(), t.TempDir() + mustNil(t, os.CopyFS(storeImage, os.DirFS(r.storage))) + mustNil(t, os.CopyFS(homeImage, os.DirFS(p.Home))) + start := r.clock.Now() + home, id, dev := p.Home, p.Identity, p.Device + return func(t *testing.T) (*memberRig, *Sync) { + t.Helper() + for _, s := range r.homes { + s.stopMember() + } + if r.svc != nil { + mustNil(t, r.svc.Close()) + r.svc = nil + } + for _, image := range [][2]string{{storeImage, r.storage}, {homeImage, home}} { + mustNil(t, os.RemoveAll(image[1])) + mustNil(t, os.CopyFS(image[1], os.DirFS(image[0]))) + } + r.clock = directorytest.NewFakeClock() + r.clock.Advance(start.Sub(r.clock.Now())) + r.svc = relayserve.New(relayserve.Config{Store: r.storage, Now: r.clock.Now, After: r.clock.After}) + p := buildAt(home, r.server.URL, id, dev, DefaultInterval, r.clock.Now) + r.homes = []*Sync{p} + t.Cleanup(p.stopMember) + personalLeader(t, r, p) + return r, p + } +} + +// R6/R25 require every conformance case to start empty, including after the +// previous identity retired, and still use a recovered actual personal leader. +func TestR6R25PersonalLeaderDirectoryIsolation(t *testing.T) { + fresh := personalLeaderDirectories(t) + t.Run("retired", func(t *testing.T) { + _, p := fresh(t) + _, err := p.Dir.Create(t.Context(), "01J0000000000000000000000A", directory.CellInit{Head: head}) + mustNil(t, err) + mustNil(t, p.Dir.PutDevice(t.Context(), p.Device.ID(), directory.Device{V: 1, Name: "c2VhbGVk"})) + mustNil(t, p.Dir.SetVault(t.Context(), "", "saved-vault")) + _, err = p.Dir.Rotate(t.Context(), directory.RotationReq{Op: directory.OpFreeze}) + mustNil(t, err) + _, err = p.Dir.Rotate(t.Context(), directory.RotationReq{Op: directory.OpRetire}) + mustNil(t, err) + }) + t.Run("empty", func(t *testing.T) { + _, p := fresh(t) + listing, err := p.Dir.List(t.Context()) + mustNil(t, err) + if len(listing.Cells) != 0 || len(listing.Devices) != 0 || listing.Identity.Vault != "" || listing.Identity.Rotation != nil { + t.Fatalf("recovered directory retained the previous case: %+v", listing) + } + }) +} diff --git a/internal/syncsetup/drive_test.go b/internal/syncsetup/drive_test.go index bbb5d9a89d..88582fc7d7 100644 --- a/internal/syncsetup/drive_test.go +++ b/internal/syncsetup/drive_test.go @@ -12,7 +12,6 @@ import ( "slices" "strings" "sync" - "sync/atomic" "testing" "time" @@ -23,6 +22,7 @@ import ( "github.com/Agent-Field/codeaf/internal/cellsync" "github.com/Agent-Field/codeaf/internal/chatlist" "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/directory/directorytest" "github.com/Agent-Field/codeaf/internal/executor" "github.com/Agent-Field/codeaf/internal/furrow" "github.com/Agent-Field/codeaf/internal/home" @@ -32,23 +32,41 @@ import ( "github.com/Agent-Field/codeaf/internal/wireauth" ) -// skewedClock is the relay's clock: the wall clock plus an offset a test moves -// to let a lease run out without waiting for it. The offset stays well inside -// the five minutes a signed request may be off by. -type skewedClock struct{ offsetMs atomic.Int64 } +// driveClock keeps lease expiry and maintenance on the same explicit timeline. +// Starting near wall time keeps ordinary client signatures within clock skew. +type driveClock struct{ *directorytest.FakeClock } -func (c *skewedClock) Now() time.Time { - return time.Now().Add(time.Duration(c.offsetMs.Load()) * time.Millisecond) -} +func (c *driveClock) advance(d time.Duration) { c.Advance(d) } -func (c *skewedClock) advance(d time.Duration) { c.offsetMs.Add(d.Milliseconds()) } +// C7 allows another computer to acquire exactly when the relay's lease expires. +func TestC7LeaseExpiresAtRelayDeadline(t *testing.T) { + r := newDriveRig(t) + ctx := t.Context() + created, err := r.a.Dir.Create(ctx, r.cell.ID, directory.CellInit{Head: head}) + mustNil(t, err) + r.clock.advance(directory.LeaseTTL - time.Millisecond) + before, err := r.b.Dir.Cell(ctx, r.cell.ID) + mustNil(t, err) + if before.Now != created.Now+directory.LeaseTTL.Milliseconds()-1 { + t.Fatalf("relay time before expiry = %d, want %d", before.Now, created.Now+directory.LeaseTTL.Milliseconds()-1) + } + if _, err := r.b.Dir.Acquire(ctx, r.cell.ID, directory.AcquireOpts{}); !errors.Is(err, directory.ErrLeaseHeld) { + t.Fatalf("acquire before expiry = %v, want lease held", err) + } + r.clock.advance(time.Millisecond) + taken, err := r.b.Dir.Acquire(ctx, r.cell.ID, directory.AcquireOpts{}) + mustNil(t, err) + if taken.Now != created.Cell.Lease.Expires || taken.Cell.Lease.Fence != created.Cell.Lease.Fence+1 || taken.Cell.Lease.Device != r.b.Device.ID() { + t.Fatalf("acquire at expiry = %+v, original lease %+v", taken, created.Cell.Lease) + } +} // driveRig is one identity on two machines and a real relay with a store, in // this process. Machine A is the one that chats. type driveRig struct { t *testing.T bin string - clock *skewedClock + clock *driveClock store string // the relay's --store directory svc *relayserve.Service url string // where the relay answers @@ -67,9 +85,10 @@ func newDriveRig(t *testing.T, wrap ...func(http.Handler) http.Handler) *driveRi if err != nil { t.Skipf("no engine binary: %v", err) } - clock := &skewedClock{} + clock := &driveClock{directorytest.NewFakeClock()} + clock.Advance(time.Since(clock.Now())) storeDir := t.TempDir() - svc := relayserve.New(relayserve.Config{Store: storeDir, Now: clock.Now, Grace: directory.GraceBounds{Min: time.Second, Max: time.Hour, Default: 10 * time.Second}}) + svc := relayserve.New(relayserve.Config{Store: storeDir, Now: clock.Now, After: clock.After, Grace: directory.GraceBounds{Min: time.Second, Max: time.Hour, Default: 10 * time.Second}}) t.Cleanup(func() { _ = svc.Close() }) handler := svc.Handler for _, one := range wrap { diff --git a/internal/syncsetup/leadership_native_test.go b/internal/syncsetup/leadership_native_test.go index 099267eb85..d8d0712ff6 100644 --- a/internal/syncsetup/leadership_native_test.go +++ b/internal/syncsetup/leadership_native_test.go @@ -45,10 +45,9 @@ func personalLeader(t *testing.T, r *memberRig, s *Sync) cs.LeadershipReceipt { // T1/T2/R6 run the existing directory contract against the REAL external // leader. The relay is only ingress, storage and A/B/C; P has its own core. func TestR6PersonalLeaderDirectoryConformance(t *testing.T) { + fresh := personalLeaderDirectories(t) directorytest.RunConsensusRigs(t, func(t *testing.T) directorytest.Rig { - r := newMemberRig(t) - p := r.home(nil) - personalLeader(t, r, p) + r, p := fresh(t) id := p.Identity type apiDevice struct { client directory.Client From 657d6242270354a71b8ff25e1ef863f31f3c8c5f Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 17:07:43 -0400 Subject: [PATCH 080/103] Check the hosted directory's paged Go policy boundary. The hosted directory now evaluates and projects through the reactor's paged Go operations, so the routing law names those operations instead of the earlier single-call frames. It still requires the original signed commit, the barrier read and no JavaScript lease writer. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/raftdirectory/routing_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/raftdirectory/routing_test.go b/internal/raftdirectory/routing_test.go index ff99ac5221..03198f6bdb 100644 --- a/internal/raftdirectory/routing_test.go +++ b/internal/raftdirectory/routing_test.go @@ -12,7 +12,11 @@ func TestR30HostedDirectoryRoutesOnlyThroughGoPolicy(t *testing.T) { b, err := os.ReadFile("../../relay/hosted/src/raft/directory.js") must(t, err) s := string(b) - for _, needle := range []string{"frame(10)", "frame(11)", "this.host.commit(originalSignedBytes)", "this.host.barrierRead(authenticatedRequest)"} { + // The hosted directory evaluates and projects through the reactor's paged + // Go operations, which keep large legacy requests inside the command and + // buffer limits: 41 starts the shared Go evaluation and 45 projects its + // public result. + for _, needle := range []string{"frame(41)", "frame(45)", "this.host.commit(originalSignedBytes)", "this.host.barrierRead(authenticatedRequest)"} { if !strings.Contains(s, needle) { t.Fatalf("missing policy boundary %s", needle) } From 5fdae16fb1dc3702cad97e2f387842dfda9cbf3d Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 20:03:37 -0400 Subject: [PATCH 081/103] Keep deferred engine durability on its captured home. Capture the same socket and engine for primary and fallback routes so a later flush cannot follow a changed home. Await captured startup work before reaping its engine. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/codeaf/continuation_work_test.go | 10 ++ internal/cellstore/transport.go | 17 +++- internal/cellstore/transport_home_test.go | 111 ++++++++++++++++++++++ 3 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 internal/cellstore/transport_home_test.go diff --git a/cmd/codeaf/continuation_work_test.go b/cmd/codeaf/continuation_work_test.go index 6eb8d9b1f7..518b609db5 100644 --- a/cmd/codeaf/continuation_work_test.go +++ b/cmd/codeaf/continuation_work_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "fmt" "os" "os/exec" @@ -79,6 +80,15 @@ func TestC31AStartupSaveIsNotAChoice(t *testing.T) { if err != nil { t.Fatal(err) } + // The grouped durability worker owns the daemon until this cell's + // records and captured files have reached their closing boundary. + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), cellstore.SealWithin) + defer cancel() + if err := cellstore.Await(ctx, c); err != nil { + t.Error(err) + } + }) if err := session.SaveMeta(c.Root, session.Meta{ID: c.ID, Workspace: work}); err != nil { t.Fatal(err) } diff --git a/internal/cellstore/transport.go b/internal/cellstore/transport.go index c176f11684..0e63f09022 100644 --- a/internal/cellstore/transport.go +++ b/internal/cellstore/transport.go @@ -150,11 +150,18 @@ func (f Fallback) Do(ctx context.Context, t Target, op Op) ([]byte, error) { // caller supplies its own Runner (which is a promise to see every spawn). func transportFor(binary string, run Runner) Transport { direct := Spawn{Binary: binary, Run: run} - if run != nil || env.Get(daemonEnv) == "0" { + if run != nil { return direct } - // With no engine to name, the daemon cannot start and the spawn reports the - // missing engine in its own words. - bin, _ := Spawn{Binary: binary}.program() - return Fallback{Primary: Daemon{Socket: socketFor(bin), Binary: bin}, Secondary: direct} + // Deferred durability retains the same daemon address on both transports; + // a fallback must never resolve a different home's address at flush time. + bin, _ := direct.program() + if bin != "" { + direct.Binary = bin + } + direct.Socket = socketFor(bin) + if env.Get(daemonEnv) == "0" { + return direct + } + return Fallback{Primary: Daemon{Socket: direct.Socket, Binary: bin}, Secondary: direct} } diff --git a/internal/cellstore/transport_home_test.go b/internal/cellstore/transport_home_test.go new file mode 100644 index 0000000000..3522a21465 --- /dev/null +++ b/internal/cellstore/transport_home_test.go @@ -0,0 +1,111 @@ +package cellstore + +import ( + "bufio" + "encoding/json" + "fmt" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/home" +) + +// C28 keeps a deferred durability operation on the home that captured its files, +// including fallback after an engine health check refuses the first connection. +func TestC28DeferredDurabilityKeepsItsHome(t *testing.T) { + for _, mode := range []string{"1", "0"} { + t.Run(mode, func(t *testing.T) { + // Short profiles avoid requiring filesystem aliases for native Unix sockets. + root, err := os.MkdirTemp("", "ch") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.RemoveAll(root); err != nil { + t.Error(err) + } + }) + first, second := filepath.Join(root, "a"), filepath.Join(root, "b") + for _, dir := range []string{first, second} { + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + } + t.Setenv(home.EnvVar, first) + t.Setenv("LOCALAPPDATA", first) + t.Setenv(daemonEnv, mode) + binary := filepath.Join(first, "furrow-fixture") + socket := socketFor(binary) + if err := os.MkdirAll(filepath.Dir(socket), 0700); err != nil { + t.Fatal(err) + } + listener, err := net.Listen("unix", socket) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + t.Cleanup(func() { _ = os.RemoveAll(filepath.Dir(socket)) }) + done := make(chan error, 1) + go func() { + for attempt := 0; ; attempt++ { + conn, err := listener.Accept() + if err != nil { + done <- err + return + } + err = func() error { + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + decoder, encoder := json.NewDecoder(bufio.NewReader(conn)), json.NewEncoder(conn) + var request wireRequest + if err := decoder.Decode(&request); err != nil { + return err + } + if request.Verb != "health" { + return fmt.Errorf("first request %s", request.Verb) + } + name := filepath.Base(binary) + reject := mode == "1" && attempt == 0 + if reject { + name = "other-engine" + } + raw, _ := json.Marshal(struct{ Engine string }{name}) + if err := encoder.Encode(wireResponse{ID: request.ID, Ok: raw}); err != nil { + return err + } + if reject { + return nil + } + if err := decoder.Decode(&request); err != nil { + return err + } + if request.Verb != "sync" || request.Target.DataDir != first { + return fmt.Errorf("durability route %+v", request) + } + return encoder.Encode(wireResponse{ID: request.ID, Ok: json.RawMessage(`{"durable":true}`)}) + }() + if err != nil || mode == "0" || attempt > 0 { + done <- err + return + } + } + }() + transport := transportFor(binary, nil) + t.Setenv(home.EnvVar, second) + t.Setenv("LOCALAPPDATA", second) + if err := syncEngine(t.Context(), transport, Target{Tree: first, DataDir: first}); err != nil { + t.Fatal(err) + } + if err := <-done; err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(second) + if err != nil || len(entries) != 0 { + t.Fatalf("durability wrote into another home: %v, %v", entries, err) + } + }) + } +} From 1d5656fb8af525ed6b534d75d764e92173d15bbc Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 20:03:37 -0400 Subject: [PATCH 082/103] Keep reopened chats on the correct ownership record. Bind each window to the record it opened and check that lease through fresh directory reads. Settle a displaced writer before opening its replacement, and retain a failed closing result for every caller. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/codeaf/syncdrive.go | 15 +- cmd/codeaf/syncdrive_retire_test.go | 82 +++++++++ docs/changes/unreleased/raft-cluster.md | 2 + .../continuing-a-chat-on-another-computer.md | 4 + internal/syncsetup/drive.go | 28 +-- internal/syncsetup/drive_lease_test.go | 167 ++++++++++++++++++ internal/syncsetup/quit_contract_test.go | 3 + 7 files changed, 288 insertions(+), 13 deletions(-) create mode 100644 cmd/codeaf/syncdrive_retire_test.go create mode 100644 internal/syncsetup/drive_lease_test.go diff --git a/cmd/codeaf/syncdrive.go b/cmd/codeaf/syncdrive.go index be2f12a6e7..5c159d5b5c 100644 --- a/cmd/codeaf/syncdrive.go +++ b/cmd/codeaf/syncdrive.go @@ -46,8 +46,19 @@ func (b *driveBook) driveOf(c cell.Cell, engine cellstore.Engine, report func(er } b.mu.Lock() defer b.mu.Unlock() - if d := b.drive[c.ID]; d != nil && !d.superseded() { - return d + if d := b.drive[c.ID]; d != nil { + if !d.superseded() { + return d + } + // Settle the displaced writer before the replacement reads its branch map + // or acquires a lease, so late uploads cannot change the new drive's target. + ctx, cancel := context.WithTimeout(context.Background(), driveCloseWithin) + err := d.Close(ctx) + cancel() + if err != nil { + report(err) + return d + } } // A drive side that no longer holds the chat's lease (another machine took it, or it was moved // back here) never drives again: the chat opened here needs a drive side of its own, or every diff --git a/cmd/codeaf/syncdrive_retire_test.go b/cmd/codeaf/syncdrive_retire_test.go new file mode 100644 index 0000000000..bc9a244241 --- /dev/null +++ b/cmd/codeaf/syncdrive_retire_test.go @@ -0,0 +1,82 @@ +package main + +import ( + "context" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/cell" + "github.com/Agent-Field/codeaf/internal/cell/celltest" + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/relayserve" + "github.com/Agent-Field/codeaf/internal/syncsetup" +) + +// C7/R2 gives a reopened chat one maintenance loop. The displaced window must +// finish its old loop before another drive can acquire and publish its copy. +func TestC7R2ReopeningChatStopsPreviousDrive(t *testing.T) { + celltest.Available(t) + t.Setenv(home.EnvVar, t.TempDir()) + service := relayserve.New(relayserve.Config{Store: t.TempDir()}) + defer service.Close() + server := httptest.NewServer(service.Handler) + defer server.Close() + t.Setenv(syncsetup.URLVar, server.URL) + _, err := identity.Ensure(home.Dir()) + if err != nil { + t.Fatal(err) + } + s, ok, err := syncsetup.Open(home.Dir()) + if err != nil || !ok { + t.Fatalf("open: %v %v", ok, err) + } + c := cell.Cell{ID: "01J0000000000000000000000A", Root: t.TempDir()} + engine := cellstore.Engine{DataRoot: t.TempDir()} + started, stopped := make(chan struct{}), make(chan struct{}) + var once sync.Once + old, err := s.Drive(t.Context(), engine, c, syncsetup.DriveOptions{Sleep: func(ctx context.Context, _ time.Duration) error { + once.Do(func() { close(started) }) + <-ctx.Done() + close(stopped) + return ctx.Err() + }}) + if err != nil { + t.Fatal(err) + } + defer old.Close(context.Background()) + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("previous drive did not begin maintenance") + } + _, err = s.Dir.Create(t.Context(), c.ID, directory.CellInit{Head: "sealed-head"}) + if err != nil { + t.Fatal(err) + } + if err = old.CheckLease(t.Context()); err != nil { + t.Fatal(err) + } + if old.Gate() == nil { + t.Fatal("fixture did not displace the old drive") + } + kept := &liveDrive{drive: old, attempted: true} + book := &driveBook{drive: map[string]*liveDrive{c.ID: kept}} + defer book.closeAll() + next := book.driveOf(c, engine, func(error) {}) + if next == kept { + t.Fatal("reopened chat retained its displaced drive") + } + select { + case <-stopped: + default: + t.Error("reopened chat left the previous drive running") + } + if next != nil { + _ = next.Gate() + } +} diff --git a/docs/changes/unreleased/raft-cluster.md b/docs/changes/unreleased/raft-cluster.md index fc061d556d..bb9ed6ef9f 100644 --- a/docs/changes/unreleased/raft-cluster.md +++ b/docs/changes/unreleased/raft-cluster.md @@ -13,6 +13,8 @@ invalidates: - "Older clients could blindly force a live chat without its observed holder. Both relays now refuse that request with another device continued this chat first; releasing the chat or updating the client permits safe continuation." - "Legal older directory requests could be refused above the smaller coordination envelope. Both relays now retain the original 1 MiB HTTP limit, stage large signed bodies and commit complete metadata atomically, recovering interruptions without exposing partial changes or deleting retained records." - "Workers previously needed personal-device trust for shared access. Delegated workers now have expiring selected-chat grants and conditional assignment claims; they cannot approve pairing, read the identity vault, rotate it, enroll devices or lead, and terminal worker setup is not available yet." + - "Reopening retained work could compare its branch fence with the original chat and falsely refuse tools. Ownership checks stay with the record each window opened, and replacing a displaced drive settles its previous writer before starting another. A failed final save remains failed for every close caller." + - "Deferred local durability could resolve a fallback daemon against a different current home. Both engine routes now retain the home and engine selected at capture." - "A single directory database was formerly the live source. First use now freezes and imports it into the three anchors while retaining the source, and the one-way upgrade cannot return to that writer after cluster changes are acknowledged." - "Self-hosted identities were described as limited only by disk space. Coordination records now have an 8 MiB default allowance, with larger imports retaining their measured allowance; capacity refuses growth without deleting saved data." --- diff --git a/internal/manual/chat/continuing-a-chat-on-another-computer.md b/internal/manual/chat/continuing-a-chat-on-another-computer.md index 3767bb4792..6c025de18e 100644 --- a/internal/manual/chat/continuing-a-chat-on-another-computer.md +++ b/internal/manual/chat/continuing-a-chat-on-another-computer.md @@ -119,6 +119,10 @@ Moving chats and pairing are available by default on Linux, macOS and native Win A chat arriving here for the first time works in its own copy. It is filed under the folder it came from, apart from conversations in this computer's own folder of the same name. Home and the keys-row label say `proj-a (copy from another machine)`. This remains a copy even if a folder with the original path happens to exist on this computer. Taking back a chat that already works in your own project restores that project in place. An unknown origin is left unnamed. +## Reopen a kept copy after another computer takes the chat + +When unsent turns were kept as a separate copy, reopening that copy checks who holds the copy itself. Another computer may still hold the original chat. The previous window finishes saving and stops syncing before the reopened copy starts its own sync, so a late save cannot change which copy it opens. Taking that copy elsewhere still stops new tool work in its older window. + ## Continue here did nothing — another device continued first or the sync service is unreachable Moving chats and pairing are available by default on Linux, macOS and native Windows on every build with a compatible engine. Pair both computers first. diff --git a/internal/syncsetup/drive.go b/internal/syncsetup/drive.go index 385e8c266f..d481673614 100644 --- a/internal/syncsetup/drive.go +++ b/internal/syncsetup/drive.go @@ -43,10 +43,14 @@ type Drive struct { opt DriveOptions batcher *cellsync.Batcher scope *Scope // what this chat's publishes are counted under + // Ownership stays on the record this window opened. Preserving displaced + // work in a branch does not transfer this window's permission to run tools. + directoryID string - cancel context.CancelFunc - done chan struct{} - once sync.Once + cancel context.CancelFunc + done chan struct{} + once sync.Once + closeErr error sent string // fingerprint of the vault as last sent; touched by the flush loop and Close only @@ -80,6 +84,7 @@ func (s *Sync) Drive(ctx context.Context, eng cellstore.Engine, c cell.Cell, opt close(d.done) return d, nil } + d.directoryID = drv.ID() if d.batcher, err = s.batcher(eng, c, drv, d); err != nil { return nil, err } @@ -355,11 +360,12 @@ func (d *Drive) HoldsLease(ctx context.Context) bool { if _, viewer := d.Viewer(); viewer || d.batcher == nil { return false } - v, err := d.sync.Dir.Cell(ctx, d.cell.ID) + v, err := d.sync.Dir.Cell(ctx, d.directoryID) if err != nil { return true } lease := v.Cell.Lease + // The first publish may establish the fence while the read is in flight. return lease.Device == d.sync.Device.ID() && lease.Fence == d.batcher.Fence() } @@ -372,7 +378,7 @@ func (d *Drive) CheckLease(ctx context.Context) error { if _, viewer := d.Viewer(); viewer || d.batcher == nil { return nil } - v, err := d.sync.Dir.Cell(ctx, d.cell.ID) + v, err := d.sync.Dir.Cell(ctx, d.directoryID) if err == nil && (v.Cell.Lease.Device != d.sync.Device.ID() || v.Cell.Lease.Fence != d.batcher.Fence()) { d.becomeViewer(cellsync.Superseded{By: v.Cell.Lease.Device}) } @@ -433,13 +439,13 @@ func (s stamping) Seal(ctx context.Context, c cell.Cell, info cellstore.TurnInfo // Close ends the drive side: what is sealed is published, and the lease is // given back so another device can take the chat at once. It is safe to call -// on every exit path, and more than once. +// on every exit path, and more than once. A failed closing boundary stays +// failed for every caller, so reopening cannot mistake it for a clean release. func (d *Drive) Close(ctx context.Context) error { - var err error d.once.Do(func() { pending := cellstore.Await(ctx, d.cell) if d.batcher == nil { - err = pending + d.closeErr = pending return } d.cancel() @@ -448,12 +454,12 @@ func (d *Drive) Close(ctx context.Context) error { // available for recovery; it is never uploaded as a successful close. if pending != nil { d.scope.Settle() - err = pending + d.closeErr = pending return } d.syncVault(ctx) - err = d.batcher.Close(ctx) + d.closeErr = d.batcher.Close(ctx) d.scope.Settle() // a publish that failed after the last flush line still counts }) - return err + return d.closeErr } diff --git a/internal/syncsetup/drive_lease_test.go b/internal/syncsetup/drive_lease_test.go new file mode 100644 index 0000000000..4946e770ab --- /dev/null +++ b/internal/syncsetup/drive_lease_test.go @@ -0,0 +1,167 @@ +package syncsetup + +import ( + "context" + "net/http" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/cellstore" + "github.com/Agent-Field/codeaf/internal/cellsync" + "github.com/Agent-Field/codeaf/internal/directory" +) + +// C7/R2 keeps ownership checks on a reopened copy's retained branch. The +// original chat may still belong to another computer without fencing the copy. +func TestC7R2ReopenedBranchChecksItsOwnLease(t *testing.T) { + r := newDriveRig(t) + ctx := t.Context() + sealed, err := r.engine.Seal(ctx, r.cell, cellstore.TurnInfo{}) + mustNil(t, err) + _, err = r.b.Dir.Create(ctx, r.cell.ID, directory.CellInit{Head: sealed.Turn.ID}) + mustNil(t, err) + const branch = "01J0000000000000000000000B" + _, err = r.a.Dir.Create(ctx, branch, directory.CellInit{Head: sealed.Turn.ID, ParentCell: r.cell.ID, OrphanTurns: 1}) + mustNil(t, err) + branches, err := branchMapOf(r.homeA) + mustNil(t, err) + mustNil(t, branches.Set(r.cell.ID, branch)) + drive, err := r.a.Drive(ctx, r.engine, r.cell, DriveOptions{DeviceName: "tower"}) + mustNil(t, err) + defer drive.Close(ctx) + mustNil(t, drive.CheckLease(ctx)) + if err := drive.Gate(); err != nil || !drive.HoldsLease(ctx) { + t.Fatalf("a retained copy was refused because its parent moved: %v", err) + } + view, err := r.b.Dir.Cell(ctx, branch) + mustNil(t, err) + _, err = r.b.Dir.Acquire(ctx, branch, directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: view.Cell.Lease.Device, Fence: view.Cell.Lease.Fence}}) + mustNil(t, err) + mustNil(t, drive.CheckLease(ctx)) + if drive.Gate() == nil || drive.HoldsLease(ctx) { + t.Fatal("a taken branch still permits tool work") + } +} + +// C7/R2 lets a chat keep working when its first publication completes while +// an ownership read is in flight. Its new lease belongs to the same window. +func TestC7R2OwnershipReadIncludesFirstPublish(t *testing.T) { + for _, method := range []string{"check", "holds"} { + t.Run(method, func(t *testing.T) { + var observe atomic.Bool + reading, published, served := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var parent string + r := newDriveRig(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + blocked := req.Method == http.MethodGet && req.URL.Path == "/v1/dir/cells/"+parent && observe.CompareAndSwap(true, false) + if blocked { + close(reading) + select { + case <-published: + case <-req.Context().Done(): + t.Error("ownership read ended before the first publication") + } + } + next.ServeHTTP(w, req) + if blocked { + close(served) + } + }) + }) + parent = r.cell.ID + c := r.startChat(¬iceLog{}) + flushed := c.drive.batcher.OnFlush + c.drive.batcher.OnFlush = func(f cellsync.Flush) { + flushed(f) + close(published) + } + observe.Store(true) + result := make(chan bool, 1) + go func() { + if method == "holds" { + result <- c.drive.HoldsLease(t.Context()) + return + } + if err := c.drive.CheckLease(t.Context()); err != nil { + t.Error(err) + } + result <- c.drive.Gate() == nil + }() + select { + case <-reading: + case <-time.After(5 * time.Second): + t.Fatal("ownership read did not start") + } + c.mustSay() + select { + case <-served: + case <-time.After(cellstore.SealWithin): + t.Fatal("first publication did not finish the ownership read") + } + if !<-result { + t.Fatal("first publication made its own window a viewer") + } + }) + } +} + +// C7/R2 keeps a displaced window fenced while its pending work becomes a copy, +// even when looking up the new holder's display name has not finished. +func TestC7R2DisplacedWindowDoesNotOwnItsPreservedBranch(t *testing.T) { + var branching atomic.Bool + var once, started sync.Once + entered, release := make(chan struct{}), make(chan struct{}) + branchStarted := make(chan struct{}) + defer close(release) + var parent string + r := newDriveRig(t, func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + id := strings.TrimPrefix(req.URL.Path, "/v1/dir/cells/") + createsBranch := req.Method == http.MethodPost && id != req.URL.Path && id != parent && !strings.Contains(id, "/") + if createsBranch { + branching.Store(true) + } + if req.Method == http.MethodGet && req.URL.Path == "/v1/dir/list" && branching.Load() { + once.Do(func() { close(entered) }) + <-release + } + next.ServeHTTP(w, req) + if createsBranch { + started.Do(func() { close(branchStarted) }) + } + }) + }) + ctx := t.Context() + parent = r.cell.ID + sealed, err := r.engine.Seal(ctx, r.cell, cellstore.TurnInfo{}) + mustNil(t, err) + _, err = r.a.Dir.Create(ctx, parent, directory.CellInit{Head: sealed.Turn.ID}) + mustNil(t, err) + c := r.startChat(¬iceLog{}) + view, err := r.b.Dir.Cell(ctx, parent) + mustNil(t, err) + _, err = r.b.Dir.Acquire(ctx, parent, directory.AcquireOpts{Force: true, Expect: &directory.LeaseExpectation{Device: view.Cell.Lease.Device, Fence: view.Cell.Lease.Fence}}) + mustNil(t, err) + c.mustSay() + mustNil(t, cellstore.Await(ctx, r.cell)) + // Upload the captured files before observing the ownership callback; its + // metadata lookup, rather than cold engine startup, is the blocked boundary. + save, cancel := context.WithTimeout(ctx, cellstore.SealWithin) + defer cancel() + select { + case <-branchStarted: + case <-save.Done(): + t.Fatal("pending work did not reach branch creation") + } + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("pending work did not reach the preserved branch") + } + if c.drive.HoldsLease(ctx) { + t.Fatal("a displaced window claimed ownership of its preserved copy") + } +} diff --git a/internal/syncsetup/quit_contract_test.go b/internal/syncsetup/quit_contract_test.go index d3c5eed8a0..47a06367c0 100644 --- a/internal/syncsetup/quit_contract_test.go +++ b/internal/syncsetup/quit_contract_test.go @@ -29,6 +29,9 @@ func TestC28C31FailedQuitKeepsFinalRecordsAndDoesNotRelease(t *testing.T) { if e = a.drive.Close(context.Background()); e == nil { t.Fatal("failed final upload reported a successful quit") } + if err := a.drive.Close(context.Background()); err == nil { + t.Fatal("a repeated close hid the failed final upload") + } v, e := h.b.Dir.Cell(context.Background(), h.cell.ID) if e != nil { t.Fatal(e) From 368478320ac713570c17ce6185d0e8f1fd00552e Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 20:03:37 -0400 Subject: [PATCH 083/103] Await native namespace readiness before watch handshakes. Complete durable namespace import and election before starting the socket handshake clock, preserving the shared watch cases and their existing deadline. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/relayserve/watch_test.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/internal/relayserve/watch_test.go b/internal/relayserve/watch_test.go index b3de6453fe..9f96acd3ec 100644 --- a/internal/relayserve/watch_test.go +++ b/internal/relayserve/watch_test.go @@ -48,6 +48,14 @@ func (r *rig) signer(d device) wireauth.Sign { return reqsign.SignFor(d, r.clock func (r *rig) watchRig(t *testing.T) directorytest.WatchRig { home := newHousehold(t) + // Namespace creation includes durable import and election. Complete that + // readiness boundary before measuring the socket handshake itself. + _, stranger := newIdentity(t) + for _, d := range []device{newDevice(t, home.id, t.TempDir()), stranger} { + if _, err := r.dir(d).List(bg); err != nil { + t.Fatal(err) + } + } return directorytest.WatchRig{ Committed: true, Rig: directorytest.Rig{ @@ -55,13 +63,10 @@ func (r *rig) watchRig(t *testing.T) directorytest.WatchRig { Devices: func(name string) directory.Client { return r.dir(home.device(name)) }, ID: func(name string) string { return home.device(name).dev.ID() }, }, - Base: r.srv.URL, - Sign: func(name string) wireauth.Sign { return r.signer(home.device(name)) }, - Stranger: func() wireauth.Sign { - _, d := newIdentity(t) - return r.signer(d) - }, - Cap: watchCap, + Base: r.srv.URL, + Sign: func(name string) wireauth.Sign { return r.signer(home.device(name)) }, + Stranger: func() wireauth.Sign { return r.signer(stranger) }, + Cap: watchCap, } } From edd6b6f324b50b0e1d85c437547240f7a3d42999 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 20:03:37 -0400 Subject: [PATCH 084/103] Reduce retained memory in the local hosted member harness. Use complete local HTTP replies and a smaller inspector client, omit unused services, and bound background compilation caches. Reap completed native hosts while keeping every hosted actor and the whole-run combined memory measurement. Co-Authored-By: Claude Opus 5.5 (1M context) --- relay/hosted/package-lock.json | 3 +- relay/hosted/package.json | 3 +- relay/hosted/test/raft/actors.test.js | 23 ++++++++--- relay/hosted/test/raft/local-host.test.js | 48 +++++++++++++++++++++++ relay/hosted/test/raft/workerd.js | 47 ++++++++++++++++------ 5 files changed, 105 insertions(+), 19 deletions(-) create mode 100644 relay/hosted/test/raft/local-host.test.js diff --git a/relay/hosted/package-lock.json b/relay/hosted/package-lock.json index 7942d6bbed..d22837d9fd 100644 --- a/relay/hosted/package-lock.json +++ b/relay/hosted/package-lock.json @@ -6,7 +6,8 @@ "": { "name": "codeaf-hosted-relay", "devDependencies": { - "wrangler": "^4.144.0" + "wrangler": "^4.144.0", + "ws": "8.21.0" } }, "node_modules/@cloudflare/kv-asset-handler": { diff --git a/relay/hosted/package.json b/relay/hosted/package.json index 796d094fc5..98a55332b4 100644 --- a/relay/hosted/package.json +++ b/relay/hosted/package.json @@ -8,6 +8,7 @@ "dev": "sh test/local.sh wrangler dev --local --var TRUST_PROXY:1 --var 'CAF_LIMITS:{\"newIdentitiesPerIpPerDay\":1000000,\"pairTtlMs\":4000}'" }, "devDependencies": { - "wrangler": "^4.144.0" + "wrangler": "^4.144.0", + "ws": "8.21.0" } } diff --git a/relay/hosted/test/raft/actors.test.js b/relay/hosted/test/raft/actors.test.js index 05d541b747..c945320273 100644 --- a/relay/hosted/test/raft/actors.test.js +++ b/relay/hosted/test/raft/actors.test.js @@ -1,5 +1,6 @@ import test from 'node:test'; import assert from 'node:assert/strict'; +import WebSocket from 'ws'; import { spawn, execFileSync } from 'node:child_process'; import { mkdtemp, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -17,9 +18,15 @@ const report = { calls: [], requirements: ['R3','R4','R6','R7','R8','R10','R12', async function sample() { const ids = [process.pid, ...(await readFile(`/proc/${process.pid}/task/${process.pid}/children`, 'utf8')).trim().split(/\s+/).filter(Boolean).map(Number)]; const processes = []; - for (const pid of ids) { try { const s = await readFile(`/proc/${pid}/status`, 'utf8'), stat = (await readFile(`/proc/${pid}/stat`, 'utf8')).split(') ').slice(1).join(') ').split(' '); processes.push({ pid, cpuSeconds: (Number(stat[11]) + Number(stat[12])) / clockTicks, rss: Number(s.match(/^VmRSS:\s+(\d+)/m)?.[1] ?? 0) * 1024 }); } catch {} } - const value = { processes, combinedRSS: processes.reduce((n, p) => n + p.rss, 0) }; report.samples.push(value); return value; + const coordinator = process.memoryUsage(); + for (const pid of ids) { try { + const s = await readFile(`/proc/${pid}/status`, 'utf8'), stat = (await readFile(`/proc/${pid}/stat`, 'utf8')).split(') ').slice(1).join(') ').split(' '); + const kb = field => Number(s.match(new RegExp(`^${field}:\\s+(\\d+)`, 'm'))?.[1] ?? 0) * 1024; + processes.push({ pid, command: s.match(/^Name:\s+(\S+)/m)?.[1], cpuSeconds: (Number(stat[11]) + Number(stat[12])) / clockTicks, rss: kb('VmRSS'), anonymous: kb('RssAnon'), file: kb('RssFile') }); + } catch {} } + const value = { coordinator, callCount: report.calls.length, processes, combinedRSS: processes.reduce((n, p) => n + p.rss, 0) }; report.samples.push(value); return value; } + class Native { constructor(dir) { this.child = spawn(join(out, 'native-peer'), ['actors', dir], { stdio: ['pipe','pipe','inherit'], env: { ...process.env, GOMEMLIMIT: '32MiB' } }); @@ -29,9 +36,10 @@ class Native { } pump() { if (!this.waiter || this.buffer.length < 4) return; const n = this.buffer.readUInt32LE(0); assert(n < 2 * 1024 * 1024); if (this.buffer.length < n + 4) return; const value = JSON.parse(this.buffer.subarray(4, n + 4)); this.buffer = this.buffer.subarray(n + 4); const p = this.waiter; this.waiter = null; clearTimeout(p.timer); p.resolve(value); } call(value) { + assert(this.child.exitCode === null && this.child.signalCode === null, 'native actor is closed'); assert(!this.waiter); return new Promise((resolve, reject) => { this.waiter = { resolve, reject, timer: setTimeout(() => reject(new Error('native actor timeout')), 5000) }; const b = Buffer.from(JSON.stringify(value)), h = Buffer.alloc(4); h.writeUInt32LE(b.length); this.child.stdin.write(h); this.child.stdin.write(b); this.pump(); }); } - async close() { this.child.stdin.end(); if (this.child.exitCode === null) await new Promise(resolve => this.child.once('exit', resolve)); } + async close() { if (this.child.exitCode !== null || this.child.signalCode !== null) return; this.child.stdin.end(); await new Promise(resolve => this.child.once('exit', resolve)); } } // This parser only identifies snapshot transport in the test scheduler. Both // production receivers use Go's strict protobuf admission, never this parser. @@ -50,7 +58,8 @@ class Group { if(op==='leadership-plan')value={...value,value:JSON.parse(encodeCanonical(value.value))}; let r; const start = performance.now(); if (this.wasm.includes(id)) { - const res = await this.worker.dispatchFetch(`http://proof/actor/${this.cluster}-${id}/${op}`, { method: 'POST', body: JSON.stringify(value), signal: AbortSignal.timeout(5000) }); assert.equal(res.status, 200, await res.clone().text()); r = await res.json(); + const res = await this.worker.dispatchFetch(`http://proof/actor/${this.cluster}-${id}/${op}`, { method: 'POST', body: JSON.stringify(value), signal: AbortSignal.timeout(5000) }); + const body = await res.text(); assert.equal(res.status, 200, body); r = JSON.parse(body); } else r = await this.native.call({ op, ...value, id: String(id) }); report.calls.push({ op, wasm: this.wasm.includes(id), ms: performance.now() - start }); if (r?.error) throw new Error(`node ${id} ${op}: ${r.error}`); @@ -191,6 +200,9 @@ test('R3 R4 R6 R7 R8 R10 R13 R14 R27 hosted actors commit, ReadIndex, crash and assert.equal((await k.call(4,'leadership-receipt',{header:b64(Buffer.from(encodeCanonical(kh)))})).node,'4'); report.phases.push('R6 WASI P promoted + elected + current-term read'); report.phases.push('two wasm + native peers; full joint checkpoint/restart/leave'); + // Native peers have completed every operation. Reap their host before + // expanding the hosted group, while keeping every Wasm member resident. + await native.close(); report.nativeClosedAt = report.calls.length; // All three anchors plus two other identities share exactly one runtime. const h = new Group(worker, native, 'co-resident', [1,2,3]); await h.init(); await h.create(); assert(await h.read(h.leader(), 'all-hosted')); await h.sameRows(); assert.equal(await g.call(1, 'row'), latest); assert.notEqual(row, latest); @@ -205,6 +217,7 @@ test('R3 R4 R6 R7 R8 R10 R13 R14 R27 hosted actors commit, ReadIndex, crash and report.metrics = metrics; report.heapSamples = heaps; report.isolateBytes = Math.max(...heaps.map(h => h.totalSize + (h.embedderHeapUsedSize ?? 0) + (h.backingStorageSize ?? 0))) + metrics.peakMemory; assert(report.isolateBytes <= 96 * 1024 * 1024); assert(metrics.peakMemory <= 64 * 1024 * 1024); assert.equal(metrics.active, 6); + assert(report.calls.slice(report.nativeClosedAt).every(call => call.wasm)); report.phases.push('three hosted anchors plus mixed identity in same isolate'); } finally { clearInterval(timer); await closeInspector(); await sample(); if (worker) await worker.dispose(); await native.close(); @@ -219,7 +232,7 @@ test('R3 R4 R6 R7 R8 R10 R13 R14 R27 hosted actors commit, ReadIndex, crash and assert(report.combinedRSS <= 256 * 1024 * 1024, `R27 combined RSS ${report.combinedRSS}`); }); async function inspect(worker) { - const base = await worker.getInspectorURL(), listing = await (await fetch(new URL('/json/list', base))).json(); + const listing = await worker.getInspectorTargets(); const target = listing.find(x => x.title?.includes('raft-proof')) ?? listing[0]; const ws = new WebSocket(target.webSocketDebuggerUrl); await new Promise((resolve, reject) => { ws.onopen = resolve; ws.onerror = reject; }); let serial = 0; const waiting = new Map(); diff --git a/relay/hosted/test/raft/local-host.test.js b/relay/hosted/test/raft/local-host.test.js new file mode 100644 index 0000000000..f054c0b0f6 --- /dev/null +++ b/relay/hosted/test/raft/local-host.test.js @@ -0,0 +1,48 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import { copyFile, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { openWorkerd } from './workerd.js'; + +// R27 preserves the complete local RPC boundary while its host stays small. +// The native denial service must also prevent an unplanned outbound request. +test('R27 local host preserves complete replies and refuses outbound traffic', { timeout: 15000 }, async () => { + const out = await mkdtemp(join(tmpdir(), 'codeaf-host-')); + const requestWithin = 5000; + let runtime, attempts = 0; + const canary = createServer((_, response) => { attempts++; response.end('unexpected'); }); + try { + await new Promise(resolve => canary.listen(0, '127.0.0.1', resolve)); + await copyFile(join(process.env.CODEAF_RAFT_PROOF_DIR ?? '/tmp/codeaf-raft-proof', 'core.wasm'), join(out, 'core.wasm')); + await copyFile('test/raft/storage-worker.js', join(out, 'storage-worker.js')); + await writeFile(join(out, 'worker.js'), `export default { async fetch(request) { + const url = new URL(request.url); + if (url.pathname === '/ready') return new Response(null, { status: 204 }); + if (url.pathname === '/binary') return new Response(await request.arrayBuffer(), { status: 201 }); + if (url.pathname === '/json') return Response.json(await request.json()); + try { await fetch(url.searchParams.get('to')); return new Response('allowed'); } + catch { return new Response('denied', { status: 403 }); } + } };`); + runtime = await openWorkerd(out, join(out, 'disk')); + const bytes = Uint8Array.from({ length: 128 * 1024 }, (_, i) => i % 256); + const binary = await runtime.dispatchFetch('http://proof/binary', { method: 'POST', body: bytes, signal: AbortSignal.timeout(requestWithin) }); + assert.equal(binary.status, 201); + assert.equal(binary.ok, true); + assert.deepEqual(new Uint8Array(await binary.arrayBuffer()), bytes); + const value = { text: 'a complete reply', fence: '18446744073709551615' }; + const json = await runtime.dispatchFetch('http://proof/json', { method: 'POST', body: JSON.stringify(value), signal: AbortSignal.timeout(requestWithin) }); + assert.equal(json.status, 200); + assert.deepEqual(await json.json(), value); + const denied = await runtime.dispatchFetch('http://proof/outbound?to=' + encodeURIComponent(`http://127.0.0.1:${canary.address().port}`), { signal: AbortSignal.timeout(requestWithin) }); + assert.equal(denied.status, 403); + assert.equal(denied.ok, false); + assert.equal(await denied.text(), 'denied'); + assert.equal(attempts, 0); + } finally { + await runtime?.dispose(); + await new Promise(resolve => canary.close(resolve)); + await rm(out, { recursive: true, force: true }); + } +}); diff --git a/relay/hosted/test/raft/workerd.js b/relay/hosted/test/raft/workerd.js index 48cca4ec76..d31273657c 100644 --- a/relay/hosted/test/raft/workerd.js +++ b/relay/hosted/test/raft/workerd.js @@ -3,6 +3,7 @@ import { spawn } from 'node:child_process'; import { createRequire } from 'node:module'; import { createServer } from 'node:net'; +import { Agent, request } from 'node:http'; import { mkdir, writeFile } from 'node:fs/promises'; import { join, dirname } from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; @@ -13,32 +14,54 @@ async function port() { const s=createServer();await new Promise((resolve,reject)=>{s.on('error',reject);s.listen(0,'127.0.0.1',resolve);}); const n=s.address().port;await new Promise(resolve=>s.close(resolve));return n; } +// Local proof RPCs own complete replies and short loopback connections. +// A small client leaves the shared memory budget for the actual member hosts. +function localRequest(url, options, agent) { + return new Promise((resolve, reject) => { + const req = request(url, { method: options?.method ?? 'GET', headers: options?.headers, signal: options?.signal, agent }, res => { + const chunks = []; + res.on('data', chunk => chunks.push(chunk)); + res.on('error', reject); + res.on('end', () => { + const body = Buffer.concat(chunks); + resolve({ status: res.statusCode, ok: res.statusCode >= 200 && res.statusCode < 300, + text: async () => body.toString(), json: async () => JSON.parse(body.toString()), + arrayBuffer: async () => body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength) }); + }); + }); + req.on('error', reject); + req.end(options?.body); + }); +} export async function openWorkerd(out,persist, actors = false) { + const deadline=Date.now()+10000; await mkdir(persist,{recursive:true}); const httpPort=await port(),inspectorPort=await port(); + const agent = new Agent({ keepAlive: false, maxSockets: 1 }); const quote=JSON.stringify; const config=join(persist,'proof.capnp'); + // Synchronous reactors share one isolate. Extra compilation workers retain + // native allocator caches without supplying member or request concurrency. await writeFile(config,`using Workerd = import ${quote(schema)}; const config :Workerd.Config = ( + v8Flags = ["--wasm-num-compilation-tasks=1"], services = [ (name = "raft-proof", worker = ( compatibilityDate = "2026-09-01", modules = [(name = "worker.js", esModule = embed ${quote(join(out,'worker.js'))}), (name = "core.wasm", wasm = embed ${quote(join(out,'core.wasm'))})], ${actors ? `durableObjectNamespaces = [(className = "ActorFixture", uniqueKey = "p08-actors", enableSql = true)], durableObjectStorage = (localDisk = "storage"),` : ""} - bindings = [${actors ? `(name = "ACTORS", durableObjectNamespace = (className = "ActorFixture")),` : ""}(name = "PROOF_STORE", durableObjectNamespace = (className = "ProofStore", serviceName = "proof-storage"))], - globalOutbound = "blocked" + bindings = [${actors ? `(name = "ACTORS", durableObjectNamespace = (className = "ActorFixture"))` : `(name = "PROOF_STORE", durableObjectNamespace = (className = "ProofStore", serviceName = "proof-storage"))`}], + globalOutbound = "internet" )), - (name = "proof-storage", worker = ( + ${actors ? "" : `(name = "proof-storage", worker = ( compatibilityDate = "2026-09-01", modules = [(name = "storage.js", esModule = embed ${quote(join(out,'storage-worker.js'))})], durableObjectNamespaces = [(className = "ProofStore", uniqueKey = "p03-local-proof", enableSql = true)], - durableObjectStorage = (localDisk = "storage"), globalOutbound = "blocked" - )), + durableObjectStorage = (localDisk = "storage"), globalOutbound = "internet" + )),`} (name = "storage", disk = (path = ${quote(persist)}, writable = true)), - (name = "blocked", worker = (compatibilityDate = "2026-09-01", modules = [ - (name = "blocked.js", esModule = "export default {fetch() {throw new Error('proof forbids outbound network')}}") - ], globalOutbound = "blocked")) + (name = "internet", network = (allow = [])) ], sockets = [(name = "http", address = "127.0.0.1:${httpPort}", http = (), service = "raft-proof")] ); @@ -50,15 +73,15 @@ const config :Workerd.Config = ( const origin=`http://127.0.0.1:${httpPort}`; const server={ origin, - dispatchFetch:(url,options)=>{const u=new URL(url);return fetch(origin+u.pathname+u.search,options);}, + dispatchFetch:(url,options)=>{const u=new URL(url);return localRequest(origin+u.pathname+u.search,options,agent);}, getInspectorURL:async()=>new URL(`http://127.0.0.1:${inspectorPort}`), - dispose:async()=>{if(!exited)child.kill('SIGKILL');await ended;}, + getInspectorTargets:async()=> (await localRequest(`http://127.0.0.1:${inspectorPort}/json/list`, {}, agent)).json(), + dispose:async()=>{agent.destroy();if(!exited)child.kill('SIGKILL');await ended;}, }; try { - const deadline=Date.now()+10000; for(;;){ if(exited)throw new Error(`local workerd exited: ${errors}`); - try{const r=await fetch(origin+'/ready',{signal:AbortSignal.timeout(500)});if(r.status===204)break;}catch{} + try{const r=await localRequest(origin+'/ready',{signal:AbortSignal.timeout(500)},agent);if(r.status===204)break;}catch{} if(Date.now()>deadline)throw new Error(`local workerd startup timed out: ${errors}`); await delay(25); } From a6f0ac83242b267627fb66cbdc9738c015be999c Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 20:26:18 -0400 Subject: [PATCH 085/103] Keep the update question reaching its answer beside the coordination pages. The coordination pages add enough text to the chat manual that "is there a newer version" fell out of its answer's top results, behind a short Windows handoff section that mentions a newer engine. The update section now says the asker's words in its body as well as in its heading, so the question reaches the section that answers it. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/manual/chat/running-from-the-terminal.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/manual/chat/running-from-the-terminal.md b/internal/manual/chat/running-from-the-terminal.md index 95e2b55eba..d95bbb8bd1 100644 --- a/internal/manual/chat/running-from-the-terminal.md +++ b/internal/manual/chat/running-from-the-terminal.md @@ -2,8 +2,8 @@ ## Is there a newer version — update a dev build — keep a dev build up to date — latest dev — /update — why does it say this every time I start -Every launch of a release build checks the newest release of the channel it -follows, and what happens next depends on the build: +Every launch of a release build checks whether there is a newer version: the +newest release of the channel it follows. What happens next depends on the build: - A released codeaf — a stable tag, in a file named `codeaf` — with `automatic updates` on (the default) installs the newest stable release BEFORE the chat From 91e612099ebf66cf9a72c27f0791cab62d7df917 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 20:26:18 -0400 Subject: [PATCH 086/103] Map the relay's coordination packages in the architecture guide. The relay now keeps its directory in a replicated log that the native relay, the hosted relay and a joined computer share. The architecture guide names the packages a contributor starts from for that work, beside the remote session row, and points at the relay guide and the manual page that explain the behaviour people see. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/ARCHITECTURE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 8576982121..cebdfa4390 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -32,6 +32,7 @@ specific action rather than reading either large package from beginning to end. | Turns, tools and task execution | [session/session.go](../internal/session/session.go), [session/task_run.go](../internal/session/task_run.go), [exec](../internal/exec/) | [Subharness contract](SUBHARNESS-CONTRACT.md) | | Provider calls and model configuration | [provider](../internal/provider/), [config](../internal/config/) | [Usage accounting](model-usage-accounting.md), [limits](LIMITS.md) | | Remote sessions and movable chats | [remote](../internal/remote/), [enginehost](../internal/enginehost/), [cell](../internal/cell/) | [Remote architecture](REMOTE.md), [remote testing](testing-anywhere.md) | +| Relay coordination across computers and workers | [relay server](../internal/relayserve/), [committed directory](../internal/raftdirectory/), [consensus core](../internal/raftcore/), [hosted relay](../relay/hosted/), [cluster command](../cmd/codeaf/cluster.go) | [Run a relay](relay.md), [coordination and workers](../internal/manual/chat/coordination-and-workers.md) | ## Find the right storage and knowledge owner From 05dc6d8a89e440dc0c309e4602f8b4d787120cf4 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 20:51:05 -0400 Subject: [PATCH 087/103] Keep hosted maintenance working while a cut-short large request's pages remain. A large request from an older client is staged as raw body pages beside the join requests in the same namespace. When such a request is cut short, its pages wait for the next request to replace them, and the maintenance turn that looks for undelivered join decisions decoded every row it scanned, failed on the raw pages and skipped the rest of its idle work. It now reads only join requests, as the delivery replay already does. Co-Authored-By: Claude Opus 5.5 (1M context) --- relay/hosted/src/raft/hosted.js | 5 ++++- relay/hosted/test/raft/v1-large.test.js | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/relay/hosted/src/raft/hosted.js b/relay/hosted/src/raft/hosted.js index ef95e196ee..ee80e7d64f 100644 --- a/relay/hosted/src/raft/hosted.js +++ b/relay/hosted/src/raft/hosted.js @@ -438,7 +438,10 @@ export class HostedIdentity { let scan=this.outboxScan??''; const app=await member.hostView(this.states.get(replica).applied,'outbox','',scan); this.outboxScan=app.rows.at(-1)?.key??''; - if(app.rows.some(row=>{const value=decodeCanonical(dec.decode(row.data));return value.decision&&!value.outcome;}))await this.effects.replay(code=>this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(code)),scan,now); + // Only join requests are delivery records. The same namespace also keeps + // upload receipts and the raw body pages of a large request that was cut + // short, and those are not canonical records to decode. + if(app.rows.some(row=>{if(!row.key.startsWith('join/'))return false;const value=decodeCanonical(dec.decode(row.data));return value.decision&&!value.outcome;}))await this.effects.replay(code=>this.env.LINK_REQUEST.get(this.env.LINK_REQUEST.idFromName(code)),scan,now); let remoteAck,remoteStatus; const schedule=async(acks=[])=>{ if(remote){ diff --git a/relay/hosted/test/raft/v1-large.test.js b/relay/hosted/test/raft/v1-large.test.js index 86b86d11b4..93c8d9f49f 100644 --- a/relay/hosted/test/raft/v1-large.test.js +++ b/relay/hosted/test/raft/v1-large.test.js @@ -55,6 +55,25 @@ test('R25 interrupted workerd staging stays atomic and bounded after restart',{t }finally{await h.close();} }); +// R25/R27 keep an abandoned staging slot inert: its raw body pages are not +// coordination records, so every maintenance turn that scans past them must +// still complete while they wait for the next request to replace them. +test('R25 abandoned workerd staging does not break hosted maintenance',{timeout:90000},async()=>{ + const h=await hostedFixture({fixture:true,rootLabel:'root',now:fixture.now}),c=fixture.cases[1]; + try{ + const cut={needle:'result=await this.commit(command);requireOK(result.value.code);',after:"if(Array.isArray(raw)&&command===raw[16])throw new Error('interrupted body staging');"}; + await h.start(cut); + const failed=await send(h,c);assert(failed.status>=500,`interruption returned ${failed.status}`);await failed.text(); + await h.start(); + const absent=await send(h,c,true);assert.equal(absent.status,404);await absent.text(); + const cluster=JSON.parse((await h.control({op:'source'})).gate).config.cluster; + const staged=await h.control({op:'staging',cluster,node:'1'});assert(staged.rows.n>0,'the interruption left no staged pages'); + const turns=await h.control({op:'leadership-ticks',ticks:8}); + assert.deepEqual(turns.errors,[],'maintenance failed while staged pages remained'); + await exact(h,c); + }finally{await h.close();} +}); + // R25 preserves the old body limit even when legal JSON needs six-byte HTML // escaping in the committed logical row and the returned original view. From 3fa9cfbb116de6829318ff99ea2fb58da3d0e21e Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 20:51:05 -0400 Subject: [PATCH 088/103] Describe the proof helpers' snapshot statements by what they check. The row probe's summary and the hosted smoke test's assertion message now say what they observe about snapshots instead of naming internal planning steps. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/codeaf-raft-probe/rows.go | 2 +- relay/hosted/test/raft/smoke.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/codeaf-raft-probe/rows.go b/cmd/codeaf-raft-probe/rows.go index bb75cbf678..15dce7ec69 100644 --- a/cmd/codeaf-raft-probe/rows.go +++ b/cmd/codeaf-raft-probe/rows.go @@ -143,5 +143,5 @@ func rowProof(origin string, result io.Writer) error { if decoded.Cell.Lease.Expires != 0 || decoded.Cell.Title != strings.Repeat("A", 768<<10) { return errors.New("R28 large row release lost state") } - return json.NewEncoder(result).Encode(map[string]any{"rows": rowCount, "bytes": total, "ceiling": max(8<<20, total), "maxRowBytes": maxRow, "digest": hex.EncodeToString(digest), "representation": "paged immutable single rows; host inventory; grandfathered ceiling; streamed snapshots remain P05"}) + return json.NewEncoder(result).Encode(map[string]any{"rows": rowCount, "bytes": total, "ceiling": max(8<<20, total), "maxRowBytes": maxRow, "digest": hex.EncodeToString(digest), "representation": "paged immutable single rows; host inventory; grandfathered ceiling; snapshots streamed separately"}) } diff --git a/relay/hosted/test/raft/smoke.test.js b/relay/hosted/test/raft/smoke.test.js index 31f00446ba..98d99bcb82 100644 --- a/relay/hosted/test/raft/smoke.test.js +++ b/relay/hosted/test/raft/smoke.test.js @@ -114,7 +114,7 @@ class Group { this.queue.push(...e.send.map(m=>({...m,from:id})));this.reads.push(...e.reads.map(r=>({...r,node:id}))); if(e.persist){ const p=e.persist;if(p.hard.length)n.d.hard=p.hard; - assert.equal(p.snapshot.length,0,'P05 snapshot lifecycle remains outside P03'); + assert.equal(p.snapshot.length,0,'this smoke run installs no snapshot'); if(p.entries.length){const first=entryIndex(p.entries[0]);n.d.entries=n.d.entries.filter(b=>entryIndex(b) Date: Sun, 25 Oct 2026 22:08:55 -0400 Subject: [PATCH 089/103] Keep relay directories usable at their content ceiling. Preserve complete older records with lossless encoding and retain their original sources. Reserve bounded initialization and retry bookkeeping so full directories can finish upgrading and persist capacity refusals without stopping committed work. Share the storage accounting between native and hosted relays while keeping content and protocol limits unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/changes/unreleased/raft-cluster.md | 3 +- docs/relay.md | 8 +- internal/clusterstate/README.md | 17 +- internal/clusterstate/apply.go | 54 ++++- internal/clusterstate/capacity.go | 30 +++ .../clusterstate/capacity_boundary_test.go | 124 +++++++++++ internal/clusterstate/legacy.go | 13 +- .../manual/chat/coordination-and-workers.md | 10 +- internal/raftabi/README.md | 14 +- internal/raftabi/abi.go | 16 ++ internal/raftdirectory/README.md | 4 +- .../raftdirectory/capacity_boundary_test.go | 67 ++++++ internal/raftdirectory/directory_test.go | 7 +- internal/raftsnapshot/README.md | 4 +- internal/raftstore/README.md | 9 +- internal/relayserve/upgrade_capacity_test.go | 205 ++++++++++++++++++ relay/hosted/src/raft/README.md | 11 +- relay/hosted/src/raft/abi.js | 4 + relay/hosted/src/raft/storage.js | 26 ++- .../test/raft/bookkeeping-storage.test.js | 57 +++++ relay/hosted/test/raft/hosted-fixture.js | 4 +- .../hosted/test/raft/upgrade-capacity.test.js | 67 ++++++ relay/hosted/test/raft/upgrade-worker.js | 27 +++ 23 files changed, 739 insertions(+), 42 deletions(-) create mode 100644 internal/clusterstate/capacity.go create mode 100644 internal/clusterstate/capacity_boundary_test.go create mode 100644 internal/raftdirectory/capacity_boundary_test.go create mode 100644 internal/relayserve/upgrade_capacity_test.go create mode 100644 relay/hosted/test/raft/bookkeeping-storage.test.js create mode 100644 relay/hosted/test/raft/upgrade-capacity.test.js create mode 100644 relay/hosted/test/raft/upgrade-worker.js diff --git a/docs/changes/unreleased/raft-cluster.md b/docs/changes/unreleased/raft-cluster.md index bb9ed6ef9f..d197d6e7d0 100644 --- a/docs/changes/unreleased/raft-cluster.md +++ b/docs/changes/unreleased/raft-cluster.md @@ -12,11 +12,12 @@ invalidates: - "Caught-up open homes previously polled shared state. Idle homes now send zero HTTP requests, peer or forwarding frames, or client attempts after all voters apply the idle barrier; presence sweeps recur only with a viewer and fresh input requires a new majority response." - "Older clients could blindly force a live chat without its observed holder. Both relays now refuse that request with another device continued this chat first; releasing the chat or updating the client permits safe continuation." - "Legal older directory requests could be refused above the smaller coordination envelope. Both relays now retain the original 1 MiB HTTP limit, stage large signed bodies and commit complete metadata atomically, recovering interruptions without exposing partial changes or deleting retained records." + - "Retained rows from legal older requests could pause upgrades after the relay added fields, and full directories could fail initialization or refusal persistence. Both relays now preserve these rows losslessly and reserve bounded bookkeeping independently of the measured content allowance." - "Workers previously needed personal-device trust for shared access. Delegated workers now have expiring selected-chat grants and conditional assignment claims; they cannot approve pairing, read the identity vault, rotate it, enroll devices or lead, and terminal worker setup is not available yet." - "Reopening retained work could compare its branch fence with the original chat and falsely refuse tools. Ownership checks stay with the record each window opened, and replacing a displaced drive settles its previous writer before starting another. A failed final save remains failed for every close caller." - "Deferred local durability could resolve a fallback daemon against a different current home. Both engine routes now retain the home and engine selected at capture." - "A single directory database was formerly the live source. First use now freezes and imports it into the three anchors while retaining the source, and the one-way upgrade cannot return to that writer after cluster changes are acknowledged." - - "Self-hosted identities were described as limited only by disk space. Coordination records now have an 8 MiB default allowance, with larger imports retaining their measured allowance; capacity refuses growth without deleting saved data." + - "Self-hosted identities were described as limited only by disk space. Directory content now has an 8 MiB default allowance, with larger imports retaining their measured allowance and bounded bookkeeping having separate room; capacity refuses content growth without deleting saved data." --- Leadership moves coordination metadata while chats keep their ordinary ownership, diff --git a/docs/relay.md b/docs/relay.md index 50707a9156..8c744237a1 100644 --- a/docs/relay.md +++ b/docs/relay.md @@ -74,9 +74,11 @@ Everything is under `--store`: - `directory/.db`: frozen original source and migration ownership receipt. - `blobs//`: encrypted chunks and their object pointers. -Keep associated WAL files and the rest of the store together. Coordination records -have an 8 MiB default allowance per identity; a larger imported identity keeps its -measured allowance. Capacity refuses growth instead of deleting existing records. +Keep associated WAL files and the rest of the store together. Retained directory content +has an 8 MiB default allowance per identity; a larger imported identity keeps its +measured allowance. Bounded retry headers and membership initialization have a +separate allowance, so a full identity can record refusals and finish upgrading. +Capacity refuses content growth instead of deleting existing records. Snapshots compact redundant coordination history, never saved chats, branches, vaults or encrypted chunks. Allow disk space for three copies, snapshots and SQLite overhead, as well as the separate encrypted content store. diff --git a/internal/clusterstate/README.md b/internal/clusterstate/README.md index 613461fda4..5076b4139b 100644 --- a/internal/clusterstate/README.md +++ b/internal/clusterstate/README.md @@ -123,7 +123,10 @@ envelope and each result byte page is at most one quarter of it. Ordinary commands retain their complete read-set cap. Staged final evaluation streams its body and dependency pages without putting them in one ABI frame. The host never yields or changes the applied view between these pages. The 64 KiB command, -2 MiB ABI buffer, 1 MiB physical row and application capacity caps are unchanged. +2 MiB ABI buffer, 1 MiB physical row and content capacity caps are unchanged. +Imported source rows include the old relay's bounded added fields; rows above +the physical envelope use the same lossless compressed representation. Impossible +source rows still refuse migration without changing their originals. Temporary request pages occupy one fixed slot outside retained application capacity. Their extent is derived from the original HTTP body and command limits. @@ -140,3 +143,15 @@ checking decimal counters and the complete canonical spelling before exposing the value. A single-cell update reuses its validated cell and typed write for visible-version comparison, preserving opaque metadata without repeatedly decoding the complete frame plan. + +## Capacity and retry bookkeeping + +The content ceiling remains 8 MiB by default or the larger measured import. +Long retry keys hash both actor and session, so caller-selected names cannot +enlarge retained headers. Ordinary short retry keys keep their existing spelling. At most 512 bytes per retained session and its count row use +`CoordinationReserveBytes`; response bytes beyond that count against content. +One existing evaluation envelope also covers membership initialization. The live +and retained session limits and membership admission limits remain unchanged. +A capacity refusal saves its exact result and permits the next entry to apply. +The persisted accounting travels with snapshots; hosted storage obtains both its +charge and maximum from the same Go function used by native application. diff --git a/internal/clusterstate/apply.go b/internal/clusterstate/apply.go index 31ca4f4b52..1c90c3ffcf 100644 --- a/internal/clusterstate/apply.go +++ b/internal/clusterstate/apply.go @@ -42,7 +42,13 @@ func DeviceKey(id string) RecordKey { return RecordKey{"state", "device/" + s func CellKey(id string) RecordKey { return RecordKey{"state", "cell/" + segment(id)} } func AuthorityKey(id string) RecordKey { return RecordKey{"enrollment", "actor/" + segment(id)} } func SessionKey(actor, session string) RecordKey { - return RecordKey{"sessions", segment(actor) + "/" + segment(session)} + key := segment(actor) + "/" + segment(session) + // Long names cannot consume the allowance needed to retain a refusal. + if len(key) > sessionBookkeepingBytes/4 { + digest := sha256.Sum256([]byte(key)) + key = hex.EncodeToString(digest[:]) + } + return RecordKey{"sessions", key} } func SessionsKey() RecordKey { return RecordKey{"sessions", "counts"} } func JoinKey(code string) RecordKey { return RecordKey{"outbox", "join/" + segment(code)} } @@ -65,8 +71,10 @@ type session struct { Result Result `json:"result"` } type sessionCounts struct { - Live uint32 `json:"live"` - Retained uint32 `json:"retained"` + Bookkeeping uint64 `json:"bookkeeping,omitempty"` + Initialization uint64 `json:"initialization,omitempty"` + Live uint32 `json:"live"` + Retained uint32 `json:"retained"` } // JoinRequest is a committed dependency from a reserved public request. @@ -423,6 +431,11 @@ func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged boo if decode(SessionKey(c.Actor, c.Session), &last) != nil || decode(SessionsKey(), &counts) != nil { return out, ErrDependency } + originalCounts := counts + originalBookkeeping, err := ApplicationBookkeeping(rows[SessionsKey()]) + if err != nil { + return out, err + } known := c.V1 == nil && rows[SessionKey(c.Actor, c.Session)] != nil refuse := func(code Code) (Evaluation, error) { out.Result.Code = code; return out, nil } if known && (last.Closed || c.Sequence < last.Sequence || c.Sequence == last.Sequence && c.RequestDigest != last.Digest) { @@ -549,12 +562,30 @@ func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged boo if c.Change.CloseSession != nil { counts.Live-- } + if c.Change.InitMembership != nil && code == OK { + for k, b := range writes { + counts.Initialization += RecordBytes(k, b) + if old := rows[k]; old != nil { + counts.Initialization -= RecordBytes(k, old) + } + } + } save := func() { if service || c.V1 != nil { return } put(SessionKey(c.Actor, c.Session), session{Sequence: c.Sequence, Digest: c.RequestDigest, Closed: c.Change.CloseSession != nil, Result: out.Result}) - put(SessionsKey(), counts) + sessionKey := SessionKey(c.Actor, c.Session) + oldCredit := bookkeepingCost(sessionKey, rows[sessionKey]) + bookkeepingCost(SessionsKey(), rows[SessionsKey()]) + base := originalCounts.Bookkeeping - min(originalCounts.Bookkeeping, oldCredit) + bookkeepingCost(sessionKey, writes[sessionKey]) + for range 8 { + put(SessionsKey(), counts) + next := base + bookkeepingCost(SessionsKey(), writes[SessionsKey()]) + if next == counts.Bookkeeping { + break + } + counts.Bookkeeping = next + } } save() @@ -580,7 +611,14 @@ func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged boo if budget > MaxEvaluationBytes { return false, nil } - if size <= ceiling || size <= meta.Bytes { + reserved := counts.Bookkeeping + counts.Initialization + if reserved > CoordinationReserveBytes { + return false, nil + } + if reserved > size || originalBookkeeping > meta.Bytes { + return false, ErrDependency + } + if size-reserved <= ceiling || size-reserved <= meta.Bytes-originalBookkeeping { return true, nil } if paged { @@ -593,7 +631,10 @@ func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged boo if scratch > meta.Bytes { return false, ErrDependency } - return size-scratch <= ceiling, nil + if scratch > size-reserved { + return false, ErrDependency + } + return size-reserved-scratch <= ceiling, nil } if ok, err := fits(); err != nil { return out, err @@ -601,6 +642,7 @@ func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged boo writes = map[RecordKey][]byte{} out.Version = meta.Version out.Result = Result{Schema: SchemaVersion, Code: Capacity, Now: out.Now, Version: meta.Version} + counts.Initialization = originalCounts.Initialization save() if ok, err := fits(); err != nil { return out, err diff --git a/internal/clusterstate/capacity.go b/internal/clusterstate/capacity.go new file mode 100644 index 0000000000..e3c0e1da65 --- /dev/null +++ b/internal/clusterstate/capacity.go @@ -0,0 +1,30 @@ +package clusterstate + +// Retry headers include a hashed key, digest, counters and a small result. +// Larger response payloads still consume the identity's content allowance. +const sessionBookkeepingBytes = 512 + +// CoordinationReserveBytes bounds retry headers for all retained sessions and +// their count row, plus one bounded membership-initialization transaction. +// The live-session and member limits remain independent admission checks. +const CoordinationReserveBytes = (MaxRetainedSessions+1)*sessionBookkeepingBytes + MaxEvaluationBytes + +func bookkeepingCost(k RecordKey, b []byte) uint64 { + if b == nil { + return 0 + } + return min(RecordBytes(k, b), sessionBookkeepingBytes) +} + +// ApplicationBookkeeping reads the same bounded accounting for native apply, +// hosted storage and streamed snapshots. It never grants content capacity. +func ApplicationBookkeeping(raw []byte) (uint64, error) { + if raw == nil { + return 0, nil + } + var counts sessionCounts + if Decode(raw, &counts) != nil || counts.Live > MaxLiveSessions || counts.Live > counts.Retained || counts.Retained > MaxRetainedSessions || counts.Bookkeeping > uint64(counts.Retained+1)*sessionBookkeepingBytes || counts.Initialization > MaxEvaluationBytes { + return 0, ErrDependency + } + return counts.Bookkeeping + counts.Initialization, nil +} diff --git a/internal/clusterstate/capacity_boundary_test.go b/internal/clusterstate/capacity_boundary_test.go new file mode 100644 index 0000000000..4942aba6d8 --- /dev/null +++ b/internal/clusterstate/capacity_boundary_test.go @@ -0,0 +1,124 @@ +package clusterstate_test + +import ( + "bytes" + "crypto/ed25519" + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +// r1.2 +func TestR12LegacySourceEnvelopeRetainsServerFields(t *testing.T) { + key := actorKey("root") + cfg := cs.LegacyConfig{Cluster: "upgrade", Generation: 1, Identity: cs.IdentityID(key.Public().(ed25519.PublicKey)), SigningKey: key.Public().(ed25519.PublicKey), EffectsKey: actorKey("effects").Public().(ed25519.PublicKey), Anchors: []uint64{1, 2, 3}} + init := cs.CellInit{Head: "saved", Title: ""} + base, _ := json.Marshal(init) + init.Title = strings.Repeat("x", cs.MaxV1BodyBytes-len(base)-len(`,"title":""`)) + body, _ := json.Marshal(init) + if len(body) != cs.MaxV1BodyBytes { + t.Fatalf("body=%d", len(body)) + } + cell := cs.Created(init, "dev_"+strings.Repeat("a", 32), 1000) + raw, _ := json.Marshal(cell) + row, err := cs.LegacyRow(cfg, "cells", "chat", raw, 0) + if err != nil { + t.Fatalf("accepted body cannot migrate after server fields: source=%d: %v", len(raw), err) + } + var got cs.Cell + if cs.Decode(row.Data, &got) != nil || !reflect.DeepEqual(got, cell) { + t.Fatal("lossy source conversion") + } + cell.Title = strings.Repeat("x", cs.MaxV1BodyBytes+1) + raw, _ = json.Marshal(cell) + saved := bytes.Clone(raw) + if _, err := cs.LegacyRow(cfg, "cells", "chat", raw, 0); err == nil || !bytes.Equal(saved, raw) { + t.Fatal("impossible source row accepted or changed") + } +} + +// r1.4 +func TestR14CapacityRefusalAtContentCeilingCommitsAndRetries(t *testing.T) { + for _, spare := range []uint64{0, 1} { + t.Run(fmt.Sprint(spare), func(t *testing.T) { + a := newApplication(t) + var size uint64 + for k, b := range a.rows { + size += cs.RecordBytes(k, b) + } + a.meta.Ceiling = size + spare + command, err := cs.Sign(cs.Command{Schema: cs.SchemaVersion, Cluster: "test", Actor: "a", Session: "new-session", Sequence: 1, Now: 1000, Change: cs.Mutation{SetVault: &cs.VaultCAS{New: strings.Repeat("v", 4000)}}}, actorKey("a")) + if err != nil { + t.Fatal(err) + } + first := a.evaluate(command) + if first.Result.Code != cs.Capacity || a.rows[cs.SessionKey("a", "new-session")] == nil { + t.Fatal("capacity was not persisted", first.Result) + } + retry := a.evaluate(command) + if !reflect.DeepEqual(first.Result, retry.Result) || len(retry.Writes) != 0 { + t.Fatal("retry changed the capacity result") + } + command.Sequence = 2 + command.Ack = 1 + command.Change = cs.Mutation{AdvanceTime: &cs.AdvanceTime{}} + command, err = cs.Sign(command, actorKey("a")) + if err != nil { + t.Fatal(err) + } + if next := a.evaluate(command); next.Result.Code != cs.OK { + t.Fatal("next committed entry cannot apply", next.Result) + } + }) + } +} + +// r1.5 +func TestR15RefusalBookkeepingRemainsBounded(t *testing.T) { + a := newApplication(t) + for k, b := range a.rows { + a.meta.Ceiling += cs.RecordBytes(k, b) + } + a.meta.Ceiling -= cs.DefaultApplicationBytes + original := a.meta.Ceiling + for i := 0; i < cs.MaxRetainedSessions; i++ { + command, err := cs.Sign(cs.Command{Schema: cs.SchemaVersion, Cluster: "test", Actor: "a", Session: fmt.Sprintf("%d-%s", i, strings.Repeat("s", 1000)), Sequence: 1, Now: 1000, Change: cs.Mutation{SetVault: &cs.VaultCAS{New: "growth"}}}, actorKey("a")) + if err != nil { + t.Fatal(err) + } + refusal := a.evaluate(command) + if refusal.Result.Code != cs.Capacity || len(refusal.Writes) == 0 { + t.Fatalf("session %d did not persist refusal: %v", i, refusal.Result) + } + command.Sequence++ + command.Ack = 1 + command.Change = cs.Mutation{CloseSession: &cs.CloseSession{}} + command, err = cs.Sign(command, actorKey("a")) + if err != nil { + t.Fatal(err) + } + if closed := a.evaluate(command); closed.Result.Code != cs.OK { + t.Fatal("cannot close refusal session", closed.Result) + } + } + if a.meta.Ceiling != original { + t.Fatal("content ceiling changed") + } + var size uint64 + for k, b := range a.rows { + size += cs.RecordBytes(k, b) + } + // The bounded result headers and membership initialization fit within one + // existing evaluation envelope plus one command envelope per live session. + if size > original+cs.MaxEvaluationBytes+cs.MaxLiveSessions*cs.MaxCommandBytes { + t.Fatalf("bookkeeping grew to %d", size) + } + extra := a.prepareBare("b", 1000, cs.Mutation{AdvanceTime: &cs.AdvanceTime{}}) + if a.evaluate(extra).Result.Code != cs.Capacity { + t.Fatal("retained session cap changed") + } +} diff --git a/internal/clusterstate/legacy.go b/internal/clusterstate/legacy.go index 4cf78aa8b7..bfdfe2a436 100644 --- a/internal/clusterstate/legacy.go +++ b/internal/clusterstate/legacy.go @@ -3,6 +3,7 @@ package clusterstate import ( "bytes" "crypto/ed25519" + "crypto/sha256" "encoding/base64" "encoding/hex" "encoding/json" @@ -105,11 +106,16 @@ func (m LegacyManifest) Validate() error { // Its exact encoded size is charged to the grandfathered inventory. const LegacyManifestReserve uint64 = 4096 +// LegacySourceRowBytes includes create-only fields, a retained title and the +// replaceable publish fields, each bounded by an original request body. JSON +// escaping and relay-owned record/lease fields can expand the stored row. +const LegacySourceRowBytes = 3*maxJSONExpansion*MaxV1BodyBytes + len(`{"V":1,"head":"","class":"","size":0,"keys":null,"durable_at":,"lease":{"device":"","fence":,"expires":,"pending":,"released_at":},"archived":true}`) + len("dev_") + sha256.Size + 5*20 + // LegacyRow converts the ORIGINAL v1 JSON bytes, never a JavaScript number // roundtrip. Unknown fields refuse migration instead of silently dropping data. // HoldUntil is sampled once while the source is frozen and recorded durably. func LegacyRow(c LegacyConfig, kind, id string, raw []byte, holdUntil int64) (RecordWrite, error) { - if c.Validate() != nil || len(raw) > MaxEvaluationBytes { + if c.Validate() != nil || len(raw) > LegacySourceRowBytes { return RecordWrite{}, ErrMigration } var key RecordKey @@ -134,7 +140,7 @@ func LegacyRow(c LegacyConfig, kind, id string, raw []byte, holdUntil int64) (Re value = v case "cells": var v Cell - if id == "" || legacyDecode(raw, &v) != nil || v.V != 1 || v.Lease.Fence > MaxSafeInteger { + if id == "" || legacyDecode(raw, &v) != nil || v.V != 1 || v.Lease.Fence > MaxSafeInteger || len(v.Title)+len(`{"title":""}`) > MaxV1BodyBytes { return RecordWrite{}, ErrMigration } if v.Lease.Expires != 0 { @@ -146,6 +152,9 @@ func LegacyRow(c LegacyConfig, kind, id string, raw []byte, holdUntil int64) (Re return RecordWrite{}, ErrMigration } data, e := Encode(value) + if e == nil && len(data)+len(key.Key) > MaxEvaluationBytes { + data, e = encodeRecord(value) + } if e != nil || len(data)+len(key.Key) > MaxEvaluationBytes { return RecordWrite{}, ErrMigration } diff --git a/internal/manual/chat/coordination-and-workers.md b/internal/manual/chat/coordination-and-workers.md index 1eec93fe47..56e175359f 100644 --- a/internal/manual/chat/coordination-and-workers.md +++ b/internal/manual/chat/coordination-and-workers.md @@ -2,7 +2,7 @@ ## Relay directory upgrade pauses and saved chats -A relay directory upgrade can briefly pause pairing and chat moves. Retry after the relay is ready. The upgrade refusal is `codeaf: directory upgrade is incomplete; retry shortly; saved chats are still here`. It does not delete saved chats, files, branches or the encrypted vault. The relay keeps the old directory as a backup and never makes it a second writer after the new directory takes over. A row that cannot be copied safely leaves the upgrade paused for the operator; it is never truncated to make it fit. +A relay directory upgrade can briefly pause pairing and chat moves. Retry after the relay is ready. The upgrade refusal is `codeaf: directory upgrade is incomplete; retry shortly; saved chats are still here`. It does not delete saved chats, files, branches or the encrypted vault. The relay keeps the old directory as a backup and never makes it a second writer after the new directory takes over. Legal older requests keep all saved fields even when the old relay added enough fields to exceed its request size. A row that cannot be copied safely leaves the upgrade paused for the operator; it is never truncated to make it fit. ## Can my phone lead, be the leader or coordinate my computers? @@ -121,7 +121,7 @@ frame hints, sealed names and key wraps are preserved; over 1 MiB is refused. Th identity’s storage capacity still applies, and existing records are never removed to make room. -## Hosted relay directory upgrade and older clients +## Hosted relay directory capacity, upgrades and older clients The hosted relay records directory changes on three persistent members. Two must agree before a take, publish or fresh listing succeeds. Pairing one or two @@ -130,7 +130,11 @@ still depend on the same hosting provider and relay connection. An existing identity upgrades on first authenticated use. An interrupted upgrade can briefly ask you to retry; it keeps the saved directory and encrypted frames. -The old directory never becomes a second writer. Older clients keep pairing, +The old directory never becomes a second writer. Directory content has an 8 MiB +default allowance; a larger imported directory keeps its measured allowance. +Initialization and bounded retry records have separate room, so reaching the +content limit still permits reads, releases and unchanged-size updates and +returns the existing capacity refusal for growth. Older clients keep pairing, listing, publishing, releasing and continuing a chat they observed. A very old client that blindly forces a live chat receives `another device continued this chat first`. Quit on the holder first, or update that client before continuing a live chat. diff --git a/internal/raftabi/README.md b/internal/raftabi/README.md index 8c5ab72e13..1cc1370ae5 100644 --- a/internal/raftabi/README.md +++ b/internal/raftabi/README.md @@ -76,6 +76,7 @@ a mutated core blindly. | 4 lease feasibility probe | command blob, optional one-row blob | next canonical row blob | | 5 row representation probe | canonical row blob | same canonical row blob | | 6 snapshot maintenance query | handle | present flag, then index/term uint64 when present | +| 48 application bookkeeping | present flag, session-count row blob | maximum allowance and retained charge, both uint64 | Step tags: 0 Poll, 1 Tick, 2 PeerMessage blob, 3 Propose blob, 4 Read string, 5 Persisted batch ID, 6 Applied batch ID, 7 Transfer target, 8 Configure blob, @@ -98,8 +99,8 @@ the opaque checkpoint fixture alone does not establish that interoperability. ## Oversized legacy representation decision -Keep each existing canonical row in host storage and pass only one immutable row -(at most the existing 1 MiB legacy envelope) to Go. Inventory/import and +Keep each physical row in host storage and pass only one immutable row +(at most the existing 1 MiB physical envelope) to Go. Inventory/import and snapshot creation enumerate sorted rows at a fixed applied index, with no whole-identity listing or JSON string in wasm. Persist the imported byte ceiling `max(8 MiB, imported application bytes)` outside the reactor alongside the @@ -114,8 +115,13 @@ with wasm and verifies every stored byte on reread, then evaluates a real release over the large row. Six production handles remain resident while the engine is sampled through its loopback inspector. This establishes representation/runtime feasibility; it does not establish migration, snapshots, -complete policy or production admission. A legacy row over 1 MiB remains a -migration refusal; a streaming row extension is not part of this build. +complete policy or production admission. The shared converter accepts the bounded +source expansion from an old 1 MiB request and the relay-added fields. When +necessary it uses the existing lossless compressed representation within the +physical row envelope. Unsupported source rows still refuse migration without +changing their originals. Bounded retry headers and membership initialization +use a separate allowance; operation 48 supplies its maximum and persisted charge +from Go for hosted application and snapshot checks. ## Local runtime measurements diff --git a/internal/raftabi/abi.go b/internal/raftabi/abi.go index c315026b6f..a1e6bbd23c 100644 --- a/internal/raftabi/abi.go +++ b/internal/raftabi/abi.go @@ -169,6 +169,22 @@ func (r *Runtime) Call(b []byte) (out []byte) { func (r *Runtime) call(op uint32, d *decoder) ([]byte, error) { e := encoder{} switch op { + case 48: + present := d.u32() + raw := d.blob() + if present > 1 || present == 0 && len(raw) != 0 || d.end() != nil { + return nil, errFrame + } + if present == 0 { + raw = nil + } + reserved, err := cs.ApplicationBookkeeping(raw) + if err != nil { + return nil, err + } + e.u64(cs.CoordinationReserveBytes) + e.u64(reserved) + return e.b, nil case 46: if d.end() != nil { return nil, errFrame diff --git a/internal/raftdirectory/README.md b/internal/raftdirectory/README.md index c2c9339af0..777332a50e 100644 --- a/internal/raftdirectory/README.md +++ b/internal/raftdirectory/README.md @@ -58,7 +58,7 @@ delivery. Provision a separate `Authority{Effects:true}` protocol key; it must not be personal. It can only record upload evidence, reserve a join dependency, or complete a delivery. These commands are idempotent by immutable effect keys and do not consume client sessions. All effect rows count against the identity's -application ceiling; capacity refuses further growth without removing receipts, +content ceiling; capacity refuses further growth without removing receipts, decisions or blobs. Before issuing a receipt, call `Effects.Upload` with the local head ID, its @@ -152,5 +152,5 @@ commands remain bounded at 64 KiB. Legal original v1 bodies up to the 1 MiB HTTP limit use bounded staged body pages; only the final application changes directory rows, after rechecking the full original proof and current authority. Interrupted staging can resume without exposing partial metadata. Both production relays use -this representation and the same consensus path. Capacity refuses retained growth +this representation and the same consensus path. Capacity refuses retained content growth without removing records or increasing the quorum. diff --git a/internal/raftdirectory/capacity_boundary_test.go b/internal/raftdirectory/capacity_boundary_test.go new file mode 100644 index 0000000000..f9ee1849e4 --- /dev/null +++ b/internal/raftdirectory/capacity_boundary_test.go @@ -0,0 +1,67 @@ +package raftdirectory_test + +import ( + "fmt" + "reflect" + "strings" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftstore" +) + +// r1.4 +func TestR14CapacityRefusalDoesNotStopMembers(t *testing.T) { + for _, spare := range []uint64{0, 1} { + t.Run(fmt.Sprint(spare), func(t *testing.T) { + g := newGroup(t, func() time.Time { return time.UnixMilli(1000) }) + _, size, err := g.stores[0].ApplicationView(ctx) + must(t, err) + g.ceiling = size + spare + c, err := cs.Sign(cs.Command{Schema: cs.SchemaVersion, Cluster: cluster, Actor: "a", Session: "new-full-session", Sequence: 1, Now: 1000, Change: cs.Mutation{SetVault: &cs.VaultCAS{New: strings.Repeat("v", 4000)}}}, key("a")) + must(t, err) + first, err := g.submit(c) + must(t, err) + if first.Code != cs.Capacity { + t.Fatal("growing command was not refused", first) + } + retry, err := g.submit(c) + must(t, err) + if !reflect.DeepEqual(first, retry) { + t.Fatal("retry changed committed result") + } + for _, store := range g.stores { + app, _, err := store.ApplicationView(ctx) + must(t, err) + k := cs.SessionKey(c.Actor, c.Session) + rows, _, err := store.Read(ctx, app.Applied, []raftstore.Key{{Namespace: raftstore.Sessions, Key: k.Key}}) + must(t, err) + if len(rows[raftstore.Key{Namespace: raftstore.Sessions, Key: k.Key}]) == 0 { + t.Fatal("member lost refusal") + } + } + c.Sequence = 2 + c.Ack = 1 + c.Change = cs.Mutation{AdvanceTime: &cs.AdvanceTime{}} + c, err = cs.Sign(c, key("a")) + must(t, err) + next, err := g.submit(c) + must(t, err) + if next.Code != cs.OK { + t.Fatal("next entry did not apply", next) + } + var applied uint64 + for i, store := range g.stores { + app, _, err := store.ApplicationView(ctx) + must(t, err) + if i == 0 { + applied = app.Applied + } + if app.Applied != applied { + t.Fatal("members stopped at different indices") + } + } + }) + } +} diff --git a/internal/raftdirectory/directory_test.go b/internal/raftdirectory/directory_test.go index 5c87e4befc..86990a3fa0 100644 --- a/internal/raftdirectory/directory_test.go +++ b/internal/raftdirectory/directory_test.go @@ -67,6 +67,7 @@ type group struct { absent map[uint64]bool blocked bool readID uint64 + ceiling uint64 } func newGroup(t *testing.T, clock func() time.Time, extra ...raftstore.Write) *group { @@ -134,7 +135,11 @@ func (g *group) drive(i int, in raftcore.Input) { if app.Applied == 0 { must(g.t, g.stores[i].Apply(ctx, *e.Apply, raftstore.Update{ExpectedApplied: 0, Writes: g.genesis})) } else { - r, err := raftdirectory.Apply(ctx, g.stores[i], *e.Apply, cluster, clusterstate.DefaultApplicationBytes) + ceiling := g.ceiling + if ceiling == 0 { + ceiling = clusterstate.DefaultApplicationBytes + } + r, err := raftdirectory.Apply(ctx, g.stores[i], *e.Apply, cluster, ceiling) must(g.t, err) g.results[i] = append(g.results[i], r...) } diff --git a/internal/raftsnapshot/README.md b/internal/raftsnapshot/README.md index 804f44a62c..fdf0cd47bb 100644 --- a/internal/raftsnapshot/README.md +++ b/internal/raftsnapshot/README.md @@ -58,8 +58,8 @@ by its eight-byte unsigned big-endian length. An application row's parts are native application's missing-row detector; the ordered stream digest separately binds the exact complete portable representation. -Hosts inventory and admit a transfer against their identity's application byte -ceiling before staging; the format does not silently impose an 8 MiB ceiling on +Hosts inventory and admit a transfer against their identity's content ceiling +and bounded bookkeeping allowances before staging; the format does not silently impose an 8 MiB ceiling on grandfathered legacy data. Only one incomplete generation may stage. Verify a new complete generation before retiring redundant consensus bytes. Keep two complete generations, including the installed one while another awaits install. diff --git a/internal/raftstore/README.md b/internal/raftstore/README.md index 1e3e366925..5036de1928 100644 --- a/internal/raftstore/README.md +++ b/internal/raftstore/README.md @@ -97,8 +97,8 @@ consensus bytes. A completed transfer awaiting installation keeps that slot. `CompleteSnapshot(id)` validates all chunks, ordering, stream digest, count and application hash and returns the portable protobuf pointer. This publishes a complete transfer, not application authority. The active pointer remains old. -The host must enforce the imported/application byte ceiling and transport quota -before staging; this adapter does not choose application/import admission policy. +The host must enforce the imported content ceiling, bounded bookkeeping +allowance and transport quota before staging; this adapter does not choose application/import admission policy. Only `Persist` of the real incoming Ready atomically replaces application rows, header/ConfState, active snapshot pointer, log prefix and hard state. Successful @@ -177,7 +177,10 @@ before checkpoint completion resumes it without replaying imported rows over later cluster work. Conflicting manifests or nonempty unimported stores refuse. The `membership/legacy-import` row retains provenance and the byte ceiling `max(8 MiB, imported rows plus exact provenance bytes)` through compaction. -`raftdirectory.Apply` reads that persisted ceiling rather than a caller override. +`raftdirectory.Apply` reads that persisted content ceiling rather than a caller +override. Bounded retry headers and membership initialization use a separate Go +allowance; they cannot consume the saved content allowance or prevent recording +a capacity refusal. Their accounting survives snapshots with the session counts. `OwnLegacy` accepts host-internal readback receipts for the same manifest from at least two distinct configured anchors; the host obtains them through a fresh diff --git a/internal/relayserve/upgrade_capacity_test.go b/internal/relayserve/upgrade_capacity_test.go new file mode 100644 index 0000000000..cbd1634d42 --- /dev/null +++ b/internal/relayserve/upgrade_capacity_test.go @@ -0,0 +1,205 @@ +package relayserve_test + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/relayserve" + "github.com/Agent-Field/codeaf/internal/reqsign" +) + +type upgradeFixture struct { + Now int64 + Cases []struct { + Method, URI, Body string + Headers map[string]string + } +} + +func loadUpgradeFixture(t *testing.T) upgradeFixture { + t.Helper() + raw, err := os.ReadFile("../raftdirectory/testdata/v1-large-requests.json") + if err != nil { + t.Fatal(err) + } + var f upgradeFixture + if err = json.Unmarshal(raw, &f); err != nil { + t.Fatal(err) + } + return f +} +func upgradeRequest(f upgradeFixture, method, uri string, body []byte) *http.Request { + r := httptest.NewRequest(method, "http://local"+uri, bytes.NewReader(body)) + for k, v := range f.Cases[1].Headers { + r.Header.Set(k, v) + } + seed := sha256.Sum256([]byte("p17-a")) + key := ed25519.NewKeyFromSeed(seed[:]) + at := fmt.Sprint(f.Now) + r.Header.Set(reqsign.HeaderTime, at) + r.Header.Set(reqsign.HeaderSig, base64.RawURLEncoding.EncodeToString(ed25519.Sign(key, cs.V1SigningMessage(method, uri, at, body)))) + return r +} +func serveUpgrade(t *testing.T, h http.Handler, f upgradeFixture, method, uri string, body []byte, want int) []byte { + t.Helper() + w := httptest.NewRecorder() + h.ServeHTTP(w, upgradeRequest(f, method, uri, body)) + if w.Code != want { + t.Fatalf("%s %s: HTTP %d, want %d: %.180s", method, uri, w.Code, want, w.Body.Bytes()) + } + return w.Body.Bytes() +} +func oldUpgradeStore(t *testing.T, f upgradeFixture) (string, *directory.SQLite, http.Handler) { + t.Helper() + store := t.TempDir() + root, err := base64.RawURLEncoding.DecodeString(f.Cases[1].Headers[reqsign.HeaderIdentity]) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(store, "directory") + if err = os.MkdirAll(path, 0700); err != nil { + t.Fatal(err) + } + old, err := directory.OpenSQLite(filepath.Join(path, reqsign.IDOf(root)+".db"), func() time.Time { return time.UnixMilli(f.Now) }) + if err != nil { + t.Fatal(err) + } + return store, old, directory.Handler(reqsign.AuthenticateAt(func() time.Time { return time.UnixMilli(f.Now) }), func(string) (directory.Directory, error) { return old, nil }) +} + +// r1.1 +func TestR11AcceptedLegacyBodySurvivesUpgrade(t *testing.T) { + f := loadUpgradeFixture(t) + store, old, handler := oldUpgradeStore(t, f) + c := f.Cases[1] + if len(c.Body) != directory.MaxBody-1 { + t.Fatal("archived body changed") + } + created := serveUpgrade(t, handler, f, c.Method, c.URI, []byte(c.Body), 200) + before := serveUpgrade(t, handler, f, "GET", c.URI, nil, 200) + if !bytes.Equal(created, before) { + t.Fatal("old create/read differ") + } + oldListRaw := serveUpgrade(t, handler, f, "GET", "/v1/dir/list", nil, 200) + var oldList directory.Listing + if json.Unmarshal(oldListRaw, &oldList) != nil { + t.Fatal("old list decode") + } + if err := old.Close(); err != nil { + t.Fatal(err) + } + svc := relayserve.New(relayserve.Config{Store: store, Now: func() time.Time { return time.UnixMilli(f.Now) }}) + defer svc.Close() + after := serveUpgrade(t, svc.Handler, f, "GET", c.URI, nil, 200) + if !bytes.Equal(after, before) { + t.Fatal("cell changed during upgrade") + } + var list directory.Listing + raw := serveUpgrade(t, svc.Handler, f, "GET", "/v1/dir/list", nil, 200) + if json.Unmarshal(raw, &list) != nil { + t.Fatal("list decode") + } + + cell, _ := json.Marshal(list.Cells["near-http-limit"]) + want, _ := json.Marshal(oldList.Cells["near-http-limit"]) + if !bytes.Equal(cell, want) { + t.Fatal("List lost retained fields") + } + take := serveUpgrade(t, svc.Handler, f, "POST", c.URI+"/acquire", []byte(`{}`), 200) + var held directory.CellView + if json.Unmarshal(take, &held) != nil || held.Cell.Lease.Fence != 2 { + t.Fatal("take did not commit") + } + serveUpgrade(t, svc.Handler, f, "POST", c.URI+"/release", []byte(`{"fence":2}`), 204) + raw = serveUpgrade(t, svc.Handler, f, "GET", c.URI, nil, 200) + if json.Unmarshal(raw, &held) != nil || held.Cell.Lease.Expires != 0 || held.Cell.Lease.Fence != 2 { + t.Fatal("release did not commit") + } +} + +// r1.3 +func TestR13ImportedCapacityKeepsDirectoryUsable(t *testing.T) { + for _, below := range []bool{false, true} { + t.Run(fmt.Sprint("below=", below), func(t *testing.T) { + f := loadUpgradeFixture(t) + store, old, handler := oldUpgradeStore(t, f) + n := 30 + if below { + n = 28 + } + sum := 0 + for i := 0; i < n; i++ { + title := 300000 + init := directory.CellInit{Head: "head", Title: strings.Repeat("s", title)} + if below && i == n-1 { + sample, _ := json.Marshal(cs.Created(init, cs.DeviceIDOfKey(actorKeyForUpgrade()), f.Now)) + title += cs.DefaultApplicationBytes - 2048 - sum - len(sample) + init.Title = strings.Repeat("s", title) + } + body, _ := json.Marshal(init) + raw := serveUpgrade(t, handler, f, "POST", fmt.Sprintf("/v1/dir/cells/chat-%d", i), body, 200) + var view directory.CellView + if json.Unmarshal(raw, &view) != nil { + t.Fatal("create decode") + } + stored, _ := json.Marshal(view.Cell) + sum += len(stored) + } + if below && (sum >= cs.DefaultApplicationBytes || sum < cs.DefaultApplicationBytes-4096) || !below && sum <= cs.DefaultApplicationBytes { + t.Fatalf("wrong source size %d", sum) + } + t.Logf("source cell bytes=%d", sum) + if err := old.Close(); err != nil { + t.Fatal(err) + } + svc := relayserve.New(relayserve.Config{Store: store, Now: func() time.Time { return time.UnixMilli(f.Now) }}) + defer svc.Close() + var list directory.Listing + raw := serveUpgrade(t, svc.Handler, f, "GET", "/v1/dir/list", nil, 200) + if json.Unmarshal(raw, &list) != nil || len(list.Cells) != n { + t.Fatal("import list") + } + uri := "/v1/dir/cells/chat-0" + var view directory.CellView + raw = serveUpgrade(t, svc.Handler, f, "GET", uri, nil, 200) + if json.Unmarshal(raw, &view) != nil { + t.Fatal("cell decode") + } + growing := directory.CellInit{Head: "growth", Title: upgradeNoise(4096)} + body, _ := json.Marshal(growing) + refused := serveUpgrade(t, svc.Handler, f, "POST", "/v1/dir/cells/growth", body, 503) + if !bytes.Contains(refused, []byte(`"err":"full"`)) { + t.Fatalf("wrong refusal %.180s", refused) + } + publish, _ := json.Marshal(directory.Publish{Fence: 1, OldHead: view.Cell.Head, Head: "next", Title: view.Cell.Title}) + serveUpgrade(t, svc.Handler, f, "POST", uri+"/publish", publish, 200) + serveUpgrade(t, svc.Handler, f, "POST", uri+"/release", []byte(`{"fence":1}`), 204) + serveUpgrade(t, svc.Handler, f, "GET", uri, nil, 200) + }) + } +} +func actorKeyForUpgrade() []byte { + seed := sha256.Sum256([]byte("p17-a")) + return ed25519.NewKeyFromSeed(seed[:]).Public().(ed25519.PublicKey) +} +func upgradeNoise(n int) string { + var b strings.Builder + for i := 0; b.Len() < n; i++ { + sum := sha256.Sum256([]byte(fmt.Sprint(i))) + b.WriteString(base64.RawURLEncoding.EncodeToString(sum[:])) + } + return b.String()[:n] +} diff --git a/relay/hosted/src/raft/README.md b/relay/hosted/src/raft/README.md index cdbdd76f95..01aaec1106 100644 --- a/relay/hosted/src/raft/README.md +++ b/relay/hosted/src/raft/README.md @@ -26,10 +26,13 @@ evaluator is synchronous with an immutable applied view and bounded 1 MiB read/write sets. It must implement committed policy, declared dependencies and request/session semantics; storage supplies atomicity, not that policy. Rows cover state, sessions, membership, enrollment, tombstones and outboxes. The -persisted application ceiling defaults to 8 MiB, including framed keys/values; -an explicit larger imported ceiling is preserved. Pressure refuses growth, never -deletes application rows. The migration host inventories and grandfathers -imported rows. +persisted content ceiling defaults to 8 MiB, including framed keys/values; +an explicit larger imported ceiling is preserved. Go supplies a separate bounded +allowance for bounded retry headers and membership initialization. Apply and +snapshot installation subtract only the persisted bookkeeping charge from content +usage. Large response payloads still count as content. Pressure refuses content +growth and never deletes application rows. The migration host inventories and +grandfathers imported rows. Snapshots use `internal/raftsnapshot` format 1 without a second JS schema. Rows stream in SQL binary order into 256 KiB chunks. The shared Go codec diff --git a/relay/hosted/src/raft/abi.js b/relay/hosted/src/raft/abi.js index 2092ee5dea..f31728ebbe 100644 --- a/relay/hosted/src/raft/abi.js +++ b/relay/hosted/src/raft/abi.js @@ -108,6 +108,10 @@ export function snapshotNeeded(bytes) { export class HostCodec { constructor(runtime) { this.runtime = runtime; } call(w) { return response(this.runtime.call(w.finish())); } + bookkeeping(raw) { + const r=this.call(frame(48).u32(raw===undefined?0:1).blob(raw??new Uint8Array())); + const value={max:Number(r.u64()),bytes:Number(r.u64())};r.end();return value; + } stagingPolicy() { if (!this.staging) { const r=this.call(frame(46)); diff --git a/relay/hosted/src/raft/storage.js b/relay/hosted/src/raft/storage.js index 82508597a6..6992acc163 100644 --- a/relay/hosted/src/raft/storage.js +++ b/relay/hosted/src/raft/storage.js @@ -9,8 +9,17 @@ const enc = new TextEncoder(), dec = new TextDecoder('utf-8', { fatal: true }); export const hex = b => Array.from(b, x => x.toString(16).padStart(2, '0')).join(''); export const equal = (a, b) => a.length === b.length && a.every((x, i) => x === b[i]); export const bytes = b => b instanceof Uint8Array ? b : new Uint8Array(b); -export const b64 = b => { let s = ''; for (const x of b) s += String.fromCharCode(x); return btoa(s); }; -export const unb64 = s => Uint8Array.from(atob(s), c => c.charCodeAt(0)); +// Bounded pieces avoid a temporary string per byte when copying large rows. +export const b64 = b => { + const parts=[],width=CHUNK_BYTES/8; + for(let at=0;at { + const text=atob(s),out=new Uint8Array(text.length); + for(let at=0;at BigInt(n).toString(16).padStart(16, '0'); const check = (v, message = 'raft storage: invalid or damaged durable state') => { if (!v) throw new Error(message); }; @@ -138,7 +147,7 @@ export class MemberStorage { read.page=(ns,k)=>{const data=this.read(ns,k);check((data?.length??0)<=ROW_BYTES,'raft storage: evaluation record-page capacity');return data;}; const change = evaluate(e.data, read, { ...app, index: e.index, term: e.term }); check(change && !change.then && Array.isArray(change.writes)); - const retainedBefore=app.size>this.identity.ceiling?app.size-this.stagingBytes():app.size; + const retainedBefore=app.size-this.stagingBytes()-this.codec.bookkeeping(this.read('sessions','counts')??undefined).bytes; for (const r of change.writes) { check(namespaces.includes(r.namespace) && typeof r.key === 'string' && enc.encode(r.key).length > 0 && enc.encode(r.key).length <= ROW_BYTES && r.data instanceof Uint8Array && r.data.length <= ROW_BYTES); writeBytes += rowSize(r.namespace, r.key, r.data); check(writeBytes <= ROW_BYTES, 'raft storage: evaluation write-set capacity'); @@ -149,8 +158,8 @@ export class MemberStorage { xor(aggregate, this.checksum('records', r.namespace, r.key, r.data)); app.size += rowSize(r.namespace, r.key, r.data); this.rows('INSERT OR REPLACE INTO records VALUES (?,?,?,?)', r.namespace, r.key, r.data, hex(this.checksum('records', r.namespace, r.key, r.data))); } - const retained=app.size>this.identity.ceiling?app.size-this.stagingBytes():app.size; - check(retained <= this.identity.ceiling || retained <= retainedBefore, 'raft storage: application capacity; retry after capacity is raised'); + const retained=app.size-this.stagingBytes()-this.codec.bookkeeping(this.read('sessions','counts')??undefined).bytes; + check(retained >= 0 && (retained <= this.identity.ceiling || retained <= retainedBefore), 'raft storage: application capacity; retry after capacity is raised'); if (change.version !== undefined) { check(BigInt(change.version) >= BigInt(app.version)); app.version = String(change.version); } if (change.now !== undefined) { check(BigInt(change.now) >= BigInt(app.now)); app.now = String(change.now); } } @@ -183,7 +192,7 @@ export class MemberStorage { } async beginSnapshot(data) { const p = this.codec.snapshot(data), s = this.codec.inspect(3, p), m = JSON.parse(dec.decode(data)), id = this.hash(data); - check(m.cluster === this.identity.cluster && m.protocol === '1' && BigInt(m.bytes) <= BigInt(this.identity.ceiling)+BigInt(this.codec.stagingPolicy().max), 'raft storage: snapshot capacity or identity mismatch'); + check(m.cluster === this.identity.cluster && m.protocol === '1' && BigInt(m.bytes) <= BigInt(this.identity.ceiling)+BigInt(this.codec.stagingPolicy().max)+BigInt(this.codec.bookkeeping().max), 'raft storage: snapshot capacity or identity mismatch'); await this.transaction(() => { const active = unb64(this.get('snapshot')), at = active.length ? this.codec.inspect(3, active).index : 0n; check(s.index >= at); @@ -206,7 +215,7 @@ export class MemberStorage { chunk(id, offset) { const r = this.one('SELECT data,hash FROM chunks WHERE generation=? AND offset=?', id, offset); check(r); const b = bytes(r.data); check(this.hash(b) === r.hash); return b; } walk(id, visit) { const { manifest: m } = this.generation(id); - let state = EMPTY, digest = this.codec.hash(EMPTY).digest, pending = EMPTY, count = 0, total = 0, size = 0, staging = 0, lastNS = '', lastKey = ''; + let state = EMPTY, digest = this.codec.hash(EMPTY).digest, pending = EMPTY, count = 0, total = 0, size = 0, staging = 0, bookkeeping = 0, lastNS = '', lastKey = ''; const aggregate = new Uint8Array(32); const chunks = Number(m.chunks); check(Number.isSafeInteger(chunks) && chunks >= 0); check(this.one('SELECT count(*) AS n FROM chunks WHERE generation=?', id).n === chunks); @@ -221,12 +230,13 @@ export class MemberStorage { const namespace = dec.decode(pending.subarray(12, 12 + n)), name = dec.decode(pending.subarray(12 + n, 12 + n + k)); check(namespaces.includes(namespace) && (count === 0 || compare(namespace, lastNS) > 0 || (namespace === lastNS && compare(name, lastKey) > 0))); const data = pending.slice(12 + n + k, length); xor(aggregate, this.checksum('records', namespace, name, data)); count++; size += rowSize(namespace, name, data); staging += this.stagingCost(namespace,name,data); + if(namespace==='sessions'&&name==='counts')bookkeeping=this.codec.bookkeeping(data).bytes; visit?.({ namespace, key: name, data }); lastNS = namespace; lastKey = name; pending = pending.slice(length); } check(pending.length <= 2 * ROW_BYTES + 28); } check(pending.length === 0 && String(count) === m.records && String(total) === m.bytes && hex(digest) === m.digest && hex(aggregate) === m.records_hash); - check(staging<=this.codec.stagingPolicy().max && size-staging <= this.identity.ceiling, 'raft storage: snapshot exceeds retained identity capacity'); + check(staging<=this.codec.stagingPolicy().max && size>=staging+bookkeeping && size-staging-bookkeeping <= this.identity.ceiling, 'raft storage: snapshot exceeds retained identity capacity'); } retain(id) { const others = this.rows('SELECT id FROM snapshots WHERE complete=1 AND id<>?', id).toArray().map(r => this.generation(r.id)).sort((a, b) => BigInt(a.manifest.index) > BigInt(b.manifest.index) ? -1 : 1); diff --git a/relay/hosted/test/raft/bookkeeping-storage.test.js b/relay/hosted/test/raft/bookkeeping-storage.test.js new file mode 100644 index 0000000000..bb86bf465a --- /dev/null +++ b/relay/hosted/test/raft/bookkeeping-storage.test.js @@ -0,0 +1,57 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {DatabaseSync} from 'node:sqlite'; +import {createHash,generateKeyPairSync,sign} from 'node:crypto'; +import {Reactor,HostCodec} from '../../src/raft/abi.js'; +import {MemberStorage} from '../../src/raft/storage.js'; +import {directoryEvaluator} from '../../src/raft/directory.js'; +import {encodeCanonical} from '../../src/raft/codec.js'; +const enc=new TextEncoder(),empty=new Uint8Array(),sha=b=>createHash('sha256').update(b).digest('hex'); +const data=v=>enc.encode(encodeCanonical(v)); +const publicBytes=key=>key.publicKey.export({format:'der',type:'spki'}).subarray(-32); +const varint=n=>{const b=[];do{b.push((n&127)|(n>127?128:0));n=Math.floor(n/128);}while(n);return b;}; +const entry=(index,raw)=>Uint8Array.from([8,0,16,1,24,...varint(index),34,...varint(raw.length),...raw]); +const hard=index=>Uint8Array.from([8,1,24,...varint(index)]),conf=Uint8Array.of(8,1,8,2,8,3); +class Storage { + constructor(){this.db=new DatabaseSync(':memory:');this.sql={exec:(q,...p)=>{const rows=this.db.prepare(q).all(...p);return {toArray:()=>rows,[Symbol.iterator]:()=>rows[Symbol.iterator]()};}};} + transactionSync(fn){this.db.exec('BEGIN');try{const result=fn();this.db.exec('COMMIT');return result;}catch(error){this.db.exec('ROLLBACK');throw error;}} + async sync(){} +} + +// r1.5 + test('r1.5 bookkeeping survives a full-content snapshot without granting content growth',{timeout:60000},async()=>{ + const runtime=new Reactor(new WebAssembly.Module(readFileSync(process.env.CODEAF_RAFT_PROOF_DIR+'/core.wasm'))),codec=new HostCodec(runtime); + const host=new Storage(),target=new Storage(),identity=id=>({cluster:'bounded-directory',id,protocol:1}); + const store=new MemberStorage(host,codec),root=generateKeyPairSync('ed25519'),actor=generateKeyPairSync('ed25519');let index=0; + const apply=async(raw,evaluate)=>{const entries=[entry(++index,raw)];await store.persist({hard:hard(index),entries,snapshot:empty});await store.apply({index:BigInt(index),entries,conf,snapshot:empty},evaluate);}; + try{ + await store.open(identity('1')); + await apply(Uint8Array.of(1),()=>({writes:[ + {namespace:'state',key:'identity',data:data({V:1,identity:'id_'+sha(publicBytes(root)).slice(0,32),signing_key:publicBytes(root).toString('base64')})}, + {namespace:'enrollment',key:'actor/61',data:data({cluster:identity('1').cluster,key:publicBytes(actor).toString('base64'),personal:true})} + ]})); + while(store.application().size0); + await apply(Uint8Array.of(1),()=>({writes:[{namespace:'state',key,data:new Uint8Array(length).fill(19)}]})); + } + const before=store.application().size; + const command={schema:1,cluster:identity('1').cluster,actor:'a',session:'bounded-retry',sequence:'1',request_digest:'',now:1000,change:{advance_time:{}},signature:null}; + command.request_digest=sha(data({cluster:command.cluster,actor:command.actor,session:command.session,sequence:command.sequence,change:command.change})); + command.signature=sign(null,Buffer.concat([Buffer.from('codeaf.directory.command.v1\0'),data(command)]),actor.privateKey).toString('base64'); + const coreEvaluate=directoryEvaluator(runtime,command.cluster,store.identity.ceiling);let result; + const evaluate=(...args)=>{const change=coreEvaluate(...args);result=JSON.parse(new TextDecoder().decode(change.result));return change;}; + await apply(data(command),evaluate);assert.equal(result.code,'ok');const firstResult=result; + const after=store.application().size,policy=codec.bookkeeping(store.read('sessions','counts'));assert(after>before);assert.equal(after-policy.bytes,before);assert(after<=before+policy.max); + await apply(data(command),evaluate);assert.deepEqual(result,firstResult);assert.equal(store.application().size,after); + const snapshot=await store.checkpoint(),info=codec.inspect(3,snapshot),manifest=JSON.parse(new TextDecoder().decode(info.data)),id=sha(info.data); + const dest=new MemberStorage(target,codec);await dest.open(identity('2'));await dest.beginSnapshot(info.data); + for(let n=0;n({writes:[{namespace:'state',key:'growth',data:Uint8Array.of(1)}]})),/application capacity/); + assert.equal(dest.application().applied,applied);assert.equal(dest.application().size,after);assert.equal(dest.read('state','growth'),null); + }finally{host.db.close();target.db.close();} +}); diff --git a/relay/hosted/test/raft/hosted-fixture.js b/relay/hosted/test/raft/hosted-fixture.js index b1b32ca5cf..aef087ae70 100644 --- a/relay/hosted/test/raft/hosted-fixture.js +++ b/relay/hosted/test/raft/hosted-fixture.js @@ -10,7 +10,7 @@ import { join } from 'node:path'; import { generateKeyPairSync, createHash, createPrivateKey, createPublicKey, sign } from 'node:crypto'; const sha=b=>createHash('sha256').update(b).digest('hex'); -export async function hostedFixture({fixture=false,base='',go=false,rootLabel,now}={}) { +export async function hostedFixture({fixture=false,base='',go=false,rootLabel,now,entryPoint}={}) { const prefix=prefixOf({BASE_PATH:base}); const out=process.env.CODEAF_RAFT_PROOF_DIR;assert(out,'build the native/WASI proof first'); const dir=await mkdtemp(join(tmpdir(),'p17b-')); @@ -41,7 +41,7 @@ export async function hostedFixture({fixture=false,base='',go=false,rootLabel,no let text=await readFile(args.path,'utf8');const needle=cut.needle;assert.equal(text.split(needle).length,2,'cut must identify exactly one boundary'); text=text.replace(needle,needle+(cut.after??"throw new Error('H2 injected crash after durable boundary');"));return {contents:text,loader:'js'}; });}}); - await build({entryPoints:[fixture?'test/raft/hosted-worker.js':'src/raft-worker.js'],bundle:true,format:'esm',target:'es2022',external:['cloudflare:*'],banner:now===undefined?undefined:{js:`const realNow=Date.now;Date.now=()=>{const value=realNow();if(value===0)return ${now};globalThis.fixtureClockOffset??=${now}-value;return value+globalThis.fixtureClockOffset;};`},plugins,outfile:join(dir,'worker.js')}); + await build({entryPoints:[entryPoint??(fixture?'test/raft/hosted-worker.js':'src/raft-worker.js')],bundle:true,format:'esm',target:'es2022',external:['cloudflare:*'],banner:now===undefined?undefined:{js:`const realNow=Date.now;Date.now=()=>{const value=realNow();if(value===0)return ${now};globalThis.fixtureClockOffset??=${now}-value;return value+globalThis.fixtureClockOffset;};`},plugins,outfile:join(dir,'worker.js')}); const saved=join(dir,'bundle-'+bundles.size+'.js'); await copyFile(join(dir,'worker.js'),saved);bundles.set(variant,saved);compiledCut=variant; }; diff --git a/relay/hosted/test/raft/upgrade-capacity.test.js b/relay/hosted/test/raft/upgrade-capacity.test.js new file mode 100644 index 0000000000..0807102418 --- /dev/null +++ b/relay/hosted/test/raft/upgrade-capacity.test.js @@ -0,0 +1,67 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {createHash,createPrivateKey,sign} from 'node:crypto'; +import {hostedFixture} from './hosted-fixture.js'; +const archive=JSON.parse(readFileSync(new URL('../../../../internal/raftdirectory/testdata/v1-large-requests.json',import.meta.url))); +const seed=createHash('sha256').update('p17-a').digest(); +const key=createPrivateKey({key:Buffer.concat([Buffer.from('302e020100300506032b657004220420','hex'),seed]),format:'der',type:'pkcs8'}); +const sha=raw=>createHash('sha256').update(raw).digest('hex'); +const fixture=()=>hostedFixture({fixture:true,rootLabel:'root',now:archive.now,entryPoint:'test/raft/upgrade-worker.js'}); +async function request(h,method,uri,body='',legacy=false,want=200){ + const headers={...archive.cases[1].headers}; + headers['Codeaf-Sig']=sign(null,Buffer.from(`codeaf-req-v1\n${method}\n${uri}\n${headers['Codeaf-Time']}\n${sha(body)}`),key).toString('base64url'); + if(legacy)headers['Fixture-Legacy']='1'; + const began=performance.now(); + const r=await h.mf.dispatchFetch('http://local'+uri,{method,headers,...(body?{body}:{})}); + const text=await r.text();if(!legacy)console.log(method+' '+uri+' HTTP '+r.status+' in '+Math.round(performance.now()-began)+' ms');assert.equal(r.status,want,method+' '+uri+': '+text.slice(0,180));return text?JSON.parse(text):null; +} + +// r1.1 + test('r1.1 accepted near-limit legacy body survives hosted upgrade byte-for-byte',{timeout:60000},async()=>{ + const h=await fixture(),c=archive.cases[1]; + try{ + await h.start();assert.equal(Buffer.byteLength(c.body),1048575); + const before=await request(h,c.method,c.uri,c.body,true); + const source=await h.control({op:'source'});assert.equal(source.gate,undefined);assert(Buffer.byteLength(source.originals.find(r=>r.kind==='cells').doc)>1048576); + const oldList=await request(h,'GET','/v1/dir/list','',true); + const after=await request(h,'GET',c.uri);assert.deepEqual(after.cell,before.cell); + const list=await request(h,'GET','/v1/dir/list');assert.deepEqual(list.cells['near-http-limit'],oldList.cells['near-http-limit']); + const held=await request(h,'POST',c.uri+'/acquire','{}');assert.equal(held.cell.lease.fence,2); + await request(h,'POST',c.uri+'/release','{"fence":2}',false,204); + const released=await request(h,'GET',c.uri);assert.equal(released.cell.lease.fence,2);assert.equal(released.cell.lease.expires,0); + await h.start();assert.deepEqual((await request(h,'GET',c.uri)).cell,released.cell); + }finally{await h.close();} +}); + +// r1.3 +for(const below of [false,true])test('r1.3 hosted imported '+(below?'nearly full':'grandfathered')+' directory completes membership and keeps saved cells usable',{timeout:60000},async()=>{ + const h=await fixture(); + try{ + await h.start();const n=below?28:30; + const body=JSON.stringify({head:'head',class:'',title:'s'.repeat(300000),size:0,keys:null});assert.equal(Buffer.byteLength(body),300058); + const created=await request(h,'POST','/v1/dir/cells/chat-0',body,true); + const rowBytes=Buffer.byteLength(JSON.stringify(created.cell)),records=[]; + let sum=n*rowBytes; + for(let i=1;i1n);} + const uri='/v1/dir/cells/chat-0',view=await request(h,'GET',uri); + let noise='';for(let i=0;noise.length<4096;i++)noise+=createHash('sha256').update(String(i)).digest('base64url'); + const refused=await request(h,'POST','/v1/dir/cells/growth',JSON.stringify({head:'growth',title:noise.slice(0,4096)}),false,507);assert.equal(refused.err,'capacity'); + await request(h,'POST',uri+'/publish',JSON.stringify({fence:1,old_head:'head',head:'next',size:0,class:'',title:view.cell.title,pending:0})); + await request(h,'POST',uri+'/release','{"fence":1}',false,204); + assert.equal((await request(h,'GET',uri)).cell.head,'next'); + assert.deepEqual((await h.control({op:'source'})).originals,source.originals); + }finally{await h.close();} +}); diff --git a/relay/hosted/test/raft/upgrade-worker.js b/relay/hosted/test/raft/upgrade-worker.js new file mode 100644 index 0000000000..a4f49815db --- /dev/null +++ b/relay/hosted/test/raft/upgrade-worker.js @@ -0,0 +1,27 @@ +export * from './hosted-worker.js'; +export { default } from './hosted-worker.js'; +import { IdentityDO as Current } from './hosted-worker.js'; +import { Directory } from '../../src/directory.js'; +import { checkCert, checkRequest } from '../../src/verify.js'; +import { matchRoute } from '../../src/routes.js'; +import { readBody } from '../../src/body.js'; +import { respond } from '../../src/wire.js'; +import { policyOf } from '../../src/rules.js'; + +// Only the loopback fixture can address the old signed handler. It writes the +// same source tables later frozen by the production first-use upgrade. +export class IdentityDO extends Current { + fetch(request) { + if(request.headers.get('Fixture-Legacy')!=='1')return super.fetch(request); + return respond(async()=>{ + const url=new URL(request.url),route=matchRoute(request.method,url.pathname); + const headers=Object.fromEntries(['identity','cert','time','sig'].map(k=>[k,request.headers.get('Codeaf-'+k)])); + const caller=await checkCert(headers),body=await readBody(request,route.limit,route.over); + const who=await checkRequest(caller,{method:request.method,uri:url.pathname},body,Date.now()); + const dir=new Directory(this.ctx.storage.sql,who.identity,Date.now,policyOf()); + await this.ctx.storage.put({identity:who.identity,admitted:true}); + const answer=await route.handler({tenant:{dir},device:who.device,body,request},route.args); + await this.ctx.storage.sync();return answer; + }); + } +} From 959ddbf82618dbbe4b12d9b852e647ed92c544a2 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 22:34:36 -0400 Subject: [PATCH 090/103] Release sleeping hosted members after saving their state. Sleeping groups free their reservations so other households can activate. Busy groups keep their reservations and admission pressure returns a retryable refusal. The next operation recovers durable followers and establishes a fresh majority. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/changes/unreleased/raft-cluster.md | 1 + .../manual/chat/coordination-and-workers.md | 4 + relay/hosted/src/raft/actor.js | 13 ++- relay/hosted/src/raft/hosted.js | 46 +++++++-- relay/hosted/src/raft/member.js | 14 ++- .../hosted/test/raft/hosted-capacity.test.js | 98 +++++++++++++++++++ relay/hosted/test/raft/hosted-worker.js | 32 +++++- 7 files changed, 196 insertions(+), 12 deletions(-) create mode 100644 relay/hosted/test/raft/hosted-capacity.test.js diff --git a/docs/changes/unreleased/raft-cluster.md b/docs/changes/unreleased/raft-cluster.md index d197d6e7d0..6bf368842f 100644 --- a/docs/changes/unreleased/raft-cluster.md +++ b/docs/changes/unreleased/raft-cluster.md @@ -10,6 +10,7 @@ invalidates: - "Coordination depended on the relay's single writer. Two remaining anchors can now elect a fallback and restore the third after a personal leader disappears, targeting about 65 seconds in healthy conditions; a returning computer remains a learner and never automatically regains leadership." - "A majority was not required by the former directory writer. Losing two of the three current voters now refuses coordination and preserves saved chats instead of creating a writable minority." - "Caught-up open homes previously polled shared state. Idle homes now send zero HTTP requests, peer or forwarding frames, or client attempts after all voters apply the idle barrier; presence sweeps recur only with a viewer and fresh input requires a new majority response." + - "Sleeping hosted groups previously retained their memory reservations indefinitely. Completed all-voter sleep now checkpoints and releases those members, and the next request recovers them through a fresh majority; busy admission pressure returns retryable HTTP 503." - "Older clients could blindly force a live chat without its observed holder. Both relays now refuse that request with another device continued this chat first; releasing the chat or updating the client permits safe continuation." - "Legal older directory requests could be refused above the smaller coordination envelope. Both relays now retain the original 1 MiB HTTP limit, stage large signed bodies and commit complete metadata atomically, recovering interruptions without exposing partial changes or deleting retained records." - "Retained rows from legal older requests could pause upgrades after the relay added fields, and full directories could fail initialization or refusal persistence. Both relays now preserve these rows losslessly and reserve bounded bookkeeping independently of the measured content allowance." diff --git a/internal/manual/chat/coordination-and-workers.md b/internal/manual/chat/coordination-and-workers.md index 56e175359f..5c9b20dfa1 100644 --- a/internal/manual/chat/coordination-and-workers.md +++ b/internal/manual/chat/coordination-and-workers.md @@ -64,6 +64,10 @@ A compatibility events viewer pings every 10 seconds. The platform answers witho A connection naming held chats still renews through committed changes every 30 seconds while fresh hold evidence is available. Presence alone never renews a chat. +## Hosted relay memory capacity and retry + +After every voter has completed the group's idle barrier, the hosted relay frees its working memory for other households and keeps the saved directory in persistent storage. The next request reloads those copies and obtains a fresh majority response. Busy groups keep their memory. When busy groups fill the relay's memory allowance, another activation receives HTTP 503 with retry guidance; retrying after a group sleeps can succeed. + ## What can a worker do — can a worker read all my chats or approve pairing? A delegated worker uses its own key and an expiring certificate naming selected chats. It receives only sealed grants for those chats, rather than the full exported identity. It cannot approve pairing, rotate the identity, read the identity vault, enroll other devices or request leadership. Richer capabilities, worker labels and execution credentials stay encrypted; only the existing platform and sandbox capability shape is readable by the relay. diff --git a/relay/hosted/src/raft/actor.js b/relay/hosted/src/raft/actor.js index e80750fd47..81d52ee441 100644 --- a/relay/hosted/src/raft/actor.js +++ b/relay/hosted/src/raft/actor.js @@ -36,12 +36,12 @@ export class MemberActor { serial(fn) { if (this.pending >= QUEUED_INPUTS) return Promise.reject(new Error('raft member: capacity; retry with backoff')); this.pending++; - const p = this.tail.then(fn); this.tail = p.catch(() => {}); + const p = this.tail.then(() => { if(this.stopped||this.store.stopped)throw new Error('raft member: stopped; recover from durable storage');return fn(); }); this.tail = p.catch(() => {}); return p.finally(() => { this.pending--; }); } init(config) { config = structuredClone(config); return this.serial(async () => { if (this.handle) throw new Error('raft member: already active'); - if (active.size * RESERVATION + RESERVATION + this.reactor.memory > CORE_MEMORY_BUDGET) throw new Error('raft member: isolate capacity; retry with backoff'); + if (active.size * RESERVATION + RESERVATION + this.reactor.memory > CORE_MEMORY_BUDGET) throw Object.assign(new Error('raft member: isolate capacity; retry with backoff'),{isolateCapacity:true}); active.add(this); try { const recovered = await this.store.open({ cluster: config.cluster, id: String(config.id), protocol: 1 }, config.ceiling ?? DEFAULT_ROWS_BYTES); @@ -180,6 +180,15 @@ export class MemberActor { begin(data) { data = data.slice(); return this.serial(() => this.store.beginSnapshot(data)); } chunk(id, offset, data, hash) { data = data.slice(); return this.serial(() => this.store.putChunk(id, offset, data, hash)); } complete(id) { return this.serial(() => this.store.completeSnapshot(id)); } + // All voters must have applied the idle barrier before a sleeping leader + // can relinquish its handle. A checkpoint precedes fencing every old caller. + releaseSleeping(now, ingressNow, acks) { acks=structuredClone(acks);return this.serial(async()=>{ + const effects=await this.scheduleInQueue(now,ingressNow,false,acks); + if(!effects.plan.sleeping||effects.send.length||effects.reads.length)throw new Error('raft member: active member cannot be evicted'); + await this.store.checkpoint(); + if(this.pending>1)throw new Error('raft member: active member cannot be evicted'); + this.replaced(); + }); } // An exclusive object replacement cannot resume its former volatile actor. // Fence pending continuations before recovering the same durable identity. replaced() { diff --git a/relay/hosted/src/raft/hosted.js b/relay/hosted/src/raft/hosted.js index ee80e7d64f..6a2de8a7a2 100644 --- a/relay/hosted/src/raft/hosted.js +++ b/relay/hosted/src/raft/hosted.js @@ -22,7 +22,7 @@ const enc=new TextEncoder(),dec=new TextDecoder('utf-8',{fatal:true}); const segment=s=>Array.from(enc.encode(s),b=>b.toString(16).padStart(2,'0')).join(''); const unsegment=s=>dec.decode(Uint8Array.from(s.match(/../g)??[],h=>parseInt(h,16))); const data=v=>enc.encode(encodeCanonical(v)); -const MIGRATION_PAGE=64,COORDINATOR_QUEUE=64; +const MIGRATION_PAGE=64,COORDINATOR_QUEUE=64,MIN_MEMBERS=2; const unavailable=()=>new Wire('unreachable',503,1); const status={not_found:404,unauthorized:401,revoked:401,rotated:410,bad_request:400,bad_grace:400,self_revoke:400,capacity:507,full:507}; function requireOK(code){if(code!=='ok')throw new Wire(code,status[code]??409);} @@ -40,7 +40,7 @@ export class HostedIdentity { this.armAlarm=arm??(async at=>{if(at===null){await ctx.storage.deleteAlarm();return;}const old=await ctx.storage.getAlarm();if(old===null||at=COORDINATOR_QUEUE)return Promise.reject(unavailable());this.pending++; @@ -48,7 +48,7 @@ export class HostedIdentity { } operation(fn){ if(this.operations>=COORDINATOR_QUEUE)return Promise.reject(unavailable());this.operations++; - const task=this.operationTail.then(fn);this.operationTail=task.catch(()=>{});return task.finally(()=>this.operations--); + const task=this.operationTail.then(fn).catch(error=>{if(error.isolateCapacity)throw unavailable();throw error;});this.operationTail=task.catch(()=>{});return task.finally(()=>this.operations--); } signal(){this.progressVersion++;for(const resolve of this.waiters)resolve();this.waiters.clear();} async progress(deadline,version=this.progressVersion){ @@ -67,7 +67,7 @@ export class HostedIdentity { // forwarding reply preserves routing, while every operation still requires // its ordinary quorum barrier and exact durable result. const hint=String(reply?.status?.Leader??'0'),term=BigInt(reply?.status?.Term??0); - const known=[...this.states.values()].reduce((max,state)=>state.term>max?state.term:max,this.routeTerm??0n); + const known=[...(this.states?.values()??[])].reduce((max,state)=>state.term>max?state.term:max,this.routeTerm??0n); if(hint!=='0'&&term>=known){this.routeHint=hint;this.routeTerm=term;} if(!reply||reply.error)throw Object.assign(unavailable(),{forwardID:token.id}); return reply; @@ -107,6 +107,17 @@ export class HostedIdentity { await this.ctx.storage.put('raft-provision',p);await this.ctx.storage.sync();return p; } async ensure(identityKey){ + // Recover inactive members without replacing live routing or sockets. + if(this.ready&&this.inactiveMembers.size){ + for(const id of [...this.inactiveMembers]){ + try{ + const member=this.member(id);if(!await member.importReceipt(this.manifest))continue; + const initialized=await member.initialize(this.config(id));if(initialized.capacity)continue; + this.members.set(id,member);this.accept(id,initialized);this.accept(id,await member.drainRecovery()); + }catch{this.members.delete(id);this.states.delete(id);} + } + if([...this.members.keys()].filter(id=>!this.inactiveMembers.has(id)).length({...m,from:String(id),to:String(m.to)})));this.reads.push(...(e.reads??[]).map(r=>({...r,node:String(id)})));this.signal();} + accept(id,e){if(!e)return;if(e.status){this.inactiveMembers?.delete(String(id));this.states.set(String(id),e.status);}this.queue.push(...(e.send??[]).map(m=>({...m,from:String(id),to:String(m.to)})));this.reads.push(...(e.reads??[]).map(r=>({...r,node:String(id)})));this.signal();} async input(id,tag,value){const result=await this.members.get(String(id)).hostInput(tag,value);this.accept(id,result);return result;} // Reserve durable sequence ranges before emission. Concurrent public and // peer turns share one allocator, and recovery skips every unused sequence. @@ -479,7 +494,22 @@ export class HostedIdentity { let sleeping=remote?(await schedule(acks)).sleeping:true; for(const [node,member]of this.members){const effects=await member.schedule(this.elapsed,now,busy,acks);this.accept(node,effects);sleeping&&=effects.plan.sleeping;} await this.pump();this.asleep=sleeping&&!this.coordinationBusy()&&wakeVersion===this.wakeVersion; - if(this.asleep){await this.peers.quiesce();await this.pump();await this.sleepMaintenance(wakeVersion);} + if(this.asleep){ + // Every sleeping replica exposes the same committed peer view. + const replica=this.replica();this.peers.viewApplied=replica+'/'+this.states.get(replica).applied; + await this.peers.quiesce();await this.pump();await this.sleepMaintenance(wakeVersion); + for(const [node,member]of this.members){ + if(!this.asleep||wakeVersion!==this.wakeVersion)break; + const state=this.states.get(node),hint=this.leaderID(); + await member.releaseSleeping(this.elapsed,now,acks); + // Durable progress still authenticates peers, but grants no authority. + if(!this.states.has(node)||this.states.get(node)===state){ + this.inactiveMembers.add(node); + if(state)this.states.set(node,{...state,role:0,leader:0n}); + } + if(state&&(this.routeTerm??0n)<=state.term){this.routeHint=hint;this.routeTerm=state.term;} + } + } }catch{this.asleep=false;} } }finally{ diff --git a/relay/hosted/src/raft/member.js b/relay/hosted/src/raft/member.js index bef1b74175..133cbdf393 100644 --- a/relay/hosted/src/raft/member.js +++ b/relay/hosted/src/raft/member.js @@ -162,7 +162,9 @@ export class RaftMemberDO extends DurableObject { return this.loading; } this.loadingConfig = JSON.stringify(config); - this.loading = this.activate(config).then(effects=>({...effects,elapsed:this.actor.elapsed??0})); + this.loading = this.activate(config).then(effects=>({...effects,elapsed:this.actor.elapsed??0})).catch(error=>{ + if(error.isolateCapacity)return {capacity:true};throw error; + }); try { return await this.loading; } finally { this.loading = null; this.loadingConfig = null; } } async activate(config) { @@ -256,6 +258,16 @@ export class RaftMemberDO extends DurableObject { } async schedule(elapsed, ingressNow, busy, acks=[]) { await this.load(); return this.actor.schedule(elapsed, ingressNow, busy, acks); } async sleepAck(index,term) { await this.load(); return this.actor.sleepAck(index,term); } + // Release runs in this object's storage context after the router completes + // all-voter sleep. The next input follows ordinary durable follower recovery. + async releaseSleeping(elapsed, now, acks) { + if(!this.actor)return; + const actor=this.actor; + if(this.recoveredEffects?.send?.length||await this.ctx.storage.get('raft-scheduled'))throw new Error('raft member: scheduled work prevents release'); + await this.ctx.storage.deleteAlarm(); + await actor.releaseSleeping(elapsed,now,acks); + this.actor=null;this.inbox=null;this.registryCache=null;this.registryIndex=null;this.results.clear();this.resultBytes=0;this.recoveredEffects=null; + } async leadershipEvidence(revision,barrier) { await this.load(); return this.actor.leadershipEvidence(revision,barrier); } async leadership(evidence) { await this.load(); return this.actor.leadership(evidence); } async leadershipReceipt() { await this.load(); return this.actor.leadershipReceipt(); } diff --git a/relay/hosted/test/raft/hosted-capacity.test.js b/relay/hosted/test/raft/hosted-capacity.test.js new file mode 100644 index 0000000000..d0545809f3 --- /dev/null +++ b/relay/hosted/test/raft/hosted-capacity.test.js @@ -0,0 +1,98 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {generateKeyPairSync,createHash,sign} from 'node:crypto'; +import {hostedFixture} from './hosted-fixture.js'; + +const sha=b=>createHash('sha256').update(b).digest('hex'); +function household(h){ + const root=generateKeyPairSync('ed25519'),key=generateKeyPairSync('ed25519'); + const pub=root.publicKey.export({format:'der',type:'spki'}).subarray(-32),dev=key.publicKey.export({format:'der',type:'spki'}).subarray(-32); + const cert={V:1,device:dev.toString('hex'),created:1000,sig:''}; + cert.sig=sign(null,Buffer.from('codeaf/device-cert/v1\n'+JSON.stringify(cert)),root.privateKey).toString('hex'); + const identity='id_'+sha(pub).slice(0,32); + return {identity,control:(op,records)=>h.control({identity,op,records}),async request(method='GET',path='/v1/dir/list',body){ + const raw=body===undefined?'':JSON.stringify(body),at=String(Date.now()); + const headers={'Codeaf-Identity':pub.toString('base64url'),'Codeaf-Cert':Buffer.from(JSON.stringify(cert)).toString('base64url'),'Codeaf-Time':at,'Codeaf-Sig':sign(null,Buffer.from('codeaf-req-v1\n'+method+'\n'+path+'\n'+at+'\n'+sha(raw)),key.privateKey).toString('base64url')}; + const r=await h.mf.dispatchFetch('http://local'+path,{method,headers,...(raw?{body:raw}:{})}),text=await r.text(); + return {status:r.status,headers:r.headers,body:text?JSON.parse(text):null}; + }}; +} +async function idle(a){const result=await a.control('idle-schedule');assert.equal(result.sleeping,true);assert.equal(result.recurringMemberCalls,0);return result;} +async function fiveHouseholds(h,save=false){ + const households=[]; + for(let n=0;n<5;n++){ + const a=household(h);households.push(a); + assert.equal((await a.request()).status,200,'first List for household '+(n+1)); + if(save&&n===0){ + assert.equal((await a.request('POST','/v1/dir/cells/saved',{head:'retained-head',class:'work',size:7,keys:{}})).status,200); + assert.equal((await a.request('POST','/v1/dir/cells/saved/release',{fence:1})).status,204); + } + await idle(a); + } + return households; +} + +test('r4.1 five sequential sleeping households fit in one production isolate',{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true}); + try{await h.start();await fiveHouseholds(h);const m=await h.control({op:'metrics'});assert(m.memory+m.reservedBytes<=m.coreBudget);} + finally{await h.close();} +}); + +test('r4.2 the earliest released household recovers saved state through a fresh majority',{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true}); + try{ + await h.start();const [a]=await fiveHouseholds(h,true),before=await a.control('retention-state'); + assert(Object.values(before).every(s=>!s.active),'sleeping members were retained'); + const cell=await a.request('GET','/v1/dir/cells/saved');assert.equal(cell.status,200);assert.equal(cell.body.cell.head,'retained-head');assert.equal(cell.body.cell.lease.expires,0); + const after=await a.control('retention-state');console.log('durable wake',JSON.stringify({before,after})); + assert(Object.values(after).filter(s=>s.active&&BigInt(s.term)>BigInt(before['1'].term)).length>=2,'read did not establish a fresh majority'); + for(const state of Object.values(after))assert(state.recoveries.length>=2&&state.recoveries.every(role=>role===0),'recovery assumed leadership'); + await idle(a);await a.control('evict-minority');assert.equal((await a.request()).status,503,'a remembered leader answered without a majority'); + }finally{await h.close();} +}); + +test('r4.3 incomplete sleep retains busy members and full admission retries after sleep',{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true}),busy=[]; + try{ + await h.start();const sleeping=household(h);assert.equal((await sleeping.request()).status,200);await idle(sleeping); + let next,refusal; + for(let n=0;n<5;n++){ + const a=household(h),reply=await a.request(); + if(reply.status!==200){next=a;refusal=reply;break;} + const states=await a.control('retention-state','remember');a.busyNodes=Object.keys(states).filter(node=>states[node].active);assert(a.busyNodes.length>=2); + busy.push(a);await a.control('pause-public'); + } + assert(next,'busy activation exceeded the admission budget');assert(busy.length>0); + const before=await h.control({op:'metrics'}); + assert.equal(before.coreBudget,64*1024*1024);assert.equal(before.reservedBytes,before.active*6*1024*1024); + assert(before.memory+before.reservedBytes+6*1024*1024>before.coreBudget); + assert.equal(refusal.status,503);assert.deepEqual(refusal.body,{err:'unreachable',retry_after:1});assert.equal(refusal.headers.get('retry-after'),'1'); + const waking=await sleeping.request();assert.equal(waking.status,503,'released household wake must retry admission');assert.deepEqual(waking.body,refusal.body); + const after=await h.control({op:'metrics'});assert.equal(after.active,before.active);assert.equal(after.reservedBytes,before.reservedBytes); + for(const a of busy){const states=await a.control('retention-state');assert(a.busyNodes.every(node=>states[node].active),'unfinished sleep was evicted');} + await busy[0].control('resume-public');await idle(busy[0]); + for(const a of busy.slice(1)){const states=await a.control('retention-state');assert(a.busyNodes.every(node=>states[node].active),'another household was evicted');} + const released=await h.control({op:'metrics'}),retry=await next.request(); + const compact=m=>({active:m.active,reservedBytes:m.reservedBytes,memory:m.memory}); + console.log('busy admission',JSON.stringify({before:compact(before),released:compact(released),retry:retry.status,final:compact(await h.control({op:'metrics'}))})); + assert.equal(retry.status,200,'released reservations did not admit the retry'); + assert(busy.length>=2);await busy[1].control('resume-public');await idle(busy[1]); + assert.equal((await sleeping.request()).status,200,'refused wake did not recover after capacity became available'); + }finally{for(const a of busy)await a.control('resume-public').catch(()=>{});await h.close();} +}); + +test('r4.4 released members stay silent, fence stale references and wake on signed input',{timeout:60000},async()=>{ + const h=await hostedFixture({fixture:true}),a=household(h); + try{ + await h.start();assert.equal((await a.request()).status,200);await a.control('retention-state','remember');await idle(a); + const stale=await a.control('retention-state','stale'); + for(const state of Object.values(stale)){assert.deepEqual(state.refused,[true,true,true]);assert.equal(state.handle,0);assert.equal(state.stopped,true);} + const before=await a.control('quiet-state');assert.equal(before.alarm,null);assert.equal(before.maintenance,false);assert.deepEqual(before.memberAlarms,{'1':null,'2':null,'3':null}); + await a.control('evict-idle');await h.control({op:'reset-metrics'}); + for(let n=0;n<24;n++)await a.control('presence-turn'); + const metrics=await h.control({op:'metrics'}),quiet=await a.control('quiet-state'); + for(const key of ['turns','http','peerIn','peerOut','writes','active','reservedBytes'])assert.equal(metrics[key],0,key); + assert.equal(quiet.generation,before.generation);assert.equal(quiet.alarm,null);assert.deepEqual(quiet.memberAlarms,before.memberAlarms); + assert.equal((await a.request()).status,200);assert.equal((await a.control('quiet-state')).maintenance,true); + }finally{await h.close();} +}); diff --git a/relay/hosted/test/raft/hosted-worker.js b/relay/hosted/test/raft/hosted-worker.js index 0c0b50cf8c..ce6ca32bc4 100644 --- a/relay/hosted/test/raft/hosted-worker.js +++ b/relay/hosted/test/raft/hosted-worker.js @@ -10,6 +10,7 @@ import {encodeCanonical} from '../../src/raft/codec.js'; import {CORE_MEMORY_BUDGET} from '../../src/raft/abi.js'; import { runtimeMetrics,sharedRuntime } from '../../src/raft/actor.js'; import {coordinationPolicy} from '../../src/raft/liveness.js'; +const recoveryHistory=new Map(); const measurements={turns:0,wall:[],membership:0,campaigns:0,http:0,peerIn:0,peerOut:0,sweeps:0,writes:0}; async function measured(fn){const start=performance.now();measurements.turns++;try{return await fn();}finally{measurements.wall.push(performance.now()-start);}} import { policyOf } from '../../src/rules.js'; @@ -32,6 +33,11 @@ export class IdentityDO extends Source { this.current.host=null;this.host=null;this.capturing=false; this.rebuilt=new Source(this.ctx,this.env);return this.quietState(); } + async retentionState(op){ + const config=await this.ctx.storage.get('raft-provision'),states={}; + for(const node of config.config.anchors)states[node]=await this.env.RAFT_MEMBERS.get(this.env.RAFT_MEMBERS.idFromName(config.config.cluster+'/'+node)).retentionState(op); + return states; + } async quietState(){ const config=await this.ctx.storage.get('raft-provision'),alarms={}; if(config)for(const node of config.config.anchors)alarms[node]=await this.env.RAFT_MEMBERS.get(this.env.RAFT_MEMBERS.idFromName(config.config.cluster+'/'+node)).alarmState(); @@ -164,7 +170,18 @@ export class IdentityDO extends Source { } } export class RaftMemberDO extends Member { - constructor(ctx,env){Date.now();if(globalThis.fixtureClockOffset){const setAlarm=ctx.storage.setAlarm.bind(ctx.storage);ctx.storage.setAlarm=at=>setAlarm(Number(at)-globalThis.fixtureClockOffset);}super(ctx,env);} + constructor(ctx,env){ + Date.now();if(globalThis.fixtureClockOffset){const setAlarm=ctx.storage.setAlarm.bind(ctx.storage);ctx.storage.setAlarm=at=>setAlarm(Number(at)-globalThis.fixtureClockOffset);}super(ctx,env); + // Recovery observations survive object replacement and sample publication + // before a queued input can change the new follower's role. + const id=ctx.id.toString(),history=recoveryHistory.get(id)??{roles:[]};recoveryHistory.set(id,history);this.recoveryHistory=history;this.recoveryRoles=history.roles; + let actor=this.actor; + Object.defineProperty(this,'actor',{get:()=>actor,set:next=>{ + if(next&&next!==actor)this.recoveryRoles.push(next.status.role); + if(!next&&actor)history.released={applied:String(actor.status.applied),term:String(actor.status.term)}; + actor=next; + }}); + } async staging(){await this.load();return this.actor.serial(()=>({rows:this.ctx.storage.sql.exec("SELECT count(*) AS n, coalesce(sum(length(data)),0) AS bytes FROM records WHERE namespace='outbox' AND key LIKE 'v1-stage/%'").toArray()[0],app:this.actor.store.application()}));} async replaceActivation(){ await this.load();const old=this.actor,before=old.store.application().applied; @@ -173,6 +190,18 @@ export class RaftMemberDO extends Member { for(const key of ['actor','config','registry','results','resultBytes','recoveredEffects'])this[key]=restored[key]; return {before,after:this.actor.store.application().applied,refused,oldHandle:old.handle,active:runtimeMetrics().active}; } + async retentionState(op){ + if(op==='remember')this.remembered=this.actor; + const actor=this.actor; + if(op==='stale'){ + const old=this.remembered,refused=[]; + for(const call of [()=>old.input(0),()=>old.sleepAck(old.status.applied,old.status.term),()=>old.joinStatus(old.status.applied)]){ + try{await call();refused.push(false);}catch{refused.push(true);} + } + return {refused,handle:old.handle,stopped:old.stopped}; + } + return {active:!!actor?.handle,stopped:actor?.stopped??false,applied:String(actor?.status?.applied??this.recoveryHistory.released?.applied??0),term:String(actor?.status?.term??this.recoveryHistory.released?.term??0),recoveries:this.recoveryRoles??[]}; + } alarmState(){return this.ctx.storage.getAlarm();} alarm(...args){return measured(()=>super.alarm(...args));} importReceipt(...args){return measured(()=>super.importReceipt(...args));} @@ -218,6 +247,7 @@ export default { if(op==='diagnostic'){console.warn('local runtime warning capture');console.error('local runtime error capture');return Response.json(true);} if(op==='evict-minority')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).evictMinority()); if(op==='presence-schedule')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).presenceSchedule()); + if(op==='retention-state')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).retentionState(records)); if(op==='quiet-state')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).quietState()); if(op==='evict-idle')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).evictIdle()); if(op==='presence-turn')return Response.json(await env.IDENTITY.get(env.IDENTITY.idFromName(identity)).presenceTurn()??true); From 4a50809f835e9014358916018f20ed473e1a2110 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 22:42:22 -0400 Subject: [PATCH 091/103] Prevent older relays from changing upgraded directories. Freeze the retained tables and install durable write guards in one transaction so older relay code cannot acknowledge divergent changes. Keep the frozen source readable for import and recovery, and document the older relay refusal. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/changes/unreleased/raft-cluster.md | 1 + docs/relay.md | 8 +- internal/directory/export.go | 21 ++- internal/directory/legacy_guard_test.go | 146 ++++++++++++++++++ internal/directory/sqlite.go | 13 +- .../manual/chat/coordination-and-workers.md | 9 ++ internal/manual/cluster_truth_test.go | 1 + relay/hosted/README.md | 7 + relay/hosted/src/directory.js | 3 +- relay/hosted/src/raft/legacy.js | 11 +- relay/hosted/test/raft/legacy-guard-worker.js | 39 +++++ relay/hosted/test/raft/legacy-guard.test.js | 84 ++++++++++ 12 files changed, 336 insertions(+), 7 deletions(-) create mode 100644 internal/directory/legacy_guard_test.go create mode 100644 relay/hosted/test/raft/legacy-guard-worker.js create mode 100644 relay/hosted/test/raft/legacy-guard.test.js diff --git a/docs/changes/unreleased/raft-cluster.md b/docs/changes/unreleased/raft-cluster.md index 6bf368842f..9c7eb172e8 100644 --- a/docs/changes/unreleased/raft-cluster.md +++ b/docs/changes/unreleased/raft-cluster.md @@ -18,6 +18,7 @@ invalidates: - "Reopening retained work could compare its branch fence with the original chat and falsely refuse tools. Ownership checks stay with the record each window opened, and replacing a displaced drive settles its previous writer before starting another. A failed final save remains failed for every close caller." - "Deferred local durability could resolve a fallback daemon against a different current home. Both engine routes now retain the home and engine selected at capture." - "A single directory database was formerly the live source. First use now freezes and imports it into the three anchors while retaining the source, and the one-way upgrade cannot return to that writer after cluster changes are acknowledged." + - "The migration gate alone did not stop older relay code from writing the retained source. Both relays now install durable database write guards with the freeze, so an older relay refuses directory startup or writes with codeaf: directory upgraded; use a current relay build. Running an older relay after the upgrade is not supported." - "Self-hosted identities were described as limited only by disk space. Directory content now has an 8 MiB default allowance, with larger imports retaining their measured allowance and bounded bookkeeping having separate room; capacity refuses content growth without deleting saved data." --- diff --git a/docs/relay.md b/docs/relay.md index 8c744237a1..1cefbf5081 100644 --- a/docs/relay.md +++ b/docs/relay.md @@ -105,9 +105,11 @@ imports its rows into three anchors, checks a fresh majority and publishes one o An interrupted import resumes the same cluster, keys and capacity allowance; the source is retained and a damaged import refuses requests instead of truncating records. -This upgrade is one-way. After the cluster acknowledges changes, an older relay binary -cannot resume the frozen writer. A code rollback must preserve the migration gate and -consensus routing. Encrypted chunks stay in their existing store. +This upgrade is one-way. Once the source is frozen, an older relay started against +that store refuses to change it. Its directory requests return HTTP 500 `internal`; +the underlying storage error is `codeaf: directory upgraded; use a current relay build`. +Running an older relay after the upgrade is not supported. Restart a current relay +with the complete store to resume. Encrypted chunks stay in their existing store. Older clients keep pairing, listing, saving, publishing and releasing. Blindly forcing a live chat without its observed holder and ownership number is refused with diff --git a/internal/directory/export.go b/internal/directory/export.go index 62c92beae2..2f2be2ca50 100644 --- a/internal/directory/export.go +++ b/internal/directory/export.go @@ -13,6 +13,8 @@ import ( "github.com/Agent-Field/codeaf/internal/clusterstate" ) +const legacyWriteRefusal = "codeaf: directory upgraded; use a current relay build" + // LegacyGate is irrevocable. A failed inventory keeps the source frozen, and // ownership publication never makes the backup writable again. type LegacyGate struct { @@ -49,6 +51,20 @@ func keepGate(tx *sql.Tx, g LegacyGate) error { _, e = tx.Exec(`INSERT OR REPLACE INTO legacy_gate VALUES(1,?)`, string(b)) return e } + +// Database guards also stop relay builds that do not know about the gate. +// They share its transaction so a failed freeze leaves the old writer usable. +func freezeLegacyWrites(tx *sql.Tx) error { + for _, table := range []string{"identity", "devices", "cells", "version"} { + for _, operation := range []string{"INSERT", "UPDATE", "DELETE"} { + if _, err := tx.Exec(`CREATE TRIGGER IF NOT EXISTS legacy_frozen_` + table + `_` + operation + ` BEFORE ` + operation + ` ON ` + table + ` BEGIN SELECT RAISE(ABORT, '` + legacyWriteRefusal + `'); END`); err != nil { + return err + } + } + } + return nil +} + func (s *SQLite) migrationTx(ctx context.Context, fn func(*sql.Tx) error) error { tx, e := s.db.BeginTx(ctx, nil) if e != nil { @@ -100,7 +116,7 @@ func (s *SQLite) freezeLegacy(ctx context.Context, c clusterstate.LegacyConfig) if !bytes.Equal(a, b) { return clusterstate.ErrMigration } - return nil + return freezeLegacyWrites(tx) } // The source rows are retained byte-for-byte. Only sampled renewal evidence // enters this separate migration table and the future committed genesis. @@ -135,6 +151,9 @@ func (s *SQLite) freezeLegacy(ctx context.Context, c clusterstate.LegacyConfig) if re != nil { return re } + if e := freezeLegacyWrites(tx); e != nil { + return e + } if e := keepGate(tx, LegacyGate{Stage: "frozen", Config: c}); e != nil { return e } diff --git a/internal/directory/legacy_guard_test.go b/internal/directory/legacy_guard_test.go new file mode 100644 index 0000000000..2a7cbee356 --- /dev/null +++ b/internal/directory/legacy_guard_test.go @@ -0,0 +1,146 @@ +package directory + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +func retainedRows(t *testing.T, db *sql.DB) []string { + t.Helper() + var result []string + for _, table := range []string{"identity", "devices", "cells", "version"} { + column := "hex(rec)" + if table == "version" { + column = "quote(n)" + } + rows, err := db.Query(`SELECT quote(id),` + column + ` FROM ` + table + ` ORDER BY id`) + legacyMust(t, err) + for rows.Next() { + var id, value string + legacyMust(t, rows.Scan(&id, &value)) + result = append(result, table+":"+id+":"+value) + } + legacyMust(t, rows.Err()) + legacyMust(t, rows.Close()) + } + return result +} + +func TestR6_1FormerSQLCannotChangeFrozenSource(t *testing.T) { + path := filepath.Join(t.TempDir(), "source.db") + s, _ := legacySeed(t, path) + defer s.Close() + old, err := sql.Open("sqlite", path) + legacyMust(t, err) + defer old.Close() + original := retainedRows(t, old) + legacyMust(t, s.FreezeLegacy(context.Background(), legacyConfig())) + statements := []struct { + query string + args []any + }{ + {`INSERT OR IGNORE INTO identity (id, rec) VALUES (1, ?)`, []any{`{"V":1}`}}, + {`INSERT OR IGNORE INTO version (id, n) VALUES (1, 0)`, nil}, + {`UPDATE version SET n = n + 1 WHERE id = 1`, nil}, + } + for _, row := range []struct{ table, id string }{{"identity", "1"}, {"devices", "b"}, {"cells", "chat"}, {"devices", "new"}, {"cells", "new"}} { + statements = append(statements, struct { + query string + args []any + }{`INSERT INTO ` + row.table + ` (id, rec) VALUES (?, ?) ON CONFLICT(id) DO UPDATE SET rec = excluded.rec`, []any{row.id, `{"V":1}`}}) + } + // The former writer has no DELETE statement, but all retained SQL operations + // must refuse changes, including writes through an already open connection. + for _, table := range []string{"identity", "devices", "cells", "version"} { + column := "rec = '{}'" + if table == "version" { + column = "n = 99" + } + statements = append(statements, struct { + query string + args []any + }{`UPDATE ` + table + ` SET ` + column, nil}, struct { + query string + args []any + }{`DELETE FROM ` + table, nil}) + } + for i, statement := range statements { + t.Run(fmt.Sprint(i), func(t *testing.T) { + _, err := old.Exec(statement.query, statement.args...) + if err == nil || !strings.Contains(err.Error(), "directory upgraded") { + t.Errorf("former write must report the upgrade: %s: %v", statement.query, err) + } + if got := retainedRows(t, old); !reflect.DeepEqual(got, original) { + t.Errorf("retained source changed after %s", statement.query) + } + }) + } +} + +func TestR6_3SourceCrashCommitsGuardWithFreeze(t *testing.T) { + for _, phase := range []string{"after-freeze-gate", "before-commit", "after-commit"} { + t.Run(phase, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "source.db") + s, _ := legacySeed(t, path) + legacyMust(t, s.Close()) + cmd := exec.Command(os.Args[0], "-test.run=^TestR24SourceCrashHelper$", "-test.count=1") + cmd.Env = append(os.Environ(), "P11_SOURCE="+path, "P11_STAGE=freeze", "P11_PHASE="+phase) + out, err := cmd.CombinedOutput() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 81 { + t.Fatalf("cut: %v %s", err, out) + } + db, err := sql.Open("sqlite", path) + legacyMust(t, err) + defer db.Close() + var gates, guards int + legacyMust(t, db.QueryRow(`SELECT count(*) FROM legacy_gate`).Scan(&gates)) + legacyMust(t, db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='trigger' AND tbl_name IN ('identity','devices','cells','version')`).Scan(&guards)) + _, err = db.Exec(`UPDATE version SET n = n + 1 WHERE id = 1`) + if phase == "after-commit" { + if gates != 1 || guards != 12 || err == nil { + t.Fatalf("committed freeze lost guards: gates=%d guards=%d write=%v", gates, guards, err) + } + } else if gates != 0 || guards != 0 || err != nil { + t.Fatalf("uncommitted freeze blocked former writer: gates=%d guards=%d write=%v", gates, guards, err) + } + }) + } +} + +func TestR6_4FrozenSourceReopensForInventoryAndResume(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "source.db") + s, _ := legacySeed(t, path) + original := retainedRows(t, s.db) + legacyMust(t, s.FreezeLegacy(ctx, legacyConfig())) + if _, err := s.db.Exec(`UPDATE version SET n = n + 1 WHERE id = 1`); err == nil { + t.Error("frozen source permits a second writer") + } + legacyMust(t, s.Close()) + s, err := OpenSQLite(path, time.Now) + legacyMust(t, err) + defer s.Close() + legacyMust(t, s.FreezeLegacy(ctx, legacyConfig())) + manifest, err := s.InventoryLegacy(ctx) + legacyMust(t, err) + _, err = s.ExportLegacy(ctx, func(_ clusterstate.RecordWrite) error { return nil }) + legacyMust(t, err) + legacyMust(t, s.OwnLegacy(ctx, []LegacyReceipt{{Node: 1, Manifest: manifest, Applied: 2}, {Node: 2, Manifest: manifest, Applied: 2}})) + again, err := s.InventoryLegacy(ctx) + legacyMust(t, err) + if !reflect.DeepEqual(manifest, again) || !reflect.DeepEqual(original, retainedRows(t, s.db)) { + t.Fatal("resume changed the retained source or inventory") + } +} diff --git a/internal/directory/sqlite.go b/internal/directory/sqlite.go index 9311f87ada..0085f1a74a 100644 --- a/internal/directory/sqlite.go +++ b/internal/directory/sqlite.go @@ -94,12 +94,23 @@ func (s *SQLite) init() error { // seed creates what is missing in tx and answers the Status the file holds. func seed(tx *sql.Tx) (Status, error) { + if _, err := tx.Exec(schema); err != nil { + return Status{}, err + } + g, err := gateIn(tx) + if err != nil { + return Status{}, err + } + // A frozen source remains readable for import and recovery without issuing + // the seed writes that its durable guards refuse from older relay builds. + if g != nil { + return statusIn(tx) + } start, _ := json.Marshal(IdentityRec{V: 1}) for _, q := range []struct { sql string args []any }{ - {schema, nil}, {`INSERT OR IGNORE INTO identity (id, rec) VALUES (1, ?)`, []any{string(start)}}, {`INSERT OR IGNORE INTO version (id, n) VALUES (1, 0)`, nil}, } { diff --git a/internal/manual/chat/coordination-and-workers.md b/internal/manual/chat/coordination-and-workers.md index 5c9b20dfa1..06a6119b2a 100644 --- a/internal/manual/chat/coordination-and-workers.md +++ b/internal/manual/chat/coordination-and-workers.md @@ -125,6 +125,15 @@ frame hints, sealed names and key wraps are preserved; over 1 MiB is refused. Th identity’s storage capacity still applies, and existing records are never removed to make room. +## Older relay binaries after a directory upgrade + +Running an older relay after the upgrade is not supported. An older relay started +against an upgraded store cannot change its frozen directory. Its directory requests +return HTTP 500 `internal`; the storage error is +`codeaf: directory upgraded; use a current relay build`. Restart a current relay with +the same complete store to resume. The hosted relay also refuses writes from older +server code. + ## Hosted relay directory capacity, upgrades and older clients The hosted relay records directory changes on three persistent members. Two must diff --git a/internal/manual/cluster_truth_test.go b/internal/manual/cluster_truth_test.go index 2b61837c95..1384af5440 100644 --- a/internal/manual/cluster_truth_test.go +++ b/internal/manual/cluster_truth_test.go @@ -57,6 +57,7 @@ func TestR30CoordinationQuotesMatchProductStrings(t *testing.T) { {"../syncsetup/cluster.go", "LeadCluster", "this computer must join"}, {"../syncsetup/member.go", "errMemberWaiting", "this computer is still joining"}, {"../syncsetup/worker.go", "ErrWorkerIncomplete", "this work may be incomplete"}, + {"../directory/export.go", "legacyWriteRefusal", "codeaf: directory upgraded"}, } { found := false for _, text := range clusterDeclarationStrings(t, source.path, source.declaration) { diff --git a/relay/hosted/README.md b/relay/hosted/README.md index c4938215f0..18bcd81b9a 100644 --- a/relay/hosted/README.md +++ b/relay/hosted/README.md @@ -169,6 +169,13 @@ namespace. A deployment's own configuration must carry the `RAFT_MEMBERS` bindin `wrangler.toml` declares; the first authenticated use of an existing identity then upgrades it in place, and that upgrade cannot be rolled back to the single-object writer. +The frozen source refuses writes even from older Worker code. Its directory +requests return HTTP 500 `internal`, with the storage error +`codeaf: directory upgraded; use a current relay build` in the Worker log. +Running an older relay after the upgrade is not supported. Restore current code +with the same complete storage to resume; the original directory and encrypted +frames remain retained. + ## Base path A deployment can share a hostname by using a path prefix, such as diff --git a/relay/hosted/src/directory.js b/relay/hosted/src/directory.js index 5f1182f378..c9048618ce 100644 --- a/relay/hosted/src/directory.js +++ b/relay/hosted/src/directory.js @@ -28,7 +28,8 @@ export class Directory { sql.exec('CREATE TABLE IF NOT EXISTS legacy_gate (one INTEGER PRIMARY KEY CHECK(one=1), doc TEXT NOT NULL)'); sql.exec('CREATE TABLE IF NOT EXISTS dir (kind TEXT NOT NULL, id TEXT NOT NULL, doc TEXT NOT NULL, PRIMARY KEY (kind, id)) WITHOUT ROWID'); sql.exec('CREATE TABLE IF NOT EXISTS dirver (one INTEGER PRIMARY KEY CHECK (one = 1), n INTEGER NOT NULL)'); - sql.exec('INSERT OR IGNORE INTO dirver VALUES (1, 0)'); + // Frozen sources reopen for inventory without attempting a guarded seed. + if(!sql.exec('SELECT doc FROM legacy_gate WHERE one=1').toArray().length) sql.exec('INSERT OR IGNORE INTO dirver VALUES (1, 0)'); this.version = sql.exec('SELECT n FROM dirver').one().n; } diff --git a/relay/hosted/src/raft/legacy.js b/relay/hosted/src/raft/legacy.js index b436aaa9be..6e564f8a64 100644 --- a/relay/hosted/src/raft/legacy.js +++ b/relay/hosted/src/raft/legacy.js @@ -11,6 +11,14 @@ const xor = (a,b) => { for(let i=0;i<32;i++) a[i]^=b[i]; }; const importKey = 'legacy-import'; const reserve = 4096; +// Guards persist with the gate, including across a rollback to older code that +// writes these tables without checking whether the directory was upgraded. +function freezeWrites(sql) { + for(const table of ['dir','dirver'])for(const operation of ['INSERT','UPDATE','DELETE']) { + sql.exec(`CREATE TRIGGER IF NOT EXISTS legacy_frozen_${table}_${operation} BEFORE ${operation} ON ${table} BEGIN SELECT RAISE(ABORT, 'codeaf: directory upgraded; use a current relay build'); END`); + } +} + // Original JSON bytes go to the shared Go conversion function, including every // numeric fence and sealed field. JavaScript never evaluates a lease rule. export function convertLegacy(runtime, config, kind, id, raw, until=0) { @@ -53,7 +61,8 @@ export class LegacySource { // Go validates the binding before the source acquires an irreversible gate. convertLegacy(this.runtime,config,'identity','',JSON.stringify({V:1,identity:config.identity})); this.storage.transactionSync(()=>{ - const old=this.gate();if(old){check(encodeCanonical(old.config)===encodeCanonical(config));return;} + const old=this.gate();if(old)check(encodeCanonical(old.config)===encodeCanonical(config)); + freezeWrites(this.sql);if(old)return; for(const {id,doc} of this.sql.exec("SELECT id,doc FROM dir WHERE kind='cells' ORDER BY id")) { let c;try{c=JSON.parse(doc);}catch{continue;} if(!c.lease)continue; diff --git a/relay/hosted/test/raft/legacy-guard-worker.js b/relay/hosted/test/raft/legacy-guard-worker.js new file mode 100644 index 0000000000..554a174adc --- /dev/null +++ b/relay/hosted/test/raft/legacy-guard-worker.js @@ -0,0 +1,39 @@ +import { ActorFixture as MigrationFixture } from './legacy-worker.js'; +import { Directory } from '../../src/directory.js'; +import { LegacySource } from '../../src/raft/legacy.js'; +import { sharedRuntime } from '../../src/raft/actor.js'; +import { policyOf } from '../../src/rules.js'; +export { default } from './legacy-worker.js'; + +export class ActorFixture extends MigrationFixture { + async control(request) { + const value = await request.clone().json(); + if (value.op === 'sql') { + try { + return Response.json({ok: this.ctx.storage.sql.exec(value.query, ...(value.args ?? [])).toArray()}); + } catch (error) { + return Response.json({error: error.message}); + } + } + if (value.op === 'rollback-freeze') { + const storage = this.ctx.storage; + const directory = new Directory(storage.sql, value.config.identity, () => value.config.now, policyOf()); + const source = new LegacySource({ + sql: storage.sql, + get: key => storage.get(key), + sync: () => storage.sync(), + transactionSync: fn => storage.transactionSync(() => { + fn(); + throw new Error('interrupted before commit'); + }), + }, directory, sharedRuntime()); + try { + await source.freeze(value.config); + return Response.json({error: 'freeze committed unexpectedly'}); + } catch (error) { + return Response.json({ok: error.message}); + } + } + return super.control(request); + } +} diff --git a/relay/hosted/test/raft/legacy-guard.test.js b/relay/hosted/test/raft/legacy-guard.test.js new file mode 100644 index 0000000000..377901d46f --- /dev/null +++ b/relay/hosted/test/raft/legacy-guard.test.js @@ -0,0 +1,84 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { build } from 'esbuild'; +import { copyFile, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createHash, generateKeyPairSync } from 'node:crypto'; +import { openWorkerd } from './workerd.js'; + +test('r6.2 r6.3 r6.4 retained hosted tables refuse former SQL atomically and remain readable', {timeout: 30000}, async t => { + const out = await mkdtemp(join(tmpdir(), 'retained-worker-')); + const persist = await mkdtemp(join(tmpdir(), 'retained-disk-')); + const proof = process.env.CODEAF_RAFT_PROOF_DIR; + const bundled = await build({entryPoints: ['test/raft/legacy-guard-worker.js'], bundle: true, format: 'esm', target: 'es2022', external: ['core.wasm', 'cloudflare:*'], write: false}); + await writeFile(join(out, 'worker.js'), bundled.outputFiles[0].contents); + await copyFile(join(proof, 'core.wasm'), join(out, 'core.wasm')); + const pub = () => generateKeyPairSync('ed25519').publicKey.export({format: 'der', type: 'spki'}).subarray(-32); + const root = pub(); + const config = {cluster: 'retained-source', generation: '1', identity: 'id_' + createHash('sha256').update(root).digest('hex').slice(0, 32), signing_key: root.toString('base64'), effects_key: pub().toString('base64'), anchors: ['1', '2', '3'], now: 1000}; + const records = [ + {kind: 'identity', id: '', doc: {V: 1, identity: config.identity, vault: 'sealed-vault'}}, + {kind: 'devices', id: 'a', doc: {V: 1, name: 'c2VhbGVk', caps: {os: 'linux'}}}, + {kind: 'cells', id: 'chat', doc: {V: 1, head: 'saved', class: 'work', size: 7, keys: {}, lease: {device: 'a', fence: 1, expires: 0, pending: 0}, frames: ['sealed-frame']}}, + ]; + let server; + const restart = async () => {if (server) await server.dispose(); server = await openWorkerd(out, persist, true);}; + const call = async (op, extra = {}) => { + const response = await server.dispatchFetch(server.origin + '/source', {method: 'POST', body: JSON.stringify({op, config, ...extra}), signal: AbortSignal.timeout(5000)}); + assert.equal(response.status, 200); + return response.json(); + }; + const ok = async (op, extra) => {const result = await call(op, extra); assert('ok' in result, JSON.stringify(result)); return result.ok;}; + const sql = (query, args) => ok('sql', {query, args}); + const original = async () => ({rows: await sql('SELECT kind,id,hex(doc) AS bytes FROM dir ORDER BY kind,id'), version: await sql('SELECT * FROM dirver')}); + try { + await restart(); + await ok('seed', {records}); + await t.test('r6.3 rollback leaves all former statements usable and no guards', async () => { + assert.equal(await ok('rollback-freeze'), 'interrupted before commit'); + assert.equal(await ok('gate'), null); + assert.deepEqual(await sql("SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name IN ('dir','dirver')"), []); + await sql('INSERT OR IGNORE INTO dirver VALUES (1, 0)'); + await sql('INSERT OR REPLACE INTO dir VALUES (?,?,?)', ['devices', 'a', JSON.stringify(records[1].doc)]); + await sql('UPDATE dirver SET n = ?', [4]); + }); + const before = await original(); + await ok('freeze'); + await restart(); + for (const [query, args] of [ + ['INSERT OR IGNORE INTO dirver VALUES (1, 0)'], + ...records.map(record => ['INSERT OR REPLACE INTO dir VALUES (?,?,?)', [record.kind, record.id, JSON.stringify({...record.doc, vault: 'changed'})]]), + ['INSERT OR REPLACE INTO dir VALUES (?,?,?)', ['cells', 'new', '{}']], + ['UPDATE dirver SET n = ?', [99]], + ["UPDATE dir SET doc = '{}'"], + ['DELETE FROM dir'], + ['DELETE FROM dirver'], + ]) { + await t.test('r6.2 ' + query + ' ' + (args?.[0] ?? ''), async () => { + const result = await call('sql', {query, args}); + assert.match(result.error ?? '', /directory upgraded/); + assert.deepEqual(await original(), before); + }); + } + await t.test('r6.3 committed freeze retains guards after process death', async () => { + assert.equal((await ok('gate')).stage, 'frozen'); + assert.equal((await sql("SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name IN ('dir','dirver')")).length, 6); + }); + await t.test('r6.4 guarded source inventories and resumes without changing retained rows', async () => { + assert('error' in await call('sql', {query: 'UPDATE dirver SET n = ?', args: [99]})); + const manifest = await ok('inventory'); + assert((await ok('rows')).length > 0); + await restart(); + assert.deepEqual(await ok('inventory'), manifest); + await ok('own', {receipts: [{node: '1', applied: '2', manifest}, {node: '2', applied: '2', manifest}]}); + await ok('freeze'); + assert.equal((await ok('gate')).stage, 'raft-owned'); + assert.deepEqual(await original(), before); + }); + } finally { + if (server) await server.dispose(); + await rm(out, {recursive: true, force: true}); + await rm(persist, {recursive: true, force: true}); + } +}); From c3836d221b08742b3c24d0e3e027d432bb403704 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 23:21:39 -0400 Subject: [PATCH 092/103] Keep signed relay paths and empty frame plans usable. Hosted requests retain the configured prefix with the original signed URI so every member interprets the same operation when a joined computer leads. Directory changes, enrollment and large rotation requests use that exact prefix. Native creates and publishes treat an empty frame array like an omitted or null plan while retaining the original actor and operation binding. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/changes/unreleased/raft-cluster.md | 2 + docs/relay.md | 6 + internal/clusterstate/personal.go | 4 +- internal/clusterstate/v1.go | 52 ++++-- internal/clusterstate/v1_prefix_test.go | 150 ++++++++++++++++++ internal/clusterstate/v1_stage.go | 6 +- .../manual/chat/coordination-and-workers.md | 8 + internal/raftdirectory/v1.go | 3 +- internal/raftdirectory/v1_frames_test.go | 58 +++++++ relay/hosted/README.md | 5 +- relay/hosted/src/raft/hosted.js | 4 +- .../hosted/test/raft/v1-prefix-parity.test.js | 79 +++++++++ relay/hosted/test/raft/v1-prefix.test.js | 82 ++++++++++ 13 files changed, 438 insertions(+), 21 deletions(-) create mode 100644 internal/clusterstate/v1_prefix_test.go create mode 100644 internal/raftdirectory/v1_frames_test.go create mode 100644 relay/hosted/test/raft/v1-prefix-parity.test.js create mode 100644 relay/hosted/test/raft/v1-prefix.test.js diff --git a/docs/changes/unreleased/raft-cluster.md b/docs/changes/unreleased/raft-cluster.md index 9c7eb172e8..a85f278bc5 100644 --- a/docs/changes/unreleased/raft-cluster.md +++ b/docs/changes/unreleased/raft-cluster.md @@ -13,6 +13,8 @@ invalidates: - "Sleeping hosted groups previously retained their memory reservations indefinitely. Completed all-voter sleep now checkpoints and releases those members, and the next request recovers them through a fresh majority; busy admission pressure returns retryable HTTP 503." - "Older clients could blindly force a live chat without its observed holder. Both relays now refuse that request with another device continued this chat first; releasing the chat or updating the client permits safe continuation." - "Legal older directory requests could be refused above the smaller coordination envelope. Both relays now retain the original 1 MiB HTTP limit, stage large signed bodies and commit complete metadata atomically, recovering interruptions without exposing partial changes or deleting retained records." + - "Hosted directory translation previously recognized only the default path prefix. Configured prefixes, including multiple segments, now remain with the signed request so directory operations and computer enrollment work under personal leadership too." + - "The native relay could reject a signed create or publish with an explicit empty frame plan. Empty arrays now have the same meaning as omitted or null frame plans while the request remains bound to its original actor and operation." - "Retained rows from legal older requests could pause upgrades after the relay added fields, and full directories could fail initialization or refusal persistence. Both relays now preserve these rows losslessly and reserve bounded bookkeeping independently of the measured content allowance." - "Workers previously needed personal-device trust for shared access. Delegated workers now have expiring selected-chat grants and conditional assignment claims; they cannot approve pairing, read the identity vault, rotate it, enroll devices or lead, and terminal worker setup is not available yet." - "Reopening retained work could compare its branch fence with the original chat and falsely refuse tools. Ownership checks stay with the record each window opened, and replacing a displaced drive settles its previous writer before starting another. A failed final save remains failed for every close caller." diff --git a/docs/relay.md b/docs/relay.md index 1cefbf5081..772908b623 100644 --- a/docs/relay.md +++ b/docs/relay.md @@ -158,3 +158,9 @@ content and the readable coordination metadata described at the top of this page and size of each request and the network address of each computer, and keeps no request log. The address has a path, and every client keeps it. `codeaf relay` (this page) always serves from the root of its host; behind your own proxy, pass the path through unchanged. + +A hosted relay can instead set `BASE_PATH` to any path prefix, including several +segments such as `/a/relay`. Use that complete address when pairing. Signatures cover +the prefix, and directory operations and computer enrollment keep working there +when a joined computer leads. Requests outside the configured prefix return +HTTP 404 `not_found`. diff --git a/internal/clusterstate/personal.go b/internal/clusterstate/personal.go index 30e304f2e0..5cb0897fda 100644 --- a/internal/clusterstate/personal.go +++ b/internal/clusterstate/personal.go @@ -5,7 +5,6 @@ import ( "crypto/ed25519" "encoding/hex" "strconv" - "strings" ) const PersonalAdmissionPath = "/v2/cluster/enroll" @@ -22,7 +21,8 @@ type PersonalAdmission struct { // for this certificate under the currently committed identity root. func VerifyPersonalAdmission(p V1Request, identity IdentityRec, now int64) (PersonalAdmission, Authority, string, error) { var request PersonalAdmission - if p.Method != "POST" || strings.TrimPrefix(p.URI, "/fabric") != PersonalAdmissionPath || !bytes.Equal(p.IdentityKey, identity.SigningKey) || legacyBody(p.Body, &request, false) != nil { + path, routeErr := p.route() + if routeErr != nil || p.Method != "POST" || path != PersonalAdmissionPath || !bytes.Equal(p.IdentityKey, identity.SigningKey) || legacyBody(p.Body, &request, false) != nil { return request, Authority{}, "", ErrCodec } token, err := hex.DecodeString(request.Store) diff --git a/internal/clusterstate/v1.go b/internal/clusterstate/v1.go index bddb4a697c..e15d05ef4b 100644 --- a/internal/clusterstate/v1.go +++ b/internal/clusterstate/v1.go @@ -19,6 +19,7 @@ import ( type V1Request struct { Method string `json:"method"` URI string `json:"uri"` + BasePath string `json:"base_path,omitempty"` IdentityKey []byte `json:"identity_key"` Certificate string `json:"certificate"` At string `json:"at"` @@ -83,7 +84,7 @@ func V1AuthorityAt(c Command, identity IdentityRec, committedNow int64) (Authori h := sha256.Sum256(c.V1.Body) auth, ok := v1ProofAuthority(c, identity, committedNow, hex.EncodeToString(h[:])) m, err := c.V1.Mutation() - if !ok || err != nil || !reflect.DeepEqual(m, c.Change) || m.PutDevice != nil && m.PutDevice.ID != c.Actor { + if !ok || err != nil || !V1MutationEqual(m, c.Change) || m.PutDevice != nil && m.PutDevice.ID != c.Actor { return Authority{}, false } return auth, true @@ -232,8 +233,8 @@ func legacyJSONKeys(raw []byte) error { // original bytes and precision; v2's decimal-string codec never parses them. func (p V1Request) Mutation() (Mutation, error) { bad := func() (Mutation, error) { return Mutation{}, ErrCodec } - path := strings.TrimPrefix(p.URI, "/fabric") - if strings.ContainsAny(path, "?#") || !strings.HasPrefix(path, "/v1/") { + path, err := p.route() + if err != nil || !strings.HasPrefix(path, "/v1/") { return bad() } if p.Method == "GET" && (path == "/v1/dir/list" || path == "/v1/dir/presence" || path == "/v1/identity/rotation" || strings.HasPrefix(path, "/v1/dir/cells/") && len(strings.Split(strings.TrimPrefix(path, "/v1/dir/cells/"), "/")) == 1) { @@ -320,12 +321,7 @@ func (p V1Request) Mutation() (Mutation, error) { if !decode(&v, false) { return bad() } - // Empty and absent frame plans both replace the previous hint. Normalize - // before signing so canonical omitempty survives original-proof replay. - if len(v.Frames) == 0 { - v.Frames = nil - } - return Mutation{Create: &CreateCell{Cell: id, Init: v, V1: true}}, nil + return normalizeV1Mutation(Mutation{Create: &CreateCell{Cell: id, Init: v, V1: true}}), nil case "acquire": var v struct { Force bool `json:"force,omitempty"` @@ -353,10 +349,7 @@ func (p V1Request) Mutation() (Mutation, error) { if !decode(&v, false) || v.Fence > MaxSafeInteger { return bad() } - if len(v.Frames) == 0 { - v.Frames = nil - } - return Mutation{Publish: &PublishCell{Cell: id, Publish: v, V1: true}}, nil + return normalizeV1Mutation(Mutation{Publish: &PublishCell{Cell: id, Publish: v, V1: true}}), nil case "release": var v struct { Quiet bool `json:"quiet,omitempty"` @@ -375,6 +368,39 @@ func (p V1Request) Mutation() (Mutation, error) { return bad() } +// The configured prefix travels with the proof so every member routes the +// original signed URI identically, without consulting its own host settings. +func (p V1Request) route() (string, error) { + if p.BasePath != "" && (!strings.HasPrefix(p.BasePath, "/") || strings.HasSuffix(p.BasePath, "/")) { + return "", ErrCodec + } + path, ok := strings.CutPrefix(p.URI, p.BasePath) + if !ok || strings.ContainsAny(p.URI, "?#") { + return "", ErrCodec + } + return path, nil +} + +// V1MutationEqual keeps route and actor binding while treating absent, null +// and empty frame plans as the same instruction to clear the previous hint. +func V1MutationEqual(a, b Mutation) bool { + return reflect.DeepEqual(normalizeV1Mutation(a), normalizeV1Mutation(b)) +} + +func normalizeV1Mutation(m Mutation) Mutation { + if m.Create != nil && m.Create.Init.Frames != nil && len(m.Create.Init.Frames) == 0 { + v := *m.Create + v.Init.Frames = nil + m.Create = &v + } + if m.Publish != nil && m.Publish.Publish.Frames != nil && len(m.Publish.Publish.Frames) == 0 { + v := *m.Publish + v.Publish.Frames = nil + m.Publish = &v + } + return m +} + func v1Segment(s string) (string, error) { var out []byte for i := 0; i < len(s); i++ { diff --git a/internal/clusterstate/v1_prefix_test.go b/internal/clusterstate/v1_prefix_test.go new file mode 100644 index 0000000000..5b4d0ff8de --- /dev/null +++ b/internal/clusterstate/v1_prefix_test.go @@ -0,0 +1,150 @@ +package clusterstate_test + +import ( + "crypto/ed25519" + "encoding/json" + "strings" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" +) + +func prefixedProof(t *testing.T, prefix, method, path, body string) cs.V1Request { + t.Helper() + root := actorKey("root").Public().(ed25519.PublicKey) + p := cs.V1Request{Method: method, URI: prefix + path, IdentityKey: root, Certificate: joinCert(t, "a"), At: "1000", Body: []byte(body)} + p.Signature = ed25519.Sign(actorKey("a"), cs.V1SigningMessage(method, p.URI, p.At, p.Body)) + raw, err := json.Marshal(p) + if err != nil { + t.Fatal(err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + t.Fatal(err) + } + if prefix != "" { + fields["base_path"], err = json.Marshal(prefix) + if err != nil { + t.Fatal(err) + } + } + raw, err = json.Marshal(fields) + if err != nil { + t.Fatal(err) + } + if err := cs.Decode(raw, &p); err != nil { + t.Fatal(err) + } + return p +} + +func TestR5_3CommittedPrefixCannotChangeSignedOperation(t *testing.T) { + root := actorKey("root").Public().(ed25519.PublicKey) + id := cs.IdentityRec{V: 1, Identity: cs.IdentityID(root), SigningKey: root} + p := prefixedProof(t, "/a/relay", "POST", "/v1/dir/cells/chat", `{"head":"saved"}`) + c, err := cs.V1Command("test", 1000, p) + if err != nil { + t.Fatal(err) + } + for _, change := range []string{"uri", "operation", "prefix"} { + t.Run(change, func(t *testing.T) { + raw, err := cs.EncodeCommand(c) + if err != nil { + t.Fatal(err) + } + bad, err := cs.DecodeCommand(raw) + if err != nil { + t.Fatal(err) + } + switch change { + case "uri": + bad.V1.URI = "/a/relay/v1/dir/cells/other" + case "operation": + bad.Change.Create.Cell = "other" + case "prefix": + var fields map[string]any + proof, _ := json.Marshal(bad.V1) + if err := json.Unmarshal(proof, &fields); err != nil { + t.Fatal(err) + } + fields["base_path"] = "/a" + proof, _ = json.Marshal(fields) + if err := cs.Decode(proof, bad.V1); err != nil { + t.Fatal(err) + } + } + if _, ok := cs.V1Authority(bad, id); ok { + t.Fatal("altered routing proof gained authority") + } + }) + } + for _, prefix := range []string{"", "/a", "/a/relay/", "/a/relayx", "a/relay"} { + candidate := prefixedProof(t, prefix, "GET", "/v1/dir/list", "") + candidate.URI = p.URI + if _, err := candidate.Mutation(); err == nil { + t.Fatalf("mismatched prefix %q accepted", prefix) + } + } + var fields map[string]any + raw, _ := json.Marshal(p) + if err := json.Unmarshal(raw, &fields); err != nil { + t.Fatal(err) + } + fields["unknown"] = true + raw, _ = json.Marshal(fields) + if cs.Decode(raw, &p) == nil { + t.Fatal("unknown proof field accepted") + } +} + +func TestR5_5PersonalAdmissionUsesCommittedPrefix(t *testing.T) { + root := actorKey("root").Public().(ed25519.PublicKey) + id := cs.IdentityRec{V: 1, Identity: cs.IdentityID(root), SigningKey: root} + for _, prefix := range []string{"", "/fabric", "/my-relay", "/a/relay"} { + t.Run(prefix, func(t *testing.T) { + p := prefixedProof(t, prefix, "POST", cs.PersonalAdmissionPath, `{"store":"`+strings.Repeat("01", 32)+`","protocol":1}`) + request, auth, actor, err := cs.VerifyPersonalAdmission(p, id, 1000) + if err != nil || request.Protocol != 1 || !auth.Personal || actor != cs.DeviceIDOfKey(actorKey("a").Public().(ed25519.PublicKey)) { + t.Fatalf("prefixed admission refused: %+v %v", request, err) + } + p.URI = prefix + "/v2/cluster/control" + if _, _, _, err := cs.VerifyPersonalAdmission(p, id, 1000); err == nil { + t.Fatal("different signed route admitted") + } + }) + } +} + +func TestR5_3StagedRotationUsesCommittedPrefix(t *testing.T) { + for _, prefix := range []string{"/my-relay", "/a/relay"} { + t.Run(prefix, func(t *testing.T) { + p := prefixedProof(t, prefix, "POST", "/v1/identity/rotation", `{"op":"thaw"}`+strings.Repeat(" ", cs.MaxCommandBytes)) + c, err := cs.V1Command("test", 1000, p) + if err != nil { + t.Fatal(err) + } + identity := cs.IdentityRec{V: 1, Identity: cs.IdentityID(p.IdentityKey), SigningKey: p.IdentityKey, Rotation: &cs.Rotation{State: "frozen", By: c.Actor, At: 1000}} + raw, err := cs.Encode(identity) + if err != nil { + t.Fatal(err) + } + rows := map[cs.RecordKey][]byte{cs.IdentityKey(): raw} + meta := cs.ApplicationMeta{Cluster: "test", Now: 1000, Ceiling: cs.DefaultApplicationBytes, Bytes: cs.RecordBytes(cs.IdentityKey(), raw)} + parts, err := cs.V1Commands(c) + if err != nil { + t.Fatal(err) + } + if len(parts) < 2 { + t.Fatal("large signed request did not stage") + } + for _, part := range parts { + if result := applyV1Page(t, part, rows, &meta); result.Code != cs.OK { + t.Fatal(result.Code) + } + } + if cs.Decode(rows[cs.IdentityKey()], &identity) != nil || identity.Rotation != nil { + t.Fatal("prefixed thaw did not commit") + } + }) + } +} diff --git a/internal/clusterstate/v1_stage.go b/internal/clusterstate/v1_stage.go index 425e55c6f0..5025129751 100644 --- a/internal/clusterstate/v1_stage.go +++ b/internal/clusterstate/v1_stage.go @@ -183,7 +183,11 @@ func evaluateV1Stage(c Command, meta ApplicationMeta, read ReadRecord) (Evaluati return out, nil } } - if identity.Rotation != nil && !(c.V1.Method == "POST" && strings.TrimPrefix(c.V1.URI, "/fabric") == "/v1/identity/rotation") { + path, err := c.V1.route() + if err != nil || !strings.HasPrefix(path, "/v1/") { + return out, nil + } + if identity.Rotation != nil && !(c.V1.Method == "POST" && path == "/v1/identity/rotation") { out.Result.Code = Rotated return out, nil } diff --git a/internal/manual/chat/coordination-and-workers.md b/internal/manual/chat/coordination-and-workers.md index 06a6119b2a..6a94917e79 100644 --- a/internal/manual/chat/coordination-and-workers.md +++ b/internal/manual/chat/coordination-and-workers.md @@ -33,6 +33,14 @@ Three voters tolerate one unavailable member. Losing a majority pauses coordinat The hosted relay keeps serving directory requests, watches, pairing decisions and worker assignments while this computer leads. It routes signed coordination requests through the computer’s outbound connection and waits for its own durable copy before replying. `codeaf cluster status` on either joined home reports the elected computer and the same three voters. +## Hosted relay address with a path prefix + +A hosted relay can serve beneath a path such as `https://relay.example.com/my-relay` or `https://relay.example.com/a/relay`. Pair using the complete address. The configured path applies to chat lists, saves, continuation, releases and joining computers, including while a joined computer leads. Request signatures include that path; requests outside it return HTTP 404 `not_found`. The native `codeaf relay` serves at the root of its host. + +## Empty frame plans from older clients + +Older clients may send an empty frame plan as an omitted field, `null` or `[]`. Both relays accept these spellings for a chat create or publish; an empty plan clears the previous frame hint. + ## Pending leadership request, retry and refusal A lost reply or restarted home keeps the original signed leadership request. Retry `codeaf cluster lead` to resume it; a pending request cannot choose a different computer. A completed refusal says `the leadership request could not complete; refresh cluster status and try again` and is acknowledged before another request is saved. Only one configuration change can run at a time. A different personal coordinator cannot directly replace an active personal coordinator; let the anchors resume, then request leadership again. diff --git a/internal/raftdirectory/v1.go b/internal/raftdirectory/v1.go index c3e520928f..c235af1c65 100644 --- a/internal/raftdirectory/v1.go +++ b/internal/raftdirectory/v1.go @@ -3,7 +3,6 @@ package raftdirectory import ( "context" "net/http" - "reflect" "time" "github.com/Agent-Field/codeaf/internal/clusterstate" @@ -78,7 +77,7 @@ func (b *v1Backend) Change(ctx context.Context, actor string, m clusterstate.Mut if b.err != nil { return clusterstate.Result{}, b.err } - if b.owner.Host == nil || actor != b.actor || !reflect.DeepEqual(m, b.command.Change) { + if b.owner.Host == nil || actor != b.actor || !clusterstate.V1MutationEqual(m, b.command.Change) { return clusterstate.Result{}, directory.ErrUnauthorized } r, err := b.owner.Host.CommitV1(ctx, b.command) diff --git a/internal/raftdirectory/v1_frames_test.go b/internal/raftdirectory/v1_frames_test.go new file mode 100644 index 0000000000..75af653018 --- /dev/null +++ b/internal/raftdirectory/v1_frames_test.go @@ -0,0 +1,58 @@ +package raftdirectory_test + +import ( + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/relayserve" + "github.com/Agent-Field/codeaf/internal/reqsign" +) + +func TestR5_6NativeExplicitEmptyFramePlans(t *testing.T) { + service := relayserve.New(relayserve.Config{Store: t.TempDir()}) + defer service.Close() + server := httptest.NewServer(service.Handler) + defer server.Close() + signer := fixtureSigner{private: key("p17-a")} + for _, one := range frozenV1(t).Cases { + if one.Name == "put-a" { + raw, err := base64.RawURLEncoding.DecodeString(one.Headers["Codeaf-Cert"]) + must(t, err) + must(t, json.Unmarshal(raw, &signer.certificate)) + } + } + send := func(t *testing.T, method, path, body string, want int) []byte { + t.Helper() + r, err := http.NewRequest(method, server.URL+path, strings.NewReader(body)) + must(t, err) + reqsign.Sign(r, []byte(body), signer, time.Now()) + response, err := server.Client().Do(r) + must(t, err) + defer response.Body.Close() + raw, err := io.ReadAll(response.Body) + must(t, err) + if response.StatusCode != want { + t.Errorf("%s %s: HTTP %d %s, want %d", method, path, response.StatusCode, raw, want) + } + return raw + } + for _, spelling := range []struct{ name, field string }{{"absent", ""}, {"null", `,"frames":null`}, {"empty", `,"frames":[]`}} { + t.Run(spelling.name, func(t *testing.T) { + path := "/v1/dir/cells/" + spelling.name + send(t, "POST", path, `{"head":"saved","class":"work","keys":{},"size":0`+spelling.field+`}`, 200) + publishPath := path + "-publish" + send(t, "POST", publishPath, `{"head":"saved","class":"work","keys":{},"size":0}`, 200) + send(t, "POST", publishPath+"/publish", `{"old_head":"saved","head":"next","fence":1,"size":0,"class":"work"`+spelling.field+`}`, 200) + raw := send(t, "GET", publishPath, "", 200) + if !strings.Contains(string(raw), `"head":"next"`) { + t.Errorf("published head missing: %s", raw) + } + }) + } +} diff --git a/relay/hosted/README.md b/relay/hosted/README.md index 18bcd81b9a..622df60831 100644 --- a/relay/hosted/README.md +++ b/relay/hosted/README.md @@ -181,7 +181,8 @@ frames remain retained. A deployment can share a hostname by using a path prefix, such as `https://relay.example.com/fabric`. `BASE_PATH = "/fabric"` makes the Worker serve every route (store, directory, the `/v1/dir/watch` upgrade, pairing, link requests, rotation) beneath that prefix. -`src/base.js` holds the whole mechanism: the Worker strips the prefix once, before dispatch, and a +Prefixes may have several segments, such as `/my-relay` or `/a/relay`. +`src/base.js` normalizes the configured prefix and strips it once before dispatch; a request outside it is the usual 404. The request signature covers the path the client sent (`internal/reqsign` signs `RequestURI()`), so a @@ -189,6 +190,8 @@ client signs `/fabric/v1/...` and the identity object checks the signature again (`signedUri`), not against the stripped path. A client that signed the bare path is `401`. Every Go client joins its routes to the relay address in one helper, `wireauth.Endpoint`, so an address with a path (`https://host/fabric`, with or without the last slash) works for all of them. +The committed request retains both the original signed URI and the configured prefix, +so every member interprets the same operation even when a joined computer leads. The Go relay (`internal/relayserve`) stays at the root of its host: it has no base path. A self-hoster who wants one puts their own proxy in front, and the proxy must pass the path on unchanged (the client diff --git a/relay/hosted/src/raft/hosted.js b/relay/hosted/src/raft/hosted.js index 6a2de8a7a2..6f8bf28542 100644 --- a/relay/hosted/src/raft/hosted.js +++ b/relay/hosted/src/raft/hosted.js @@ -11,7 +11,7 @@ import { v1Command } from './v1.js'; import { projectRecord } from './directory.js'; import { RaftEffectsFacade } from './effects.js'; import { Wire, empty, json, unwrap } from '../wire.js'; -import { signedUri } from '../base.js'; +import { prefixOf, signedUri } from '../base.js'; import { fromBase64Url } from '../codec.js'; import { presenceOf, parseBeat } from '../presence.js'; import { parseHolds } from '../vouch.js'; @@ -362,7 +362,7 @@ export class HostedIdentity { return '{"now":'+app.now+',"identity":'+identity+',"devices":{'+devices.join(',')+'},"cells":{'+cells.join(',')+'}}'; } jsonBytes(raw){return new Response(raw,{headers:{'content-type':'application/json'}});} - proof(request,body){return {method:request.method,uri:signedUri(this.env,new URL(request.url)),identity_key:b64(fromBase64Url(request.headers.get('codeaf-identity'))),certificate:request.headers.get('codeaf-cert'),at:request.headers.get('codeaf-time'),signature:b64(fromBase64Url(request.headers.get('codeaf-sig'))),body:b64(body)};} + proof(request,body){const base=prefixOf(this.env);return {...(base?{base_path:base}:{}),method:request.method,uri:signedUri(this.env,new URL(request.url)),identity_key:b64(fromBase64Url(request.headers.get('codeaf-identity'))),certificate:request.headers.get('codeaf-cert'),at:request.headers.get('codeaf-time'),signature:b64(fromBase64Url(request.headers.get('codeaf-sig'))),body:b64(body)};} configuration(raw){const r=response(this.runtime.call(frame(35).blob(unb64(raw)).finish())),conf=decodeCanonical(dec.decode(r.blob()));r.end();return conf;} async anchorCommand(change,now){ const id=this.manifest.config.anchors[0],actor='anchor/'+id; diff --git a/relay/hosted/test/raft/v1-prefix-parity.test.js b/relay/hosted/test/raft/v1-prefix-parity.test.js new file mode 100644 index 0000000000..1c6cc9eeed --- /dev/null +++ b/relay/hosted/test/raft/v1-prefix-parity.test.js @@ -0,0 +1,79 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {spawn} from 'node:child_process'; +import {DatabaseSync} from 'node:sqlite'; +import {generateKeyPairSync,sign,createHash} from 'node:crypto'; +import {Reactor} from '../../src/raft/abi.js'; +import {MemberActor} from '../../src/raft/actor.js'; +import {HostedIdentity} from '../../src/raft/hosted.js'; +import {directoryEvaluator} from '../../src/raft/directory.js'; +import {encodeCanonical,decodeCanonical} from '../../src/raft/codec.js'; +import {v1Command} from '../../src/raft/v1.js'; + +test('r5.4 committed hosted prefix proofs apply identically in the native personal evaluator', {timeout:30000}, async () => { + const out=process.env.CODEAF_RAFT_PROOF_DIR; + const runtime=new Reactor(new WebAssembly.Module(readFileSync(out+'/core.wasm'))); + const calls=[],traced={call(input){const result=runtime.call(input);calls.push([input.slice(),result.slice()]);return result;}}; + const enc=new TextEncoder(),dec=new TextDecoder(),bytes=v=>enc.encode(encodeCanonical(v)); + const root=generateKeyPairSync('ed25519'),device=generateKeyPairSync('ed25519'); + const pub=key=>key.export({format:'der',type:'spki'}).subarray(-32); + const sha=b=>createHash('sha256').update(b).digest('hex'); + const cert={V:1,device:pub(device.publicKey).toString('hex'),created:1000,sig:''}; + cert.sig=sign(null,Buffer.from('codeaf/device-cert/v1\n'+JSON.stringify(cert)),root.privateKey).toString('hex'); + for (const prefix of ['/my-relay','/a/relay']) { + const body=Buffer.from('{"head":"saved","class":"work","size":0,"keys":{}}'); + const route='/v1/dir/cells/chat',uri=prefix+route,at='1000'; + const request=new Request('http://local'+route,{method:'POST',headers:{ + 'Codeaf-Identity':pub(root.publicKey).toString('base64url'), + 'Codeaf-Cert':Buffer.from(JSON.stringify(cert)).toString('base64url'), + 'Codeaf-Time':at, + 'Codeaf-Sig':sign(null,Buffer.from('codeaf-req-v1\nPOST\n'+uri+'\n'+at+'\n'+sha(body)),device.privateKey).toString('base64url'), + },body}); + const proof=HostedIdentity.prototype.proof.call({env:{BASE_PATH:prefix}},request,body); + const committed=v1Command(traced,'cluster',1000,proof); + const command=decodeCanonical(dec.decode(committed)); + assert.equal(command.v1_request.uri,uri); + assert.equal(command.v1_request.base_path,prefix); + const identity=bytes({V:1,identity:'id_'+sha(pub(root.publicKey)).slice(0,32),signing_key:pub(root.publicKey).toString('base64')}); + const actors=[],stores=[],queue=[],results=[]; + const accept=effects=>queue.push(...effects.send); + const pump=async()=>{for(let n=0;queue.length;n++){assert(n<3000);const message=queue.shift();accept(await actors[Number(message.to)-1].input(2,message.bytes));}}; + try { + for(let n=0;n<3;n++) { + const db=new DatabaseSync(':memory:');stores.push(db); + const storage={sql:{exec(sql,...args){const rows=db.prepare(sql).all(...args);return {toArray:()=>rows,[Symbol.iterator]:()=>rows[Symbol.iterator]()};}},transactionSync(fn){db.exec('BEGIN');try{const value=fn();db.exec('COMMIT');return value;}catch(error){db.exec('ROLLBACK');throw error;}},async sync(){}}; + const evaluate=(raw,read,app)=>{ + const initial=read('state','identity')?[]:[{namespace:'state',key:'identity',data:identity}]; + const size=initial.length?12+'state'.length+'identity'.length+identity.length:0; + const result=directoryEvaluator(traced,'cluster',8*1024*1024)(raw,(ns,key)=>initial.find(row=>row.namespace===ns&&row.key===key)?.data??read(ns,key),{...app,size:app.size+size}); + results.push(decodeCanonical(dec.decode(result.result)));return {...result,writes:[...initial,...result.writes]}; + }; + const actor=new MemberActor(storage,runtime,evaluate);actors.push(actor); + accept(await actor.init({cluster:'cluster',id:String(n+1),seed:String(n+20),bootstrap:['1','2','3']})); + } + await pump(); + for(let n=0;n<30&&!actors.some(actor=>actor.status.role===2);n++){for(const actor of actors)accept(await actor.input(1));await pump();} + const leader=actors.find(actor=>actor.status.role===2);assert(leader); + accept(await leader.input(3,committed));await pump(); + assert.equal(results.length,3);assert(results.every(result=>result.code==='ok')); + const cells=actors.map(actor=>actor.store.read('state','cell/'+Buffer.from('chat').toString('hex'))); + assert.deepEqual(cells[0],cells[1]);assert.deepEqual(cells[0],cells[2]); + const saved=decodeCanonical(dec.decode(cells[0])); + assert.equal(saved.head,'saved');assert.equal(saved.lease.fence,'1'); + } finally {for(const db of stores)db.close();} + } + const child=spawn(out+'/native-peer',[],{stdio:['pipe','pipe','inherit'],env:{...process.env,GOMEMLIMIT:'32MiB'}}); + let buffer=Buffer.alloc(0),waiter; + const pump=()=>{if(!waiter||buffer.length<4)return;const n=buffer.readUInt32LE();assert(n<=2*1024*1024);if(buffer.length{buffer=Buffer.concat([buffer,raw]);pump();}); + try { + for (const [input,expected] of calls) { + const actual=await new Promise((resolve,reject)=>{assert(!waiter);waiter={resolve,timer:setTimeout(()=>reject(Error('native ABI timeout')),5000)};const header=Buffer.alloc(4);header.writeUInt32LE(input.length);child.stdin.write(header);child.stdin.write(input);pump();}); + assert.deepEqual(actual,expected); + } + } finally { + if(waiter)clearTimeout(waiter.timer); + child.stdin.end();if(child.exitCode===null)await new Promise(resolve=>child.once('exit',resolve)); + } +}); diff --git a/relay/hosted/test/raft/v1-prefix.test.js b/relay/hosted/test/raft/v1-prefix.test.js new file mode 100644 index 0000000000..15be4e2eaf --- /dev/null +++ b/relay/hosted/test/raft/v1-prefix.test.js @@ -0,0 +1,82 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {sign, createHash} from 'node:crypto'; +import {hostedFixture} from './hosted-fixture.js'; + +for (const base of ['', '/fabric', '/my-relay', '/a/relay']) { + test(`${base === '' || base === '/fabric' ? 'r5.2' : 'r5.1'} signed directory operations at ${base || '/'}`, {timeout:30000}, async () => { + const h=await hostedFixture({base}),a=h.client(),b=h.client(); + try { + await h.start(); + const list=await h.call(a,'GET','/v1/dir/list'); + assert.equal(list.status,200,JSON.stringify(list)); + assert.deepEqual(list.body.cells,{}); + const created=await h.call(a,'POST','/v1/dir/cells/chat',{head:'saved',class:'work',size:0,keys:{}}); + assert.equal(created.status,200,JSON.stringify(created)); + assert.equal(created.body.cell.lease.fence,1); + assert.equal(created.body.cell.lease.device,a.device); + const acquired=await h.call(b,'POST','/v1/dir/cells/chat/acquire',{force:true,expect:{device:a.device,fence:1}}); + assert.equal(acquired.status,200,JSON.stringify(acquired)); + assert.equal(acquired.body.cell.head,'saved'); + assert.equal(acquired.body.cell.lease.fence,2); + assert.equal(acquired.body.cell.lease.device,b.device); + assert.equal((await h.call(b,'POST','/v1/dir/cells/chat/release',{fence:2})).status,204); + const released=await h.call(a,'GET','/v1/dir/cells/chat'); + assert.equal(released.status,200); + assert.equal(released.body.cell.lease.expires,0); + assert.equal(released.body.cell.lease.fence,2); + } finally {await h.close();} + }); +} + +test('r5.3 signatures bind the complete configured path and outside paths stay absent', {timeout:30000}, async () => { + const base='/a/relay',h=await hostedFixture({base}),a=h.client(); + try { + await h.start(); + const dispatch=h.mf.dispatchFetch.bind(h.mf); + let captured; + h.mf.dispatchFetch=(url,init)=>{captured={url,init};return dispatch(url,init);}; + assert.equal((await h.call(a,'GET','/v1/dir/list')).status,200); + const {headers}=captured.init; + let r=await dispatch('http://local'+base+'/v1/dir/cells/other',{headers}); + assert.equal(r.status,401);assert.deepEqual(await r.json(),{err:'unauthorized'}); + const at=headers['Codeaf-Time'],hash=createHash('sha256').update('').digest('hex'); + const bare={...headers,'Codeaf-Sig':sign(null,Buffer.from('codeaf-req-v1\nGET\n/v1/dir/list\n'+at+'\n'+hash),a.key.privateKey).toString('base64url')}; + r=await dispatch('http://local'+base+'/v1/dir/list',{headers:bare}); + assert.equal(r.status,401);assert.deepEqual(await r.json(),{err:'unauthorized'}); + for (const path of ['/v1/dir/list',base+'x/v1/dir/list','/a/v1/dir/list']) { + r=await dispatch('http://local'+path,{headers}); + assert.equal(r.status,404);assert.deepEqual(await r.json(),{err:'not_found'}); + } + } finally {await h.close();} +}); + +test('r5.5 a paired computer enrolls under a custom prefix', {timeout:30000}, async () => { + const h=await hostedFixture({base:'/a/relay'}),a=h.client(); + try { + await h.start(); + const body={store:'01'.repeat(32),protocol:1}; + for (let n=0;n<2;n++) { + const r=await h.request(a,'POST','/v2/cluster/enroll',body),text=await r.text(); + assert.equal(r.status,200,text); + assert.equal(JSON.parse(text).node,'4'); + } + } finally {await h.close();} +}); + +test('r5.6 hosted create and publish accept absent, null and empty frame plans', {timeout:30000}, async () => { + const h=await hostedFixture(),a=h.client(); + try { + await h.start(); + for (const [name,plan] of [['absent',{}],['null',{frames:null}],['empty',{frames:[]}]]) { + const path='/v1/dir/cells/'+name; + const created=await h.call(a,'POST',path,{head:'saved',class:'work',keys:{},size:0,...plan}); + assert.equal(created.status,200,JSON.stringify(created)); + const published=await h.call(a,'POST',path+'/publish',{old_head:'saved',head:'next',fence:1,size:0,class:'work',...plan}); + assert.equal(published.status,200,JSON.stringify(published)); + const read=await h.call(a,'GET',path); + assert.equal(read.status,200);assert.equal(read.body.cell.head,'next'); + assert.equal(read.body.cell.frames?.length??0,0); + } + } finally {await h.close();} +}); From 93e29eaff4f1436993f60b0dc3627344d43f25dc Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 23:27:48 -0400 Subject: [PATCH 093/103] Release worker retry sessions when assignments end. Finished, cancelled and reassigned work frees claim retry records so later assignments and personal leadership requests retain session capacity. Retained assignment receipts keep retries idempotent and prevent old requests from acquiring work again. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/changes/unreleased/raft-cluster.md | 1 + internal/clusterstate/apply.go | 91 ++++-- internal/clusterstate/assignment.go | 62 +++- internal/clusterstate/worker_sessions_test.go | 274 ++++++++++++++++++ .../manual/chat/coordination-and-workers.md | 2 + internal/syncsetup/worker.go | 4 +- relay/hosted/test/raft/assignment.test.js | 21 +- 7 files changed, 427 insertions(+), 28 deletions(-) create mode 100644 internal/clusterstate/worker_sessions_test.go diff --git a/docs/changes/unreleased/raft-cluster.md b/docs/changes/unreleased/raft-cluster.md index a85f278bc5..abf0324a16 100644 --- a/docs/changes/unreleased/raft-cluster.md +++ b/docs/changes/unreleased/raft-cluster.md @@ -17,6 +17,7 @@ invalidates: - "The native relay could reject a signed create or publish with an explicit empty frame plan. Empty arrays now have the same meaning as omitted or null frame plans while the request remains bound to its original actor and operation." - "Retained rows from legal older requests could pause upgrades after the relay added fields, and full directories could fail initialization or refusal persistence. Both relays now preserve these rows losslessly and reserve bounded bookkeeping independently of the measured content allowance." - "Workers previously needed personal-device trust for shared access. Delegated workers now have expiring selected-chat grants and conditional assignment claims; they cannot approve pairing, read the identity vault, rotate it, enroll devices or lead, and terminal worker setup is not available yet." + - "Completed worker assignments previously accumulated retry sessions and could block later work or personal leadership requests. Assignment retry records now release on finish, cancellation or reassignment while retained assignments prevent old requests from starting work again." - "Reopening retained work could compare its branch fence with the original chat and falsely refuse tools. Ownership checks stay with the record each window opened, and replacing a displaced drive settles its previous writer before starting another. A failed final save remains failed for every close caller." - "Deferred local durability could resolve a fallback daemon against a different current home. Both engine routes now retain the home and engine selected at capture." - "A single directory database was formerly the live source. First use now freezes and imports it into the three anchors while retaining the source, and the one-way upgrade cannot return to that writer after cluster changes are acknowledged." diff --git a/internal/clusterstate/apply.go b/internal/clusterstate/apply.go index 1c90c3ffcf..543218577e 100644 --- a/internal/clusterstate/apply.go +++ b/internal/clusterstate/apply.go @@ -428,7 +428,25 @@ func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged boo } var last session var counts sessionCounts - if decode(SessionKey(c.Actor, c.Session), &last) != nil || decode(SessionsKey(), &counts) != nil { + sessionKey := SessionKey(c.Actor, c.Session) + scopedRetry := auth.Worker != nil && workerAssignmentSession(c) + persistRetry := true + if scopedRetry { + // Only a live claim needs a retry session. Completed assignments keep + // their claim receipt and acknowledge matching finishes themselves. + var a Assignment + r := assignmentRef(c.Change) + matching := rows[AssignmentKey(r.ID)] != nil && Decode(rows[AssignmentKey(r.ID)], &a) == nil && a.Ref() == r + if matching && c.Change.ClaimAssignment != nil && a.Status == "finished" && a.Claim != nil && a.Claim.Digest == c.RequestDigest { + out.Result = a.claimResult() + return out, nil + } + persistRetry = c.Change.ClaimAssignment != nil && matching && (a.Status == "pending" || a.Status == "claimed") + if c.Change.ClaimAssignment != nil { + sessionKey = assignmentSessionKey(r.ID) + } + } + if decode(sessionKey, &last) != nil || decode(SessionsKey(), &counts) != nil { return out, ErrDependency } originalCounts := counts @@ -436,7 +454,7 @@ func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged boo if err != nil { return out, err } - known := c.V1 == nil && rows[SessionKey(c.Actor, c.Session)] != nil + known := c.V1 == nil && rows[sessionKey] != nil refuse := func(code Code) (Evaluation, error) { out.Result.Code = code; return out, nil } if known && (last.Closed || c.Sequence < last.Sequence || c.Sequence == last.Sequence && c.RequestDigest != last.Digest) { return refuse(Conflict) @@ -448,7 +466,7 @@ func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged boo if !service && c.V1 == nil && (!known && (c.Sequence != 1 || c.Ack != 0) || known && (last.Sequence == ^uint64(0) || c.Sequence != last.Sequence+1 || c.Ack != last.Sequence)) { return refuse(Conflict) } - if !service && c.V1 == nil && !known && (counts.Live >= MaxLiveSessions || counts.Retained >= MaxRetainedSessions) { + if !service && c.V1 == nil && persistRetry && !known && (counts.Live >= MaxLiveSessions || counts.Retained >= MaxRetainedSessions) { return refuse(Capacity) } if c.Now < 0 || c.Now > int64(MaxSafeInteger)-LeaseTTLMillis { @@ -555,13 +573,6 @@ func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged boo if rows[QuiescenceKey()] != nil && c.Change.PrepareQuiescence == nil { put(QuiescenceKey(), Quiescence{}) } - if !known { - counts.Live++ - counts.Retained++ - } - if c.Change.CloseSession != nil { - counts.Live-- - } if c.Change.InitMembership != nil && code == OK { for k, b := range writes { counts.Initialization += RecordBytes(k, b) @@ -570,14 +581,55 @@ func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged boo } } } + initialization := counts.Initialization save := func() { if service || c.V1 != nil { return } - put(SessionKey(c.Actor, c.Session), session{Sequence: c.Sequence, Digest: c.RequestDigest, Closed: c.Change.CloseSession != nil, Result: out.Result}) - sessionKey := SessionKey(c.Actor, c.Session) - oldCredit := bookkeepingCost(sessionKey, rows[sessionKey]) + bookkeepingCost(SessionsKey(), rows[SessionsKey()]) - base := originalCounts.Bookkeeping - min(originalCounts.Bookkeeping, oldCredit) + bookkeepingCost(sessionKey, writes[sessionKey]) + counts = originalCounts + counts.Initialization = initialization + if persistRetry { + put(sessionKey, session{Sequence: c.Sequence, Digest: c.RequestDigest, Closed: c.Change.CloseSession != nil, Result: out.Result}) + } + if out.Result.Code == OK && (c.Change.FinishAssignment != nil || c.Change.CancelAssignment != nil || c.Change.Reassign != nil) { + k := assignmentSessionKey(assignmentRef(c.Change).ID) + if rows[k] != nil { + if c.Change.FinishAssignment != nil { + var claim session + var a Assignment + _ = Decode(rows[k], &claim) + _ = Decode(writes[AssignmentKey(assignmentRef(c.Change).ID)], &a) + if claim.Result.Code == OK && claim.Result.View != nil { + a.Claim = &AssignmentClaimReceipt{Digest: claim.Digest, View: *claim.Result.View, Now: claim.Result.Now, Version: claim.Result.Version} + put(AssignmentKey(a.Spec.ID), a) + } + } + writes[k] = nil + } + } + base := originalCounts.Bookkeeping - min(originalCounts.Bookkeeping, bookkeepingCost(SessionsKey(), rows[SessionsKey()])) + for k, b := range writes { + if k.Namespace != "sessions" || k == SessionsKey() { + continue + } + var old, next session + if rows[k] != nil { + _ = Decode(rows[k], &old) + counts.Retained-- + if !old.Closed { + counts.Live-- + } + } + if b != nil { + _ = Decode(b, &next) + counts.Retained++ + if !next.Closed { + counts.Live++ + } + } + base -= min(base, bookkeepingCost(k, rows[k])) + base += bookkeepingCost(k, b) + } for range 8 { put(SessionsKey(), counts) next := base + bookkeepingCost(SessionsKey(), writes[SessionsKey()]) @@ -593,8 +645,11 @@ func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged boo size := meta.Bytes budget := 0 for k, b := range writes { - cost := RecordBytes(k, b) - budget += int(cost) + cost := uint64(0) + if b != nil { + cost = RecordBytes(k, b) + } + budget += int(RecordBytes(k, b)) if old := rows[k]; old != nil { oldSize := RecordBytes(k, old) if oldSize > size { @@ -642,7 +697,7 @@ func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged boo writes = map[RecordKey][]byte{} out.Version = meta.Version out.Result = Result{Schema: SchemaVersion, Code: Capacity, Now: out.Now, Version: meta.Version} - counts.Initialization = originalCounts.Initialization + initialization = originalCounts.Initialization save() if ok, err := fits(); err != nil { return out, err @@ -651,7 +706,7 @@ func evaluateCommand(c Command, meta ApplicationMeta, read ReadRecord, paged boo } } for k, b := range writes { - out.Writes = append(out.Writes, RecordWrite{RecordKey: k, Data: b}) + out.Writes = append(out.Writes, RecordWrite{RecordKey: k, Data: b, Delete: b == nil}) } sort.Slice(out.Writes, func(i, j int) bool { a, b := out.Writes[i].RecordKey, out.Writes[j].RecordKey diff --git a/internal/clusterstate/assignment.go b/internal/clusterstate/assignment.go index 5b884b77ff..50f5acd9ca 100644 --- a/internal/clusterstate/assignment.go +++ b/internal/clusterstate/assignment.go @@ -1,5 +1,10 @@ package clusterstate +import ( + "crypto/sha256" + "encoding/hex" +) + // Assignment routing is public; instructions, credentials and task descriptors // stay inside SealedPayload. A target is always selected by an authorized client. type Assign struct { @@ -37,10 +42,29 @@ type CancelAssignment struct { } type Assignment struct { - Spec Assign `json:"spec"` - Status string `json:"status"` - Fence uint64 `json:"fence"` - Head string `json:"head"` + Spec Assign `json:"spec"` + Status string `json:"status"` + Fence uint64 `json:"fence"` + Head string `json:"head"` + Claim *AssignmentClaimReceipt `json:"claim,omitempty"` +} + +// AssignmentClaimReceipt preserves a completed claim's reply without retaining +// a live session or duplicating the assignment's immutable specification. +type AssignmentClaimReceipt struct { + Digest string `json:"digest"` + View CellView `json:"view"` + Now int64 `json:"now"` + Version uint64 `json:"version"` +} + +func (a Assignment) claimResult() Result { + r := a.Claim + a.Claim = nil + a.Status = "claimed" + a.Fence = r.View.Cell.Lease.Fence + a.Head = r.View.Cell.Head + return Result{Schema: SchemaVersion, Code: OK, Now: r.Now, Version: r.Version, View: &r.View, Assignment: &a} } func (a Assignment) Ref() AssignmentRef { @@ -60,6 +84,29 @@ type WorkerHold struct { } func AssignmentKey(id string) RecordKey { return RecordKey{"state", "assignment/" + segment(id)} } + +// WorkerAssignmentSession binds a saved request to one assignment attempt. +func WorkerAssignmentSession(kind string, ref AssignmentRef) string { + b, _ := Encode(ref) + h := sha256.Sum256(b) + return "worker-" + kind + "-" + hex.EncodeToString(h[:]) +} + +// Assignment-owned retry rows let replacement and cancellation release the +// previous claimant without discovering its process or session name. +func assignmentSessionKey(id string) RecordKey { + return RecordKey{"sessions", "assignment/" + segment(id)} +} + +func workerAssignmentSession(c Command) bool { + kind := "claim" + if c.Change.FinishAssignment != nil { + kind = "finish" + } else if c.Change.ClaimAssignment == nil { + return false + } + return c.Session == WorkerAssignmentSession(kind, assignmentRef(c.Change)) +} func AssignmentSlotKey(cell string) RecordKey { return RecordKey{"state", "assignment-slot/" + segment(cell)} } @@ -91,7 +138,7 @@ func assignmentDependencies(c Command) []RecordKey { var keys []RecordKey if assignmentMutation(c.Change) { r := assignmentRef(c.Change) - keys = append(keys, AssignmentKey(r.ID), AssignmentSlotKey(r.Cell), CellKey(r.Cell), WorkerKey(r.Target), AuthorityKey(r.Target), DeviceKey(r.Target), WorkerHoldKey(r.Target, r.Cell)) + keys = append(keys, AssignmentKey(r.ID), AssignmentSlotKey(r.Cell), CellKey(r.Cell), WorkerKey(r.Target), AuthorityKey(r.Target), DeviceKey(r.Target), WorkerHoldKey(r.Target, r.Cell), assignmentSessionKey(r.ID)) v := c.Change.Assign if v == nil { v = c.Change.Reassign @@ -235,6 +282,11 @@ func applyAssignment(c Command, now int64, identity IdentityRec, rows map[Record if c.Actor != r.Target || c.WorkerIncarnation != r.Incarnation { return Unauthorized } + // Completion is retained on the assignment after its retry row is freed. + if workerAssignmentSession(c) && a.Status == "finished" && a.Fence == v.Fence && a.Head == v.Head { + result.Assignment = &a + return OK + } if a.Status != "claimed" || !slot.Active { return Conflict } diff --git a/internal/clusterstate/worker_sessions_test.go b/internal/clusterstate/worker_sessions_test.go new file mode 100644 index 0000000000..d9bd906b27 --- /dev/null +++ b/internal/clusterstate/worker_sessions_test.go @@ -0,0 +1,274 @@ +package clusterstate_test + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "reflect" + "sort" + "testing" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/raftsnapshot" +) + +type retryCounts struct { + Live uint32 `json:"live"` + Retained uint32 `json:"retained"` + Bookkeeping uint64 `json:"bookkeeping,omitempty"` + Initialization uint64 `json:"initialization,omitempty"` +} + +type assignmentRetries struct { + f *workerFixture + v cs.Assign + base retryCounts +} + +func newAssignmentRetries(t *testing.T) *assignmentRetries { + f, v := assignmentFixture(t) + expectWorker(t, f.a.run("a", 1003, cs.Mutation{AdvanceTime: &cs.AdvanceTime{}}), cs.OK) + return &assignmentRetries{f: f, v: v, base: readRetryCounts(t, f.a)} +} + +func readRetryCounts(t *testing.T, a *application) retryCounts { + t.Helper() + var counts retryCounts + if err := cs.Decode(a.rows[cs.SessionsKey()], &counts); err != nil { + t.Fatal(err) + } + var retained, credit uint64 + for k, b := range a.rows { + if k.Namespace == "sessions" && b != nil { + credit += min(cs.RecordBytes(k, b), 512) + if k != cs.SessionsKey() { + retained++ + } + } + } + if uint64(counts.Retained) != retained || counts.Bookkeeping != credit { + t.Fatalf("retry accounting disagrees with rows: %+v, rows=%d credit=%d", counts, retained, credit) + } + if _, err := cs.ApplicationBookkeeping(a.rows[cs.SessionsKey()]); err != nil { + t.Fatal(err) + } + return counts +} + +func (r *assignmentRetries) owner(m cs.Mutation) cs.Result { + return r.f.a.run("a", 1004, m) +} + +func (r *assignmentRetries) command(kind string, v cs.Assign, m cs.Mutation) cs.Command { + b, err := cs.Encode(ref(v)) + if err != nil { + r.f.a.t.Fatal(err) + } + h := sha256.Sum256(b) + c, err := cs.Sign(cs.Command{Schema: 1, Cluster: "test", Actor: r.f.id, WorkerIncarnation: v.Incarnation, Session: "worker-" + kind + "-" + hex.EncodeToString(h[:]), Sequence: 1, Now: 1004, Change: m}, r.f.key) + if err != nil { + r.f.a.t.Fatal(err) + } + return c +} + +func (r *assignmentRetries) start(id string) (cs.Assign, cs.Command, cs.Result) { + v := r.v + v.ID = id + cell := r.f.a.cell() + v.Expect = cs.LeaseExpectation{Device: cell.Lease.Device, Fence: cell.Lease.Fence} + a := r.owner(cs.Mutation{Assign: &v}) + if a.Code != cs.OK { + return v, cs.Command{}, a + } + c := r.command("claim", v, cs.Mutation{ClaimAssignment: &cs.ClaimAssignment{Ref: ref(v), Expect: v.Expect}}) + return v, c, r.f.eval(c) +} + +func (r *assignmentRetries) finish(v cs.Assign, claimed cs.Result) cs.Command { + return r.command("finish", v, cs.Mutation{FinishAssignment: &cs.FinishAssignment{Ref: ref(v), Fence: claimed.View.Cell.Lease.Fence, Head: claimed.View.Cell.Head}}) +} + +func (r *assignmentRetries) complete(n int) int { + for i := 0; i < n; i++ { + v, _, claimed := r.start(fmt.Sprintf("run-%d", i)) + if claimed.Code != cs.OK { + r.f.a.t.Errorf("assignment %d claim: %s", i+1, claimed.Code) + return i + } + finished := r.f.eval(r.finish(v, claimed)) + if finished.Code != cs.OK { + r.f.a.t.Errorf("assignment %d finish: %s", i+1, finished.Code) + return i + } + } + return n +} + +// r3.1 +func TestR31WorkerSessionsStayBoundedAcrossFiveThousandAssignments(t *testing.T) { + r := newAssignmentRetries(t) + if n := r.complete(5000); n != 5000 { + t.Fatalf("completed %d assignments", n) + } + got := readRetryCounts(t, r.f.a) + if got.Live != r.base.Live || got.Retained != r.base.Retained || got.Bookkeeping > r.base.Bookkeeping+512 { + t.Fatalf("completed assignments retained retry sessions: before=%+v after=%+v", r.base, got) + } +} + +// r3.2 +func TestR32WorkerHistoryLeavesRoomForPersonalLeadershipSession(t *testing.T) { + r := newAssignmentRetries(t) + r.f.a.meta.Term = 1 + expectWorker(t, enroll(r.f.a, "b"), cs.OK) + expectWorker(t, settle(r.f.a), cs.OK) + r.complete(5000) + h := memberHeader(r.f.a) + c, err := cs.Sign(cs.Command{Schema: 1, Cluster: "test", Actor: "b", Session: "leadership", Sequence: 1, Now: 1004, Change: cs.Mutation{RequestLeadership: &cs.RequestLeadership{Revision: h.Revision, Node: 4, Device: "b"}}}, actorKey("b")) + if err != nil { + t.Fatal(err) + } + expectWorker(t, r.f.eval(c), cs.OK) +} + +// r3.3 +func TestR33WorkerLostRepliesAndReleasedSessionReplays(t *testing.T) { + r := newAssignmentRetries(t) + v, claim, claimed := r.start("original") + expectWorker(t, claimed, cs.OK) + if retry := r.f.eval(claim); !reflect.DeepEqual(retry, claimed) { + t.Fatal("lost claim reply changed result or fence", retry, claimed) + } + finish := r.finish(v, claimed) + expectWorker(t, r.f.eval(finish), cs.OK) + before := r.f.a.cell() + expectWorker(t, r.f.eval(finish), cs.OK) + if !reflect.DeepEqual(before, r.f.a.cell()) { + t.Fatal("finish retry changed the chat") + } + got := readRetryCounts(t, r.f.a) + if got.Live != r.base.Live || got.Retained != r.base.Retained { + t.Errorf("finished assignment retained worker sessions: before=%+v after=%+v", r.base, got) + } + _, _, next := r.start("replacement") + expectWorker(t, next, cs.OK) + before = r.f.a.cell() + for _, c := range []cs.Command{claim, finish} { + r.f.eval(c) + if !reflect.DeepEqual(before, r.f.a.cell()) { + t.Fatal("released request changed a replacement's fence or head") + } + } +} + +// r3.4 +func TestR34WorkerSessionReleaseOnWithdrawalDrainAndReassignment(t *testing.T) { + for _, event := range []string{"withdraw", "drain-withdraw", "reassign", "new-incarnation"} { + t.Run(event, func(t *testing.T) { + r := newAssignmentRetries(t) + v, oldClaim, claimed := r.start("original") + expectWorker(t, claimed, cs.OK) + oldFinish := r.finish(v, claimed) + if event == "drain-withdraw" { + expectWorker(t, r.owner(cs.Mutation{DrainWorker: &cs.DrainWorker{Worker: r.f.id, Incarnation: 1}}), cs.OK) + } + if event == "withdraw" || event == "drain-withdraw" { + expectWorker(t, r.owner(cs.Mutation{CancelAssignment: &cs.CancelAssignment{Ref: ref(v)}}), cs.OK) + } else { + next := v + next.Revision++ + next.Expect = cs.LeaseExpectation{Device: r.f.id, Fence: claimed.View.Cell.Lease.Fence} + if event == "new-incarnation" { + expectWorker(t, r.f.register(1, 1004), cs.OK) + expectWorker(t, r.f.beat(r.f.proof(2, 1, 1004), 1004), cs.OK) + next.Incarnation = 2 + } + before := readRetryCounts(t, r.f.a) + expectWorker(t, r.owner(cs.Mutation{Reassign: &next}), cs.OK) + after := readRetryCounts(t, r.f.a) + if after.Live >= before.Live || after.Retained >= before.Retained { + t.Errorf("reassignment retained previous worker sessions: before=%+v after=%+v", before, after) + } + c := r.command("claim", next, cs.Mutation{ClaimAssignment: &cs.ClaimAssignment{Ref: ref(next), Expect: next.Expect}}) + current := r.f.eval(c) + expectWorker(t, current, cs.OK) + beforeCell := r.f.a.cell() + r.f.eval(oldClaim) + r.f.eval(oldFinish) + if !reflect.DeepEqual(beforeCell, r.f.a.cell()) || !reflect.DeepEqual(current, r.f.eval(c)) { + t.Fatal("released requests interfered with the new assignment") + } + expectWorker(t, r.f.eval(r.finish(next, current)), cs.OK) + } + got := readRetryCounts(t, r.f.a) + allowance := uint32(0) + if event == "new-incarnation" { + allowance = 2 + } + if got.Live != r.base.Live+allowance || got.Retained != r.base.Retained+allowance { + t.Fatalf("released assignment left worker sessions: before=%+v after=%+v", r.base, got) + } + }) + } +} + +// r3.5 +func TestR35WorkerSessionReleaseIsDeterministicAndSnapshotPortable(t *testing.T) { + r := newAssignmentRetries(t) + v, _, claimed := r.start("original") + expectWorker(t, claimed, cs.OK) + finish := r.finish(v, claimed) + raw, err := cs.EncodeCommand(finish) + if err != nil { + t.Fatal(err) + } + r.f.a.meta.Bytes = 0 + for k, b := range r.f.a.rows { + r.f.a.meta.Bytes += cs.RecordBytes(k, b) + } + var first cs.Evaluation + for i := 0; i < 3; i++ { + got, err := cs.Evaluate(raw, r.f.a.meta, func(k cs.RecordKey) ([]byte, error) { return bytes.Clone(r.f.a.rows[k]), nil }) + if err != nil { + t.Fatal(err) + } + if i > 0 && !reflect.DeepEqual(first, got) { + t.Fatal("members disagree on release result or write order") + } + first = got + } + expectWorker(t, r.f.eval(finish), cs.OK) + keys := make([]cs.RecordKey, 0, len(r.f.a.rows)) + for k, b := range r.f.a.rows { + if b != nil { + keys = append(keys, k) + } + } + sort.Slice(keys, func(i, j int) bool { return keys[i].Namespace+"/"+keys[i].Key < keys[j].Namespace+"/"+keys[j].Key }) + var stream bytes.Buffer + for _, k := range keys { + if err := raftsnapshot.WriteRow(&stream, raftsnapshot.Row{Namespace: k.Namespace, Key: k.Key, Value: r.f.a.rows[k]}); err != nil { + t.Fatal(err) + } + } + restored := map[cs.RecordKey][]byte{} + for { + row, err := raftsnapshot.ReadRow(&stream) + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + restored[cs.RecordKey{Namespace: row.Namespace, Key: row.Key}] = row.Value + } + r.f.a.rows = restored + got := readRetryCounts(t, r.f.a) + if got.Live != r.base.Live || got.Retained != r.base.Retained || got.Bookkeeping > r.base.Bookkeeping+512 { + t.Fatalf("restored state retained completed worker sessions: before=%+v after=%+v", r.base, got) + } + expectWorker(t, r.f.eval(finish), cs.OK) +} diff --git a/internal/manual/chat/coordination-and-workers.md b/internal/manual/chat/coordination-and-workers.md index 6a94917e79..04b2b995b9 100644 --- a/internal/manual/chat/coordination-and-workers.md +++ b/internal/manual/chat/coordination-and-workers.md @@ -101,6 +101,8 @@ Worker setup and assignment execution are not available as a terminal command ye No. The worker saves its claim request before sending it and records an execution intent before invoking tools. A lost coordination reply can retry that same request, retaining its original fence. Once execution may have started, reopening keeps the records and reports `this work may be incomplete; its saved records have been kept`. It never invokes that attempt again automatically. A recorded finish can retry its coordination acknowledgment without running tools. +Finished, cancelled and reassigned worker attempts free their retry space for later work and your own leadership requests. Retrying an old assignment cannot start that work again. + Reassignment changes the run revision and requires a new conditional claim. The replacement can recover saved records, but the scripted adapter refuses automatic execution of a reassigned run. Missing or incomplete calls are not instructions to execute them. Cancellation prevents future claims; stopping a claimed assignment does not cancel an already running shell process. Another take changes the fence and blocks the displaced holder from publishing. Already downloaded bytes and uploaded frames stay retained. ## Why can an older computer list chats but not continue a chat that is still running? diff --git a/internal/syncsetup/worker.go b/internal/syncsetup/worker.go index b38e1e0ace..50076bebac 100644 --- a/internal/syncsetup/worker.go +++ b/internal/syncsetup/worker.go @@ -186,9 +186,7 @@ func (w WorkerAdapter) Run(ctx context.Context, a cs.Assignment, execute func(co return errors.Join(saveWorkerAttempt(path, attempt), saveWorkerAttempt(cellPath, attempt)) } func (w WorkerAdapter) command(kind string, ref cs.AssignmentRef, m cs.Mutation) (cs.Command, error) { - b, _ := cs.Encode(ref) - h := sha256.Sum256(b) - c := cs.Command{Schema: cs.SchemaVersion, Cluster: w.Credential.Certificate.Cluster, Actor: w.Credential.ID(), WorkerIncarnation: w.Incarnation, Session: "worker-" + kind + "-" + hex.EncodeToString(h[:]), Sequence: 1, Now: w.Now(), Change: m} + c := cs.Command{Schema: cs.SchemaVersion, Cluster: w.Credential.Certificate.Cluster, Actor: w.Credential.ID(), WorkerIncarnation: w.Incarnation, Session: cs.WorkerAssignmentSession(kind, ref), Sequence: 1, Now: w.Now(), Change: m} var err error c.RequestDigest, err = cs.Digest(c) if err != nil { diff --git a/relay/hosted/test/raft/assignment.test.js b/relay/hosted/test/raft/assignment.test.js index 8570419641..5dd8d88084 100644 --- a/relay/hosted/test/raft/assignment.test.js +++ b/relay/hosted/test/raft/assignment.test.js @@ -54,8 +54,8 @@ test('R18 R19 R21 assignment claims, delayed decisions and uploaded heads match const r=directoryEvaluator(runtime,cluster,8*1024*1024)(raw,(ns,key)=>base.find(r=>r.namespace===ns&&r.key===key)?.data??read(ns,key),{...app,size:app.size+(base.length?initialSize:0)}); results[i].push(decodeCanonical(dec.decode(r.result)));return {...r,writes:[...base,...r.writes]}; }; - const command=(change,{signer='owner',inc=0n,now=1000}={})=>{ - const actor=signer==='worker'?worker:signer, session='req-'+(++serial), sequence='1'; + const command=(change,{signer='owner',inc=0n,now=1000,session='req-'+(++serial)}={})=>{ + const actor=signer==='worker'?worker:signer, sequence='1'; const fields={cluster,actor,session,sequence,change,...(inc?{worker_incarnation:String(inc)}:{})}; const c={schema:1,...fields,request_digest:sha(bytes(fields)),now,signature:null}; c.signature=sign(null,Buffer.concat([Buffer.from('codeaf.directory.command.v1\0'),bytes(c)]),keys[signer].privateKey).toString('base64');return bytes(c); @@ -103,8 +103,25 @@ test('R18 R19 R21 assignment claims, delayed decisions and uploaded heads match await send({finish_assignment:{ref:ref(first),fence:'2',head:'result'}},{signer:'worker',inc:1n}); const parent=actors[leader()].store.read('state','cell/'+segment('parent')); const p=decodeCanonical(dec.decode(parent));assert.equal(p.head,'parent-head');assert.equal(p.lease.fence,'2'); + // r3.5 + const released={...spec('child-two'),id:'released-run'}; + await send({assign:released}); + const counts=a=>decodeCanonical(dec.decode(a.store.read('sessions','counts'))); + const beforeRelease=counts(actors[leader()]); + const scoped=(kind,change)=>command(change,{signer:'worker',inc:1n,session:'worker-'+kind+'-'+sha(bytes(ref(released)))}); + const releasedClaim=scoped('claim',{claim_assignment:{ref:ref(released),expect:released.expect}}); + const releaseClaimed=await submit(releasedClaim);assert.equal(releaseClaimed.code,'ok'); + assert.deepEqual(await submit(releasedClaim),releaseClaimed); + const releasedFinish=scoped('finish',{finish_assignment:{ref:ref(released),fence:releaseClaimed.view.cell.lease.fence,head:releaseClaimed.view.cell.head}}); + assert.equal((await submit(releasedFinish)).code,'ok'); + assert.equal((await submit(releasedFinish)).code,'ok'); + const releasedCell=actors[leader()].store.read('state','cell/'+segment(released.cell)); + assert.deepEqual(await submit(releasedClaim),releaseClaimed); + assert.deepEqual(actors[leader()].store.read('state','cell/'+segment(released.cell)),releasedCell); + for(const a of actors)assert.deepEqual(counts(a),beforeRelease); for(const a of actors)await a.checkpoint(); const reopened=(leader()+1)%3;await actors[reopened].close();actors[reopened]=new MemberActor(hosts[reopened],runtime,evaluator(reopened));accept(await actors[reopened].init(configs[reopened]));await pump(); + assert.deepEqual(counts(actors[reopened]),beforeRelease); assert.deepEqual(await submit(claim),claimed); assert.equal((await readRun()).assignment.status,'finished'); await send({revoke:{device:worker}});assert.equal((await readRun()).code,'revoked'); From ea0495cb1dab5fa78c370385cf2364713f447803 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 23:52:19 -0400 Subject: [PATCH 094/103] Batch native hold renewals on the shared thirty-second cycle. Sharing the existing cycle avoids one replicated renewal per presence ping. Each bounded batch still requires fresh evidence for the exact holder and fence, and receipts remain available only after durable commitment. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/changes/unreleased/raft-cluster.md | 1 + internal/directory/feed.go | 29 +-- internal/directory/renewal.go | 85 ++++++++ internal/directory/renewal_period_test.go | 118 +++++++++++ .../manual/chat/coordination-and-workers.md | 2 +- internal/relayserve/renewal_period_test.go | 195 ++++++++++++++++++ 6 files changed, 415 insertions(+), 15 deletions(-) create mode 100644 internal/directory/renewal.go create mode 100644 internal/directory/renewal_period_test.go create mode 100644 internal/relayserve/renewal_period_test.go diff --git a/docs/changes/unreleased/raft-cluster.md b/docs/changes/unreleased/raft-cluster.md index abf0324a16..1484eb18a1 100644 --- a/docs/changes/unreleased/raft-cluster.md +++ b/docs/changes/unreleased/raft-cluster.md @@ -11,6 +11,7 @@ invalidates: - "A majority was not required by the former directory writer. Losing two of the three current voters now refuses coordination and preserves saved chats instead of creating a writable minority." - "Caught-up open homes previously polled shared state. Idle homes now send zero HTTP requests, peer or forwarding frames, or client attempts after all voters apply the idle barrier; presence sweeps recur only with a viewer and fresh input requires a new majority response." - "Sleeping hosted groups previously retained their memory reservations indefinitely. Completed all-voter sleep now checkpoints and releases those members, and the next request recovers them through a fresh majority; busy admission pressure returns retryable HTTP 503." + - "Native holding connections previously committed a lease renewal on every 10-second presence ping. They now share bounded renewal batches every 30 seconds, preserving fresh exact-ownership evidence and committed receipts." - "Older clients could blindly force a live chat without its observed holder. Both relays now refuse that request with another device continued this chat first; releasing the chat or updating the client permits safe continuation." - "Legal older directory requests could be refused above the smaller coordination envelope. Both relays now retain the original 1 MiB HTTP limit, stage large signed bodies and commit complete metadata atomically, recovering interruptions without exposing partial changes or deleting retained records." - "Hosted directory translation previously recognized only the default path prefix. Configured prefixes, including multiple segments, now remain with the signed request so directory operations and computer enrollment work under personal leadership too." diff --git a/internal/directory/feed.go b/internal/directory/feed.go index c6ad22def6..0bfef460a0 100644 --- a/internal/directory/feed.go +++ b/internal/directory/feed.go @@ -62,10 +62,13 @@ type Watchable interface { type Feed struct { cur atomic.Pointer[Status] - mu sync.Mutex - cap int - subs map[*Sub]struct{} - closed chan struct{} + mu sync.Mutex + cap int + subs map[*Sub]struct{} + closed chan struct{} + renewMu sync.Mutex + renewAt int64 + renewPage []holdSample // clock is the directory's own clock, so a sign of life and a lease expiry // are read on one timeline (a test's fake clock moves both). @@ -176,8 +179,10 @@ func (f *Feed) SubscribeCommitted(device string, holds []Hold, beat time.Duratio if err != nil { return nil, err } + f.mu.Lock() s.committed = true s.renew = renew + f.mu.Unlock() if err = s.PingCommitted(); err != nil { s.leave() return nil, err @@ -312,20 +317,16 @@ func (s *Sub) Ping() { _ = s.PingCommitted() } // PingCommitted withholds a pong when a consensus renewal cannot commit. func (s *Sub) PingCommitted() error { at := s.f.clock().UnixMilli() - if s.evidence != nil || s.renew != nil { + if s.renew != nil { + if err := s.f.renewCommitted(s, at); err != nil { + return err + } + } else if s.evidence != nil { holds := make([]Hold, len(s.keys)) for i, k := range s.keys { holds[i] = Hold{Cell: k.cell, Fence: k.fence} } - if s.renew != nil { - receipts, err := s.renew(s.device, holds, at) - if err != nil { - return err - } - s.receiptMu.Lock() - s.receipts = receipts - s.receiptMu.Unlock() - } else if err := s.evidence(s.device, holds, at); err != nil { + if err := s.evidence(s.device, holds, at); err != nil { return err } } diff --git a/internal/directory/renewal.go b/internal/directory/renewal.go new file mode 100644 index 0000000000..f0ca1bce17 --- /dev/null +++ b/internal/directory/renewal.go @@ -0,0 +1,85 @@ +package directory + +import "github.com/Agent-Field/codeaf/internal/clusterstate" + +// A cycle retains one sample per admitted holding socket. Its immutable tuples +// and original renewal callback disappear when the bounded page is consumed. +type holdSample struct { + sub *Sub + at int64 + offset int +} + +// renewCommitted shares one period across the identity. Presence between cycles +// updates the socket's latest sample without retaining a history of pings. +func (f *Feed) renewCommitted(current *Sub, at int64) error { + f.renewMu.Lock() + defer f.renewMu.Unlock() + if len(current.keys) == 0 || at < f.renewAt { + return nil + } + if f.renewPage == nil { + f.mu.Lock() + for s := range f.subs { + if s.renew == nil || len(s.keys) == 0 { + continue + } + seen := s.alive.Load() + if s == current { + seen = at + } + f.renewPage = append(f.renewPage, holdSample{sub: s, at: seen}) + } + f.mu.Unlock() + } + remaining := clusterstate.MaxRenewHolds + var currentErr error + for len(f.renewPage) > 0 && remaining > 0 { + sample := &f.renewPage[0] + s := sample.sub + st := f.Status() + if !s.shut.Load() && !s.lapsed.Load() && !st.Stopped[s.device] && st.Rotation == nil && sample.at > 0 && sample.at <= at && at-sample.at <= clusterstate.EvidenceWindowMillis { + end := min(len(s.keys), sample.offset+remaining) + holds := make([]Hold, 0, end-sample.offset) + for _, key := range s.keys[sample.offset:end] { + holds = append(holds, Hold{Cell: key.cell, Fence: key.fence}) + } + remaining -= len(holds) + sample.offset = end + receipts, err := s.renew(s.device, holds, sample.at) + if err != nil { + if s == current { + currentErr = err + } else { + s.lapse() + } + } else { + s.receiptMu.Lock() + for _, receipt := range receipts { + found := false + for i, old := range s.receipts { + if old.Cell == receipt.Cell && old.Fence == receipt.Fence { + s.receipts[i] = receipt + found = true + break + } + } + if !found { + s.receipts = append(s.receipts, receipt) + } + } + s.receiptMu.Unlock() + if end < len(s.keys) { + continue + } + } + } + f.renewPage[0] = holdSample{} + f.renewPage = f.renewPage[1:] + } + if len(f.renewPage) == 0 { + f.renewPage = nil + f.renewAt = at + clusterstate.RenewEveryMillis + } + return currentErr +} diff --git a/internal/directory/renewal_period_test.go b/internal/directory/renewal_period_test.go new file mode 100644 index 0000000000..cb32ed9777 --- /dev/null +++ b/internal/directory/renewal_period_test.go @@ -0,0 +1,118 @@ +package directory + +import ( + "fmt" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/dirwatch" +) + +func TestR7_3CommittedWatchesShareBoundedCycle(t *testing.T) { + clock := &fakeNow{ms: 1_000_000} + f := NewFeed(clock.now) + defer f.Close() + var batches [][]Hold + var samples []int64 + renew := func(_ string, holds []Hold, at int64) ([]dirwatch.HoldReceipt, error) { + batches = append(batches, append([]Hold(nil), holds...)) + samples = append(samples, at) + if len(holds) > cs.MaxRenewHolds { + t.Fatal("r7.3 renewal exceeded batch bound", len(holds)) + } + var receipts []dirwatch.HoldReceipt + for _, h := range holds { + receipts = append(receipts, dirwatch.HoldReceipt{Cell: h.Cell, Fence: h.Fence, Index: 1, At: at, Until: at + cs.LeaseTTLMillis}) + } + return receipts, nil + } + var subs []*Sub + for socket := range cs.MaxRenewHolds/MaxHolds + 1 { + holds := make([]Hold, MaxHolds) + for i := range holds { + holds[i] = Hold{Cell: fmt.Sprintf("chat-%d-%d", socket, i), Fence: 1} + } + s, err := f.SubscribeCommitted("device", holds, DefaultBeat, renew) + if err != nil { + t.Fatal(err) + } + defer s.Close() + s.takeReceipts() + subs = append(subs, s) + } + for period := range 6 { + batches, samples = nil, nil + for beat := 1; beat <= int(cs.RenewEveryMillis/cs.PresenceEveryMillis); beat++ { + clock.ms += cs.PresenceEveryMillis + before := len(batches) + if err := subs[0].PingCommitted(); err != nil { + t.Fatal(err) + } + items := 0 + for _, batch := range batches[before:] { + items += len(batch) + } + if items > cs.MaxRenewHolds { + t.Fatalf("r7.3 one turn renewed %d holds, cap %d", items, cs.MaxRenewHolds) + } + for _, s := range subs[1:] { + if err := s.PingCommitted(); err != nil { + t.Fatal(err) + } + } + if beat*int(cs.PresenceEveryMillis) < int(cs.RenewEveryMillis) && len(batches) != 0 { + t.Fatalf("r7.3 period %d: %d renewals before the shared deadline", period, len(batches)) + } + } + seen := map[string]bool{} + for i, batch := range batches { + if clock.ms-samples[i] > cs.EvidenceWindowMillis { + t.Fatal("r7.3 stale sample", samples[i]) + } + for _, h := range batch { + if seen[h.Cell] { + t.Fatal("r7.3 duplicate renewal in a period", h) + } + seen[h.Cell] = true + } + } + if len(seen) != len(subs)*MaxHolds { + t.Fatalf("r7.3 later sockets starved: renewed=%d, want %d", len(seen), len(subs)*MaxHolds) + } + for _, s := range subs { + if receipts := s.takeReceipts(); len(receipts) != MaxHolds { + t.Fatalf("r7.3 pending receipts=%d, want %d", len(receipts), MaxHolds) + } + } + } + for _, s := range subs[1:] { + s.Close() + } + batches = nil + clock.ms += cs.RenewEveryMillis + if err := subs[0].PingCommitted(); err != nil { + t.Fatal(err) + } + if len(batches) != 1 || len(batches[0]) != MaxHolds { + t.Fatal("r7.3 closed subscriptions retained evidence", batches) + } + clock.ms += cs.EvidenceWindowMillis + 1 + other, err := f.SubscribeCommitted("other", []Hold{{Cell: "other-chat", Fence: 2}}, time.Second, renew) + if err != nil { + t.Fatal(err) + } + defer other.Close() + batches = nil + clock.ms += cs.RenewEveryMillis + if err = other.PingCommitted(); err != nil { + t.Fatal(err) + } + for _, batch := range batches { + for _, h := range batch { + if h.Cell != "other-chat" { + t.Fatal("r7.3 silent socket retained fresh evidence", h) + } + } + } +} diff --git a/internal/manual/chat/coordination-and-workers.md b/internal/manual/chat/coordination-and-workers.md index 04b2b995b9..0e97503d79 100644 --- a/internal/manual/chat/coordination-and-workers.md +++ b/internal/manual/chat/coordination-and-workers.md @@ -70,7 +70,7 @@ With its three anchors voting, or with a joined computer leading, a hosted group A compatibility events viewer pings every 10 seconds. The platform answers without running coordination. While somebody watches presence, the relay checks at the next 25-second silence deadline so a stopped computer appears offline within 30 seconds. With two viewers pinging together, the simulated ten-minute measurement uses 29 short sweeps; those sweeps make no member calls or data writes. With no events viewer, no presence sweep recurs. Home follows the local learner once joined, so its ordinary display needs no remote presence viewer. -A connection naming held chats still renews through committed changes every 30 seconds while fresh hold evidence is available. Presence alone never renews a chat. +Connections naming held chats share a renewal cycle every 30 seconds on both relays while fresh hold evidence is available. Each cycle uses bounded batches; the 10-second presence pings between cycles do not commit another renewal. Presence alone never renews a chat. ## Hosted relay memory capacity and retry diff --git a/internal/relayserve/renewal_period_test.go b/internal/relayserve/renewal_period_test.go new file mode 100644 index 0000000000..ee106bdd5e --- /dev/null +++ b/internal/relayserve/renewal_period_test.go @@ -0,0 +1,195 @@ +package relayserve_test + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + cs "github.com/Agent-Field/codeaf/internal/clusterstate" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/dirwatch" + "github.com/Agent-Field/codeaf/internal/raftcore" + "github.com/Agent-Field/codeaf/internal/raftstore" + pb "go.etcd.io/raft/v3/raftpb" +) + +func renewalEntries(t *testing.T, r *rig, identity string) []cs.RenewHolds { + t.Helper() + path := filepath.Join(r.store, "raft", identity) + raw, err := os.ReadFile(filepath.Join(path, "provision.json")) + if err != nil { + t.Fatal(err) + } + var provision struct{ Cluster string } + if err = json.Unmarshal(raw, &provision); err != nil { + t.Fatal(err) + } + store, err := raftstore.Open(filepath.Join(path, "1.db"), raftstore.Identity{ClusterUUID: provision.Cluster, NodeID: 1, Protocol: raftcore.ProtocolVersion}) + if err != nil { + t.Fatal(err) + } + defer store.Close() + recovered, err := store.Recover(bg) + if err != nil { + t.Fatal(err) + } + if recovered.Core.Snapshot.GetMetadata().GetIndex() > 1 { + t.Fatal("renewal count requires the complete retained log") + } + var renewals []cs.RenewHolds + for _, entry := range recovered.Core.Entries { + if entry.GetIndex() > recovered.Application.Applied || entry.GetType() != pb.EntryNormal || len(entry.GetData()) == 0 { + continue + } + command, err := cs.DecodeCommand(entry.GetData()) + if err != nil { + t.Fatal(err) + } + if command.Change.RenewHolds != nil { + renewals = append(renewals, *command.Change.RenewHolds) + } + } + return renewals +} + +func pingRenewalWatch(t *testing.T, stream dirwatch.Stream) []dirwatch.HoldReceipt { + t.Helper() + ctx, cancel := context.WithTimeout(bg, 10*time.Second) + defer cancel() + if err := stream.Ping(ctx); err != nil { + t.Fatal(err) + } + var receipts []dirwatch.HoldReceipt + for { + frame, err := stream.Next(ctx) + if err != nil { + t.Fatal(err) + } + receipts = append(receipts, frame.Holds...) + if frame.Pong { + return receipts + } + } +} + +func TestR7_1NativeWatchRenewalPeriod(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + created, err := r.dir(a).Create(bg, cell, directory.CellInit{Head: head1}) + if err != nil { + t.Fatal(err) + } + start := r.clock.Now().UnixMilli() + stream, err := r.dir(a).WatchHolding(bg, []directory.Hold{{Cell: cell, Fence: created.Cell.Lease.Fence}}) + if err != nil { + t.Fatal(err) + } + defer stream.Close() + for elapsed := cs.PresenceEveryMillis; elapsed <= 6*cs.RenewEveryMillis; elapsed += cs.PresenceEveryMillis { + r.clock.Advance(time.Duration(cs.PresenceEveryMillis) * time.Millisecond) + receipts := pingRenewalWatch(t, stream) + view, err := r.dir(a).Cell(bg, cell) + if err != nil || view.Cell.Lease.Device != a.dev.ID() || view.Cell.Lease.Fence != 1 || view.Cell.Lease.Expires <= view.Now { + t.Fatalf("r7.1 lease lapsed at +%d: %+v, %v", elapsed, view, err) + } + entries := renewalEntries(t, r, id.ID()) + if got, want := len(entries), int(elapsed/cs.RenewEveryMillis); got != want { + t.Errorf("r7.1 +%d ms: committed renewals=%d, want %d", elapsed, got, want) + } + wantExpiry := start + elapsed/cs.RenewEveryMillis*cs.RenewEveryMillis + cs.LeaseTTLMillis + if view.Cell.Lease.Expires != wantExpiry { + t.Errorf("r7.1 +%d ms: expiry=%d, want %d", elapsed, view.Cell.Lease.Expires, wantExpiry) + } + if elapsed%cs.RenewEveryMillis == 0 && (len(receipts) != 1 || receipts[0].Index == 0 || receipts[0].Until != view.Cell.Lease.Expires) { + t.Errorf("r7.1 missing exact committed receipt: %+v", receipts) + } + } + t.Logf("r7.1 six renewal periods, 18 presence pings, %d committed renewals", len(renewalEntries(t, r, id.ID()))) +} + +func TestR7_2NativeWatchNeedsFreshFenceEvidence(t *testing.T) { + r := newRig(t) + _, a := newIdentity(t) + if _, err := r.dir(a).Create(bg, cell, directory.CellInit{Head: head1}); err != nil { + t.Fatal(err) + } + stream, err := r.dir(a).WatchHolding(bg, []directory.Hold{{Cell: cell, Fence: 1}}) + if err != nil { + t.Fatal(err) + } + defer stream.Close() + if err = r.dir(a).Release(bg, cell, 1); err != nil { + t.Fatal(err) + } + r.clock.Advance(time.Duration(cs.PresenceEveryMillis) * time.Millisecond) + if got := pingRenewalWatch(t, stream); len(got) != 0 { + t.Fatal("r7.2 released hold received a renewal", got) + } + view, err := r.dir(a).Cell(bg, cell) + if err != nil || view.Cell.Lease.Expires != 0 { + t.Fatal("r7.2 released hold revived", view, err) + } + view, err = r.dir(a).Acquire(bg, cell, directory.AcquireOpts{}) + if err != nil || view.Cell.Lease.Fence != 2 { + t.Fatal("r7.2 new fence", view, err) + } + expires := view.Cell.Lease.Expires + for range 4 { + r.clock.Advance(time.Duration(cs.PresenceEveryMillis) * time.Millisecond) + if got := pingRenewalWatch(t, stream); len(got) != 0 { + t.Fatal("r7.2 old socket renewed a new fence", got) + } + } + view, err = r.dir(a).Cell(bg, cell) + if err != nil || view.Cell.Lease.Expires != expires { + t.Fatal("r7.2 old evidence extended new fence", view, err) + } +} + +func TestR7_3NativeSocketsShareRenewalPeriod(t *testing.T) { + r := newRig(t) + id, a := newIdentity(t) + var streams []dirwatch.Stream + var cells []string + for i := range 3 { + name := cell + string(rune('a'+i)) + if _, err := r.dir(a).Create(bg, name, directory.CellInit{Head: head1}); err != nil { + t.Fatal(err) + } + stream, err := r.dir(a).WatchHolding(bg, []directory.Hold{{Cell: name, Fence: 1}}) + if err != nil { + t.Fatal(err) + } + defer stream.Close() + streams = append(streams, stream) + cells = append(cells, name) + } + for period := 1; period <= 5; period++ { + for beat := 1; beat <= int(cs.RenewEveryMillis/cs.PresenceEveryMillis); beat++ { + r.clock.Advance(time.Duration(cs.PresenceEveryMillis) * time.Millisecond) + pingRenewalWatch(t, streams[0]) + if beat == int(cs.RenewEveryMillis/cs.PresenceEveryMillis) { + if got, want := len(renewalEntries(t, r, id.ID())), period*len(streams); got != want { + t.Errorf("r7.3 first ping at shared deadline: commits=%d, want %d", got, want) + } + } + for _, stream := range streams[1:] { + pingRenewalWatch(t, stream) + } + } + for _, name := range cells { + view, err := r.dir(a).Cell(bg, name) + if err != nil || view.Cell.Lease.Device != a.dev.ID() || view.Cell.Lease.Expires <= view.Now { + t.Fatal("r7.3 held socket lease lapsed", name, view, err) + } + } + } + for _, entry := range renewalEntries(t, r, id.ID()) { + if len(entry.Items) > cs.MaxRenewHolds { + t.Fatal("r7.3 committed batch exceeded cap", len(entry.Items)) + } + } +} From 9be594e7ae17ef02ba05195ae25700a76474da7a Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 23:56:11 -0400 Subject: [PATCH 095/103] Measure the coordination core's binary weight on the current dev base. The coordination core now adds 3,379,200 bytes to the stripped linux/amd64 binary: 75,755,785 bytes on the current dev base and 79,134,985 with the core linked. The size cap moves by the same amount, from dev's 75,811,848 to 79,191,048, so the build keeps the 56,063 bytes of headroom the base has. Co-Authored-By: Claude Opus 5.5 (1M context) --- PERF.md | 6 +++--- SIZE-BUDGET | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/PERF.md b/PERF.md index cf48fceda5..01bd6d4275 100644 --- a/PERF.md +++ b/PERF.md @@ -124,6 +124,6 @@ the feature and its measurements; keep `SIZE-BUDGET` explicit and reviewable. Every build links the Raft coordination core: the modified Raft copy, protobuf, the signed state machine and its SQLite snapshot host. The relay and devices always coordinate through it, so no build leaves it out. On linux/amd64 it adds -3,260,416 bytes to the stripped binary (75,071,753 to 78,332,169 on the same dev -base), and the cap moves by the same amount, from 75,811,848 to 79,072,264, so -the 740,095 bytes of headroom the base had are kept. +3,379,200 bytes to the stripped binary (75,755,785 to 79,134,985 on the same dev +base), and the cap moves by the same amount, from 75,811,848 to 79,191,048, so +the 56,063 bytes of headroom the base had are kept. diff --git a/SIZE-BUDGET b/SIZE-BUDGET index 3420c0cc28..4a76e6f9f4 100644 --- a/SIZE-BUDGET +++ b/SIZE-BUDGET @@ -1 +1 @@ -79072264 +79191048 From c50fc3f1b93c27721be819d094e6e981455f569c Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 23:56:55 -0400 Subject: [PATCH 096/103] Keep the coordination change entry to what changed against dev. The entry lists what a reader of dev now believes wrongly. Problems that existed only while this feature was being built never reached dev, so their lines go, and the upgrade line now says what an operator sees when an older relay is started on an upgraded store. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/changes/unreleased/raft-cluster.md | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/docs/changes/unreleased/raft-cluster.md b/docs/changes/unreleased/raft-cluster.md index 1484eb18a1..8c1939fda7 100644 --- a/docs/changes/unreleased/raft-cluster.md +++ b/docs/changes/unreleased/raft-cluster.md @@ -10,19 +10,12 @@ invalidates: - "Coordination depended on the relay's single writer. Two remaining anchors can now elect a fallback and restore the third after a personal leader disappears, targeting about 65 seconds in healthy conditions; a returning computer remains a learner and never automatically regains leadership." - "A majority was not required by the former directory writer. Losing two of the three current voters now refuses coordination and preserves saved chats instead of creating a writable minority." - "Caught-up open homes previously polled shared state. Idle homes now send zero HTTP requests, peer or forwarding frames, or client attempts after all voters apply the idle barrier; presence sweeps recur only with a viewer and fresh input requires a new majority response." - - "Sleeping hosted groups previously retained their memory reservations indefinitely. Completed all-voter sleep now checkpoints and releases those members, and the next request recovers them through a fresh majority; busy admission pressure returns retryable HTTP 503." - - "Native holding connections previously committed a lease renewal on every 10-second presence ping. They now share bounded renewal batches every 30 seconds, preserving fresh exact-ownership evidence and committed receipts." - "Older clients could blindly force a live chat without its observed holder. Both relays now refuse that request with another device continued this chat first; releasing the chat or updating the client permits safe continuation." - "Legal older directory requests could be refused above the smaller coordination envelope. Both relays now retain the original 1 MiB HTTP limit, stage large signed bodies and commit complete metadata atomically, recovering interruptions without exposing partial changes or deleting retained records." - - "Hosted directory translation previously recognized only the default path prefix. Configured prefixes, including multiple segments, now remain with the signed request so directory operations and computer enrollment work under personal leadership too." - - "The native relay could reject a signed create or publish with an explicit empty frame plan. Empty arrays now have the same meaning as omitted or null frame plans while the request remains bound to its original actor and operation." - - "Retained rows from legal older requests could pause upgrades after the relay added fields, and full directories could fail initialization or refusal persistence. Both relays now preserve these rows losslessly and reserve bounded bookkeeping independently of the measured content allowance." - "Workers previously needed personal-device trust for shared access. Delegated workers now have expiring selected-chat grants and conditional assignment claims; they cannot approve pairing, read the identity vault, rotate it, enroll devices or lead, and terminal worker setup is not available yet." - - "Completed worker assignments previously accumulated retry sessions and could block later work or personal leadership requests. Assignment retry records now release on finish, cancellation or reassignment while retained assignments prevent old requests from starting work again." - "Reopening retained work could compare its branch fence with the original chat and falsely refuse tools. Ownership checks stay with the record each window opened, and replacing a displaced drive settles its previous writer before starting another. A failed final save remains failed for every close caller." - "Deferred local durability could resolve a fallback daemon against a different current home. Both engine routes now retain the home and engine selected at capture." - - "A single directory database was formerly the live source. First use now freezes and imports it into the three anchors while retaining the source, and the one-way upgrade cannot return to that writer after cluster changes are acknowledged." - - "The migration gate alone did not stop older relay code from writing the retained source. Both relays now install durable database write guards with the freeze, so an older relay refuses directory startup or writes with codeaf: directory upgraded; use a current relay build. Running an older relay after the upgrade is not supported." + - "A single directory database was formerly the live source. First use now freezes and imports it into the three anchors while retaining the source. The upgrade is one-way: an older relay started on an upgraded store refuses to start or write with codeaf: directory upgraded; use a current relay build." - "Self-hosted identities were described as limited only by disk space. Directory content now has an 8 MiB default allowance, with larger imports retaining their measured allowance and bounded bookkeeping having separate room; capacity refuses content growth without deleting saved data." --- From a6ba01a986a0aa5e0c4e7117b8e75550302384dd Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Sun, 25 Oct 2026 23:58:50 -0400 Subject: [PATCH 097/103] Number the coordination change entry. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/changes/unreleased/{raft-cluster.md => 1950-raft-cluster.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/changes/unreleased/{raft-cluster.md => 1950-raft-cluster.md} (100%) diff --git a/docs/changes/unreleased/raft-cluster.md b/docs/changes/unreleased/1950-raft-cluster.md similarity index 100% rename from docs/changes/unreleased/raft-cluster.md rename to docs/changes/unreleased/1950-raft-cluster.md From 67a369640c40b1cea013b8090def061aec67293c Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 26 Oct 2026 00:06:29 -0400 Subject: [PATCH 098/103] Give the coordination change entry its pull request number. The entry was renamed for this pull request, but its pr field still said 0, which the change-entry check rejects. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/changes/unreleased/1950-raft-cluster.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changes/unreleased/1950-raft-cluster.md b/docs/changes/unreleased/1950-raft-cluster.md index 8c1939fda7..3aa5097597 100644 --- a/docs/changes/unreleased/1950-raft-cluster.md +++ b/docs/changes/unreleased/1950-raft-cluster.md @@ -1,7 +1,7 @@ --- kind: changed title: Relay coordination uses three voters and joined computers can lead -pr: 0 +pr: 1950 surface: [chat, remote, build, docs] invalidates: - "One relay directory writer previously decided each identity's changes. The hosted relay and codeaf relay --store now use a three-voter Raft group per identity, with majority commitment and fresh authority reads." From 63a7d292b5613718b3d723be2d929670bacd62f2 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 26 Oct 2026 01:57:20 -0400 Subject: [PATCH 099/103] Measure the hosted learner's command boundary on the relay's own command. A hosted relay under a prefix commits that prefix beside the signed URI, and members now route the proof by it. The maximum-command proof captured its request on the client without the prefix, so its size probe could no longer translate the request at all. It now carries the request's prefix, measuring exactly the command the relay commits; the boundary and delivery assertions are unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/syncsetup/member_hosted_learner_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/syncsetup/member_hosted_learner_test.go b/internal/syncsetup/member_hosted_learner_test.go index ff5129866a..98c40cf1f3 100644 --- a/internal/syncsetup/member_hosted_learner_test.go +++ b/internal/syncsetup/member_hosted_learner_test.go @@ -297,6 +297,9 @@ func TestR9R27HostedLearnerMaximumCommand(t *testing.T) { reqsign.Sign(req, body, deviceSigner{a.Identity, a.Device}, a.now()) proof, e := reqsign.Capture(req, body) mustNil(t, e) + // The hosted relay commits its configured prefix beside the signed URI, + // so the boundary is measured on the command the relay itself commits. + proof.BasePath = strings.TrimSuffix(req.URL.Path, "/v1/dir/devices/"+a.Device.ID()) command, e := cs.V1Command(status.Cluster, a.now().UnixMilli(), proof) mustNil(t, e) raw, e := cs.EncodeCommand(command) From 8aa842330128880402c069d3f3d79e00c5471264 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 26 Oct 2026 06:19:46 -0400 Subject: [PATCH 100/103] Show a joined computer as online only while its relay connection is open. A joined home's Home took presence from its local learner, which counted any computer holding an unexpired chat lease as online. A computer that stopped without goodbye therefore stayed online until its 90-second lease lapsed, and the other computer could not offer to continue its chat for over a minute. Home now takes online and offline only from the relay watch, with the usual fifteen-second offline debounce, while versions, device news and directory reads still come from the local learner. Lease expiry stays the authority for ownership decisions and never for the display. New tests stop the holding process without goodbye and require the offer within 30 seconds while the lease is still valid, on the native relay and the hosted Worker. The open-home idle tests now allow Home's watch socket its platform ping and pong, bounded per ten minutes from the ping interval, and still require zero HTTP requests, client attempts, peer and forward frames, recurring alarms and presence-driven coordination wakes. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/syncsetup/leadership_native_test.go | 35 ++- internal/syncsetup/member_client_test.go | 89 +++++-- internal/syncsetup/member_display.go | 227 +++++++++--------- .../member_hosted_leader_quiet_test.go | 11 +- .../syncsetup/member_hosted_quiet_test.go | 75 +++++- internal/syncsetup/presence_c31_test.go | 170 +++++++++++++ internal/syncsetup/quiet.go | 13 +- relay/hosted/test/raft/hosted-c31-go.test.js | 30 +++ 8 files changed, 495 insertions(+), 155 deletions(-) create mode 100644 internal/syncsetup/presence_c31_test.go create mode 100644 relay/hosted/test/raft/hosted-c31-go.test.js diff --git a/internal/syncsetup/leadership_native_test.go b/internal/syncsetup/leadership_native_test.go index d8d0712ff6..6e4a58954b 100644 --- a/internal/syncsetup/leadership_native_test.go +++ b/internal/syncsetup/leadership_native_test.go @@ -308,6 +308,9 @@ func TestR6R7R23R26PersonalLeaderWholeHomeIdle(t *testing.T) { streams = append(streams, stream) defer stream.Close() } + for _, follower := range followers { + c31AwaitFeed(t, follower, requestTimeout, func(st dirwatch.State) bool { return st.Up }) + } round := func() { for _, tick := range ticks { tick() @@ -318,6 +321,14 @@ func TestR6R7R23R26PersonalLeaderWholeHomeIdle(t *testing.T) { } for tick := 0; tick < 24; tick++ { r.clock.Advance(DefaultInterval) + if time.Duration(tick+1)*DefaultInterval%dirwatch.KeepEvery == 0 { + for _, stream := range streams { + pingHomeStream(t, stream) + } + for _, follower := range followers { + follower.Probe() + } + } round() } attempts := func() uint64 { @@ -327,18 +338,14 @@ func TestR6R7R23R26PersonalLeaderWholeHomeIdle(t *testing.T) { } return n } + watchBefore := r.watchIn.count.Load() + r.watchOut.count.Load() httpBefore, framesBefore, attemptBefore := r.requests.Load(), r.in.count.Load()+r.out.count.Load(), attempts() for tick := 0; tick < 120; tick++ { r.clock.Advance(DefaultInterval) round() for i, screen := range screens { - mustNil(t, streams[i].Ping(t.Context())) - for { - frame, err := streams[i].Next(t.Context()) - mustNil(t, err) - if frame.Pong { - break - } + if time.Duration(tick+1)*DefaultInterval%dirwatch.KeepEvery == 0 { + pingHomeStream(t, streams[i]) } _, _, err := screen.RowsAt(t.Context()) mustNil(t, err) @@ -346,7 +353,9 @@ func TestR6R7R23R26PersonalLeaderWholeHomeIdle(t *testing.T) { mustNil(t, err) _, err = screen.FleetSize(t.Context()) mustNil(t, err) - followers[i].Probe() + if time.Duration(tick+1)*DefaultInterval%dirwatch.KeepEvery == 0 { + followers[i].Probe() + } } } gotHTTP, gotFrames, gotAttempts := r.requests.Load()-httpBefore, r.in.count.Load()+r.out.count.Load()-framesBefore, attempts()-attemptBefore @@ -354,6 +363,16 @@ func TestR6R7R23R26PersonalLeaderWholeHomeIdle(t *testing.T) { if gotHTTP != 0 || gotFrames != 0 || gotAttempts != 0 { t.Fatal("R23/R26 whole idle Home generated traffic") } + // Advisory presence permits 488 watch frames with two Homes or 244 with + // one: two sockets per Home times two frames per KeepEvery in ten minutes, + // plus two initial snapshot frames per socket. C31 take-back after a + // computer closes its lid depends on this advisory connection. + watchBudget := uint64(2*len(live)) * uint64(2*(10*time.Minute/dirwatch.KeepEvery)+2) + watchFrames := r.watchIn.count.Load() + r.watchOut.count.Load() - watchBefore + t.Logf("Home watch frames=%d, ten-minute bound=%d", watchFrames, watchBudget) + if watchFrames > watchBudget { + t.Fatal("Home presence exceeded the ten-minute watch frame budget") + } _, err := b.Dir.List(t.Context()) mustNil(t, err) mustNil(t, b.Dir.PutDevice(t.Context(), b.Device.ID(), directory.Device{V: 1, Name: "c2VhbGVk"})) diff --git a/internal/syncsetup/member_client_test.go b/internal/syncsetup/member_client_test.go index a0b1157223..214a81db85 100644 --- a/internal/syncsetup/member_client_test.go +++ b/internal/syncsetup/member_client_test.go @@ -115,17 +115,18 @@ func (w peerCountWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { } type memberRig struct { - departures chan struct{} - parkExchanges atomic.Bool - storage string - t *testing.T - clock *directorytest.FakeClock - svc *relayserve.Service - server *httptest.Server - requests atomic.Uint64 - in, out peerFrameCounter - homes []*Sync - interrupt atomic.Bool + departures chan struct{} + parkExchanges atomic.Bool + storage string + t *testing.T + clock *directorytest.FakeClock + svc *relayserve.Service + server *httptest.Server + requests atomic.Uint64 + in, out peerFrameCounter + watchIn, watchOut peerFrameCounter + homes []*Sync + interrupt atomic.Bool } func newMemberRig(t *testing.T) *memberRig { @@ -170,7 +171,11 @@ func newMemberRig(t *testing.T) *memberRig { return } if strings.EqualFold(q.Header.Get("Upgrade"), "websocket") { - r.svc.Handler.ServeHTTP(peerCountWriter{w, &r.in, &r.out}, q) + in, out := &r.in, &r.out + if q.URL.Path == "/v1/dir/watch" { + in, out = &r.watchIn, &r.watchOut + } + r.svc.Handler.ServeHTTP(peerCountWriter{w, in, out}, q) } else { r.svc.Handler.ServeHTTP(w, q) } @@ -446,11 +451,6 @@ func TestR26R27C30OpenLearnerIdleAndStateWake(t *testing.T) { settle(a) settle(b) } - for tick := 0; tick < 20; tick++ { - r.clock.Advance(DefaultInterval) - settle(a) - settle(b) - } // Exercise the same owner used by the real terminal during the entire // measured window. Host ticks advance explicitly; no wall sleeps are used. tickHome := func(s *Sync) func() { @@ -493,22 +493,34 @@ func TestR26R27C30OpenLearnerIdleAndStateWake(t *testing.T) { defer stream.Close() streams = append(streams, stream) } + c31AwaitFeed(t, followA, requestTimeout, func(st dirwatch.State) bool { return st.Up }) + c31AwaitFeed(t, followB, requestTimeout, func(st dirwatch.State) bool { return st.Up }) countAttempts := func() uint64 { return a.relayAttempts.Load() + a.clusterFrames.Load() + b.relayAttempts.Load() + b.clusterFrames.Load() + screenA.relayAttempts.Load() + screenA.clusterFrames.Load() + screenB.relayAttempts.Load() + screenB.clusterFrames.Load() } + // Settle socket admission before measuring idle, and service the real + // presence sockets at their advertised cadence as the relay clock advances. + for tick := 0; tick < 20; tick++ { + r.clock.Advance(DefaultInterval) + if time.Duration(tick+1)*DefaultInterval%dirwatch.KeepEvery == 0 { + for _, stream := range streams { + pingHomeStream(t, stream) + } + followA.Probe() + followB.Probe() + } + settle(a) + settle(b) + } + watchBefore := r.watchIn.count.Load() + r.watchOut.count.Load() requests, frames, attempts := r.requests.Load(), r.in.count.Load()+r.out.count.Load(), countAttempts() for tick := 0; tick < 120; tick++ { r.clock.Advance(DefaultInterval) tickA() tickB() for i, screen := range []*Sync{screenA, screenB} { - mustNil(t, streams[i].Ping(t.Context())) - for { - frame, err := streams[i].Next(t.Context()) - mustNil(t, err) - if frame.Pong { - break - } + if time.Duration(tick+1)*DefaultInterval%dirwatch.KeepEvery == 0 { + pingHomeStream(t, streams[i]) } _, _, err := screen.RowsAt(t.Context()) mustNil(t, err) @@ -517,8 +529,10 @@ func TestR26R27C30OpenLearnerIdleAndStateWake(t *testing.T) { _, err = screen.FleetSize(t.Context()) mustNil(t, err) } - followA.Probe() - followB.Probe() + if time.Duration(tick+1)*DefaultInterval%dirwatch.KeepEvery == 0 { + followA.Probe() + followB.Probe() + } } gotHTTP, gotFrames := r.requests.Load()-requests, r.in.count.Load()+r.out.count.Load()-frames gotAttempts := countAttempts() - attempts @@ -526,6 +540,17 @@ func TestR26R27C30OpenLearnerIdleAndStateWake(t *testing.T) { if gotHTTP != 0 || gotFrames != 0 || gotAttempts != 0 { t.Fatal("R27 idle home generated traffic") } + // Advisory presence permits at most 488 watch frames in ten minutes: four + // sockets times two frames times (ten minutes / KeepEvery), plus two initial + // snapshot frames per socket. C31 take-back after a computer closes its lid + // depends on keeping this advisory socket alive. + const watchSockets = 4 + const watchBudget = watchSockets * (2*(10*time.Minute/dirwatch.KeepEvery) + 2) + watchFrames := r.watchIn.count.Load() + r.watchOut.count.Load() - watchBefore + t.Logf("Home watch frames=%d, ten-minute bound=%d", watchFrames, watchBudget) + if watchFrames > uint64(watchBudget) { + t.Fatal("Home presence exceeded the ten-minute watch frame budget") + } // The first public read after idle and the next state write retain ordinary // ReadIndex/proposal authority. Local learner state is only a change signal. _, e := b.Dir.List(t.Context()) @@ -918,3 +943,15 @@ func TestR9R23LocalHomeFollowerSharesTheOwnedLearner(t *testing.T) { t.Fatal("R23 local follower sent after off") } } + +func pingHomeStream(t *testing.T, stream dirwatch.Stream) { + t.Helper() + mustNil(t, stream.Ping(t.Context())) + for { + frame, err := stream.Next(t.Context()) + mustNil(t, err) + if frame.Pong { + return + } + } +} diff --git a/internal/syncsetup/member_display.go b/internal/syncsetup/member_display.go index ee3bd9eba7..6c8afd7e8b 100644 --- a/internal/syncsetup/member_display.go +++ b/internal/syncsetup/member_display.go @@ -3,12 +3,12 @@ package syncsetup import ( "context" "errors" - "github.com/Agent-Field/codeaf/internal/identity" "sort" "sync" "github.com/Agent-Field/codeaf/internal/directory" "github.com/Agent-Field/codeaf/internal/dirwatch" + "github.com/Agent-Field/codeaf/internal/identity" "github.com/Agent-Field/codeaf/internal/raftdirectory" ) @@ -67,125 +67,106 @@ func (s *Sync) displayListing(ctx context.Context) (directory.Listing, bool, err return l, true, err } -var errMemberDisplayChanged = errors.New("the local directory follower changed") - -// switchingDisplay ends the old advisory socket at its next use once the -// terminal's learner is available. Its replacement is entirely local. -type switchingDisplay struct { - dirwatch.Stream - owner *Sync -} - -func (s switchingDisplay) Ping(ctx context.Context) error { - if !s.owner.memberAllowed() { - s.Stream.Close() - return ErrQuiet - } - if s.owner.memberAllowed() && s.owner.displayMember() != nil { - s.Stream.Close() - return errMemberDisplayChanged - } - s.owner.clusterFrames.Add(1) - return s.Stream.Ping(ctx) -} - +// followStream keeps advisory presence on the relay watch even while Home's +// directory reads use the local learner. A held lease cannot show that its +// computer is connected; only the relay's watch sockets supply that evidence. func (s *Sync) followStream(ctx context.Context, w directory.Watcher) (dirwatch.Stream, error) { if !s.memberAllowed() { return nil, ErrQuiet } - if m := s.displayMember(); m != nil { - m.mu.Lock() - valid := !m.closed && m.failure == nil - m.mu.Unlock() - if valid { - return &localDisplayStream{owner: s, member: m, pong: make(chan struct{}, 1), closed: make(chan struct{}), devices: map[string]directory.Device{}, online: map[string]bool{}}, nil - } - } stream, err := w.Watch(ctx) if err != nil { return nil, err } - return switchingDisplay{Stream: stream, owner: s}, nil + ctx, cancel := context.WithCancel(ctx) + display := &memberDisplayStream{ + Stream: stream, owner: s, cancel: cancel, closed: ctx.Done(), replies: make(chan displayReply), + } + // Capture the initial roster before returning, so a device committed after + // this open is still announced even if the first read happens later. + if _, _, err := display.refresh(ctx); err != nil { + display.Close() + return nil, err + } + go display.readRelay(ctx) + return quietStream{Stream: display, home: s.Home, owner: s}, nil } -type localDisplayStream struct { +var errMemberDisplayChanged = errors.New("the local directory follower changed") + +type displayReply struct { + frame dirwatch.Frame + err error +} + +// memberDisplayStream combines committed learner changes with the relay's +// presence and pongs. Local state never supplies online evidence or a pong. +type memberDisplayStream struct { + dirwatch.Stream owner *Sync - member *personalMember - mu sync.Mutex - pong chan struct{} - closed chan struct{} + cancel context.CancelFunc + closed <-chan struct{} once sync.Once - initialized bool + replies chan displayReply + member *personalMember applied uint64 - cached directory.Listing + initialized bool version uint64 devices map[string]directory.Device - online map[string]bool frames []dirwatch.Frame } -func (s *localDisplayStream) Vouching() bool { return false } -func (s *localDisplayStream) Close() { s.once.Do(func() { close(s.closed) }) } -func (s *localDisplayStream) Ping(context.Context) error { - if !s.owner.memberAllowed() { - s.Close() - return ErrQuiet - } - select { - case s.pong <- struct{}{}: - default: - } - return nil +func (s *memberDisplayStream) Close() { + s.once.Do(func() { + s.cancel() + s.Stream.Close() + }) } -func (s *localDisplayStream) Next(ctx context.Context) (dirwatch.Frame, error) { + +func (s *memberDisplayStream) readRelay(ctx context.Context) { for { - if !s.owner.memberAllowed() { - s.Close() - return dirwatch.Frame{}, ErrQuiet - } - m := s.member - m.mu.Lock() - changed := m.changed - applied := m.status.Applied - closed, failed := m.closed, m.failure - m.mu.Unlock() - if closed || failed != nil { - return dirwatch.Frame{}, errMemberDisplayChanged + frame, err := s.Stream.Next(ctx) + select { + case s.replies <- displayReply{frame: frame, err: err}: + case <-ctx.Done(): + return } - listing, local, err := s.cached, true, error(nil) - if s.applied != applied || !s.initialized { - listing, local, err = s.owner.displayListing(ctx) - if err == nil && local { - s.cached = listing - s.applied = applied - } + if err != nil { + return } - listing.Now = max(listing.Now, s.owner.now().UnixMilli()) + } +} + +func (s *memberDisplayStream) Next(ctx context.Context) (dirwatch.Frame, error) { + for { + changed, local, err := s.refresh(ctx) if err != nil { return dirwatch.Frame{}, err } - if !local { - return dirwatch.Frame{}, errMemberDisplayChanged - } - if listing.Identity.Rotation != nil { - return dirwatch.Frame{}, dirwatch.ErrRotated - } - if listing.Devices[s.owner.Device.ID()].Revoked { - return dirwatch.Frame{}, dirwatch.ErrRevoked - } - s.mu.Lock() - s.project(listing) if len(s.frames) > 0 { frame := s.frames[0] + s.frames[0] = dirwatch.Frame{} s.frames = s.frames[1:] - s.mu.Unlock() return frame, nil } - s.mu.Unlock() select { + case reply := <-s.replies: + if reply.err != nil { + return dirwatch.Frame{}, reply.err + } + frame := reply.frame + if local { + // The learner supplies versions and committed device news once + // available, avoiding duplicate join announcements from the relay. + if frame.Event == nil && !frame.Pong && frame.Holds == nil { + continue + } + if frame.Event != nil && (frame.Event.T == dirwatch.EventJoined || frame.Event.T == dirwatch.EventRevoked) { + continue + } + } + return frame, nil case <-changed: - case <-s.pong: - return dirwatch.Frame{Pong: true}, nil case <-s.closed: return dirwatch.Frame{}, context.Canceled case <-ctx.Done(): @@ -194,10 +175,55 @@ func (s *localDisplayStream) Next(ctx context.Context) (dirwatch.Frame, error) { } } -// project produces only display hints and committed joined/revoked news. A -// quiet neighbour's presence ages normally; a live unexpired held chat is -// also recent evidence. Neither hint can suppress an ownership heartbeat. -func (s *localDisplayStream) project(l directory.Listing) { +// refresh follows the terminal-owned learner, including one admitted after +// Home opened. Its notification is captured before the view to avoid lost news. +func (s *memberDisplayStream) refresh(ctx context.Context) (<-chan struct{}, bool, error) { + if !s.owner.memberAllowed() { + return nil, false, ErrQuiet + } + m := s.owner.displayMember() + if m == nil { + if s.member != nil { + return nil, false, errMemberDisplayChanged + } + return nil, false, nil + } + m.mu.Lock() + changed, applied := m.changed, m.status.Applied + valid := !m.closed && m.failure == nil + m.mu.Unlock() + if !valid { + if s.member != nil { + return nil, false, errMemberDisplayChanged + } + return nil, false, nil + } + if s.member != m { + s.member, s.initialized = m, false + } + if !s.initialized || s.applied != applied { + listing, local, err := s.owner.displayListing(ctx) + if err != nil { + return nil, false, err + } + if !local { + return nil, false, errMemberDisplayChanged + } + if listing.Identity.Rotation != nil { + return nil, false, dirwatch.ErrRotated + } + if listing.Devices[s.owner.Device.ID()].Revoked { + return nil, false, dirwatch.ErrRevoked + } + s.project(listing) + s.applied = applied + } + return changed, true, nil +} + +// project retains committed display versions and joined/revoked news. Neither +// leases nor device last-seen records can announce a computer online. +func (s *memberDisplayStream) project(l directory.Listing) { if !s.initialized || l.Version != s.version { s.frames = append(s.frames, dirwatch.Frame{Version: l.Version}) } @@ -206,19 +232,8 @@ func (s *localDisplayStream) project(l directory.Listing) { ids = append(ids, id) } sort.Strings(ids) - online := map[string]bool{s.owner.Device.ID(): true} - for _, cell := range l.Cells { - if cell.Lease.Expires > l.Now { - online[cell.Lease.Device] = true - } - } for _, id := range ids { d := l.Devices[id] - active := !d.Revoked && (online[id] || d.LastSeen != 0 && l.Now-d.LastSeen < dirwatch.DeadAfter.Milliseconds()) - if active != s.online[id] { - e := dirwatch.PresenceEvent(id, active, l.Now) - s.frames = append(s.frames, dirwatch.Frame{Event: &e}) - } old, existed := s.devices[id] if s.initialized && !existed && !d.Revoked && id != s.owner.Device.ID() { e := dirwatch.JoinedEvent(id, d.Name, d.Platform, d.Created) @@ -228,12 +243,8 @@ func (s *localDisplayStream) project(l directory.Listing) { e := dirwatch.RevokedEvent(id, l.Now) s.frames = append(s.frames, dirwatch.Frame{Event: &e}) } - online[id] = active } - s.initialized = true - s.version = l.Version - s.devices = l.Devices - s.online = online + s.initialized, s.version, s.devices = true, l.Version, l.Devices } // currentDisplaySync follows a pairing or relay adoption made while Home is diff --git a/internal/syncsetup/member_hosted_leader_quiet_test.go b/internal/syncsetup/member_hosted_leader_quiet_test.go index 5f0aec4c71..6636c89ba1 100644 --- a/internal/syncsetup/member_hosted_leader_quiet_test.go +++ b/internal/syncsetup/member_hosted_leader_quiet_test.go @@ -145,7 +145,10 @@ func hostedLeadingHomesQuiet(t *testing.T, fallback bool) { if len(alarms) != 3 || string(before["maintenance"]) != "false" { t.Fatalf("R26 sleeping group retained maintenance: %s", before) } - if !maps.Equal(hostedSocketCounts(t, before), map[string]int{"peer": len(live), "forward": len(live), "watch": len(live)}) { + // Presence is advisory; C31's take-back after a computer closes its lid + // depends on it. Each Home retains its follower and direct watch alongside + // the compatibility watch, so the inventory is exactly three per computer. + if !maps.Equal(hostedSocketCounts(t, before), map[string]int{"peer": len(live), "forward": len(live), "watch": 3 * len(live)}) { t.Fatalf("R26 missing retained sockets: %s", before) } generation := recordOf(t, a).Grant.Generation @@ -218,6 +221,12 @@ func hostedLeadingHomesQuiet(t *testing.T, fallback bool) { t.Fatal("R26 cold presence lost a retained socket") } t.Logf("R26 R27 real-time presence: duration=%s sweeps=%d member calls=0 writes=0 peer frames=0", time.Since(started), hostedNumber(t, real, "sweeps")) + for _, follow := range follows { + follow.Close() + } + for _, stream := range streams { + stream.Close() + } for _, watch := range watches { watch.Close() } diff --git a/internal/syncsetup/member_hosted_quiet_test.go b/internal/syncsetup/member_hosted_quiet_test.go index 700a5d1fae..f2c64e627b 100644 --- a/internal/syncsetup/member_hosted_quiet_test.go +++ b/internal/syncsetup/member_hosted_quiet_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strings" + "sync/atomic" "testing" "time" @@ -19,6 +20,41 @@ import ( "github.com/Agent-Field/codeaf/internal/store" ) +// presenceCountDirectory counts the actual Home watch writes and reads, including +// platform auto-responses that never enter the Worker's application callbacks. +type presenceCountDirectory struct { + directory.Client + frames *atomic.Uint64 +} + +func (d presenceCountDirectory) Watch(ctx context.Context) (dirwatch.Stream, error) { + stream, err := d.Client.(directory.Watcher).Watch(ctx) + if err != nil { + return nil, err + } + return presenceCountStream{Stream: stream, frames: d.frames}, nil +} + +type presenceCountStream struct { + dirwatch.Stream + frames *atomic.Uint64 +} + +func (s presenceCountStream) Ping(ctx context.Context) error { + err := s.Stream.Ping(ctx) + if err == nil { + s.frames.Add(1) + } + return err +} +func (s presenceCountStream) Next(ctx context.Context) (dirwatch.Frame, error) { + frame, err := s.Stream.Next(ctx) + if err == nil { + s.frames.Add(1) + } + return frame, err +} + func hostedQuietControl(t *testing.T, endpoint, identity, op string) map[string]json.RawMessage { t.Helper() raw, _ := json.Marshal(map[string]string{"identity": identity, "op": op}) @@ -142,6 +178,7 @@ func TestR23R26R27C30HostedOpenHomesIdleEvictionAndWake(t *testing.T) { hostedSettled(t, a) hostedSettled(t, b) } + var watchFrames atomic.Uint64 screens := []*Sync{} follows := []dirwatch.Follower{} streams := []dirwatch.Stream{} @@ -152,11 +189,12 @@ func TestR23R26R27C30HostedOpenHomesIdleEvictionAndWake(t *testing.T) { if !ok { t.Fatal("screen open") } + screen.Dir = presenceCountDirectory{Client: screen.Dir, frames: &watchFrames} screens = append(screens, screen) follow := screen.Follow() defer follow.Close() - // The local observer and the compatibility watch both remain open. Only - // the latter uses platform auto-response for advisory presence. + // Home and the compatibility watch both retain advisory presence through + // platform auto-response without waking coordination. stream, e := screen.followStream(t.Context(), screen.Dir.(directory.Watcher)) mustNil(t, e) defer stream.Close() @@ -167,6 +205,9 @@ func TestR23R26R27C30HostedOpenHomesIdleEvictionAndWake(t *testing.T) { watches = append(watches, watch) follows = append(follows, follow) } + for _, follow := range follows { + c31AwaitFeed(t, follow, requestTimeout, func(st dirwatch.State) bool { return st.Up }) + } idle := hostedQuietControl(t, endpoint, a.Identity.ID(), "idle-schedule") if string(idle["sleeping"]) != "true" || hostedNumber(t, idle, "recurringMemberCalls") != 0 { t.Fatalf("R26 idle schedule: %s", idle) @@ -202,7 +243,10 @@ func TestR23R26R27C30HostedOpenHomesIdleEvictionAndWake(t *testing.T) { if len(alarms) != 3 || string(before["maintenance"]) != "false" { t.Fatalf("R26 sleeping group retained maintenance: %s", before) } - if !maps.Equal(hostedSocketCounts(t, before), map[string]int{"peer": 2, "forward": 2, "watch": 2}) { + // Advisory presence retains exactly six watches: a Home follower, a direct + // Home stream and a compatibility watch per computer. C31 take-back after + // a computer closes its lid depends on these advisory Home connections. + if !maps.Equal(hostedSocketCounts(t, before), map[string]int{"peer": 2, "forward": 2, "watch": 6}) { t.Fatalf("R26 missing retained sockets: %s", before) } generation := recordOf(t, a).Grant.Generation @@ -212,17 +256,22 @@ func TestR23R26R27C30HostedOpenHomesIdleEvictionAndWake(t *testing.T) { } hostedQuietControl(t, endpoint, a.Identity.ID(), "reset-metrics") attempts := count() + watchBefore := watchFrames.Load() pongs := 0 for n := 0; n < 120; n++ { tickA() tickB() - for _, follow := range follows { - follow.Probe() + if time.Duration(n+1)*DefaultInterval%dirwatch.KeepEvery == 0 { + for _, follow := range follows { + follow.Probe() + } } for i, screen := range screens { - ping(streams[i]) - ping(watches[i]) - pongs++ + if time.Duration(n+1)*DefaultInterval%dirwatch.KeepEvery == 0 { + ping(streams[i]) + ping(watches[i]) + pongs += 2 + } _, _, e := screen.RowsAt(t.Context()) mustNil(t, e) _, e = screen.Roster(t.Context()) @@ -243,6 +292,16 @@ func TestR23R26R27C30HostedOpenHomesIdleEvictionAndWake(t *testing.T) { if httpCount != 0 || frames != 0 || clientAttempts != 0 || hostedNumber(t, metrics, "writes") != 0 || hostedNumber(t, metrics, "turns") != 0 { t.Fatalf("R27 idle activity: %s", metrics) } + // Advisory presence permits 732 watch frames in ten minutes: six sockets + // times two frames per KeepEvery, plus two initial snapshot frames each. + // C31 take-back after a computer closes its lid depends on these advisory + // sockets, whose platform pongs must leave coordination asleep. + const watchBudget = 6 * (2*(10*time.Minute/dirwatch.KeepEvery) + 2) + measuredWatchFrames := watchFrames.Load() - watchBefore + t.Logf("Home watch frames=%d, ten-minute bound=%d", measuredWatchFrames, watchBudget) + if measuredWatchFrames > uint64(watchBudget) { + t.Fatal("Home presence exceeded the ten-minute watch frame budget") + } // R26/R27 retain real hibernating sockets across two complete presence // windows. Auto-response timestamps, rather than a mocked renewal, age them. hostedQuietControl(t, endpoint, a.Identity.ID(), "reset-metrics") diff --git a/internal/syncsetup/presence_c31_test.go b/internal/syncsetup/presence_c31_test.go new file mode 100644 index 0000000000..3f42ebfd04 --- /dev/null +++ b/internal/syncsetup/presence_c31_test.go @@ -0,0 +1,170 @@ +package syncsetup + +import ( + "bufio" + "context" + "errors" + "fmt" + "net/http/httptest" + "os" + "os/exec" + "slices" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/chatlist" + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/directory" + "github.com/Agent-Field/codeaf/internal/dirwatch" + "github.com/Agent-Field/codeaf/internal/identity" + "github.com/Agent-Field/codeaf/internal/relayserve" +) + +const c31Chat = "01J000000000000000000C31A" + +// The holder runs in another process so stopping it closes its sockets without +// releasing its chat or sending a watch goodbye. +func TestC31HolderProcessHelper(t *testing.T) { + home := os.Getenv("CODEAF_TEST_C31_HOME") + if home == "" { + return + } + s, ok, err := Open(home) + mustNil(t, err) + if !ok { + t.Fatal("holder home unavailable") + } + mustNil(t, s.putDevice(t.Context(), "holder")) + hostedOwner(t, s) + title, err := directory.SealTitle(directory.MetadataKey(s.Identity.CellKey()), "Take back the saved chat") + mustNil(t, err) + _, err = s.Dir.Create(t.Context(), c31Chat, directory.CellInit{Head: strings.Repeat("1", 64), Class: "chat", Title: title}) + mustNil(t, err) + feed := s.Follow() + c31AwaitFeed(t, feed, requestTimeout, func(st dirwatch.State) bool { return st.Up }) + fmt.Println("C31 holder ready") + select {} +} + +func c31AwaitFeed(t *testing.T, feed dirwatch.Follower, within time.Duration, accept func(dirwatch.State) bool) dirwatch.State { + t.Helper() + timer := time.NewTimer(within) + defer timer.Stop() + for { + st := feed.State() + if accept(st) { + return st + } + select { + case <-feed.Changes(): + case <-timer.C: + t.Fatalf("Home feed did not reach the required presence within %s: %+v", within, feed.State()) + } + } +} + +func TestC31TakeBackOfferAfterHolderStopsWithoutGoodbye(t *testing.T) { + svc := relayserve.New(relayserve.Config{Store: t.TempDir()}) + defer svc.Close() + relay := httptest.NewServer(svc.Handler) + defer relay.Close() + c31TakeBack(t, relay.URL) +} + +func TestC31HostedTakeBackOfferAfterHolderStopsWithoutGoodbye(t *testing.T) { + c31TakeBack(t, hostedLearnerEndpoint(t)) +} + +// Home must offer the saved chat while its holder's lease is still valid. The +// public Home feed and listing are the same inputs the resume band reads. +func c31TakeBack(t *testing.T, endpoint string) { + t.Helper() + t.Setenv(config.ProfileDirEnv, "") + homeA := machine(t, endpoint) + id, err := identity.Load(homeA) + mustNil(t, err) + deviceA, err := identity.Device(homeA) + mustNil(t, err) + homeB := machine(t, endpoint) + _, err = identity.Adopt(homeB, id, true) + mustNil(t, err) + child := exec.Command(os.Args[0], "-test.run=^TestC31HolderProcessHelper$", "-test.v", "-test.timeout=15m") + child.Env = append(os.Environ(), "CODEAF_TEST_C31_HOME="+homeA) + stdout, err := child.StdoutPipe() + mustNil(t, err) + child.Stderr = os.Stderr + mustNil(t, child.Start()) + defer func() { + if child.ProcessState == nil { + _ = child.Process.Kill() + _ = child.Wait() + } + }() + ready := make(chan bool, 1) + go func() { + scan := bufio.NewScanner(stdout) + for scan.Scan() { + if scan.Text() == "C31 holder ready" { + ready <- true + return + } + } + ready <- false + }() + select { + case ok := <-ready: + if !ok { + t.Fatal("holder exited before opening Home") + } + case <-time.After(requestTimeout): + t.Fatal("holder did not open Home") + } + b, ok, err := Open(homeB) + mustNil(t, err) + if !ok { + t.Fatal("second Home unavailable") + } + mustNil(t, b.putDevice(t.Context(), "observer")) + hostedOwner(t, b) + feed := b.Follow() + defer feed.Close() + c31AwaitFeed(t, feed, requestTimeout, func(st dirwatch.State) bool { return st.Up && slices.Contains(st.Online, deviceA.ID()) }) + before, err := b.Dir.Cell(t.Context(), c31Chat) + mustNil(t, err) + if before.Cell.Lease.Device != deviceA.ID() || before.Cell.Lease.Expires <= time.Now().UnixMilli() { + t.Fatalf("holder has no live lease: %+v", before) + } + t.Logf("C31 before SIGKILL: holder online, lease expires in %s", time.Until(time.UnixMilli(before.Cell.Lease.Expires))) + started := time.Now() + mustNil(t, child.Process.Kill()) + if err := child.Wait(); err == nil { + t.Fatal("holder exited without SIGKILL") + } + st := c31AwaitFeed(t, feed, 30*time.Second, func(st dirwatch.State) bool { return st.Up && !slices.Contains(st.Online, deviceA.ID()) }) + elapsed := time.Since(started) + rows, _, err := b.RowsAt(t.Context()) + mustNil(t, err) + offered := false + for _, row := range rows { + if row.Cell == c31Chat { + if row.Status != chatlist.Running || row.DeviceID != deviceA.ID() || row.DurableAgo > 24*time.Hour { + t.Fatalf("resume row is not a recent held chat: %+v", row) + } + offered = st.Up && !slices.Contains(st.Online, row.DeviceID) + } + } + if !offered { + t.Fatalf("Home cannot offer the holder's chat: %+v, %+v", rows, st) + } + after, err := b.Dir.Cell(t.Context(), c31Chat) + mustNil(t, err) + if after.Cell.Lease != before.Cell.Lease || after.Cell.Lease.Expires <= time.Now().UnixMilli() { + t.Fatalf("offline presence changed or outwaited ownership: before=%+v after=%+v", before, after) + } + _, err = b.Dir.Acquire(context.Background(), c31Chat, directory.AcquireOpts{}) + if !errors.Is(err, directory.ErrLeaseHeld) { + t.Fatalf("offline hint authorized ordinary acquisition: %v", err) + } + t.Logf("C31 resume offer on B: true after %s, holder lease remains unexpired", elapsed) +} diff --git a/internal/syncsetup/quiet.go b/internal/syncsetup/quiet.go index facac01903..978000b86d 100644 --- a/internal/syncsetup/quiet.go +++ b/internal/syncsetup/quiet.go @@ -90,24 +90,29 @@ func (s *Sync) ClusterHTTP(client *http.Client) *http.Client { // decision cannot retain a second transport path around quietWire. type quietStream struct { dirwatch.Stream - home string + home string + owner *Sync +} + +func (s quietStream) allowed() bool { + return !Quiet(s.home) && (s.owner == nil || s.owner.memberAllowed()) } func (s quietStream) Next(ctx context.Context) (dirwatch.Frame, error) { - if Quiet(s.home) { + if !s.allowed() { s.Stream.Close() return dirwatch.Frame{}, ErrQuiet } return s.Stream.Next(ctx) } func (s quietStream) Ping(ctx context.Context) error { - if Quiet(s.home) { + if !s.allowed() { s.Stream.Close() return ErrQuiet } return s.Stream.Ping(ctx) } -func (s quietStream) Vouching() bool { return !Quiet(s.home) && s.Stream.Vouching() } +func (s quietStream) Vouching() bool { return s.allowed() && s.Stream.Vouching() } // relayCountingWire counts actual relay attempts inside the quiet gate. // Frame attempts are counted at their separate guarded socket write boundary. diff --git a/relay/hosted/test/raft/hosted-c31-go.test.js b/relay/hosted/test/raft/hosted-c31-go.test.js new file mode 100644 index 0000000000..e1d33180b1 --- /dev/null +++ b/relay/hosted/test/raft/hosted-c31-go.test.js @@ -0,0 +1,30 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {spawn} from 'node:child_process'; +import {join} from 'node:path'; +import {hostedFixture} from './hosted-fixture.js'; + +const root = new URL('../../../../', import.meta.url); +const footprint = new URL('../../../../scripts/raft-proof-footprint.sh', import.meta.url).pathname; +async function run(args, env = {}) { + const child = spawn('sh', [footprint, ...args], {cwd: root, env: {...process.env, ...env}, stdio: ['ignore', 'pipe', 'pipe']}); + let output = ''; + for (const stream of [child.stdout, child.stderr]) stream.on('data', bytes => { output += bytes; process.stdout.write(bytes); }); + const code = await new Promise((resolve, reject) => { child.once('error', reject); child.once('exit', resolve); }); + return {code, output}; +} + +test('C31 Home offers a chat after its holder is killed without goodbye', {timeout: 180000}, async () => { + const binary = join(process.env.CODEAF_RAFT_PROOF_DIR, 'hosted-c31.test'); + const compiled = await run(['go', 'test', '-c', '-p=1', '-o', binary, './internal/syncsetup']); + assert.equal(compiled.code, 0, compiled.output); + const relay = await hostedFixture({fixture: true}); + try { + await relay.start(); + const url = await relay.mf.ready; + const {code, output} = await run([binary, '-test.run=^TestC31HostedTakeBackOfferAfterHolderStopsWithoutGoodbye$', '-test.count=1', '-test.timeout=15m', '-test.v'], {CODEAF_HOSTED_PROOF_URL: url.origin}); + assert.equal(code, 0, output); + assert.doesNotMatch(output, /--- SKIP:/); + assert.match(output, /C31 resume offer on B: true after/); + } finally { await relay.close(); } +}); From d0eaeb3cb5ca7cb8d8259ef48ec86ce8db354617 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 26 Oct 2026 06:19:46 -0400 Subject: [PATCH 101/103] Say that Home keeps a relay watch for presence after joining. The manual and the change entry described a joined Home with no remote presence viewer. Home keeps its relay watch with ten-second pings, so a stopped computer appears offline within 30 seconds and its chat can be continued elsewhere. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/changes/unreleased/1950-raft-cluster.md | 2 +- internal/manual/chat/coordination-and-workers.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changes/unreleased/1950-raft-cluster.md b/docs/changes/unreleased/1950-raft-cluster.md index 3aa5097597..45450745fc 100644 --- a/docs/changes/unreleased/1950-raft-cluster.md +++ b/docs/changes/unreleased/1950-raft-cluster.md @@ -9,7 +9,7 @@ invalidates: - "Joined computers had no leadership commands. codeaf cluster status now reports committed membership and the current leader, and codeaf cluster lead safely replaces one anchor vote, transfers leadership and returns an elected read receipt; saved requests retry unchanged." - "Coordination depended on the relay's single writer. Two remaining anchors can now elect a fallback and restore the third after a personal leader disappears, targeting about 65 seconds in healthy conditions; a returning computer remains a learner and never automatically regains leadership." - "A majority was not required by the former directory writer. Losing two of the three current voters now refuses coordination and preserves saved chats instead of creating a writable minority." - - "Caught-up open homes previously polled shared state. Idle homes now send zero HTTP requests, peer or forwarding frames, or client attempts after all voters apply the idle barrier; presence sweeps recur only with a viewer and fresh input requires a new majority response." + - "Caught-up open homes previously polled shared state. Idle homes now send zero HTTP requests, peer or forwarding frames, or client attempts after all voters apply the idle barrier; Home retains a relay watch with ten-second advisory pings, presence sweeps recur only with a viewer and fresh input requires a new majority response." - "Older clients could blindly force a live chat without its observed holder. Both relays now refuse that request with another device continued this chat first; releasing the chat or updating the client permits safe continuation." - "Legal older directory requests could be refused above the smaller coordination envelope. Both relays now retain the original 1 MiB HTTP limit, stage large signed bodies and commit complete metadata atomically, recovering interruptions without exposing partial changes or deleting retained records." - "Workers previously needed personal-device trust for shared access. Delegated workers now have expiring selected-chat grants and conditional assignment claims; they cannot approve pairing, read the identity vault, rotate it, enroll devices or lead, and terminal worker setup is not available yet." diff --git a/internal/manual/chat/coordination-and-workers.md b/internal/manual/chat/coordination-and-workers.md index 0e97503d79..a5ff3317ed 100644 --- a/internal/manual/chat/coordination-and-workers.md +++ b/internal/manual/chat/coordination-and-workers.md @@ -62,13 +62,13 @@ The sync connection carries presence hints and committed lease acknowledgments. ## Does syncing keep sending requests when nothing is happening? -A home that has never paired sends no sync traffic. `CODEAF_SYNC_URL=off` also blocks new sync requests from clients that were opened before the setting changed. Presence pings on an open view are hints; they do not write chat ownership. On the hosted relay and the native relay, an open idle home uses its learner’s applied vault pointer to notice shared memory and preference changes. Local edits are detected locally and pushed on the next state round. Over ten idle minutes, caught-up open homes make zero HTTP requests, zero peer or forwarding frames and zero client attempts, including while one home leads and after it sleeps and the anchors resume. A caught-up idle home sends no recurring state polls or peer keepalives. Home also follows its local applied directory for display changes; its roster and backstop read that same local hint. Quiet neighbours’ presence can age out without forcing a network ping. Continuing a chat and all ownership decisions still use fresh majority reads. A committed change wakes peer delivery and the next state round pulls the changed vault. If an older hosted relay has no learner door, the home keeps its existing five-second state polling. On the consensus directory, a group with no work or held chats can stop its periodic coordination timer after an idle barrier has reached every voter. The first operation wakes coordination and must obtain a fresh majority response. A saved leader address cannot answer that operation by itself. Losing the majority leaves syncing unavailable and keeps local work intact. +A home that has never paired sends no sync traffic. `CODEAF_SYNC_URL=off` also blocks new sync requests from clients that were opened before the setting changed. Presence pings on an open view are hints; they do not write chat ownership. On the hosted relay and the native relay, an open idle home uses its learner’s applied vault pointer to notice shared memory and preference changes. Local edits are detected locally and pushed on the next state round. Over ten idle minutes, caught-up open homes make zero HTTP requests, zero peer or forwarding frames and zero client attempts, including while one home leads and after it sleeps and the anchors resume. A caught-up idle home sends no recurring state polls or peer keepalives. Home reads its roster and backstop from the local applied directory. Its separate relay watch supplies advisory online presence, pinging every 10 seconds. A computer whose last socket closes is marked offline after a 15-second reconnect grace period; a silent connection is marked offline within 30 seconds. Home can then offer its recent chat before the 90-second lease expires. Continuing a chat and all ownership decisions still use fresh majority reads. A committed change wakes peer delivery and the next state round pulls the changed vault. If an older hosted relay has no learner door, the home keeps its existing five-second state polling. On the consensus directory, a group with no work or held chats can stop its periodic coordination timer after an idle barrier has reached every voter. The first operation wakes coordination and must obtain a fresh majority response. A saved leader address cannot answer that operation by itself. Losing the majority leaves syncing unavailable and keeps local work intact. ## Hosted idle presence sweeps and sleeping connections With its three anchors voting, or with a joined computer leading, a hosted group with no work or held chats stops its maintenance alarm after every voter applies the idle barrier. Its three anchors then schedule no recurring alarms. Reconstructing the sleeping coordinator leaves its learner and forwarding sockets open; a presence sweep neither wakes the anchors nor reconnects those computers. The first real request obtains a fresh majority response before answering. -A compatibility events viewer pings every 10 seconds. The platform answers without running coordination. While somebody watches presence, the relay checks at the next 25-second silence deadline so a stopped computer appears offline within 30 seconds. With two viewers pinging together, the simulated ten-minute measurement uses 29 short sweeps; those sweeps make no member calls or data writes. With no events viewer, no presence sweep recurs. Home follows the local learner once joined, so its ordinary display needs no remote presence viewer. +An open Home or compatibility events viewer pings every 10 seconds. The platform answers without running coordination. While somebody watches presence, the relay checks at the next 25-second silence deadline so a stopped computer appears offline within 30 seconds. With two viewers pinging together, the simulated ten-minute measurement uses 29 short sweeps; those sweeps make no member calls or data writes. With no events viewer, no presence sweep recurs. A joined Home keeps its relay presence watch while its directory reads use the local learner. Presence is advisory and cannot acquire, release or renew chat ownership. Connections naming held chats share a renewal cycle every 30 seconds on both relays while fresh hold evidence is available. Each cycle uses bounded batches; the 10-second presence pings between cycles do not commit another renewal. Presence alone never renews a chat. From a0a7870714659af920e2e76c5603871d8e9c8cce Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 26 Oct 2026 06:49:42 -0400 Subject: [PATCH 102/103] Prepare the take-back proof's holder home like every other test home. The holder runs in a re-executed test process, which did not mark the cell engine available the way the test's other homes are prepared. On a computer without an installed engine binary the holder could not open its home and exited at once, so the proof depended on the machine it ran on. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/syncsetup/presence_c31_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/syncsetup/presence_c31_test.go b/internal/syncsetup/presence_c31_test.go index 3f42ebfd04..5afa1c0337 100644 --- a/internal/syncsetup/presence_c31_test.go +++ b/internal/syncsetup/presence_c31_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/Agent-Field/codeaf/internal/cell/celltest" "github.com/Agent-Field/codeaf/internal/chatlist" "github.com/Agent-Field/codeaf/internal/config" "github.com/Agent-Field/codeaf/internal/directory" @@ -30,6 +31,9 @@ func TestC31HolderProcessHelper(t *testing.T) { if home == "" { return } + // The holder's home is prepared like every other test home, so the proof + // does not depend on an engine binary being installed on this computer. + celltest.Available(t) s, ok, err := Open(home) mustNil(t, err) if !ok { From 113e2b1e825d50b066cedf5445185da79635e05b Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 26 Oct 2026 06:49:42 -0400 Subject: [PATCH 103/103] Link the hosted take-back proof's client before its time limit starts. Linking the client tests can take minutes on a busy disk. Doing it inside the timed test let the link consume the proof's own limit, as the other hosted client launchers already avoid by linking first. Co-Authored-By: Claude Opus 5.5 (1M context) --- relay/hosted/test/raft/hosted-c31-go.test.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/relay/hosted/test/raft/hosted-c31-go.test.js b/relay/hosted/test/raft/hosted-c31-go.test.js index e1d33180b1..36cb31d74b 100644 --- a/relay/hosted/test/raft/hosted-c31-go.test.js +++ b/relay/hosted/test/raft/hosted-c31-go.test.js @@ -14,10 +14,13 @@ async function run(args, env = {}) { return {code, output}; } +// Linking the client tests can leave a disk backlog, so it finishes before +// the Worker starts and cannot spend the proof's own time limit. +const binary = join(process.env.CODEAF_RAFT_PROOF_DIR, 'hosted-c31.test'); +const compiled = await run(['go', 'test', '-c', '-p=1', '-o', binary, './internal/syncsetup']); +assert.equal(compiled.code, 0, compiled.output); + test('C31 Home offers a chat after its holder is killed without goodbye', {timeout: 180000}, async () => { - const binary = join(process.env.CODEAF_RAFT_PROOF_DIR, 'hosted-c31.test'); - const compiled = await run(['go', 'test', '-c', '-p=1', '-o', binary, './internal/syncsetup']); - assert.equal(compiled.code, 0, compiled.output); const relay = await hostedFixture({fixture: true}); try { await relay.start();